From a3057de51e9ca91057ba573cfc77416073a3d72f Mon Sep 17 00:00:00 2001 From: bsodfather Date: Fri, 30 Oct 2026 12:21:57 +0000 Subject: [PATCH] =?UTF-8?q?feat(epic4):=20=D0=B1=D0=B5=D0=B7=D0=BE=D0=BF?= =?UTF-8?q?=D0=B0=D1=81=D0=BD=D1=8B=D0=B5=20=D0=BF=D0=BE=D0=B2=D1=82=D0=BE?= =?UTF-8?q?=D1=80=D1=8B=20=D0=B8=20=D0=BE=D1=82=D0=BC=D0=B5=D0=BD=D0=B0=20?= =?UTF-8?q?durable=20Task=20(Story=204.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Durable per-outbound attempt до отправки запроса, bounded retry/read, server-owned verified writes, честный cancel/unknown, deadline/fencing, additive schema v29. Staged renames task.rs/tasks.rs разложены на модули. Includes disclosed cosmetic parity exception: единственная правка вне принятой дельты — удалена лишняя пустая строка в конце scripts/ci-story-checks.sh (EOF blank line); story-код не менялся. --- .../durable_tasks/attempt_guards.rs | 555 ++++++++++++ .../durable_tasks/attempt_trace.rs | 43 +- .../integration/durable_tasks/cancel_chain.rs | 846 ++++++++++++++++++ .../tests/integration/durable_tasks/mod.rs | 47 + .../durable_tasks/shutdown_window.rs | 66 +- .../integration/durable_tasks/task_worker.rs | 187 ++-- crates/crank-community-mcp/src/app.rs | 2 +- .../src/app/task_worker.rs | 254 +++--- crates/crank-community-mcp/src/app/tasks.rs | 55 ++ crates/crank-community-mcp/src/lib.rs | 7 +- crates/crank-core/src/lib.rs | 28 +- crates/crank-core/src/task/attempt.rs | 285 ++++++ .../crank-core/src/{task.rs => task/mod.rs} | 100 ++- crates/crank-registry/src/migrations.rs | 2 + .../src/migrations/authority.rs | 14 +- .../src/migrations/durable_tasks_v29.rs | 45 + .../src/migrations/durable_tasks_v29.sql | 52 ++ .../src/migrations/schema_guard.rs | 20 +- .../src/migrations/schema_guard_v27.rs | 2 +- .../src/migrations/schema_guard_v28.rs | 10 +- .../src/migrations/schema_guard_v29.rs | 87 ++ .../src/postgres/master_key_tasks.rs | 1 + .../src/postgres/tasks/acceptance.rs | 195 ++++ .../src/postgres/tasks/attempts.rs | 583 ++++++++++++ .../src/postgres/{tasks.rs => tasks/mod.rs} | 406 ++++----- .../tests/integration/migrations.rs | 17 +- .../migrations/artifact_metadata.rs | 12 +- .../artifact_metadata/exhaustive_v12.rs | 16 +- .../migrations/durable_tasks_guard.rs | 6 +- .../integration/migrations/import_plan.rs | 4 +- .../integration/migrations/legacy_adoption.rs | 16 +- .../resource_security_revocations.rs | 10 +- .../tests/integration/migrations/rollback.rs | 2 +- .../tests/integration/tasks/mod.rs | 49 +- .../tests/integration/tasks/task_attempts.rs | 534 +++++++++++ .../tests/integration/tasks/task_worker.rs | 16 +- crates/crank-runtime/src/executor.rs | 372 ++++++-- crates/crank-runtime/src/lib.rs | 8 +- crates/crank-runtime/src/task_attempt.rs | 113 +++ crates/crank-runtime/src/task_worker.rs | 233 +---- crates/crank-runtime/src/tasks.rs | 33 + docs/execution-boundary.md | 20 + docs/schemas/migration-sequence.json | 2 +- scripts/ci-story-checks.sh | 7 + 44 files changed, 4547 insertions(+), 815 deletions(-) create mode 100644 apps/mcp-server/tests/integration/durable_tasks/attempt_guards.rs create mode 100644 apps/mcp-server/tests/integration/durable_tasks/cancel_chain.rs create mode 100644 crates/crank-core/src/task/attempt.rs rename crates/crank-core/src/{task.rs => task/mod.rs} (88%) create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v29.rs create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v29.sql create mode 100644 crates/crank-registry/src/migrations/schema_guard_v29.rs create mode 100644 crates/crank-registry/src/postgres/tasks/acceptance.rs create mode 100644 crates/crank-registry/src/postgres/tasks/attempts.rs rename crates/crank-registry/src/postgres/{tasks.rs => tasks/mod.rs} (74%) create mode 100644 crates/crank-registry/tests/integration/tasks/task_attempts.rs create mode 100644 crates/crank-runtime/src/task_attempt.rs diff --git a/apps/mcp-server/tests/integration/durable_tasks/attempt_guards.rs b/apps/mcp-server/tests/integration/durable_tasks/attempt_guards.rs new file mode 100644 index 0000000..041044e --- /dev/null +++ b/apps/mcp-server/tests/integration/durable_tasks/attempt_guards.rs @@ -0,0 +1,555 @@ +//! Story 4.4 ATDD: no outbound send exists without a committed durable attempt, and an elapsed +//! accepted deadline fails before dispatch. +//! +//! Both oracles drive the production composition (application entry → PG store → runtime executor → +//! real local HTTP upstream); only the upstream and the checkpoint failure are test-controlled. + +use super::*; + +use axum::Router; +use axum::http::StatusCode; +use axum::routing::post; +use crank_community_mcp::TaskWorkerHandler; +use crank_core::{ + AgentOperationBinding, PlatformApiKeyScope, TaskState, ToolExecutionMode, ToolExecutionPolicy, +}; +use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use time::OffsetDateTime; + +async fn composition( + registry: &PostgresRegistry, + upstream: &str, + policy: ToolExecutionPolicy, + caller_name: &str, + scoped_key: &str, +) -> ( + Arc, + crank_community_mcp::TaskApplication, + SecretCrypto, + crank_community_mcp::TaskPrincipal, +) { + let mut operation = common::test_operation(upstream, TOOL_NAME); + operation.execution_config.execution_policy = Some(policy); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish(registry, &operation.id, 1).await; + common::publish_agent_with_bindings( + registry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; + let caller = principal( + registry, + caller_name, + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &OffsetDateTime::now_utc(), + }) + .await + .expect("test master-key identity registered"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let tasks = TaskApplication::new( + registry.clone(), + Duration::ZERO, + Arc::new(InMemoryCoordinationStateStore::default()), + TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())), + ); + let _ = scoped_key; + (store, tasks, crypto, caller) +} + +fn worker_for( + registry: &PostgresRegistry, + store: &Arc, + tasks: &TaskApplication, + crypto: &SecretCrypto, + owner: &str, +) -> TaskWorker { + let runtime = crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .build(); + let handler = Arc::new(TaskWorkerHandler::from_application( + tasks, + registry.clone(), + runtime, + crypto.clone(), + )); + let config = TaskWorkerConfig { + owner: owner.to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + ..TaskWorkerConfig::default() + }; + TaskWorker::new(store.clone(), handler, config).expect("valid worker") +} + +async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option { + sqlx::query_scalar("select state from tasks where task_id = $1") + .bind(task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap() +} + +/// AC1: a dispatch checkpoint that cannot commit produces zero outbound calls and no attempt row. +#[tokio::test] +async fn checkpoint_commit_failure_never_reaches_upstream() { + let registry = common::test_registry().await; + let calls = Arc::new(AtomicUsize::new(0)); + let upstream_app = Router::new().route( + "/crm/leads", + post({ + let calls = Arc::clone(&calls); + move || { + let calls = Arc::clone(&calls); + async move { + calls.fetch_add(1, Ordering::SeqCst); + (StatusCode::OK, axum::Json(json!({"id": "lead-1"}))) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let (store, tasks, crypto, caller) = composition( + ®istry, + &upstream, + background_policy(), + "attempt-guards-checkpoint", + "guards-checkpoint-key", + ) + .await; + let acceptance = tasks + .accept_background( + &caller, + TOOL_NAME, + None, + "guards-checkpoint-key", + &json!({"email": "user@example.com"}), + &unsampled_correlation(), + ) + .await + .expect("durable acceptance"); + + // The checkpoint is the only statement that commits an attempt; it is refused here. + sqlx::raw_sql(sqlx::AssertSqlSafe( + "create function refuse_task_checkpoint() returns trigger language plpgsql as $$ + begin + if new.current_attempt_id is not null and old.current_attempt_id is null then + raise exception 'durable checkpoint refused for test'; + end if; + return new; + end $$; + create trigger refuse_task_checkpoint before update on tasks + for each row execute function refuse_task_checkpoint();", + )) + .execute(registry.pool()) + .await + .expect("checkpoint refusal trigger"); + + let worker = worker_for( + ®istry, + &store, + &tasks, + &crypto, + "worker-guards-checkpoint", + ); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + tokio::time::sleep(Duration::from_secs(2)).await; + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "an uncommitted durable checkpoint must never produce an outbound call" + ); + let attempts: i64 = + sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!( + attempts, 0, + "no attempt history without a committed checkpoint" + ); + assert_ne!( + state_of(®istry, acceptance.task_id).await.as_deref(), + Some("completed"), + "the refused attempt cannot complete the Task" + ); + + sqlx::raw_sql(sqlx::AssertSqlSafe( + "drop trigger refuse_task_checkpoint on tasks; drop function refuse_task_checkpoint();", + )) + .execute(registry.pool()) + .await + .expect("checkpoint refusal trigger removed"); + server.abort(); +} + +/// AC6: an accepted absolute deadline that already elapsed fails the Task before any dispatch. +#[tokio::test] +async fn elapsed_deadline_fails_before_dispatch_without_upstream_call() { + let registry = common::test_registry().await; + let calls = Arc::new(AtomicUsize::new(0)); + let upstream_app = Router::new().route( + "/crm/leads", + post({ + let calls = Arc::clone(&calls); + move || { + let calls = Arc::clone(&calls); + async move { + calls.fetch_add(1, Ordering::SeqCst); + (StatusCode::OK, axum::Json(json!({"id": "lead-1"}))) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let short_deadline = ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 300, + no_tasks_fallback: crank_core::NoTasksFallback::Unsupported, + }; + let (store, tasks, crypto, caller) = composition( + ®istry, + &upstream, + short_deadline, + "attempt-guards-deadline", + "guards-deadline-key", + ) + .await; + let acceptance = tasks + .accept_background( + &caller, + TOOL_NAME, + None, + "guards-deadline-key", + &json!({"email": "user@example.com"}), + &unsampled_correlation(), + ) + .await + .expect("durable acceptance"); + + // The accepted deadline elapses before any worker claims the Task. + tokio::time::sleep(Duration::from_millis(500)).await; + + let worker = worker_for(®istry, &store, &tasks, &crypto, "worker-guards-deadline"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + let mut state = None; + for _ in 0..100 { + tokio::time::sleep(Duration::from_millis(50)).await; + state = state_of(®istry, acceptance.task_id).await; + if state.as_deref() == Some("failed") { + break; + } + } + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + + assert_eq!(state.as_deref(), Some("failed")); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "an elapsed deadline must fail before dispatch" + ); + let attempts: i64 = + sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(attempts, 0, "no attempt is checkpointed after the deadline"); + let code: Option = + sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(code.as_deref(), Some("task_deadline_elapsed")); + assert_eq!( + sqlx::query_scalar::<_, String>("select state from tasks where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(), + TaskState::Failed.as_str() + ); + + server.abort(); +} + +/// AC2: a write whose upstream idempotency is backed by server-owned evidence is retried with the +/// same stable key, in a bounded number of distinct durable attempts, and applies exactly one effect. +#[tokio::test] +async fn verified_idempotent_write_retries_with_stable_key_and_one_effect() { + use crank_community_mcp::VerifiedIdempotentWrite; + use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy}; + use std::collections::HashSet; + use std::sync::Mutex; + + let registry = common::test_registry().await; + let requests = Arc::new(AtomicUsize::new(0)); + let effects = Arc::new(AtomicUsize::new(0)); + let keys = Arc::new(Mutex::new(Vec::::new())); + let seen = Arc::new(Mutex::new(HashSet::::new())); + let upstream_app = Router::new().route( + "/crm/leads", + post({ + let requests = Arc::clone(&requests); + let effects = Arc::clone(&effects); + let keys = Arc::clone(&keys); + let seen = Arc::clone(&seen); + move |headers: axum::http::HeaderMap| { + let requests = Arc::clone(&requests); + let effects = Arc::clone(&effects); + let keys = Arc::clone(&keys); + let seen = Arc::clone(&seen); + async move { + let key = headers + .get("idempotency-key") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned(); + keys.lock().expect("keys").push(key.clone()); + let first_application = seen.lock().expect("seen").insert(key); + if first_application { + effects.fetch_add(1, Ordering::SeqCst); + } + let attempt = requests.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + // The first application is stored but its acknowledgement is lost. + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error": "acknowledgement lost"})), + ); + } + ( + StatusCode::OK, + axum::Json(json!({"id": "lead-1", "deduped": !first_application})), + ) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let mut operation = common::test_operation(&upstream, TOOL_NAME); + operation.execution_config.execution_policy = Some(background_policy()); + operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 }); + operation.execution_config.idempotency = Some(IdempotencyPolicy { + mode: IdempotencyMode::Required, + ttl_ms: 60_000, + input_field: Some("idempotency_key".to_owned()), + header_name: Some("Idempotency-Key".to_owned()), + }); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish(®istry, &operation.id, 1).await; + common::publish_agent_with_bindings( + ®istry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; + let caller = principal( + ®istry, + "attempt-guards-verified", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &OffsetDateTime::now_utc(), + }) + .await + .expect("test master-key identity registered"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let tasks = TaskApplication::new( + registry.clone(), + Duration::ZERO, + Arc::new(InMemoryCoordinationStateStore::default()), + TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())), + ); + let acceptance = tasks + .accept_background( + &caller, + TOOL_NAME, + None, + "guards-verified-key", + &json!({"email": "user@example.com", "idempotency_key": "k-1"}), + &unsampled_correlation(), + ) + .await + .expect("durable acceptance"); + + let runtime = crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let handler = Arc::new( + TaskWorkerHandler::from_application(&tasks, registry.clone(), runtime, crypto.clone()) + .with_verified_writes(vec![VerifiedIdempotentWrite { + operation_id: operation.id.as_str().to_owned(), + operation_version: 1, + key_header: "Idempotency-Key".to_owned(), + validity_ms: 60_000, + }]), + ); + let config = TaskWorkerConfig { + owner: "worker-guards-verified".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + ..TaskWorkerConfig::default() + }; + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + let mut state = None; + for _ in 0..200 { + tokio::time::sleep(Duration::from_millis(50)).await; + state = state_of(®istry, acceptance.task_id).await; + if state.as_deref() == Some("completed") { + break; + } + } + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + server.abort(); + + assert_eq!(state.as_deref(), Some("completed")); + assert_eq!( + requests.load(Ordering::SeqCst), + 2, + "exactly one retry of the write" + ); + assert_eq!( + effects.load(Ordering::SeqCst), + 1, + "the upstream applied the write exactly once" + ); + assert_eq!( + keys.lock().expect("keys").as_slice(), + ["k-1".to_owned(), "k-1".to_owned()], + "the same stable key is delivered on both attempts" + ); + + let attempts = store + .attempts( + &crank_core::TaskAccess { + workspace_id: common::test_workspace_id().to_string(), + owner_kind: crank_core::TaskOwnerKind::Agent, + owner_id: "agent_attempt_guards".to_owned(), + authorization_scope: String::new(), + }, + acceptance.task_id, + ) + .await; + // Owner scope of this principal is the agent of the published binding; read the history through + // the application entry instead of guessing the authorization scope string. + assert!(attempts.is_err(), "a foreign scope never reads the history"); + let attempts = tasks + .attempts(&caller, TOOL_NAME, acceptance.task_id) + .await + .expect("owner-scoped attempt history"); + assert_eq!(attempts.len(), 2, "two distinct durable attempts"); + assert_eq!(attempts[0].ordinal, 1); + assert_eq!(attempts[1].ordinal, 2); + assert!(attempts.iter().all(|attempt| attempt.retry_allowed)); + assert_eq!( + attempts[0].outcome, + Some(crank_core::AttemptOutcomeClass::RetryScheduled) + ); + assert_eq!( + attempts[1].outcome, + Some(crank_core::AttemptOutcomeClass::Completed) + ); + assert_eq!( + attempts[0].upstream_key_digest, attempts[1].upstream_key_digest, + "the same stable key digest is recorded on both attempts" + ); + assert!(attempts[0].upstream_key_digest.is_some()); + let record = tasks + .status(&caller, TOOL_NAME, acceptance.task_id) + .await + .expect("status"); + assert_eq!(record.pins.retry.max_attempts, 2); +} diff --git a/apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs b/apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs index 8be2e7d..330dfa1 100644 --- a/apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs +++ b/apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs @@ -215,13 +215,15 @@ async fn adapter_attempt_span_carries_durable_attempt_id() { runtime, crypto, )); - let mut config = TaskWorkerConfig::default(); - config.owner = "runtime-worker-attempt-trace".to_owned(); - config.batch_limit = 1; - config.lease_ms = 10_000; - config.poll_interval = std::time::Duration::from_millis(50); - config.max_inflight = 1; - config.shutdown_timeout = std::time::Duration::from_secs(5); + let config = TaskWorkerConfig { + owner: "runtime-worker-attempt-trace".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: std::time::Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: std::time::Duration::from_secs(5), + ..TaskWorkerConfig::default() + }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); @@ -263,6 +265,24 @@ async fn adapter_attempt_span_carries_durable_attempt_id() { assert_eq!(state.as_deref(), Some("completed")); + // Story 4.4: every outbound send is its own durable attempt, so the two retries of one lease carry + // two distinct attempt identities, both recorded in the immutable attempt history. + let attempt_rows: Vec<(String, i32)> = sqlx::query_as( + "select attempt_id, ordinal from task_attempts where task_id = $1 order by ordinal", + ) + .bind(acceptance.task_id.as_uuid()) + .fetch_all(registry.pool()) + .await + .unwrap(); + assert_eq!(attempt_rows.len(), 2, "two durable attempts were committed"); + assert_eq!(attempt_rows[0].1, 1); + assert_eq!(attempt_rows[1].1, 2); + assert_eq!( + Some(attempt_rows[1].0.as_str()), + durable_attempt_id.as_deref(), + "the Task points at its latest durable attempt" + ); + let spans = capture.snapshot(); let attempt_spans: Vec<_> = spans .iter() @@ -276,17 +296,16 @@ async fn adapter_attempt_span_carries_durable_attempt_id() { !aid.contains("traceparent"), "attempt_id must not contain traceparent" ); - assert_eq!( - Some(aid.as_str()), - durable_attempt_id.as_deref(), - "span attempt_id must equal durable attempt id" + assert!( + attempt_rows.iter().any(|(id, _)| id == &aid), + "every attempt span names an attempt that was durably committed" ); } let ids: Vec = attempt_spans .iter() .map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default()) .collect(); - assert_eq!(ids[0], ids[1], "same durable id on both retries"); + assert_ne!(ids[0], ids[1], "each retry has its own durable attempt id"); assert_eq!( spans .iter() diff --git a/apps/mcp-server/tests/integration/durable_tasks/cancel_chain.rs b/apps/mcp-server/tests/integration/durable_tasks/cancel_chain.rs new file mode 100644 index 0000000..c28e6d1 --- /dev/null +++ b/apps/mcp-server/tests/integration/durable_tasks/cancel_chain.rs @@ -0,0 +1,846 @@ +//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream. + +use super::common; +use super::unsampled_correlation; + +use axum::Router; +use axum::http::StatusCode; +use axum::routing::post; +use crank_community_mcp::{TaskApplication, TaskWorkerHandler}; +use crank_core::{ + AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState, + TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential, +}; +use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest}; +use crank_runtime::{ + InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig, +}; +use serde_json::json; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; +use time::OffsetDateTime; + +const AGENT_SLUG: &str = "reports"; +const TOOL_NAME: &str = "report_gen"; +const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000"; + +async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option { + sqlx::query_scalar("select state from tasks where task_id = $1") + .bind(task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap() +} + +async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) { + let mut operation = common::test_operation(upstream, TOOL_NAME); + operation.execution_config.execution_policy = Some(policy); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + registry + .publish_operation(PublishRequest { + workspace_id: &common::test_workspace_id(), + operation_id: &operation.id, + version: 1, + published_at: &OffsetDateTime::parse( + "2026-03-26T10:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap(), + published_by: Some("alice"), + }) + .await + .unwrap(); + common::publish_agent_with_bindings( + registry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; +} + +async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal { + let token = common::create_platform_api_key( + registry, + AGENT_SLUG, + name, + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let secret_hash = common::hash_access_secret(&token); + let api_key = registry + .get_platform_api_key_by_secret_for_agent_slug( + common::test_workspace_slug(), + AGENT_SLUG, + &secret_hash, + ) + .await + .unwrap() + .expect("active key"); + registry + .touch_platform_api_key( + &api_key.api_key.workspace_id, + &api_key.api_key.id, + &OffsetDateTime::now_utc(), + ) + .await + .unwrap(); + crank_community_mcp::TaskPrincipal::Agent { + workspace_slug: common::test_workspace_slug().to_owned(), + agent_slug: AGENT_SLUG.to_owned(), + credential: VerifiedMachineCredential { + machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey, + max_security_level: crank_core::OperationSecurityLevel::Standard, + scopes: api_key.api_key.scopes.clone(), + platform_api_key_id: Some(api_key.api_key.id.clone()), + }, + } +} + +async fn application(registry: &PostgresRegistry) -> TaskApplication { + let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); + let observed_at = OffsetDateTime::now_utc(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &observed_at, + }) + .await + .expect("master key"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("store"), + ); + TaskApplication::new( + registry.clone(), + Duration::ZERO, + Arc::new(InMemoryCoordinationStateStore::default()), + TaskService::with_secret_crypto(store, Arc::new(crypto)), + ) +} + +#[tokio::test] +async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() { + let registry = common::test_registry().await; + let calls = Arc::new(AtomicUsize::new(0)); + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + let signal = Arc::new(std::sync::Mutex::new(Some(tx))); + let upstream_app = Router::new().route( + "/crm/leads", + post({ + let calls = Arc::clone(&calls); + let signal = Arc::clone(&signal); + move || { + let calls = Arc::clone(&calls); + let signal = Arc::clone(&signal); + async move { + calls.fetch_add(1, Ordering::SeqCst); + if let Some(tx) = signal.lock().unwrap().take() { + let _ = tx.send(()); + } + tokio::time::sleep(Duration::from_millis(1500)).await; + (StatusCode::OK, axum::Json(json!({"id":"lead"}))) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let policy = ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 30_000, + no_tasks_fallback: NoTasksFallback::Unsupported, + }; + seed_tool(®istry, &upstream, policy).await; + let caller = principal(®istry, "cancel-s1").await; + let foreign = principal(®istry, "cancel-s1-foreign").await; + let app = application(®istry).await; + + let acceptance = app + .accept_background( + &caller, + TOOL_NAME, + None, + "s1-key", + &json!({"email":"u"}), + &unsampled_correlation(), + ) + .await + .expect("accept"); + + let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &OffsetDateTime::now_utc(), + }) + .await + .expect("master key"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("store"), + ); + let runtime = crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .build(); + let handler = Arc::new(TaskWorkerHandler::from_application( + &app, + registry.clone(), + runtime, + crypto.clone(), + )); + let worker = TaskWorker::new( + store, + handler, + TaskWorkerConfig { + owner: "worker-s1".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + renewal_interval: Duration::from_millis(100), + shutdown_timeout: Duration::from_secs(5), + }, + ) + .expect("worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + let _ = tokio::time::timeout(Duration::from_secs(5), rx).await; + + let cancel = app + .cancel(&caller, TOOL_NAME, acceptance.task_id, None) + .await + .expect("cancel"); + match cancel { + crank_core::TaskCancelOutcome::OutcomeUnknown(record) => { + assert_eq!(record.state, TaskState::OutcomeUnknown); + } + other => panic!("expected OutcomeUnknown got {other:?}"), + } + + let foreign_cancel = app + .cancel(&foreign, TOOL_NAME, acceptance.task_id, None) + .await; + assert!(matches!( + foreign_cancel, + Err(crank_community_mcp::TaskApplicationError::NotFound) + )); + + tokio::time::sleep(Duration::from_millis(1600)).await; + + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!( + state_of(®istry, acceptance.task_id).await.as_deref(), + Some("outcome_unknown") + ); + let certainty: Option = sqlx::query_scalar( + "select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1", + ) + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + assert_eq!(certainty.as_deref(), Some("unknown")); + let code: Option = + sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(code.as_deref(), Some("cancelled_after_dispatch")); + let attempts_count: i64 = + sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(attempts_count, 1); + + let cancel2 = app + .cancel(&caller, TOOL_NAME, acceptance.task_id, None) + .await + .expect("cancel2"); + assert!(matches!( + cancel2, + crank_core::TaskCancelOutcome::OutcomeUnknown(_) + )); + + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + server.abort(); +} + +#[tokio::test] +async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() { + let registry = common::test_registry().await; + let calls = Arc::new(AtomicUsize::new(0)); + let upstream_app = Router::new().route( + "/crm/leads", + post({ + let calls = Arc::clone(&calls); + move || { + let calls = Arc::clone(&calls); + async move { + calls.fetch_add(1, Ordering::SeqCst); + (StatusCode::OK, axum::Json(json!({"id":"lead"}))) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let policy = ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 30_000, + no_tasks_fallback: NoTasksFallback::Unsupported, + }; + seed_tool(®istry, &upstream, policy).await; + let caller = principal(®istry, "cancel-s3").await; + let app = application(®istry).await; + + let acceptance = app + .accept_background( + &caller, + TOOL_NAME, + None, + "s3-key", + &json!({"email":"u"}), + &unsampled_correlation(), + ) + .await + .expect("accept"); + + let cancel = app + .cancel(&caller, TOOL_NAME, acceptance.task_id, None) + .await + .expect("cancel"); + match cancel { + crank_core::TaskCancelOutcome::Cancelled(record) => { + assert_eq!(record.state, TaskState::Cancelled) + } + other => panic!("expected Cancelled got {other:?}"), + } + assert_eq!( + state_of(®istry, acceptance.task_id).await.as_deref(), + Some("cancelled") + ); + let attempts = app + .attempts(&caller, TOOL_NAME, acceptance.task_id) + .await + .expect("attempts"); + assert!(attempts.is_empty()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + + server.abort(); +} + +#[tokio::test] +async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() { + use crank_community_mcp::VerifiedIdempotentWrite; + use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy}; + + let registry = common::test_registry().await; + let requests = Arc::new(AtomicUsize::new(0)); + let upstream_app = Router::new().route( + "/crm/leads", + post({ + let requests = Arc::clone(&requests); + move |headers: axum::http::HeaderMap| { + let requests = Arc::clone(&requests); + async move { + let _ = headers.get("idempotency-key"); + let attempt = requests.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + // Applied upstream, acknowledgement lost after the caller had time to observe that + // the request really left the worker. + tokio::time::sleep(Duration::from_millis(600)).await; + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error":"acknowledgement lost"})), + ); + } + (StatusCode::OK, axum::Json(json!({"id":"lead-1"}))) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let mut operation = common::test_operation(&upstream, TOOL_NAME); + operation.execution_config.execution_policy = Some(ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 30_000, + no_tasks_fallback: NoTasksFallback::Unsupported, + }); + operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 }); + operation.execution_config.idempotency = Some(IdempotencyPolicy { + mode: IdempotencyMode::Required, + ttl_ms: 60_000, + input_field: Some("idempotency_key".to_owned()), + header_name: Some("Idempotency-Key".to_owned()), + }); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + registry + .publish_operation(PublishRequest { + workspace_id: &common::test_workspace_id(), + operation_id: &operation.id, + version: 1, + published_at: &OffsetDateTime::parse( + "2026-03-26T10:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap(), + published_by: Some("alice"), + }) + .await + .unwrap(); + common::publish_agent_with_bindings( + ®istry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; + let caller = principal(®istry, "cancel-s2").await; + let app = application(®istry).await; + let acceptance = app + .accept_background( + &caller, + TOOL_NAME, + None, + "s2-key", + &json!({"email":"u","idempotency_key":"k-2"}), + &unsampled_correlation(), + ) + .await + .expect("accept"); + + let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &OffsetDateTime::now_utc(), + }) + .await + .expect("master key"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("store"), + ); + let runtime = crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let handler = Arc::new( + TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone()) + .with_verified_writes(vec![VerifiedIdempotentWrite { + operation_id: operation.id.as_str().to_owned(), + operation_version: 1, + key_header: "Idempotency-Key".to_owned(), + validity_ms: 60_000, + }]), + ); + // Reproduce deterministically the durable state the executor commits after an ambiguous attempt + // whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and + // `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a + // running executor against the cancel would not be a repeatable oracle. + let lease = store + .claim_batch("worker-s2-prep", 1, 10_000) + .await + .expect("claim") + .pop() + .expect("one claimed lease"); + let attempt = store + .checkpoint_dispatch(crank_core::BeginAttempt { + task_id: acceptance.task_id, + owner: lease.owner.clone(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: true, + }) + .await + .expect("durable checkpoint"); + let resolution = store + .resolve_attempt( + &attempt, + crank_core::AttemptOutcome::Uncertain { + error: crank_core::TaskEnvelope::try_redacted( + &json!({"code": "upstream_ack_lost", "phase": "dispatched"}), + ) + .expect("bounded attempt error"), + repeatable: true, + phase: crank_core::AttemptPhase::Dispatched, + }, + ) + .await + .expect("attempt resolution"); + assert!( + matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }), + "an ambiguous but safely repeatable attempt arms the pinned retry" + ); + + let cancel = app + .cancel(&caller, TOOL_NAME, acceptance.task_id, None) + .await + .expect("cancel"); + match cancel { + crank_core::TaskCancelOutcome::OutcomeUnknown(record) => { + assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown) + } + other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"), + } + + // A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not + // claimable and no second terminal write is possible. + let worker = TaskWorker::new( + store.clone(), + handler, + TaskWorkerConfig { + owner: "worker-s2".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + ..TaskWorkerConfig::default() + }, + ) + .expect("worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + tokio::time::sleep(Duration::from_secs(1)).await; + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + server.abort(); + + assert_eq!( + requests.load(Ordering::SeqCst), + 0, + "no outbound send after the cancel" + ); + assert_eq!( + state_of(®istry, acceptance.task_id).await.as_deref(), + Some("outcome_unknown") + ); + let attempts_count: i64 = + sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(attempts_count, 1, "no second durable attempt"); + let certainty: Option = sqlx::query_scalar( + "select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1", + ) + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + assert_eq!(certainty.as_deref(), Some("unknown")); + let code: Option = + sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(code.as_deref(), Some("cancelled_after_dispatch")); +} + +#[tokio::test] +async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() { + use crank_community_mcp::VerifiedIdempotentWrite; + use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy}; + + let registry = common::test_registry().await; + let requests = Arc::new(AtomicUsize::new(0)); + let upstream_app = Router::new().route( + "/crm/leads", + post({ + let requests = Arc::clone(&requests); + move |headers: axum::http::HeaderMap| { + let requests = Arc::clone(&requests); + async move { + let _ = headers.get("idempotency-key"); + let attempt = requests.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + // Applied upstream, acknowledgement lost after the caller had time to observe that + // the request really left the worker. + tokio::time::sleep(Duration::from_millis(600)).await; + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error":"acknowledgement lost"})), + ); + } + (StatusCode::OK, axum::Json(json!({"id":"lead-1"}))) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let mut operation = common::test_operation(&upstream, TOOL_NAME); + operation.execution_config.execution_policy = Some(ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 30_000, + no_tasks_fallback: NoTasksFallback::Unsupported, + }); + operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 }); + operation.execution_config.idempotency = Some(IdempotencyPolicy { + mode: IdempotencyMode::Required, + ttl_ms: 60_000, + input_field: Some("idempotency_key".to_owned()), + header_name: Some("Idempotency-Key".to_owned()), + }); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + registry + .publish_operation(PublishRequest { + workspace_id: &common::test_workspace_id(), + operation_id: &operation.id, + version: 1, + published_at: &OffsetDateTime::parse( + "2026-03-26T10:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap(), + published_by: Some("alice"), + }) + .await + .unwrap(); + common::publish_agent_with_bindings( + ®istry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; + let caller = principal(®istry, "cancel-s4").await; + let app = application(®istry).await; + let acceptance = app + .accept_background( + &caller, + TOOL_NAME, + None, + "s4-key", + &json!({"email":"u","idempotency_key":"k-4"}), + &unsampled_correlation(), + ) + .await + .expect("accept"); + + let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &OffsetDateTime::now_utc(), + }) + .await + .expect("master key"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("store"), + ); + let runtime = crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let handler = Arc::new( + TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone()) + .with_verified_writes(vec![VerifiedIdempotentWrite { + operation_id: operation.id.as_str().to_owned(), + operation_version: 1, + key_header: "Idempotency-Key".to_owned(), + validity_ms: 60_000, + }]), + ); + let worker = TaskWorker::new( + store.clone(), + handler, + TaskWorkerConfig { + owner: "worker-s4".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + ..TaskWorkerConfig::default() + }, + ) + .expect("worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + // Revoke the key the Task was accepted with while the first outbound write is provably in flight: + // that attempt still resolves as an uncertain write, and the next attempt is refused by the + // authority before its checkpoint. + for _ in 0..200 { + if requests.load(Ordering::SeqCst) >= 1 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!( + requests.load(Ordering::SeqCst), + 1, + "the first attempt is dispatched before the refusal" + ); + registry + .revoke_platform_api_key_for_agent( + &common::test_workspace_id(), + &common::test_agent_id(AGENT_SLUG), + &PlatformApiKeyId::new("pk_cancel-s4".to_owned()), + &OffsetDateTime::now_utc(), + ) + .await + .expect("revoke"); + + let mut state = None; + for _ in 0..200 { + tokio::time::sleep(Duration::from_millis(50)).await; + state = state_of(®istry, acceptance.task_id).await; + if state.as_deref() == Some("outcome_unknown") { + break; + } + } + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + server.abort(); + + assert_eq!( + state.as_deref(), + Some("outcome_unknown"), + "a refused attempt after a lost acknowledgement must stay uncertain" + ); + assert_eq!( + requests.load(Ordering::SeqCst), + 1, + "the refused attempt is never dispatched" + ); + let attempts_count: i64 = + sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(attempts_count, 1, "no second durable attempt"); + let certainty: Option = sqlx::query_scalar( + "select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1", + ) + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + assert_eq!(certainty.as_deref(), Some("unknown")); + let phase: Option = sqlx::query_scalar( + "select phase from task_attempts where task_id = $1 order by ordinal desc limit 1", + ) + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + assert_eq!(phase.as_deref(), Some("dispatched")); + let error: Option<(Option, Option)> = sqlx::query_as( + "select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1", + ) + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + let (code, error_phase) = error.expect("the refused Task carries its bounded error"); + assert_eq!(code.as_deref(), Some("task_authority_rejected")); + assert_eq!( + error_phase.as_deref(), + Some("dispatched"), + "the error keeps the phase of the attempt that may have reached the upstream" + ); +} diff --git a/apps/mcp-server/tests/integration/durable_tasks/mod.rs b/apps/mcp-server/tests/integration/durable_tasks/mod.rs index c8ff9fb..98332c9 100644 --- a/apps/mcp-server/tests/integration/durable_tasks/mod.rs +++ b/apps/mcp-server/tests/integration/durable_tasks/mod.rs @@ -605,6 +605,53 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() { server.abort(); } +/// Commits and resolves exactly one durable attempt of the current lease through the real store. +/// +/// Story 4.4 moved the dispatch checkpoint and the attempt resolution into the runtime seam, so a +/// worker-level test double persists exactly one attempt instead of inventing a second retry loop. +pub(crate) async fn commit_attempt( + context: &crank_runtime::TaskAttemptContext, + outcome: crank_core::AttemptOutcome, +) -> crank_runtime::TaskAttemptOutcome { + use crank_core::{AttemptResolution, BeginAttempt, TaskState}; + + let lease = context.lease(); + let begin = BeginAttempt { + task_id: lease.record.task_id, + owner: lease.owner.clone(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: lease.record.pins.retry.max_attempts > 1, + }; + match context.store().checkpoint_dispatch(begin).await { + Ok(attempt) => match context.store().resolve_attempt(&attempt, outcome).await { + Ok(AttemptResolution::Finished(task)) if task.state == TaskState::Completed => { + crank_runtime::TaskAttemptOutcome::Completed + } + Ok(AttemptResolution::Finished(_)) => crank_runtime::TaskAttemptOutcome::Failed { + code: "test_failure".to_owned(), + }, + Ok(AttemptResolution::Unknown(_)) => crank_runtime::TaskAttemptOutcome::OutcomeUnknown, + Ok(AttemptResolution::RetryAt { .. }) | Err(_) => { + crank_runtime::TaskAttemptOutcome::Stopped + } + }, + Err(_) => crank_runtime::TaskAttemptOutcome::Stopped, + } +} + +/// Bounded success outcome used by the worker-level test doubles. +pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::AttemptOutcome { + crank_core::AttemptOutcome::Completed( + crank_core::TaskEnvelope::try_redacted(&value).expect("test result is bounded"), + ) +} + +mod attempt_guards; mod attempt_trace; +mod cancel_chain; mod shutdown_window; mod task_worker; diff --git a/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs b/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs index 37d794c..ecca025 100644 --- a/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs +++ b/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs @@ -15,8 +15,7 @@ use super::*; async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() { use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; use crank_runtime::{ - TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, - TaskWorkerError, + TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -67,6 +66,7 @@ async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, + retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: scoped_key.repeat(64), input_fingerprint: scoped_key.repeat(64), @@ -101,15 +101,44 @@ async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream let _ = self.release.acquire().await; Ok(()) } - async fn execute( - &self, - _lease: &TaskLease, - _attempt_id: &str, - _cancelled: Arc, - _permit: TaskDispatchPermit, - ) -> TaskAttemptOutcome { + async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome { + // The trigger delays the durable checkpoint; a stop that lands while it is in flight must + // finalize failed(shutdown_before_dispatch) without an outbound send. + let begin = crank_core::BeginAttempt { + task_id: context.lease().record.task_id, + owner: context.lease().owner.clone(), + fence: context.lease().fence, + expected_version: context.lease().record.version, + attempt_id: "attempt-stop-window".to_owned(), + input_fingerprint: context.lease().record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }; + let Ok(attempt) = context.store().checkpoint_dispatch(begin).await else { + return TaskAttemptOutcome::Stopped; + }; + if context.is_stopping() { + let _ = context + .store() + .resolve_attempt( + &attempt, + crank_core::AttemptOutcome::Failed { + error: crank_core::TaskEnvelope::try_redacted(&json!({ + "code": "shutdown_before_dispatch", + "phase": "before_dispatch", + })) + .expect("bounded error envelope"), + retryable: false, + phase: crank_core::AttemptPhase::BeforeDispatch, + }, + ) + .await; + return TaskAttemptOutcome::Failed { + code: "shutdown_before_dispatch".to_owned(), + }; + } self.executed.fetch_add(1, Ordering::SeqCst); - TaskAttemptOutcome::Completed(json!({ "ok": true })) + commit_attempt(&context, completed_outcome(json!({ "ok": true }))).await } } let handler: Arc = Arc::new(GatedHandler { @@ -117,14 +146,15 @@ async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream release: revalidate_release.clone(), executed: executed.clone(), }); - let mut config = TaskWorkerConfig::default(); - config.owner = "runtime-worker-stop-window".to_owned(); - config.batch_limit = 1; - config.lease_ms = 30_000; - config.poll_interval = Duration::from_millis(50); - config.renewal_interval = Duration::from_secs(20); - config.max_inflight = 1; - config.shutdown_timeout = Duration::from_secs(20); + let config = TaskWorkerConfig { + owner: "runtime-worker-stop-window".to_owned(), + batch_limit: 1, + lease_ms: 30_000, + poll_interval: Duration::from_millis(50), + renewal_interval: Duration::from_secs(20), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(20), + }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); diff --git a/apps/mcp-server/tests/integration/durable_tasks/task_worker.rs b/apps/mcp-server/tests/integration/durable_tasks/task_worker.rs index cc8b85c..4cce421 100644 --- a/apps/mcp-server/tests/integration/durable_tasks/task_worker.rs +++ b/apps/mcp-server/tests/integration/durable_tasks/task_worker.rs @@ -38,6 +38,7 @@ async fn task_worker_store_lease_recovery_contract() { execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, + retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "a".repeat(64), input_fingerprint: "b".repeat(64), @@ -85,29 +86,37 @@ async fn task_worker_store_lease_recovery_contract() { // A stale fencing token can never checkpoint dispatch, even under the issuing owner. assert!( store - .checkpoint_dispatch( - "worker_1", - lease.record.task_id, - lease.fence + 1, - "attempt-stale".to_owned() - ) + .checkpoint_dispatch(crank_core::BeginAttempt { + task_id: lease.record.task_id, + owner: "worker_1".to_owned(), + fence: lease.fence + 1, + expected_version: lease.record.version, + attempt_id: "attempt-stale".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) .await .is_err() ); // The valid owner still commits the dispatch checkpoint before any outbound work. - assert!( - store - .checkpoint_dispatch( - "worker_1", - lease.record.task_id, - lease.fence, - "attempt-1".to_owned(), - ) - .await - .unwrap() - .current_attempt_id - .as_deref() - == Some("attempt-1") + let committed = store + .checkpoint_dispatch(crank_core::BeginAttempt { + task_id: lease.record.task_id, + owner: "worker_1".to_owned(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: "attempt-1".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) + .await + .unwrap(); + assert_eq!(committed.attempt.attempt_id, "attempt-1"); + assert_eq!( + committed.task.current_attempt_id.as_deref(), + Some("attempt-1") ); } @@ -116,8 +125,7 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() { use async_trait::async_trait; use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; use crank_runtime::{ - TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, - TaskWorkerError, + TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -154,6 +162,7 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() { execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, + retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "r".repeat(64), input_fingerprint: "b".repeat(64), @@ -180,15 +189,9 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() { async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { Ok(()) } - async fn execute( - &self, - _lease: &TaskLease, - _attempt_id: &str, - _cancelled: Arc, - _permit: TaskDispatchPermit, - ) -> TaskAttemptOutcome { + async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome { self.observed.fetch_add(1, Ordering::SeqCst); - TaskAttemptOutcome::Completed(serde_json::json!({"ok": true})) + commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await } } let handler: Arc = Arc::new(RuntimeHandler { @@ -196,14 +199,15 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() { }); // PG-only authoritative dispatch: no Valkey or upstream process is involved. - let mut config = TaskWorkerConfig::default(); - config.owner = "runtime-worker-1".to_owned(); - config.batch_limit = 1; - config.lease_ms = 10_000; - config.poll_interval = Duration::from_millis(100); - config.renewal_interval = Duration::from_millis(1_000); - config.max_inflight = 1; - config.shutdown_timeout = Duration::from_secs(5); + let config = TaskWorkerConfig { + owner: "runtime-worker-1".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(100), + renewal_interval: Duration::from_millis(1_000), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); @@ -238,10 +242,12 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() { assert_eq!(record.state, TaskState::Completed); // Bounded shutdown with no fresh work returns promptly. - let mut idle_config = TaskWorkerConfig::default(); - idle_config.owner = "runtime-worker-idle".to_owned(); - idle_config.poll_interval = Duration::from_millis(50); - idle_config.shutdown_timeout = Duration::from_secs(2); + let idle_config = TaskWorkerConfig { + owner: "runtime-worker-idle".to_owned(), + poll_interval: Duration::from_millis(50), + shutdown_timeout: Duration::from_secs(2), + ..TaskWorkerConfig::default() + }; let idle_worker = TaskWorker::new( store.clone(), handler_for_no_work(store.clone()), @@ -259,8 +265,7 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() { async fn task_worker_revalidation_denial_never_dispatches() { use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind}; use crank_runtime::{ - TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, - TaskWorkerError, + TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -293,6 +298,7 @@ async fn task_worker_revalidation_denial_never_dispatches() { execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, + retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "d".repeat(64), input_fingerprint: "b".repeat(64), @@ -320,28 +326,24 @@ async fn task_worker_revalidation_denial_never_dispatches() { "publication or credential revoked", ))) } - async fn execute( - &self, - _lease: &TaskLease, - _attempt_id: &str, - _cancelled: Arc, - _permit: TaskDispatchPermit, - ) -> TaskAttemptOutcome { + async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome { self.observed.fetch_add(1, Ordering::SeqCst); - TaskAttemptOutcome::Completed(json!({"ok": true})) + commit_attempt(&context, completed_outcome(json!({"ok": true}))).await } } let observed = Arc::new(AtomicUsize::new(0)); let handler: Arc = Arc::new(DeniedHandler { observed: observed.clone(), }); - let mut config = TaskWorkerConfig::default(); - config.owner = "runtime-worker-denied".to_owned(); - config.batch_limit = 1; - config.lease_ms = 10_000; - config.poll_interval = Duration::from_millis(50); - config.max_inflight = 1; - config.shutdown_timeout = Duration::from_secs(5); + let config = TaskWorkerConfig { + owner: "runtime-worker-denied".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + ..TaskWorkerConfig::default() + }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); @@ -387,12 +389,9 @@ fn handler_for_no_work( } async fn execute( &self, - _lease: &crank_core::TaskLease, - _attempt_id: &str, - _cancelled: std::sync::Arc, - _permit: crank_runtime::TaskDispatchPermit, + context: crank_runtime::TaskAttemptContext, ) -> crank_runtime::TaskAttemptOutcome { - crank_runtime::TaskAttemptOutcome::Completed(serde_json::json!({"ok": true})) + commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await } } std::sync::Arc::new(NoWorkHandler) @@ -522,13 +521,15 @@ async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() { runtime, crypto, )); - let mut config = TaskWorkerConfig::default(); - config.owner = "runtime-worker-e2e".to_owned(); - config.batch_limit = 1; - config.lease_ms = 10_000; - config.poll_interval = Duration::from_millis(50); - config.max_inflight = 1; - config.shutdown_timeout = Duration::from_secs(5); + let config = TaskWorkerConfig { + owner: "runtime-worker-e2e".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + ..TaskWorkerConfig::default() + }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); @@ -663,13 +664,15 @@ async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() { runtime, crypto, )); - let mut config = TaskWorkerConfig::default(); - config.owner = "runtime-worker-uncertain".to_owned(); - config.batch_limit = 1; - config.lease_ms = 10_000; - config.poll_interval = Duration::from_millis(50); - config.max_inflight = 1; - config.shutdown_timeout = Duration::from_secs(5); + let config = TaskWorkerConfig { + owner: "runtime-worker-uncertain".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(5), + ..TaskWorkerConfig::default() + }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); @@ -709,8 +712,7 @@ async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() { async fn task_worker_graceful_stop_drains_active_attempt() { use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; use crank_runtime::{ - TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, - TaskWorkerError, + TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -743,6 +745,7 @@ async fn task_worker_graceful_stop_drains_active_attempt() { execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, + retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "s".repeat(64), input_fingerprint: "d".repeat(64), @@ -768,29 +771,25 @@ async fn task_worker_graceful_stop_drains_active_attempt() { async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { Ok(()) } - async fn execute( - &self, - _lease: &TaskLease, - _attempt_id: &str, - _cancelled: Arc, - _permit: TaskDispatchPermit, - ) -> TaskAttemptOutcome { + async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome { self.started.fetch_add(1, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(500)).await; - TaskAttemptOutcome::Completed(json!({"ok": true})) + commit_attempt(&context, completed_outcome(json!({"ok": true}))).await } } let started = Arc::new(AtomicUsize::new(0)); let handler: Arc = Arc::new(SlowHandler { started: started.clone(), }); - let mut config = TaskWorkerConfig::default(); - config.owner = "runtime-worker-drain".to_owned(); - config.batch_limit = 1; - config.lease_ms = 10_000; - config.poll_interval = Duration::from_millis(50); - config.max_inflight = 1; - config.shutdown_timeout = Duration::from_secs(20); + let config = TaskWorkerConfig { + owner: "runtime-worker-drain".to_owned(), + batch_limit: 1, + lease_ms: 10_000, + poll_interval: Duration::from_millis(50), + max_inflight: 1, + shutdown_timeout: Duration::from_secs(20), + ..TaskWorkerConfig::default() + }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); diff --git a/crates/crank-community-mcp/src/app.rs b/crates/crank-community-mcp/src/app.rs index 96e3c74..e23bb6f 100644 --- a/crates/crank-community-mcp/src/app.rs +++ b/crates/crank-community-mcp/src/app.rs @@ -58,7 +58,7 @@ use self::approvals::{ }; pub(super) use self::auth_resolution::resolve_operation_auth; use self::endpoint::{mcp_delete, mcp_get, mcp_post}; -pub use self::task_worker::TaskWorkerHandler; +pub use self::task_worker::{TaskWorkerHandler, VerifiedIdempotentWrite}; pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal}; // Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two // through `super`, and the plain lib build does not use them (same pattern as diff --git a/crates/crank-community-mcp/src/app/task_worker.rs b/crates/crank-community-mcp/src/app/task_worker.rs index e4fefc7..b60c4f8 100644 --- a/crates/crank-community-mcp/src/app/task_worker.rs +++ b/crates/crank-community-mcp/src/app/task_worker.rs @@ -1,6 +1,10 @@ -//! Application-level durable Task handler for Story 4.3. +//! Application-level durable Task handler and attempt authority (Story 4.3/4.4). +//! +//! The handler owns everything that touches product authority: the exact published pin, the typed +//! Platform API key, the approval policy, the encrypted executable input and the resolved upstream +//! auth. The runtime owns the single bounded retry loop and the durable attempt checkpoint/resolution; +//! this module only revalidates and prepares one attempt at a time. -use std::sync::Arc; use std::time::{Duration, Instant}; use async_trait::async_trait; @@ -12,9 +16,11 @@ use crank_core::{ use crank_registry::PostgresRegistry; use crank_registry::PublishedAgentTool; use crank_runtime::{ - ExecutionAuthorization, ExecutionRequestError, RuntimeExecutionRequest, RuntimeExecutor, - SecretCrypto, TaskDispatchPermit, TaskHandler, TaskService, TaskWorkerError, + ExecutionAuthorization, RuntimeExecutor, RuntimeOperation, SecretCrypto, TaskAttemptAuthority, + TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan, TaskAttemptRefusal, TaskHandler, + TaskService, TaskWorkerError, }; +use sha2::{Digest, Sha256}; use super::TaskApplication; use super::auth_resolution::resolve_operation_auth_with; @@ -22,12 +28,31 @@ use super::tool_resolution::runtime_operation; use crate::access::{allows_scope, credential_allows_security_level}; use crate::catalog::PublishedToolCatalog; +/// Server-owned evidence that one published operation deduplicates repeated sends of the same stable +/// idempotency key. +/// +/// The descriptor is constructed by the application for an integration whose upstream contract was +/// actually qualified (key delivery, dedup scope, canonical input binding and a dedup window that +/// covers the pinned retry budget). It is never deserialized from Task input, so a caller key or +/// header alone can never make a write retryable. Production wiring passes an empty set until such a +/// contract exists. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VerifiedIdempotentWrite { + pub operation_id: String, + pub operation_version: u32, + /// Header through which the stable key is delivered to this exact upstream. + pub key_header: String, + /// Dedup window of the upstream contract, which must cover the pinned retry budget. + pub validity_ms: i64, +} + pub struct TaskWorkerHandler { registry: PostgresRegistry, catalog: PublishedToolCatalog, runtime: RuntimeExecutor, secret_crypto: SecretCrypto, service: TaskService, + verified_writes: Vec, } impl TaskWorkerHandler { @@ -46,9 +71,16 @@ impl TaskWorkerHandler { runtime, secret_crypto, service: tasks.task_service(), + verified_writes: Vec::new(), } } + /// Registers server-owned verified-idempotency evidence of qualified integrations. + pub fn with_verified_writes(mut self, verified_writes: Vec) -> Self { + self.verified_writes = verified_writes; + self + } + /// Authoritative pre-dispatch authorization of one leased Task (Story 4.3). /// /// Resolves the exact pinned Published Version in the current publication, repeats the approval @@ -143,6 +175,74 @@ impl TaskWorkerHandler { } Ok(tool) } + + /// Safe-retry classification of one attempt and the stable upstream key digest. + /// + /// A read is always safe to repeat. A write is only safe to repeat when this process holds + /// server-owned evidence for the exact published operation and that evidence's dedup window still + /// covers the accepted retry budget; the presence of a client key or an idempotency header alone + /// never qualifies. + fn retry_classification( + &self, + operation: &RuntimeOperation, + input: &serde_json::Value, + retry_window_ms: i64, + ) -> (bool, Option) { + if matches!( + &operation.target, + crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get + ) { + return (true, None); + } + let Some(policy) = operation.execution_config.idempotency.as_ref() else { + return (false, None); + }; + let Some(field) = policy + .input_field + .as_deref() + .filter(|field| !field.trim().is_empty()) + else { + return (false, None); + }; + let Some(header) = policy + .header_name + .as_deref() + .filter(|value| !value.is_empty()) + else { + return (false, None); + }; + let key = match input.get(field) { + Some(serde_json::Value::String(value)) if !value.trim().is_empty() => value.clone(), + Some(serde_json::Value::Number(value)) => value.to_string(), + _ => return (false, None), + }; + let evidence = self.verified_writes.iter().find(|evidence| { + evidence.operation_id == operation.operation_id.as_str() + && evidence.operation_version == operation.operation_version + && evidence.key_header.eq_ignore_ascii_case(header) + && evidence.validity_ms >= retry_window_ms + }); + let Some(evidence) = evidence else { + return (false, None); + }; + // Only a digest of the stable key is persisted; the key itself stays in the encrypted input. + let mut digest = Sha256::new(); + digest.update(evidence.key_header.as_bytes()); + digest.update([0u8]); + digest.update(key.as_bytes()); + (true, Some(format!("{:x}", digest.finalize()))) + } + + /// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store + /// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal. + fn refusal(error: TaskError) -> TaskAttemptRefusal { + match error { + TaskError::Unavailable { .. } => TaskAttemptRefusal::Stopped, + _ => TaskAttemptRefusal::Failed { + code: "task_authority_rejected".to_owned(), + }, + } + } } #[async_trait] @@ -154,74 +254,54 @@ impl TaskHandler for TaskWorkerHandler { Ok(()) } - async fn execute( + async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome { + self.runtime.execute_task_outcome(&context, self).await + } +} + +#[async_trait] +impl TaskAttemptAuthority for TaskWorkerHandler { + async fn prepare_attempt( &self, lease: &TaskLease, - _attempt_id: &str, - cancelled: Arc, - permit: TaskDispatchPermit, - ) -> crank_runtime::TaskAttemptOutcome { - use crank_runtime::TaskAttemptOutcome; - use std::sync::atomic::Ordering; - - // The lease may have been lost between the checkpoint and the outbound call; never dispatch - // on behalf of a worker that no longer owns the Task. - if cancelled.load(Ordering::Acquire) { - return TaskAttemptOutcome::OutcomeUnknown; - } - - let tool = match self.authorize(lease).await { - Ok(tool) => tool, - Err(error) => { - return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()})); - } - }; - - let input = match self.service.executable_input(lease).await { - Ok(input) => input, - Err(error) => { - return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()})); - } - }; - + ) -> Result { + let tool = self.authorize(lease).await.map_err(Self::refusal)?; + let input = self + .service + .executable_input(lease) + .await + .map_err(Self::refusal)?; let operation = runtime_operation(&tool); let now_ms = current_unix_ms(); let deadline_ms = lease.record.pins.execution_deadline_unix_ms; if deadline_ms <= now_ms { - return TaskAttemptOutcome::Failed( - serde_json::json!({"code": "task deadline elapsed"}), - ); + // The accepted absolute deadline already elapsed: no further attempt may be dispatched, + // and any previous uncertainty was already recorded as such. + return Err(TaskAttemptRefusal::Failed { + code: "task_deadline_elapsed".to_owned(), + }); } - let deadline = Instant::now() - + Duration::from_millis((deadline_ms - now_ms).min(i64::MAX as i64) as u64); + let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64); let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str()); let agent_id = AgentId::new(lease.record.owner_id.as_str()); - let resolved_auth = match resolve_operation_auth_with( + let resolved_auth = resolve_operation_auth_with( &self.registry, &self.secret_crypto, &workspace_id, &operation.execution_config, ) .await - { - Ok(auth) => auth, - Err(error) => { - return TaskAttemptOutcome::Failed( - serde_json::json!({ "code": error.to_string() }), - ); - } - }; - - // Continue the parent trace persisted with the accepted Task; the worker's attempt span is a - // child of it, and the executor records the attempt identity. Trace data stays out of labels. - let trace_context = match TraceContext::parse(&lease.record.support.traceparent) { - Ok(context) => context, - Err(_) => { - return TaskAttemptOutcome::Failed( - serde_json::json!({ "code": "task trace parent is invalid" }), - ); - } - }; + .map_err(|_| TaskAttemptRefusal::Failed { + code: "task_auth_resolution_failed".to_owned(), + })?; + // Continue the parent trace persisted with the accepted Task; every attempt adds its own + // child `adapter_attempt` span carrying the committed durable attempt ID. + let trace_context = + TraceContext::parse(&lease.record.support.traceparent).map_err(|_| { + TaskAttemptRefusal::Failed { + code: "task_trace_parent_invalid".to_owned(), + } + })?; let request_context = crank_runtime::RuntimeRequestContext::new( lease.record.support.request_id.clone(), trace_context, @@ -231,52 +311,20 @@ impl TaskHandler for TaskWorkerHandler { Some(agent_id.clone()), crank_core::InvocationSource::AgentToolCall, ); - - let request = match RuntimeExecutionRequest::try_new( - &workspace_id, - ExecutionOrigin::AgentSnapshot, - Some(&agent_id), - &operation, - &input, - ExecutionAuthorization::Authorized, - resolved_auth.as_ref(), - &request_context, + let (retry_allowed, upstream_key_digest) = + self.retry_classification(&operation, &input, deadline_ms - now_ms); + Ok(TaskAttemptPlan { + workspace_id, + origin: ExecutionOrigin::AgentSnapshot, + agent_id: Some(agent_id), + operation, + input, + authorization: ExecutionAuthorization::Authorized, + resolved_auth, + request_context, deadline, - ) { - Ok(request) => request, - Err(error) => { - return TaskAttemptOutcome::Failed(match error { - ExecutionRequestError::InvalidOrigin => { - serde_json::json!({"code": "invalid execution origin"}) - } - ExecutionRequestError::InvalidOperationVersion => { - serde_json::json!({"code": "invalid operation version"}) - } - ExecutionRequestError::DeadlineElapsed => { - serde_json::json!({"code": "task deadline elapsed"}) - } - ExecutionRequestError::InvalidMeteringScope => { - serde_json::json!({"code": "invalid metering scope"}) - } - }); - } - }; - - if cancelled.load(Ordering::Acquire) { - return TaskAttemptOutcome::OutcomeUnknown; - } - match self.runtime.execute_task_outcome(request, permit).await { - Ok(success) => TaskAttemptOutcome::Completed(success.output), - Err(failure) => { - // An ambiguous outbound failure is recorded as uncertainty, never as a retryable - // failure: the durable checkpoint already exists, so Story 4.4 may reconcile it. - if failure.outcome_certainty() == crank_core::OutcomeCertainty::OutcomeUnknown { - return TaskAttemptOutcome::OutcomeUnknown; - } - TaskAttemptOutcome::Failed( - serde_json::json!({ "code": format!("{:?}", failure.error_code()) }), - ) - } - } + retry_allowed, + upstream_key_digest, + }) } } diff --git a/crates/crank-community-mcp/src/app/tasks.rs b/crates/crank-community-mcp/src/app/tasks.rs index 697709c..57354a6 100644 --- a/crates/crank-community-mcp/src/app/tasks.rs +++ b/crates/crank-community-mcp/src/app/tasks.rs @@ -213,6 +213,7 @@ impl TaskApplication { deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)), retention_ms: TaskService::default_retention_ms(), approval_reference: None, + retry: task_retry_policy(&tool), } }; @@ -274,6 +275,45 @@ impl TaskApplication { } } + /// Owner-scoped cancellation of one Task of the addressed published Tool (Story 4.4). + /// + /// Cancellation is decided by the durable store under a row lock: before the dispatch checkpoint + /// it is an honest cancellation with zero outbound calls, after it the Task becomes the + /// non-terminal uncertain outcome instead of a plain cancel or a retry. + pub async fn cancel( + &self, + principal: &TaskPrincipal, + tool_name: &str, + task_id: TaskId, + expected_version: Option, + ) -> Result { + let access = self.task_access(principal, tool_name).await?; + match self + .service + .cancel(&access, task_id, expected_version, "user_cancelled") + .await + { + Ok(outcome) => Ok(outcome), + Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound), + Err(error) => Err(TaskApplicationError::Task(error)), + } + } + + /// Owner-scoped attempt history of one Task of the addressed published Tool. + pub async fn attempts( + &self, + principal: &TaskPrincipal, + tool_name: &str, + task_id: TaskId, + ) -> Result, TaskApplicationError> { + let access = self.task_access(principal, tool_name).await?; + match self.service.attempts(&access, task_id).await { + Ok(attempts) => Ok(attempts), + Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound), + Err(error) => Err(TaskApplicationError::Task(error)), + } + } + async fn task_access( &self, principal: &TaskPrincipal, @@ -320,3 +360,18 @@ impl TaskApplication { Ok(tool) } } + +/// Bounded retry snapshot pinned from the exact published operation at acceptance. A later +/// publication, a restart or a new binary never changes the numbers an accepted Task was admitted +/// with; a missing or oversized policy is clamped to the closed story bound. +fn task_retry_policy(tool: &PublishedAgentTool) -> crank_core::TaskRetryPolicy { + let max_attempts = tool + .operation + .execution_config + .retry_policy + .as_ref() + .map_or(1, |policy| { + policy.max_attempts.clamp(1, crank_core::TASK_MAX_ATTEMPTS) + }); + crank_core::TaskRetryPolicy::new(max_attempts, 1) +} diff --git a/crates/crank-community-mcp/src/lib.rs b/crates/crank-community-mcp/src/lib.rs index 217d031..c693ac9 100644 --- a/crates/crank-community-mcp/src/lib.rs +++ b/crates/crank-community-mcp/src/lib.rs @@ -16,7 +16,8 @@ mod tool_search; mod transport; pub use app::{ - TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler, build_app, - build_app_with_artifacts, build_app_with_artifacts_and_tasks, - build_app_with_background_workers, build_app_with_background_workers_and_limits, + TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler, + VerifiedIdempotentWrite, build_app, build_app_with_artifacts, + build_app_with_artifacts_and_tasks, build_app_with_background_workers, + build_app_with_background_workers_and_limits, }; diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index b688ca6..51cd156 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -75,13 +75,16 @@ pub mod domain { pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion}; pub use crate::task::{ - TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, + AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase, + AttemptResolution, BeginAttempt, TASK_APPROVAL_REFERENCE_MAX_BYTES, + TASK_CANCEL_REASON_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES, - TASK_IDENTITY_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, - TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, - TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, - TaskState, TaskSupportIds, + TASK_IDENTITY_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS, + TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, + TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome, + TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, + TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskSupportIds, }; pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget}; pub use crate::tool_quality::{ @@ -204,12 +207,15 @@ pub use product_event::{ pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion}; pub use task::{ - TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, - TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, - TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, - TaskError, TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, - TaskRecord, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys, - redact_task_envelope, sensitive_task_key, validate_task_envelope, + AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase, + AttemptResolution, BeginAttempt, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, + TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_ATTEMPTS, + TASK_MAX_BACKOFF_STEP_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, + TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, + TaskCancelOutcome, TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, + TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, + TaskStore, TaskSupportIds, contains_sensitive_task_keys, redact_task_envelope, + sensitive_task_key, validate_task_envelope, }; pub use tool_catalog::{ RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError, diff --git a/crates/crank-core/src/task/attempt.rs b/crates/crank-core/src/task/attempt.rs new file mode 100644 index 0000000..7d46bb7 --- /dev/null +++ b/crates/crank-core/src/task/attempt.rs @@ -0,0 +1,285 @@ +//! Durable attempt, retry-snapshot and cancellation vocabulary of a Task (Story 4.4). + +use serde::{Deserialize, Serialize}; + +use super::{TASK_IDENTITY_MAX_BYTES, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskRecord}; + +/// Upper bound of the retry budget a Task may pin at acceptance (Story 4.4). +pub const TASK_MAX_ATTEMPTS: u32 = 5; +/// Upper bound of the pinned backoff step between two durable attempts (Story 4.4). +pub const TASK_MAX_BACKOFF_STEP_MS: i64 = 1_000; +/// Bound of a stored cancellation reason. +pub const TASK_CANCEL_REASON_MAX_BYTES: usize = 160; + +/// Immutable retry snapshot of one accepted Task. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TaskRetryPolicy { + /// Total durable outbound attempts (1 = a single attempt, never retried). + pub max_attempts: u32, + /// Pinned base backoff between two attempts; the delay grows by this step per attempt. + pub backoff_step_ms: i64, +} + +impl TaskRetryPolicy { + /// A Task that may be dispatched exactly once (legacy Tasks and unverified writes). + pub const fn single_attempt() -> Self { + Self { + max_attempts: 1, + backoff_step_ms: 0, + } + } + + pub fn new(max_attempts: u32, backoff_step_ms: i64) -> Self { + Self { + max_attempts, + backoff_step_ms, + } + } + + pub fn validate(&self) -> Result<(), TaskError> { + if !(1..=TASK_MAX_ATTEMPTS).contains(&self.max_attempts) { + return Err(TaskError::invalid("task retry budget is invalid")); + } + if !(0..=TASK_MAX_BACKOFF_STEP_MS).contains(&self.backoff_step_ms) { + return Err(TaskError::invalid("task retry backoff is invalid")); + } + Ok(()) + } + + /// Delay before attempt number `next_ordinal` (2..=max_attempts), capped so a pinned snapshot can + /// never stretch the accepted execution deadline on its own. + pub fn backoff_before(&self, next_ordinal: u32) -> i64 { + let steps = i64::from(next_ordinal.saturating_sub(1)).max(1); + self.backoff_step_ms + .saturating_mul(steps) + .min(self.backoff_step_ms.saturating_mul(4)) + } +} + +/// Certainty of one durable attempt outcome. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AttemptCertainty { + /// The outcome of the outbound operation is proven (success or a definite failure). + Known, + /// The operation may have been applied upstream; the effect cannot be proven locally. + Unknown, +} + +impl AttemptCertainty { + pub const fn as_str(self) -> &'static str { + match self { + Self::Known => "known", + Self::Unknown => "unknown", + } + } + + pub fn parse(value: &str) -> Result { + match value { + "known" => Ok(Self::Known), + "unknown" => Ok(Self::Unknown), + _ => Err(TaskError::invalid("unknown attempt certainty")), + } + } +} + +/// Phase of one durable attempt at the moment it was resolved. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AttemptPhase { + /// The durable checkpoint exists but no outbound send was observed. + BeforeDispatch, + /// The request was handed to the transport; its effect may exist upstream. + Dispatched, +} + +impl AttemptPhase { + pub const fn as_str(self) -> &'static str { + match self { + Self::BeforeDispatch => "before_dispatch", + Self::Dispatched => "dispatched", + } + } + + pub fn parse(value: &str) -> Result { + match value { + "before_dispatch" => Ok(Self::BeforeDispatch), + "dispatched" => Ok(Self::Dispatched), + _ => Err(TaskError::invalid("unknown attempt phase")), + } + } +} + +/// Closed outcome vocabulary of one resolved attempt. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AttemptOutcomeClass { + Completed, + Failed, + /// A bounded further attempt was scheduled instead of a terminal outcome. + RetryScheduled, + OutcomeUnknown, +} + +impl AttemptOutcomeClass { + pub const fn as_str(self) -> &'static str { + match self { + Self::Completed => "completed", + Self::Failed => "failed", + Self::RetryScheduled => "retry_scheduled", + Self::OutcomeUnknown => "outcome_unknown", + } + } + + pub fn parse(value: &str) -> Result { + match value { + "completed" => Ok(Self::Completed), + "failed" => Ok(Self::Failed), + "retry_scheduled" => Ok(Self::RetryScheduled), + "outcome_unknown" => Ok(Self::OutcomeUnknown), + _ => Err(TaskError::invalid("unknown attempt outcome")), + } + } +} + +/// One durable outbound attempt. The row is created by the dispatch checkpoint before any network +/// use and becomes immutable once resolved, so a stale worker can never append to the history. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskAttempt { + pub attempt_id: String, + pub task_id: TaskId, + pub ordinal: u32, + pub owner: String, + pub fence: u64, + /// Canonical fingerprint of the sealed executable input this attempt was admitted for. + pub input_fingerprint: String, + /// Digest of the stable upstream idempotency key (never the raw key or headers). + pub upstream_key_digest: Option, + /// Whether a retry of this attempt is safe (read or server-verified idempotent write). + pub retry_allowed: bool, + pub checkpoint_at_unix_ms: i64, + pub resolved_at_unix_ms: Option, + pub outcome: Option, + pub certainty: Option, + pub phase: Option, +} + +/// Committed dispatch intent of one durable attempt (Story 4.4). The store owns the transaction: the +/// attempt row, the incremented budget and the new Task version commit together before the network is +/// touched, so an uncommitted checkpoint can never produce an outbound call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BeginAttempt { + pub task_id: TaskId, + pub owner: String, + pub fence: u64, + /// Version the worker read from the lease; a concurrent change rejects the checkpoint. + pub expected_version: u32, + pub attempt_id: String, + pub input_fingerprint: String, + pub upstream_key_digest: Option, + /// The runtime's decision that a retry of this attempt is safe; the store still enforces the + /// pinned budget and the absolute deadline. + pub retry_allowed: bool, +} + +impl BeginAttempt { + pub fn validate(&self) -> Result<(), TaskError> { + if self.owner.is_empty() || self.owner.len() > TASK_IDENTITY_MAX_BYTES { + return Err(TaskError::invalid("task attempt owner is invalid")); + } + if self.attempt_id.is_empty() || self.attempt_id.len() > TASK_IDENTITY_MAX_BYTES { + return Err(TaskError::invalid("task attempt identity is invalid")); + } + if self.input_fingerprint.len() != 64 { + return Err(TaskError::invalid("task attempt fingerprint is invalid")); + } + if let Some(digest) = self.upstream_key_digest.as_deref() + && (digest.is_empty() || digest.len() > TASK_IDENTITY_MAX_BYTES) + { + return Err(TaskError::invalid( + "task attempt upstream key digest is invalid", + )); + } + Ok(()) + } +} + +/// Leased, committed view of one in-flight attempt and the Task version it was admitted under. +#[derive(Clone, Debug, PartialEq)] +pub struct AttemptLease { + pub attempt: TaskAttempt, + pub task: TaskRecord, +} + +/// Typed outcome of one attempt, handed to the store for a single atomic resolution. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AttemptOutcome { + /// Proven success; the Task becomes `completed` with this result. + Completed(TaskEnvelope), + /// Proven failure. `retryable` only schedules another attempt when the store's pinned budget, + /// the attempt's safe-retry decision and the absolute deadline all still allow it; otherwise the + /// Task fails with this error. + Failed { + error: TaskEnvelope, + retryable: bool, + phase: AttemptPhase, + }, + /// The outbound effect cannot be proven. `repeatable` only schedules another bounded attempt when + /// repeating it is safe and the pinned budget/deadline still allow it; otherwise the Task becomes + /// the non-terminal `outcome_unknown`, never a plain failure. + Uncertain { + error: TaskEnvelope, + repeatable: bool, + phase: AttemptPhase, + }, +} + +/// Single-winner result of resolving one attempt. +#[derive(Clone, Debug, PartialEq)] +pub enum AttemptResolution { + /// A terminal transition committed (completed or failed). + Finished(TaskRecord), + /// The attempt failed in a known way and another bounded attempt was armed. + RetryAt { + task: TaskRecord, + next_attempt_at_unix_ms: i64, + }, + /// The attempt resolved to the non-terminal uncertain state. + Unknown(TaskRecord), +} + +/// Owner-side cancellation request. `expected_version` is optional so a repeated cancel of the same +/// accepted Task stays idempotent while a stale version is reported as a conflict. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskCancelRequest { + pub access: TaskAccess, + pub task_id: TaskId, + pub expected_version: Option, + pub reason: String, +} + +impl TaskCancelRequest { + pub fn validate(&self) -> Result<(), TaskError> { + self.access.validate()?; + if self.reason.is_empty() + || self.reason.len() > TASK_CANCEL_REASON_MAX_BYTES + || self.reason.bytes().any(|byte| byte.is_ascii_control()) + { + return Err(TaskError::invalid("task cancellation reason is invalid")); + } + Ok(()) + } +} + +/// Typed result of an owner-scoped cancellation. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskCancelOutcome { + /// Cancelled before the dispatch checkpoint: no outbound call can have happened. + Cancelled(TaskRecord), + /// The Task already had a checkpointed attempt: non-terminal uncertain, never a plain cancel. + OutcomeUnknown(TaskRecord), + /// The Task was already in a terminal state; its accepted result is unchanged. + AlreadyTerminal(TaskRecord), + /// The caller's expected version is stale; `current` is the authorized current state. + Conflict(TaskRecord), +} diff --git a/crates/crank-core/src/task.rs b/crates/crank-core/src/task/mod.rs similarity index 88% rename from crates/crank-core/src/task.rs rename to crates/crank-core/src/task/mod.rs index 813514c..5c84482 100644 --- a/crates/crank-core/src/task.rs +++ b/crates/crank-core/src/task/mod.rs @@ -9,7 +9,9 @@ //! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a //! Task was executed. A terminal state is written by a conditional transition only. +mod attempt; use async_trait::async_trait; +pub use attempt::*; use serde::{Deserialize, Serialize}; use serde_json::Value; use uuid::Uuid; @@ -127,9 +129,9 @@ impl TaskState { | Self::Cancelled ), Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled), - Self::OutcomeUnknown => { - matches!(next, Self::Completed | Self::Failed | Self::Cancelled) - } + // Story 4.4: an uncertain outbound outcome is never resolved by a generic transition. + // Reconciliation is Story 4.6, so no ordinary finish/cancel/retry may leave this state. + Self::OutcomeUnknown => false, Self::Completed | Self::Failed | Self::Cancelled => false, } } @@ -206,6 +208,9 @@ pub struct TaskPins { pub retain_until_unix_ms: i64, /// Verified approval evidence reference (identifier only, never a bearer token). pub approval_reference: Option, + /// Bounded retry budget pinned at acceptance (Story 4.4). A later publication, a restart or a + /// new binary never changes the numbers an accepted Task was admitted with. + pub retry: TaskRetryPolicy, } impl TaskPins { @@ -233,6 +238,7 @@ impl TaskPins { { return Err(TaskError::invalid("task approval reference is invalid")); } + self.retry.validate()?; Ok(()) } } @@ -534,6 +540,12 @@ pub struct TaskRecord { pub current_attempt_id: Option, /// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed. pub dispatch_started_at_unix_ms: Option, + /// Earliest moment the next durable attempt may be dispatched (pinned backoff). + pub next_attempt_at_unix_ms: Option, + /// Durable cancellation request that arrived while an attempt was already checkpointed. + pub cancel_requested_at_unix_ms: Option, + /// Bounded reason of the cancellation request. + pub cancel_reason: Option, /// Monotonic lifecycle version used by the terminal CAS. pub version: u32, pub created_at_unix_ms: i64, @@ -688,14 +700,28 @@ pub trait TaskStore: Send + Sync { progress: TaskEnvelope, ) -> Result; - /// Commits an outbound attempt identity and dispatch timestamp before network use. - async fn checkpoint_dispatch( + /// Commits one durable outbound attempt before network use: attempt row, incremented budget and + /// the new Task version commit in one transaction. A returned lease is the only authority to + /// dispatch that attempt; an unknown commit means zero outbound calls. + async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result; + + /// Resolves exactly one committed attempt under its lease/fence/version. Only the current attempt + /// of the current owner may be resolved, and the attempt becomes immutable afterwards. + async fn resolve_attempt( &self, - owner: &str, + lease: &AttemptLease, + outcome: AttemptOutcome, + ) -> Result; + + /// Owner-scoped atomic cancellation. It serializes with claim and checkpoint on the same Task. + async fn cancel(&self, request: TaskCancelRequest) -> Result; + + /// Owner-scoped bounded history of the durable attempts of one Task. + async fn attempts( + &self, + access: &TaskAccess, task_id: TaskId, - fence: u64, - attempt_id: String, - ) -> Result; + ) -> Result, TaskError>; /// Applies a terminal transition only under a valid lease and releases intent atomically. async fn finish_leased( @@ -704,6 +730,11 @@ pub trait TaskStore: Send + Sync { owner: &str, fence: u64, ) -> Result; + + /// Resolves a leased Task whose next attempt was refused before its dispatch checkpoint (elapsed + /// deadline, rejected authority). A Task that still carries a proven ambiguity is never downgraded + /// to a plain failure: it ends non-terminal `outcome_unknown` with `code`; otherwise `failed`. + async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result; } #[cfg(test)] @@ -726,6 +757,7 @@ mod tests { execution_deadline_unix_ms: 1_790_000_000_000, retain_until_unix_ms: 1_790_000_000_000 + 1_000, approval_reference: Some("appr_1".to_owned()), + retry: TaskRetryPolicy::new(3, 1), }, scoped_key: "a".repeat(64), input_fingerprint: "b".repeat(64), @@ -840,6 +872,56 @@ mod tests { assert!(empty_key_reference.validate().is_err()); } + #[test] + fn outcome_unknown_has_no_generic_successor() { + // Story 4.4: reconciliation of an uncertain outcome is Story 4.6. Until then no ordinary + // finish/cancel/retry may leave the non-terminal uncertain state. + for next in [ + TaskState::Queued, + TaskState::Working, + TaskState::InputRequired, + TaskState::OutcomeUnknown, + TaskState::Completed, + TaskState::Failed, + TaskState::Cancelled, + ] { + assert!( + !TaskState::OutcomeUnknown.allows_transition_to(next), + "outcome_unknown must not transition to {next:?}" + ); + } + assert!(!TaskState::OutcomeUnknown.is_terminal()); + } + + #[test] + fn retry_policy_bounds_are_pinned_and_bounded() { + assert!(TaskRetryPolicy::single_attempt().validate().is_ok()); + assert!( + TaskRetryPolicy::new(TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS) + .validate() + .is_ok() + ); + assert!(TaskRetryPolicy::new(0, 0).validate().is_err()); + assert!( + TaskRetryPolicy::new(TASK_MAX_ATTEMPTS + 1, 0) + .validate() + .is_err() + ); + assert!( + TaskRetryPolicy::new(1, TASK_MAX_BACKOFF_STEP_MS + 1) + .validate() + .is_err() + ); + // The pinned step grows the delay but never beyond four steps. + assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(2), 2); + assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(5), 8); + assert_eq!(TaskRetryPolicy::single_attempt().backoff_before(2), 0); + // A pin whose retry snapshot is invalid is rejected at acceptance. + let mut invalid = request(); + invalid.pins.retry = TaskRetryPolicy::new(0, 0); + assert!(invalid.validate().is_err()); + } + #[test] fn finish_requires_a_terminal_reachable_transition() { let access = TaskAccess { diff --git a/crates/crank-registry/src/migrations.rs b/crates/crank-registry/src/migrations.rs index 8e7ad1b..2274d69 100644 --- a/crates/crank-registry/src/migrations.rs +++ b/crates/crank-registry/src/migrations.rs @@ -10,6 +10,7 @@ mod durable_tasks_v25; mod durable_tasks_v26; mod durable_tasks_v27; mod durable_tasks_v28; +mod durable_tasks_v29; mod execution_admission_v24; mod execution_outcome_v5; mod import_plan_v14; @@ -45,6 +46,7 @@ mod schema_guard_v25; mod schema_guard_v26; mod schema_guard_v27; mod schema_guard_v28; +mod schema_guard_v29; mod schema_guard_v7; mod schema_guard_v8; mod schema_guard_v9; diff --git a/crates/crank-registry/src/migrations/authority.rs b/crates/crank-registry/src/migrations/authority.rs index abb4594..2ec20a0 100644 --- a/crates/crank-registry/src/migrations/authority.rs +++ b/crates/crank-registry/src/migrations/authority.rs @@ -8,6 +8,7 @@ use super::durable_tasks_v25; use super::durable_tasks_v26; use super::durable_tasks_v27; use super::durable_tasks_v28; +use super::durable_tasks_v29; use super::execution_admission_v24; use super::execution_outcome_v5; use super::import_plan_v14; @@ -44,10 +45,10 @@ use preflight::*; use contract::baseline_source_digest; use contract::validate_descriptors; const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947; -const CURRENT_VERSION: i64 = 28; +const CURRENT_VERSION: i64 = 29; const IMPLEMENTED_VERSIONS: &[i64] = &[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, + 27, 28, 29, ]; const BASELINE_SOURCE_SHA256: &str = "eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675"; @@ -366,6 +367,12 @@ impl MigrationAuthority { durable_tasks_v28::SOURCE_SHA256, 27, ), + expand_descriptor( + 29, + durable_tasks_v29::NAME, + durable_tasks_v29::SOURCE_SHA256, + 28, + ), ] } pub fn validate_sequence() -> Result<(), MigrationError> { @@ -540,6 +547,9 @@ impl MigrationAuthority { if from < 28 { durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?; } + if from < 29 { + durable_tasks_v29::apply(&mut transaction, &Self::sequence()[28]).await?; + } match inspect(&mut transaction).await? { MigrationPreflight::Current { version: CURRENT_VERSION, diff --git a/crates/crank-registry/src/migrations/durable_tasks_v29.rs b/crates/crank-registry/src/migrations/durable_tasks_v29.rs new file mode 100644 index 0000000..b226cdf --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v29.rs @@ -0,0 +1,45 @@ +use sqlx::{Postgres, Transaction, query}; + +use super::authority::{MigrationDescriptor, MigrationError}; + +pub(super) const NAME: &str = "durable-tasks-v29"; +pub(super) const SOURCE: &str = include_str!("durable_tasks_v29.sql"); +pub(super) const SOURCE_SHA256: &str = + "0666c7d2aa440e6aae2170ceaf2fce2549118612e1505ef78817be60436a75dd"; + +pub(super) async fn apply( + transaction: &mut Transaction<'_, Postgres>, + descriptor: &MigrationDescriptor, +) -> Result<(), MigrationError> { + sqlx::raw_sql(SOURCE) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v29", + Some(29), + "restore_known_good_backup", + ) + })?; + query( + "insert into __crank_migrations (version, name, checksum, phase, compatibility) + values ($1, $2, $3, $4, $5)", + ) + .bind(descriptor.version) + .bind(descriptor.name) + .bind(&descriptor.checksum) + .bind(descriptor.phase) + .bind(descriptor.compatibility) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v29", + Some(29), + "restore_known_good_backup", + ) + })?; + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/durable_tasks_v29.sql b/crates/crank-registry/src/migrations/durable_tasks_v29.sql new file mode 100644 index 0000000..d9a30ac --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v29.sql @@ -0,0 +1,52 @@ +-- V29: durable per-outbound attempt history, pinned retry policy and cancellation (Story 4.4). +-- An accepted Task keeps the retry snapshot it was admitted with; legacy Tasks get a single attempt. +alter table tasks add column if not exists max_attempts integer not null default 1; +alter table tasks add column if not exists retry_backoff_step_ms bigint not null default 0; +alter table tasks add column if not exists next_attempt_at timestamptz; +alter table tasks add column if not exists cancel_requested_at timestamptz; +alter table tasks add column if not exists cancel_reason text; + +alter table tasks add constraint tasks_max_attempts_check check (max_attempts between 1 and 5); +alter table tasks add constraint tasks_retry_backoff_step_check + check (retry_backoff_step_ms between 0 and 1000); +alter table tasks add constraint tasks_cancel_reason_check check ( + cancel_reason is null or octet_length(cancel_reason) between 1 and 160 +); + +-- One row per committed outbound attempt. The authoritative outcome of a Task stays on `tasks`; +-- this history only records what was attempted and how each attempt resolved. +create table if not exists task_attempts ( + attempt_id text primary key, + task_id uuid not null references tasks (task_id) on delete cascade, + ordinal integer not null, + owner text not null, + fence bigint not null default 0, + input_fingerprint text not null, + upstream_key_digest text, + retry_allowed boolean not null default false, + checkpoint_at timestamptz not null default now(), + resolved_at timestamptz, + outcome text, + certainty text, + phase text, + constraint task_attempts_task_ordinal_unique unique (task_id, ordinal), + constraint task_attempts_ordinal_check check (ordinal between 1 and 5), + constraint task_attempts_outcome_check check ( + outcome is null + or outcome in ('completed', 'failed', 'retry_scheduled', 'outcome_unknown') + ), + constraint task_attempts_certainty_check check ( + certainty is null or certainty in ('known', 'unknown') + ), + constraint task_attempts_phase_check check ( + phase is null or phase in ('before_dispatch', 'dispatched') + ), + -- A resolved attempt is complete: an unresolved row carries no resolution at all, so a stale + -- writer can never append a second outcome to the same attempt. + constraint task_attempts_resolution_check check ( + (resolved_at is null and outcome is null and certainty is null and phase is null) + or (resolved_at is not null and outcome is not null and certainty is not null) + ) +); + +create index if not exists task_attempts_task_id_idx on task_attempts (task_id, ordinal); diff --git a/crates/crank-registry/src/migrations/schema_guard.rs b/crates/crank-registry/src/migrations/schema_guard.rs index d5df30e..2937375 100644 --- a/crates/crank-registry/src/migrations/schema_guard.rs +++ b/crates/crank-registry/src/migrations/schema_guard.rs @@ -489,6 +489,9 @@ pub(super) async fn validate_schema_fingerprint( if !super::schema_guard_v28::validate_v28_absent(connection).await? { return Err(schema_error(current_version)); } + if !super::schema_guard_v29::validate_v29_absent(connection).await? { + return Err(schema_error(current_version)); + } } else if current_version < 26 { super::schema_guard_v25::validate_v25_durable_tasks(connection).await?; if !super::schema_guard_v26::validate_v26_absent(connection).await? { @@ -497,6 +500,9 @@ pub(super) async fn validate_schema_fingerprint( if !super::schema_guard_v28::validate_v28_absent(connection).await? { return Err(schema_error(current_version)); } + if !super::schema_guard_v29::validate_v29_absent(connection).await? { + return Err(schema_error(current_version)); + } } else if current_version < 27 { super::schema_guard_v26::validate_v26_durable_tasks(connection).await?; if !super::schema_guard_v27::validate_v27_absent(connection).await? { @@ -505,15 +511,27 @@ pub(super) async fn validate_schema_fingerprint( if !super::schema_guard_v28::validate_v28_absent(connection).await? { return Err(schema_error(current_version)); } + if !super::schema_guard_v29::validate_v29_absent(connection).await? { + return Err(schema_error(current_version)); + } } else if current_version < 28 { super::schema_guard_v26::validate_v26_base_columns(connection).await?; super::schema_guard_v27::validate_v27_durable_tasks(connection).await?; if !super::schema_guard_v28::validate_v28_absent(connection).await? { return Err(schema_error(current_version)); } - } else { + if !super::schema_guard_v29::validate_v29_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else if current_version < 29 { super::schema_guard_v26::validate_v26_base_columns(connection).await?; super::schema_guard_v28::validate_v28_durable_tasks(connection).await?; + if !super::schema_guard_v29::validate_v29_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else { + super::schema_guard_v26::validate_v26_base_columns(connection).await?; + super::schema_guard_v29::validate_v29_durable_attempts(connection).await?; } Ok(()) } diff --git a/crates/crank-registry/src/migrations/schema_guard_v27.rs b/crates/crank-registry/src/migrations/schema_guard_v27.rs index 04638af..9495d03 100644 --- a/crates/crank-registry/src/migrations/schema_guard_v27.rs +++ b/crates/crank-registry/src/migrations/schema_guard_v27.rs @@ -73,7 +73,7 @@ pub(super) async fn validate_v27_durable_tasks( .map_err(|_| MigrationError::storage("preflight.schema"))? .try_get("count") .map_err(|_| MigrationError::storage("preflight.schema"))?; - if count != i64::try_from(37).unwrap_or_default() { + if count != 37_i64 { return Err(MigrationError::new( "partial_sequence", "v27.columns", diff --git a/crates/crank-registry/src/migrations/schema_guard_v28.rs b/crates/crank-registry/src/migrations/schema_guard_v28.rs index bd529fd..a2ea063 100644 --- a/crates/crank-registry/src/migrations/schema_guard_v28.rs +++ b/crates/crank-registry/src/migrations/schema_guard_v28.rs @@ -10,10 +10,10 @@ const TASKS: &str = "tasks"; pub(super) async fn validate_v28_absent( connection: &mut PgConnection, ) -> Result { - if relation_exists(connection, TASKS).await? { - if super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await? { - return Ok(false); - } + if relation_exists(connection, TASKS).await? + && super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await? + { + return Ok(false); } Ok(true) } @@ -55,7 +55,7 @@ pub(super) async fn validate_v28_durable_tasks( .map_err(|_| MigrationError::storage("preflight.schema"))? .try_get("count") .map_err(|_| MigrationError::storage("preflight.schema"))?; - if count != i64::try_from(38).unwrap_or_default() { + if count != 38_i64 { return Err(MigrationError::new( "partial_sequence", "v28.columns", diff --git a/crates/crank-registry/src/migrations/schema_guard_v29.rs b/crates/crank-registry/src/migrations/schema_guard_v29.rs new file mode 100644 index 0000000..30e5859 --- /dev/null +++ b/crates/crank-registry/src/migrations/schema_guard_v29.rs @@ -0,0 +1,87 @@ +//! V29 schema guard: durable per-outbound attempt history, pinned retry policy and cancellation +//! (Story 4.4). + +use sqlx::{PgConnection, Row, query}; + +use super::authority::MigrationError; +use super::schema_guard::{column_exists, relation_exists}; + +const TASKS: &str = "tasks"; +const ATTEMPTS: &str = "task_attempts"; + +pub(super) async fn validate_v29_absent( + connection: &mut PgConnection, +) -> Result { + if relation_exists(connection, TASKS).await? { + if column_exists(connection, TASKS, "max_attempts").await? { + return Ok(false); + } + if relation_exists(connection, ATTEMPTS).await? { + return Ok(false); + } + } + Ok(true) +} +pub(super) async fn validate_v29_durable_attempts( + connection: &mut PgConnection, +) -> Result<(), MigrationError> { + // The V28 column set is still required and unchanged; V29 only appends columns and one table. + for column in [ + "platform_api_key_id", + "max_attempts", + "retry_backoff_step_ms", + "next_attempt_at", + "cancel_requested_at", + "cancel_reason", + ] { + if !column_exists(connection, TASKS, column).await? { + return Err(invalid(29, "v29.columns")); + } + } + // Exact total column count for version 29: the 38 V28 columns + 5 V29 columns. + let count: i64 = query( + "select count(*)::bigint as count from information_schema.columns + where table_schema = current_schema() and table_name = $1", + ) + .bind(TASKS) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get("count") + .map_err(|_| MigrationError::storage("preflight.schema"))?; + if count != 43_i64 { + return Err(invalid(29, "v29.columns")); + } + if !relation_exists(connection, ATTEMPTS).await? { + return Err(invalid(29, "v29.attempts")); + } + for column in [ + "attempt_id", + "task_id", + "ordinal", + "owner", + "fence", + "input_fingerprint", + "upstream_key_digest", + "retry_allowed", + "checkpoint_at", + "resolved_at", + "outcome", + "certainty", + "phase", + ] { + if !column_exists(connection, ATTEMPTS, column).await? { + return Err(invalid(29, "v29.attempts")); + } + } + Ok(()) +} + +fn invalid(version: i64, stage: &'static str) -> MigrationError { + MigrationError::new( + "partial_sequence", + stage, + Some(version), + "restore_known_good_backup", + ) +} diff --git a/crates/crank-registry/src/postgres/master_key_tasks.rs b/crates/crank-registry/src/postgres/master_key_tasks.rs index 81ba671..ba4aa33 100644 --- a/crates/crank-registry/src/postgres/master_key_tasks.rs +++ b/crates/crank-registry/src/postgres/master_key_tasks.rs @@ -69,6 +69,7 @@ impl PostgresRegistry { rows.into_iter().map(map_master_key_task_version).collect() } + #[allow(clippy::too_many_arguments)] pub async fn stage_master_key_rotation_task_ciphertext( &self, rotation_id: &str, diff --git a/crates/crank-registry/src/postgres/tasks/acceptance.rs b/crates/crank-registry/src/postgres/tasks/acceptance.rs new file mode 100644 index 0000000..a2c4ddf --- /dev/null +++ b/crates/crank-registry/src/postgres/tasks/acceptance.rs @@ -0,0 +1,195 @@ +//! Acceptance half of the PostgreSQL Task store (Story 4.2/4.3): the one-transaction acceptance +//! path, its bounded unique-conflict reread and the typed miss classification. + +use crank_core::{ + AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback, + PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskError, TaskFinishRequest, TaskState, +}; +use sqlx::{Postgres, Row, Transaction}; + +use super::super::admission::{ + AttemptError, bounded_queue_len, cleanup_expired_unlinked, insert_admission_entry, + lock_admission_in_tx, +}; +use super::{ + Accepted, PostgresTaskStore, TASK_PROJECTION, acceptance_from_record, policy_mode_str, + record_from_row, task_sql_error, +}; + +impl PostgresTaskStore { + pub(super) async fn accept_locked( + &self, + transaction: &mut Transaction<'_, Postgres>, + request: &TaskAcceptRequest, + ) -> Result { + // Replay first: an already accepted Task is returned without capacity mutation, so it works + // with a full queue and after the original execution deadline while the record is retained. + if let Some((acceptance, fingerprint)) = + self.stored_acceptance(transaction, request).await? + { + return Ok(if fingerprint == request.input_fingerprint { + Accepted::Replayed(acceptance) + } else { + Accepted::Conflict + }); + } + + // Serialize with synchronous admission and any concurrent acceptance of the same scope. + lock_admission_in_tx(transaction, &request.workspace_id).await?; + + // The competitor may have committed while this transaction waited for the controller lock. + if let Some((acceptance, fingerprint)) = + self.stored_acceptance(transaction, request).await? + { + return Ok(if fingerprint == request.input_fingerprint { + Accepted::Replayed(acceptance) + } else { + Accepted::Conflict + }); + } + + cleanup_expired_unlinked(transaction).await?; + let queued = bounded_queue_len(transaction).await?; + if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) { + return Err(AttemptError::Overloaded); + } + + let created_ms: i64 = sqlx::query( + "insert into tasks ( + task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id, + operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent, + trace_sampled, policy_mode, approval_reference, state, input_envelope, + execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version, + executable_input_master_key_epoch, platform_api_key_id, max_attempts, retry_backoff_step_ms + ) values ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16, + to_timestamp($17::double precision / 1000.0), + to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22, $23, $24 + ) + returning (extract(epoch from created_at) * 1000)::bigint as created_ms", + ) + .bind(request.task_id.as_uuid()) + .bind(&request.workspace_id) + .bind(request.owner_kind.as_str()) + .bind(&request.owner_id) + .bind(&request.authorization_scope) + .bind(&request.operation_id) + .bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX)) + .bind(&request.scoped_key) + .bind(&request.input_fingerprint) + .bind(request.support.request_id.as_str()) + .bind(request.support.trace_id.as_str()) + .bind(&request.support.traceparent) + .bind(request.support.trace_sampled) + .bind(policy_mode_str(request.pins.policy_mode)) + .bind(request.pins.approval_reference.as_deref()) + .bind(request.input.value().clone()) + .bind(request.pins.execution_deadline_unix_ms) + .bind(request.pins.retain_until_unix_ms) + .bind(request.executable_input_ciphertext.as_deref()) + .bind(request.executable_input_key_version.as_deref()) + .bind(request.executable_input_master_key_epoch) + .bind( + request + .platform_api_key_id + .as_ref() + .map(PlatformApiKeyId::as_str), + ) + .bind(i32::try_from(request.pins.retry.max_attempts).unwrap_or(1)) + .bind(request.pins.retry.backoff_step_ms) + .fetch_one(&mut **transaction) + .await + .map_err(task_sql_error)? + .try_get("created_ms") + .map_err(AttemptError::from_sql)?; + + let admission_request = AdmissionRequest { + scope: AdmissionScope { + workspace_id: request.workspace_id.clone(), + agent_key: request.owner_id.clone(), + operation_id: request.operation_id.clone(), + operation_version: request.pins.operation_version, + }, + mode: AdmissionMode::Background, + policy_mode: request.pins.policy_mode, + no_tasks_fallback: NoTasksFallback::Unsupported, + // Never reached: the intent row is inserted by this transaction, not by `admit`. + approval: ApprovalEvidence::NotRequired, + fingerprint: request.input_fingerprint.clone(), + expires_at_unix_ms: request.pins.execution_deadline_unix_ms, + }; + let owner_token = format!("task_{}", request.task_id); + insert_admission_entry( + transaction, + &admission_request, + &owner_token, + "reserved", + Some(request.task_id.as_uuid()), + ) + .await?; + + Ok(Accepted::Created(TaskAcceptance { + task_id: request.task_id, + state: TaskState::Queued, + operation_version: request.pins.operation_version, + support: request.support.clone(), + created_at_unix_ms: created_ms, + retain_until_unix_ms: request.pins.retain_until_unix_ms, + replayed: false, + })) + } + + /// Bounded scoped reread after a foreign unique conflict aborted the transaction. + pub(super) async fn reread_after_unique( + &self, + request: &TaskAcceptRequest, + ) -> Result { + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks + where workspace_id = $1 and owner_kind = $2 and owner_id = $3 + and authorization_scope = $4 and operation_id = $5 and scoped_key = $6" + ))) + .bind(&request.workspace_id) + .bind(request.owner_kind.as_str()) + .bind(&request.owner_id) + .bind(&request.authorization_scope) + .bind(&request.operation_id) + .bind(&request.scoped_key) + .fetch_optional(&self.pool) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + return Err(TaskError::unavailable( + "durable task acceptance ended with an unresolved unique conflict", + )); + }; + let record = record_from_row(&row)?; + if record.input_fingerprint != request.input_fingerprint { + return Err(TaskError::FingerprintConflict); + } + Ok(acceptance_from_record(&record)) + } + + /// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are + /// reported without payload, and a foreign Task is indistinguishable from an absent one. + pub(super) async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError { + let access = &request.access; + match sqlx::query( + "select state, version from tasks + where task_id = $1 and workspace_id = $2 and owner_kind = $3 + and owner_id = $4 and authorization_scope = $5", + ) + .bind(request.task_id.as_uuid()) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .fetch_optional(&self.pool) + .await + { + Ok(Some(_)) => TaskError::StateConflict, + Ok(None) => TaskError::NotFound, + Err(error) => Self::unavailable(error), + } + } +} diff --git a/crates/crank-registry/src/postgres/tasks/attempts.rs b/crates/crank-registry/src/postgres/tasks/attempts.rs new file mode 100644 index 0000000..b447c1e --- /dev/null +++ b/crates/crank-registry/src/postgres/tasks/attempts.rs @@ -0,0 +1,583 @@ +//! Durable per-outbound attempt persistence of a Task (Story 4.4): the dispatch checkpoint, the +//! single-winner attempt resolution and the owner-scoped cancellation live here so the acceptance +//! and retention half of the store stays readable. + +use crank_core::{ + AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase, + AttemptResolution, BeginAttempt, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest, + TaskEnvelope, TaskError, TaskId, TaskRecord, TaskState, +}; +use sqlx::Row; +use uuid::Uuid; + +use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row, unavailable_row}; + +const ATTEMPT_PROJECTION: &str = "attempt_id, task_id, ordinal, owner, fence, input_fingerprint, + upstream_key_digest, retry_allowed, + (extract(epoch from checkpoint_at) * 1000)::bigint as checkpoint_at_ms, + (extract(epoch from resolved_at) * 1000)::bigint as resolved_at_ms, + outcome, certainty, phase"; + +/// Bounded attempt-resolution helper: `retry` may only be armed when the attempt itself is marked +/// safe-to-retry, the pinned budget still has room and the accepted absolute deadline still does. +impl PostgresTaskStore { + fn retry_delay( + &self, + record: &TaskRecord, + lease: &AttemptLease, + safe: bool, + now: i64, + ) -> Option { + let delay = record.pins.retry.backoff_before(lease.attempt.ordinal + 1); + (safe + && lease.attempt.retry_allowed + && lease.attempt.ordinal < record.pins.retry.max_attempts + && now.saturating_add(delay) < record.pins.execution_deadline_unix_ms) + .then_some(delay) + } +} + +impl PostgresTaskStore { + /// Authoritative word on whether an earlier resolved attempt really reached upstream despite + /// the current one not having dispatched. Returns Some(phase) only when the latest resolved + /// attempt that actually dispatched (phase = 'dispatched') is still of unknown certainty. + pub(super) async fn unproven_dispatch_phase( + &self, + task_id: TaskId, + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result, TaskError> { + let row = sqlx::query( + "select certainty, phase from task_attempts where task_id = $1 and resolved_at is not null and phase = 'dispatched' order by ordinal desc limit 1", + ) + .bind(task_id.as_uuid()) + .fetch_optional(&mut **transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + return Ok(None); + }; + let certainty: Option = row.try_get("certainty").map_err(Self::unavailable)?; + if certainty.as_deref() != Some(AttemptCertainty::as_str(AttemptCertainty::Unknown)) { + return Ok(None); + } + let phase: Option = row.try_get("phase").map_err(Self::unavailable)?; + Ok(Some(phase.unwrap_or_else(|| "dispatched".to_owned()))) + } +} + +fn attempt_from_row(row: &sqlx::postgres::PgRow) -> Result { + let attempt_id: String = row.try_get("attempt_id").map_err(unavailable_row)?; + let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?; + let outcome: Option = row.try_get("outcome").map_err(unavailable_row)?; + let certainty: Option = row.try_get("certainty").map_err(unavailable_row)?; + let phase: Option = row.try_get("phase").map_err(unavailable_row)?; + Ok(TaskAttempt { + attempt_id, + task_id: TaskId::parse(&task_id.to_string())?, + ordinal: u32::try_from(row.try_get::("ordinal").map_err(unavailable_row)?) + .unwrap_or_default(), + owner: row.try_get("owner").map_err(unavailable_row)?, + fence: u64::try_from(row.try_get::("fence").map_err(unavailable_row)?) + .unwrap_or_default(), + input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?, + upstream_key_digest: row + .try_get("upstream_key_digest") + .map_err(unavailable_row)?, + retry_allowed: row.try_get("retry_allowed").map_err(unavailable_row)?, + checkpoint_at_unix_ms: row.try_get("checkpoint_at_ms").map_err(unavailable_row)?, + resolved_at_unix_ms: row.try_get("resolved_at_ms").map_err(unavailable_row)?, + outcome: outcome + .as_deref() + .map(AttemptOutcomeClass::parse) + .transpose()?, + certainty: certainty + .as_deref() + .map(AttemptCertainty::parse) + .transpose()?, + phase: phase.as_deref().map(AttemptPhase::parse).transpose()?, + }) +} + +impl PostgresTaskStore { + pub(super) async fn checkpoint_dispatch_attempt( + &self, + request: BeginAttempt, + ) -> Result { + request.validate()?; + let mut transaction = self.start().await?; + // One row lock serializes this checkpoint with claim, cancellation and the resolution of the + // same Task, so a cancel that wins first can never be followed by an outbound call. + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks where task_id = $1 for update" + ))) + .bind(request.task_id.as_uuid()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::NotFound); + }; + let record = record_from_row(&row)?; + if record.input_fingerprint != request.input_fingerprint { + transaction.rollback().await.ok(); + return Err(TaskError::FingerprintConflict); + } + let now = crank_core::current_unix_ms(); + let eligible = record.state == TaskState::Working + && record.lease_owner.as_deref() == Some(request.owner.as_str()) + && record.fence == request.fence + && record.version == request.expected_version + && record.current_attempt_id.is_none() + && record.cancel_requested_at_unix_ms.is_none() + && record.lease_until_unix_ms.is_some_and(|until| until > now) + && record.attempts < record.pins.retry.max_attempts + && record.pins.execution_deadline_unix_ms > now + && record.next_attempt_at_unix_ms.is_none_or(|at| at <= now); + if !eligible { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + } + let ordinal = record.attempts.saturating_add(1); + sqlx::query( + "insert into task_attempts ( + attempt_id, task_id, ordinal, owner, fence, input_fingerprint, + upstream_key_digest, retry_allowed + ) values ($1, $2, $3, $4, $5, $6, $7, $8)", + ) + .bind(&request.attempt_id) + .bind(request.task_id.as_uuid()) + .bind(i32::try_from(ordinal).unwrap_or(i32::MAX)) + .bind(&request.owner) + .bind(i64::try_from(request.fence).unwrap_or_default()) + .bind(&request.input_fingerprint) + .bind(request.upstream_key_digest.as_deref()) + .bind(request.retry_allowed) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set current_attempt_id = $1, + dispatch_started_at = now(), + attempts = attempts + 1, + next_attempt_at = null, + version = version + 1, + updated_at = now() + where task_id = $2 and current_attempt_id is null and version = $3 + returning {TASK_PROJECTION}" + ))) + .bind(&request.attempt_id) + .bind(request.task_id.as_uuid()) + .bind(i64::from(record.version)) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let task = record_from_row(&row)?; + transaction.commit().await.map_err(Self::unavailable)?; + let attempt = TaskAttempt { + attempt_id: request.attempt_id, + task_id: request.task_id, + ordinal, + owner: request.owner, + fence: request.fence, + input_fingerprint: request.input_fingerprint, + upstream_key_digest: request.upstream_key_digest, + retry_allowed: request.retry_allowed, + checkpoint_at_unix_ms: task.dispatch_started_at_unix_ms.unwrap_or(now), + resolved_at_unix_ms: None, + outcome: None, + certainty: None, + phase: None, + }; + Ok(AttemptLease { attempt, task }) + } + + pub(super) async fn resolve_attempt_outcome( + &self, + lease: &AttemptLease, + outcome: AttemptOutcome, + ) -> Result { + let mut transaction = self.start().await?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks where task_id = $1 for update" + ))) + .bind(lease.attempt.task_id.as_uuid()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::NotFound); + }; + let record = record_from_row(&row)?; + let now = crank_core::current_unix_ms(); + let prior_uncertain_phase = self + .unproven_dispatch_phase(lease.attempt.task_id, &mut transaction) + .await?; + // Only the current attempt of the current owner, under the version it was admitted with, may + // be resolved. A stale worker can never append a second outcome to the history or the Task. + let current = record.state == TaskState::Working + && record.lease_owner.as_deref() == Some(lease.attempt.owner.as_str()) + && record.fence == lease.attempt.fence + && record.version == lease.task.version + && record.current_attempt_id.as_deref() == Some(lease.attempt.attempt_id.as_str()) + && record.lease_until_unix_ms.is_some_and(|until| until > now); + if !current { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + } + let (class, certainty, phase) = match &outcome { + AttemptOutcome::Completed(_) => ( + AttemptOutcomeClass::Completed, + AttemptCertainty::Known, + AttemptPhase::Dispatched, + ), + AttemptOutcome::Failed { phase, .. } => { + (AttemptOutcomeClass::Failed, AttemptCertainty::Known, *phase) + } + AttemptOutcome::Uncertain { phase, .. } => ( + AttemptOutcomeClass::OutcomeUnknown, + AttemptCertainty::Unknown, + *phase, + ), + }; + // A failed or uncertain attempt that may be safely repeated keeps the lease and arms the + // pinned backoff. Exhausted budget or deadline never turns uncertainty into a plain failure. + let retry = match &outcome { + AttemptOutcome::Failed { + retryable, + phase: _, + error: _, + } => self.retry_delay(&record, lease, *retryable, now), + AttemptOutcome::Uncertain { + repeatable, + phase: _, + error: _, + } => self.retry_delay(&record, lease, *repeatable, now), + AttemptOutcome::Completed(_) => None, + }; + let attempt_class = if retry.is_some() { + AttemptOutcomeClass::RetryScheduled + } else { + class + }; + let closed = sqlx::query( + "update task_attempts + set resolved_at = now(), outcome = $1, certainty = $2, phase = $3 + where attempt_id = $4 and resolved_at is null", + ) + .bind(attempt_class.as_str()) + .bind(certainty.as_str()) + .bind(phase.as_str()) + .bind(&lease.attempt.attempt_id) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + if closed.rows_affected() != 1 { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + } + let (state, result, error, keep_lease, next_attempt_at) = match outcome { + AttemptOutcome::Completed(result) => { + (TaskState::Completed, Some(result), None, false, None) + } + AttemptOutcome::Failed { + error, + retryable: _, + phase, + } => match retry { + Some(delay) => ( + TaskState::Working, + None, + None, + true, + Some(now.saturating_add(delay)), + ), + None => { + if phase == AttemptPhase::BeforeDispatch { + if let Some(prior) = prior_uncertain_phase.as_ref() { + let mut value = error.value().clone(); + if let serde_json::Value::Object(map) = &mut value { + map.insert( + "phase".to_string(), + serde_json::Value::String(prior.clone()), + ); + } + let replaced = TaskEnvelope::try_redacted(&value)?; + (TaskState::OutcomeUnknown, None, Some(replaced), false, None) + } else { + (TaskState::Failed, None, Some(error), false, None) + } + } else { + (TaskState::Failed, None, Some(error), false, None) + } + } + }, + AttemptOutcome::Uncertain { + error, + repeatable: _, + phase: _, + } => match retry { + Some(delay) => ( + TaskState::Working, + None, + None, + true, + Some(now.saturating_add(delay)), + ), + None => (TaskState::OutcomeUnknown, None, Some(error), false, None), + }, + }; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set state = $1, + result_envelope = $2, + error_envelope = $3, + lease_owner = case when $4 then lease_owner else null end, + lease_until = case when $4 then lease_until else null end, + -- A retry-scheduled attempt is already resolved, so it no longer holds the Task: + -- the next durable attempt needs its own checkpoint under the same lease. + current_attempt_id = case when $8 then null else current_attempt_id end, + next_attempt_at = case when $5::bigint is null then null + else to_timestamp($5::double precision / 1000.0) end, + version = version + 1, + updated_at = now() + where task_id = $6 and version = $7 + returning {TASK_PROJECTION}" + ))) + .bind(state.as_str()) + .bind(result.as_ref().map(|value| value.value().clone())) + .bind(error.as_ref().map(|value| value.value().clone())) + .bind(keep_lease) + .bind(next_attempt_at) + .bind(lease.attempt.task_id.as_uuid()) + .bind(i64::from(record.version)) + .bind(retry.is_some()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let task = record_from_row(&row)?; + if !keep_lease { + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(lease.attempt.task_id.as_uuid()) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + } + transaction.commit().await.map_err(Self::unavailable)?; + Ok(match retry { + Some(_) => AttemptResolution::RetryAt { + next_attempt_at_unix_ms: next_attempt_at.unwrap_or(now), + task, + }, + None if state == TaskState::OutcomeUnknown => AttemptResolution::Unknown(task), + None => AttemptResolution::Finished(task), + }) + } + + pub(super) async fn cancel_task( + &self, + request: TaskCancelRequest, + ) -> Result { + request.validate()?; + let access = &request.access; + let mut transaction = self.start().await?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks + where task_id = $1 and workspace_id = $2 and owner_kind = $3 + and owner_id = $4 and authorization_scope = $5 + for update" + ))) + .bind(request.task_id.as_uuid()) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::NotFound); + }; + let record = record_from_row(&row)?; + if let Some(expected) = request.expected_version + && record.version != expected + { + transaction.rollback().await.ok(); + return Ok(TaskCancelOutcome::Conflict(record)); + } + if record.state.is_terminal() { + transaction.rollback().await.ok(); + return Ok(TaskCancelOutcome::AlreadyTerminal(record)); + } + if record.state == TaskState::OutcomeUnknown { + transaction.rollback().await.ok(); + return Ok(TaskCancelOutcome::OutcomeUnknown(record)); + } + // A checkpointed attempt may already have reached the upstream: the honest answer is the + // non-terminal uncertain state, never a plain cancellation and never a retry. + let checkpointed = record.current_attempt_id.clone(); + // Preserve an uncertainty recorded by a previously resolved attempt even after RetryAt + // cleared current_attempt_id: the request already reached the upstream in that case. + let prior_uncertain = self + .unproven_dispatch_phase(request.task_id, &mut transaction) + .await? + .is_some(); + let (state, code) = match (&checkpointed, prior_uncertain) { + (Some(_), _) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"), + (None, true) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"), + (None, false) => (TaskState::Cancelled, "cancelled_before_dispatch"), + }; + if let Some(attempt_id) = checkpointed { + let closed = sqlx::query( + "update task_attempts + set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown', + phase = 'dispatched' + where task_id = $1 and attempt_id = $2 and resolved_at is null", + ) + .bind(request.task_id.as_uuid()) + .bind(&attempt_id) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let _ = closed; + } + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set state = $1, + error_envelope = jsonb_build_object('code', $2), + cancel_requested_at = now(), + cancel_reason = $3, + lease_owner = null, + lease_until = null, + version = version + 1, + updated_at = now() + where task_id = $4 and version = $5 + returning {TASK_PROJECTION}" + ))) + .bind(state.as_str()) + .bind(code) + .bind(&request.reason) + .bind(request.task_id.as_uuid()) + .bind(i64::from(record.version)) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let task = record_from_row(&row)?; + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(request.task_id.as_uuid()) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(match state { + TaskState::Cancelled => TaskCancelOutcome::Cancelled(task), + _ => TaskCancelOutcome::OutcomeUnknown(task), + }) + } + + pub(super) async fn refuse_attempt_resolution( + &self, + lease: &crank_core::TaskLease, + code: &str, + ) -> Result { + let mut transaction = self.start().await?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks\n where task_id = $1\n for update" + ))) + .bind(lease.record.task_id.as_uuid()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::NotFound); + }; + let record = record_from_row(&row).map_err(Self::unavailable)?; + let now = crank_core::current_unix_ms(); + let eligible = matches!(record.state, TaskState::Queued | TaskState::Working) + && record.lease_owner.as_deref() == Some(lease.owner.as_str()) + && record.fence == lease.fence + && record.version == lease.record.version + && record.current_attempt_id.is_none() + && record.lease_until_unix_ms.is_some_and(|until| until > now); + if !eligible { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + } + let prior_phase = self + .unproven_dispatch_phase(lease.record.task_id, &mut transaction) + .await?; + let uncertain = prior_phase.is_some(); + let state = if uncertain { + TaskState::OutcomeUnknown + } else { + TaskState::Failed + }; + let phase = prior_phase.unwrap_or_else(|| "before_dispatch".to_owned()); + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks\n set state = $1,\n error_envelope = jsonb_build_object('code', $2, 'phase', $3),\n lease_owner = null,\n lease_until = null,\n next_attempt_at = null,\n version = version + 1,\n updated_at = now()\n where task_id = $4 and version = $5\n returning {TASK_PROJECTION}" + ))) + .bind(state.as_str()) + .bind(code) + .bind(&phase) + .bind(lease.record.task_id.as_uuid()) + .bind(i64::from(record.version)) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let task = record_from_row(&row).map_err(Self::unavailable)?; + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(lease.record.task_id.as_uuid()) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(task) + } + + pub(super) async fn attempt_history( + &self, + access: &TaskAccess, + task_id: TaskId, + ) -> Result, TaskError> { + access.validate()?; + let rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {ATTEMPT_PROJECTION} from task_attempts + where task_id = $1 and exists ( + select 1 from tasks + where tasks.task_id = task_attempts.task_id and tasks.workspace_id = $2 + and tasks.owner_kind = $3 and tasks.owner_id = $4 + and tasks.authorization_scope = $5 + ) + order by ordinal" + ))) + .bind(task_id.as_uuid()) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .fetch_all(&self.pool) + .await + .map_err(Self::unavailable)?; + rows.iter().map(attempt_from_row).collect() + } +} diff --git a/crates/crank-registry/src/postgres/tasks.rs b/crates/crank-registry/src/postgres/tasks/mod.rs similarity index 74% rename from crates/crank-registry/src/postgres/tasks.rs rename to crates/crank-registry/src/postgres/tasks/mod.rs index 63dd71f..c724389 100644 --- a/crates/crank-registry/src/postgres/tasks.rs +++ b/crates/crank-registry/src/postgres/tasks/mod.rs @@ -12,19 +12,20 @@ use async_trait::async_trait; use crank_core::{ - AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback, - PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, - TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, - TaskState, TaskStore, TaskSupportIds, ToolExecutionMode, + AttemptLease, AttemptOutcome, AttemptResolution, BeginAttempt, PlatformApiKeyId, + TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome, + TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskLease, + TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskStore, + TaskSupportIds, ToolExecutionMode, }; use serde_json::Value; use sqlx::{PgPool, Postgres, Row, Transaction}; use uuid::Uuid; -use super::admission::{ - AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked, - insert_admission_entry, lock_admission_in_tx, -}; +use super::admission::{AdmissionLimits, AttemptError}; + +mod acceptance; +mod attempts; const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id, authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version, @@ -35,6 +36,10 @@ const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_i (extract(epoch from lease_until) * 1000)::bigint as lease_until_ms, current_attempt_id, dispatch_started_at, (extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version, + max_attempts, retry_backoff_step_ms, + (extract(epoch from next_attempt_at) * 1000)::bigint as next_attempt_at_ms, + (extract(epoch from cancel_requested_at) * 1000)::bigint as cancel_requested_at_ms, + cancel_reason, (extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms, (extract(epoch from retain_until) * 1000)::bigint as retain_until_ms, (extract(epoch from created_at) * 1000)::bigint as created_ms, @@ -276,6 +281,7 @@ impl TaskStore for PostgresTaskStore { and owner_id = $7 and authorization_scope = $8 and state = $9 and version = $10 and (lease_owner is null or fence = 0) + and current_attempt_id is null returning {TASK_PROJECTION}" ))) .bind(request.target_state.as_str()) @@ -371,39 +377,117 @@ and lease_owner is null lease_ms: i64, ) -> Result, TaskError> { let batch = (batch_limit.clamp(1, 64)) as usize; - // A checkpointed attempt is never redispatched: once its lease expires the outbound outcome - // is uncertain, so it resolves to outcome_unknown and releases its admission slot. Story 4.4 - // owns any later retry/cancellation of that uncertain outcome. let mut transaction = self.start().await?; - sqlx::query( - "with candidates as ( - select task_id from tasks - where state = 'working' and lease_owner is not null - and current_attempt_id is not null - and lease_until < clock_timestamp() - order by task_id - limit 64 - ), resolved as ( - update tasks - set state = 'outcome_unknown', - lease_owner = null, - lease_until = null, - version = version + 1, - updated_at = now() - where task_id in (select task_id from candidates) - and state = 'working' - returning task_id - ) - delete from execution_admission_entries - where task_id in (select task_id from resolved)", + // A checkpointed attempt is never redispatched blindly: once its lease expires another worker + // must decide per Task whether the pinned safe-retry evidence, the budget and the deadline + // still allow exactly one bounded new attempt (Story 4.4). Everything else resolves to the + // non-terminal uncertain state and releases its admission slot. + let candidates = sqlx::query( + "select t.task_id, t.current_attempt_id, t.attempts, t.max_attempts, + t.execution_deadline > clock_timestamp() as deadline_left, + coalesce(a.retry_allowed, false) as retry_allowed, + a.resolved_at is null as attempt_open + from tasks t + left join task_attempts a + on a.task_id = t.task_id and a.attempt_id = t.current_attempt_id + where t.state = 'working' and t.lease_owner is not null + and t.current_attempt_id is not null + and t.lease_until < clock_timestamp() + order by t.task_id + limit 64 + for update of t skip locked", ) - .execute(&mut *transaction) + .fetch_all(&mut *transaction) .await .map_err(Self::unavailable)?; + for candidate in candidates { + let task_id: Uuid = candidate.try_get("task_id").map_err(Self::unavailable)?; + let attempt_id: String = candidate + .try_get("current_attempt_id") + .map_err(Self::unavailable)?; + let attempts: i32 = candidate.try_get("attempts").map_err(Self::unavailable)?; + let max_attempts: i32 = candidate + .try_get("max_attempts") + .map_err(Self::unavailable)?; + let deadline_left: bool = candidate + .try_get("deadline_left") + .map_err(Self::unavailable)?; + let retry_allowed: bool = candidate + .try_get("retry_allowed") + .map_err(Self::unavailable)?; + let attempt_open: bool = candidate + .try_get("attempt_open") + .map_err(Self::unavailable)?; + let safe_recovery = + retry_allowed && attempt_open && attempts < max_attempts && deadline_left; + let close = sqlx::query( + "update task_attempts + set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown', + phase = 'dispatched' + where task_id = $1 and attempt_id = $2 and resolved_at is null", + ) + .bind(task_id) + .bind(&attempt_id) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let _ = close; + if safe_recovery { + // Interrupted, not failed: re-arm the same admission intent and let the bounded claim + // hand the Task to a new owner with a new fence and exactly one new attempt. + sqlx::query( + "update tasks + set state = 'queued', + lease_owner = null, + lease_until = null, + current_attempt_id = null, + next_attempt_at = null, + version = version + 1, + updated_at = now() + where task_id = $1 and state = 'working' + and lease_until < clock_timestamp()", + ) + .bind(task_id) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + sqlx::query( + "update execution_admission_entries + set state = 'reserved', granted_at = null + where task_id = $1 and state = 'executing'", + ) + .bind(task_id) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + } else { + sqlx::query( + "update tasks + set state = 'outcome_unknown', + lease_owner = null, + lease_until = null, + error_envelope = jsonb_build_object('code', 'outcome_unknown', 'phase', 'dispatched'), + version = version + 1, + updated_at = now() + where task_id = $1 and state = 'working' + and lease_until < clock_timestamp()", + ) + .bind(task_id) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(task_id) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + } + } // A crash before the durable dispatch checkpoint left the Task queued for execution: its // admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is - // bounded by the <=30s lease expiry, inside the 60s story budget. + // bounded by the <=30s lease expiry, inside the 60s story budget. The expiry condition is + // re-checked in the UPDATE so a concurrent renewal can never be clobbered by recovery. sqlx::query( "with candidates as ( select task_id from tasks @@ -412,15 +496,19 @@ and lease_owner is null and lease_until < clock_timestamp() order by task_id limit 64 + for update skip locked ), requeued as ( update tasks set state = 'queued', lease_owner = null, lease_until = null, + next_attempt_at = null, version = version + 1, updated_at = now() where task_id in (select task_id from candidates) and state = 'working' + and current_attempt_id is null + and lease_until < clock_timestamp() returning task_id ) update execution_admission_entries @@ -476,6 +564,7 @@ and lease_owner is null or (state = 'working' and lease_owner is not null and lease_until < clock_timestamp() and current_attempt_id is null) ) + and (next_attempt_at is null or next_attempt_at <= clock_timestamp()) returning {TASK_PROJECTION}" ))) .bind(owner) @@ -607,10 +696,10 @@ and lease_owner is null .fetch_optional(&self.pool) .await .map_err(Self::unavailable)?; - if let Some(active) = active { - if active != epoch { - return Err(TaskError::StateConflict); - } + if let Some(active) = active + && active != epoch + { + return Err(TaskError::StateConflict); } } match (ciphertext, key_version, epoch) { @@ -621,38 +710,28 @@ and lease_owner is null } } - async fn checkpoint_dispatch( + async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result { + self.checkpoint_dispatch_attempt(request).await + } + + async fn resolve_attempt( &self, - owner: &str, + lease: &AttemptLease, + outcome: AttemptOutcome, + ) -> Result { + self.resolve_attempt_outcome(lease, outcome).await + } + + async fn cancel(&self, request: TaskCancelRequest) -> Result { + self.cancel_task(request).await + } + + async fn attempts( + &self, + access: &TaskAccess, task_id: TaskId, - fence: u64, - attempt_id: String, - ) -> Result { - let mut transaction = self.start().await?; - let row = sqlx::query(sqlx::AssertSqlSafe(format!( - "update tasks - set current_attempt_id = $1, - dispatch_started_at = now(), - attempts = attempts + 1, - updated_at = now() - where task_id = $2 and lease_owner = $3 and fence = $4 - and lease_until > clock_timestamp() and current_attempt_id is null - returning {TASK_PROJECTION}" - ))) - .bind(&attempt_id) - .bind(task_id.as_uuid()) - .bind(owner) - .bind(i64::try_from(fence).unwrap_or_default()) - .fetch_optional(&mut *transaction) - .await - .map_err(Self::unavailable)?; - let Some(row) = row else { - transaction.rollback().await.ok(); - return Err(TaskError::StateConflict); - }; - let record = record_from_row(&row).map_err(Self::unavailable)?; - transaction.commit().await.map_err(Self::unavailable)?; - Ok(record) + ) -> Result, TaskError> { + self.attempt_history(access, task_id).await } async fn finish_leased( @@ -678,6 +757,7 @@ and lease_owner is null and state = $9 and version = $10 and lease_owner = $11 and fence = $12 and lease_until > clock_timestamp() + and current_attempt_id is null returning {TASK_PROJECTION}" ))) .bind(request.target_state.as_str()) @@ -708,181 +788,9 @@ and lease_owner is null transaction.commit().await.map_err(Self::unavailable)?; Ok(record) } -} -impl PostgresTaskStore { - async fn accept_locked( - &self, - transaction: &mut Transaction<'_, Postgres>, - request: &TaskAcceptRequest, - ) -> Result { - // Replay first: an already accepted Task is returned without capacity mutation, so it works - // with a full queue and after the original execution deadline while the record is retained. - if let Some((acceptance, fingerprint)) = - self.stored_acceptance(transaction, request).await? - { - return Ok(if fingerprint == request.input_fingerprint { - Accepted::Replayed(acceptance) - } else { - Accepted::Conflict - }); - } - - // Serialize with synchronous admission and any concurrent acceptance of the same scope. - lock_admission_in_tx(transaction, &request.workspace_id).await?; - - // The competitor may have committed while this transaction waited for the controller lock. - if let Some((acceptance, fingerprint)) = - self.stored_acceptance(transaction, request).await? - { - return Ok(if fingerprint == request.input_fingerprint { - Accepted::Replayed(acceptance) - } else { - Accepted::Conflict - }); - } - - cleanup_expired_unlinked(transaction).await?; - let queued = bounded_queue_len(transaction).await?; - if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) { - return Err(AttemptError::Overloaded); - } - - let created_ms: i64 = sqlx::query( - "insert into tasks ( - task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id, - operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent, - trace_sampled, policy_mode, approval_reference, state, input_envelope, - execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version, - executable_input_master_key_epoch, platform_api_key_id - ) values ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16, - to_timestamp($17::double precision / 1000.0), - to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22 - ) - returning (extract(epoch from created_at) * 1000)::bigint as created_ms", - ) - .bind(request.task_id.as_uuid()) - .bind(&request.workspace_id) - .bind(request.owner_kind.as_str()) - .bind(&request.owner_id) - .bind(&request.authorization_scope) - .bind(&request.operation_id) - .bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX)) - .bind(&request.scoped_key) - .bind(&request.input_fingerprint) - .bind(request.support.request_id.as_str()) - .bind(request.support.trace_id.as_str()) - .bind(&request.support.traceparent) - .bind(request.support.trace_sampled) - .bind(policy_mode_str(request.pins.policy_mode)) - .bind(request.pins.approval_reference.as_deref()) - .bind(request.input.value().clone()) - .bind(request.pins.execution_deadline_unix_ms) - .bind(request.pins.retain_until_unix_ms) - .bind(request.executable_input_ciphertext.as_deref()) - .bind(request.executable_input_key_version.as_deref()) - .bind(request.executable_input_master_key_epoch) - .bind( - request - .platform_api_key_id - .as_ref() - .map(PlatformApiKeyId::as_str), - ) - .fetch_one(&mut **transaction) - .await - .map_err(task_sql_error)? - .try_get("created_ms") - .map_err(AttemptError::from_sql)?; - - let admission_request = AdmissionRequest { - scope: AdmissionScope { - workspace_id: request.workspace_id.clone(), - agent_key: request.owner_id.clone(), - operation_id: request.operation_id.clone(), - operation_version: request.pins.operation_version, - }, - mode: AdmissionMode::Background, - policy_mode: request.pins.policy_mode, - no_tasks_fallback: NoTasksFallback::Unsupported, - // Never reached: the intent row is inserted by this transaction, not by `admit`. - approval: ApprovalEvidence::NotRequired, - fingerprint: request.input_fingerprint.clone(), - expires_at_unix_ms: request.pins.execution_deadline_unix_ms, - }; - let owner_token = format!("task_{}", request.task_id); - insert_admission_entry( - transaction, - &admission_request, - &owner_token, - "reserved", - Some(request.task_id.as_uuid()), - ) - .await?; - - Ok(Accepted::Created(TaskAcceptance { - task_id: request.task_id, - state: TaskState::Queued, - operation_version: request.pins.operation_version, - support: request.support.clone(), - created_at_unix_ms: created_ms, - retain_until_unix_ms: request.pins.retain_until_unix_ms, - replayed: false, - })) - } - - /// Bounded scoped reread after a foreign unique conflict aborted the transaction. - async fn reread_after_unique( - &self, - request: &TaskAcceptRequest, - ) -> Result { - let row = sqlx::query(sqlx::AssertSqlSafe(format!( - "select {TASK_PROJECTION} from tasks - where workspace_id = $1 and owner_kind = $2 and owner_id = $3 - and authorization_scope = $4 and operation_id = $5 and scoped_key = $6" - ))) - .bind(&request.workspace_id) - .bind(request.owner_kind.as_str()) - .bind(&request.owner_id) - .bind(&request.authorization_scope) - .bind(&request.operation_id) - .bind(&request.scoped_key) - .fetch_optional(&self.pool) - .await - .map_err(Self::unavailable)?; - let Some(row) = row else { - return Err(TaskError::unavailable( - "durable task acceptance ended with an unresolved unique conflict", - )); - }; - let record = record_from_row(&row)?; - if record.input_fingerprint != request.input_fingerprint { - return Err(TaskError::FingerprintConflict); - } - Ok(acceptance_from_record(&record)) - } - - /// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are - /// reported without payload, and a foreign Task is indistinguishable from an absent one. - async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError { - let access = &request.access; - match sqlx::query( - "select state, version from tasks - where task_id = $1 and workspace_id = $2 and owner_kind = $3 - and owner_id = $4 and authorization_scope = $5", - ) - .bind(request.task_id.as_uuid()) - .bind(&access.workspace_id) - .bind(access.owner_kind.as_str()) - .bind(&access.owner_id) - .bind(&access.authorization_scope) - .fetch_optional(&self.pool) - .await - { - Ok(Some(_)) => TaskError::StateConflict, - Ok(None) => TaskError::NotFound, - Err(error) => Self::unavailable(error), - } + async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result { + self.refuse_attempt_resolution(lease, code).await } } @@ -948,6 +856,15 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result .map_err(unavailable_row)?, retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?, approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?, + retry: TaskRetryPolicy::new( + u32::try_from( + row.try_get::("max_attempts") + .map_err(unavailable_row)?, + ) + .unwrap_or(1), + row.try_get::("retry_backoff_step_ms") + .map_err(unavailable_row)?, + ), }, support: TaskSupportIds { request_id: crank_core::RequestId::parse(&request_id) @@ -972,6 +889,11 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result dispatch_started_at_unix_ms: row .try_get("dispatch_started_at_ms") .map_err(unavailable_row)?, + next_attempt_at_unix_ms: row.try_get("next_attempt_at_ms").map_err(unavailable_row)?, + cancel_requested_at_unix_ms: row + .try_get("cancel_requested_at_ms") + .map_err(unavailable_row)?, + cancel_reason: row.try_get("cancel_reason").map_err(unavailable_row)?, version: u32::try_from(row.try_get::("version").map_err(unavailable_row)?) .unwrap_or_default(), created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?, diff --git a/crates/crank-registry/tests/integration/migrations.rs b/crates/crank-registry/tests/integration/migrations.rs index 886c0e6..a922b56 100644 --- a/crates/crank-registry/tests/integration/migrations.rs +++ b/crates/crank-registry/tests/integration/migrations.rs @@ -76,6 +76,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() { (26, "durable-tasks-v26"), (27, "durable-tasks-v27"), (28, "durable-tasks-v28"), + (29, "durable-tasks-v29"), ]; assert_eq!(rows.len(), expected.len()); for (row, (version, name)) in rows.iter().zip(expected) { @@ -186,7 +187,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() { ); assert_eq!( MigrationAuthority::preflight(first.pool()).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } #[tokio::test] @@ -234,7 +235,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() { .expect("the V18-local guard must be restorable"); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "restoring the exact V18 contract must return the current verdict" ); } @@ -360,7 +361,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -473,13 +474,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 2, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); let trace_column: bool = sqlx::query_scalar( "select exists ( @@ -522,7 +523,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 3, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -582,7 +583,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 4, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -836,6 +837,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } diff --git a/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs b/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs index 548beb5..4e269dc 100644 --- a/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs +++ b/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs @@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 11, - target: 28, + target: 29, } ); @@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "restore: {restore}" ); } @@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "restore: {restore}" ); } @@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) { .unwrap(); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "restore: {restore}" ); } @@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); for (drift, restore) in [ @@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "restore: {restore}" ); } diff --git a/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs b/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs index 42d8886..eb3a0b7 100644 --- a/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs +++ b/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs @@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column ); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "a rejected primary-key nullability change must leave the schema current" ); @@ -473,7 +473,7 @@ async fn assert_definition_only_drift( assert_eq!(super::catalog_constraint_definitions(pool).await, before); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } @@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "the baseline before the drift matrix must be the current V16 schema" ); let mut observations: Vec = Vec::new(); @@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift .await .expect("restoring the exact definition must apply"); let restored = MigrationAuthority::preflight(&pool).await; - let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 28 })); + let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 29 })); // Точное восстановление доказывается сравнением снятого снимка с baseline, а не // мнением того же guard-а, который принимает неправильные определения (F4). let restored_exact = catalog_definition_snapshot(&pool).await == before; @@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "baseline before inserting the shadow schema" ); @@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() // Тень не меняет вердикт guard-а по текущей схеме. assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "a same-named object in another schema must not change the verdict for the current schema" ); @@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() ); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 }, + MigrationPreflight::Current { version: 29 }, "the exact restoration must return the current V16 schema" ); @@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() .unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); assert!( diff --git a/crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs b/crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs index 1b92dd2..a2c657d 100644 --- a/crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs +++ b/crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs @@ -22,16 +22,16 @@ async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 24, - target: 28, + target: 29, } ); assert_eq!( MigrationAuthority::apply(&pool).await.unwrap(), - MigrationApplyResult::Applied { from: 24, to: 28 } + MigrationApplyResult::Applied { from: 24, to: 29 } ); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } diff --git a/crates/crank-registry/tests/integration/migrations/import_plan.rs b/crates/crank-registry/tests/integration/migrations/import_plan.rs index 1a101b2..80c682f 100644 --- a/crates/crank-registry/tests/integration/migrations/import_plan.rs +++ b/crates/crank-registry/tests/integration/migrations/import_plan.rs @@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); let tables = sqlx::query( "select table_name @@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 13, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); diff --git a/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs b/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs index 5beb44c..bf613f2 100644 --- a/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs +++ b/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs @@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() { assert_eq!(operation_count, 1); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } @@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los ); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } @@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() { assert!(supports_elicitation); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } @@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 28, + target: 29, } ); MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } diff --git a/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs b/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs index 8745973..ecc39bc 100644 --- a/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs +++ b/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs @@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations") .fetch_one(&pool) @@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 18, - target: 28, + target: 29, } ); assert_eq!( MigrationAuthority::apply(&pool).await.unwrap(), - MigrationApplyResult::Applied { from: 18, to: 28 } + MigrationApplyResult::Applied { from: 18, to: 29 } ); let after: Vec<(i64, String, String)> = sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version") @@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() { assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten"); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 28 } + MigrationPreflight::Current { version: 29 } ); } @@ -161,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 18, - target: 28, + target: 29, } ); } diff --git a/crates/crank-registry/tests/integration/migrations/rollback.rs b/crates/crank-registry/tests/integration/migrations/rollback.rs index 329ab07..445ad72 100644 --- a/crates/crank-registry/tests/integration/migrations/rollback.rs +++ b/crates/crank-registry/tests/integration/migrations/rollback.rs @@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 11, - target: 28, + target: 29, } ); } diff --git a/crates/crank-registry/tests/integration/tasks/mod.rs b/crates/crank-registry/tests/integration/tasks/mod.rs index a478a33..b5c8de4 100644 --- a/crates/crank-registry/tests/integration/tasks/mod.rs +++ b/crates/crank-registry/tests/integration/tasks/mod.rs @@ -9,10 +9,11 @@ use super::common; use crank_core::{ AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, AgentId, ApprovalEvidence, - ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind, - PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest, TaskAccess, - TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, - TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId, current_unix_ms, + BeginAttempt, ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId, + PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest, + TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, + TaskState, TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId, + current_unix_ms, }; use crank_registry::{ AdmissionLimits, CreatePlatformApiKeyRequest, PostgresExecutionAdmission, PostgresTaskStore, @@ -60,6 +61,7 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta execution_deadline_unix_ms: deadline, retain_until_unix_ms: deadline + 3_600_000, approval_reference: Some("appr_1".to_owned()), + retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: key.to_owned(), input_fingerprint: fingerprint.to_owned(), @@ -708,28 +710,38 @@ async fn task_worker_lease_and_fencing_contract() { // checkpoint dispatch once let checkpointed = store - .checkpoint_dispatch( - "worker_1", - lease.record.task_id, - lease.fence, - "attempt-1".to_owned(), - ) + .checkpoint_dispatch(BeginAttempt { + task_id: lease.record.task_id, + owner: "worker_1".to_owned(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: "attempt-1".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: true, + }) .await .unwrap(); + assert_eq!(checkpointed.attempt.attempt_id, "attempt-1"); + assert_eq!(checkpointed.attempt.ordinal, 1); + assert!(checkpointed.task.dispatch_started_at_unix_ms.is_some()); assert_eq!( - checkpointed.current_attempt_id.as_deref(), + checkpointed.task.current_attempt_id.as_deref(), Some("attempt-1") ); - assert!(checkpointed.dispatch_started_at_unix_ms.is_some()); // second checkpoint rejected let second = store - .checkpoint_dispatch( - "worker_1", - lease.record.task_id, - lease.fence, - "attempt-2".to_owned(), - ) + .checkpoint_dispatch(BeginAttempt { + task_id: lease.record.task_id, + owner: "worker_1".to_owned(), + fence: lease.fence, + expected_version: checkpointed.task.version, + attempt_id: "attempt-2".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: true, + }) .await; assert!(second.is_err()); @@ -902,4 +914,5 @@ fn authority_key( } } +mod task_attempts; mod task_worker; diff --git a/crates/crank-registry/tests/integration/tasks/task_attempts.rs b/crates/crank-registry/tests/integration/tasks/task_attempts.rs new file mode 100644 index 0000000..1b092cf --- /dev/null +++ b/crates/crank-registry/tests/integration/tasks/task_attempts.rs @@ -0,0 +1,534 @@ +use super::*; +use crank_core::{ + AttemptOutcome, AttemptOutcomeClass, AttemptPhase, AttemptResolution, BeginAttempt, + TaskCancelOutcome, TaskCancelRequest, TaskRetryPolicy, +}; +use serde_json::json; + +#[tokio::test] +async fn cancel_before_checkpoint_wins_race_with_claim_and_never_dispatches() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let created = store + .accept(request( + &key("cancel_before_1"), + &digest("fb1"), + AGENT, + 30_000, + )) + .await + .unwrap(); + + let outcome = store + .cancel(TaskCancelRequest { + access: access(AGENT), + task_id: created.task_id, + expected_version: None, + reason: "cancel before dispatch".to_owned(), + }) + .await + .unwrap(); + match outcome { + TaskCancelOutcome::Cancelled(record) => assert_eq!(record.state, TaskState::Cancelled), + other => panic!("expected Cancelled, got {other:?}"), + } + + let claims = store.claim_batch("worker", 4, 10_000).await.unwrap(); + assert!(claims.is_empty(), "cancelled task must not be claimed"); + + let attempts = store + .attempts(&access(AGENT), created.task_id) + .await + .unwrap(); + assert!(attempts.is_empty(), "no attempts for cancelled task"); + + let intents: i64 = sqlx::query_scalar( + "select count(*)::bigint from execution_admission_entries where task_id = $1", + ) + .bind(created.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(intents, 0, "dispatch intent must be removed"); + + database.cleanup().await; +} + +#[tokio::test] +async fn cancel_after_checkpoint_is_uncertain_and_never_cancelled_or_retried() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let created = store + .accept(request( + &key("cancel_after_1"), + &digest("fa1"), + AGENT, + 30_000, + )) + .await + .unwrap(); + let leases = store.claim_batch("worker", 1, 10_000).await.unwrap(); + let lease = leases.into_iter().next().expect("claim"); + + let leased = store + .checkpoint_dispatch(BeginAttempt { + task_id: lease.record.task_id, + owner: "worker".to_owned(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: "attempt-cancel-after".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: true, + }) + .await + .expect("checkpoint"); + + let outcome = store + .cancel(TaskCancelRequest { + access: access(AGENT), + task_id: created.task_id, + expected_version: None, + reason: "cancel after checkpoint".to_owned(), + }) + .await + .unwrap(); + match outcome { + TaskCancelOutcome::OutcomeUnknown(record) => { + assert_eq!(record.state, TaskState::OutcomeUnknown) + } + other => panic!("expected OutcomeUnknown, got {other:?}"), + } + + let record = store.read(&access(AGENT), created.task_id).await.unwrap(); + assert_eq!(record.state, TaskState::OutcomeUnknown); + + let attempts = store + .attempts(&access(AGENT), created.task_id) + .await + .unwrap(); + assert_eq!(attempts.len(), 1); + assert_eq!( + attempts[0].outcome, + Some(AttemptOutcomeClass::OutcomeUnknown) + ); + + // Repeated cancel stays uncertain. + let outcome2 = store + .cancel(TaskCancelRequest { + access: access(AGENT), + task_id: created.task_id, + expected_version: None, + reason: "cancel after checkpoint".to_owned(), + }) + .await + .unwrap(); + assert!(matches!(outcome2, TaskCancelOutcome::OutcomeUnknown(_))); + + // finish_leased by the original lease must be rejected. + let finish = store + .finish_leased( + TaskFinishRequest { + access: access(AGENT), + task_id: created.task_id, + expected_state: leased.task.state, + expected_version: leased.task.version, + target_state: TaskState::Completed, + result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()), + error: None, + }, + "worker", + leased.attempt.fence, + ) + .await; + assert!(matches!(finish, Err(TaskError::StateConflict))); + + let claims2 = store.claim_batch("worker2", 4, 10_000).await.unwrap(); + assert!( + !claims2.iter().any(|l| l.record.task_id == created.task_id), + "outcome unknown task must never be claimed" + ); + + database.cleanup().await; +} + +#[tokio::test] +async fn checkpoint_and_resolve_are_cas_guarded_and_history_is_immutable() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let created = store + .accept(request(&key("cas_1"), &digest("fc1"), AGENT, 30_000)) + .await + .unwrap(); + let leases = store.claim_batch("worker", 1, 10_000).await.unwrap(); + let lease = leases.into_iter().next().expect("claim"); + let first_version = lease.record.version; + + let committed = store + .checkpoint_dispatch(BeginAttempt { + task_id: lease.record.task_id, + owner: "worker".to_owned(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: "attempt-cas".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) + .await + .expect("first checkpoint"); + let committed_lease = committed; + + // Repeated checkpoint with the same old version -> StateConflict. + let second = store + .checkpoint_dispatch(BeginAttempt { + task_id: lease.record.task_id, + owner: "worker".to_owned(), + fence: lease.fence, + expected_version: first_version, + attempt_id: "attempt-cas-2".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) + .await; + assert!(matches!(second, Err(TaskError::StateConflict))); + + // resolve with a modified lease task version -> StateConflict. + let mut bad_lease = committed_lease.clone(); + bad_lease.task.version = first_version; + let bad_resolve = store + .resolve_attempt( + &bad_lease, + AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()), + ) + .await; + assert!(matches!(bad_resolve, Err(TaskError::StateConflict))); + + // Correct resolve -> Finished(Completed). + let ok = store + .resolve_attempt( + &committed_lease, + AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()), + ) + .await + .expect("resolve"); + match ok { + AttemptResolution::Finished(rec) => assert_eq!(rec.state, TaskState::Completed), + other => panic!("expected Finished, got {other:?}"), + } + + // Repeated resolve -> StateConflict. + let repeat = store + .resolve_attempt( + &committed_lease, + AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()), + ) + .await; + assert!(matches!(repeat, Err(TaskError::StateConflict))); + + let attempts = store + .attempts(&access(AGENT), created.task_id) + .await + .unwrap(); + assert_eq!(attempts.len(), 1, "exactly one immutable attempt record"); + assert_eq!(attempts[0].outcome, Some(AttemptOutcomeClass::Completed)); + + let _ = created; + database.cleanup().await; +} + +#[tokio::test] +async fn expired_checkpointed_attempt_recovers_only_with_safe_retry_evidence() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + // Task A: retry policy allows 2 attempts, retry_allowed=true. + let mut req_a = request(&key("rec_a"), &digest("fa"), AGENT, 30_000); + req_a.pins.retry = TaskRetryPolicy::new(2, 0); + let created_a = store.accept(req_a).await.unwrap(); + + // Task B: single attempt, retry_allowed=false. + let created_b = store + .accept(request(&key("rec_b"), &digest("fb"), AGENT, 30_000)) + .await + .unwrap(); + + let claims = store.claim_batch("worker_dead", 2, 150).await.unwrap(); + assert_eq!(claims.len(), 2); + + let lease_a = claims + .iter() + .find(|l| l.record.task_id == created_a.task_id) + .cloned() + .expect("claim A"); + let lease_b = claims + .iter() + .find(|l| l.record.task_id == created_b.task_id) + .cloned() + .expect("claim B"); + + store + .checkpoint_dispatch(BeginAttempt { + task_id: lease_b.record.task_id, + owner: "worker_dead".to_owned(), + fence: lease_b.fence, + expected_version: lease_b.record.version, + attempt_id: "attempt-b".to_owned(), + input_fingerprint: lease_b.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) + .await + .expect("checkpoint B"); + + store + .checkpoint_dispatch(BeginAttempt { + task_id: lease_a.record.task_id, + owner: "worker_dead".to_owned(), + fence: lease_a.fence, + expected_version: lease_a.record.version, + attempt_id: "attempt-a".to_owned(), + input_fingerprint: lease_a.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: true, + }) + .await + .expect("checkpoint A"); + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + + let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap(); + let lease_a2 = recovered + .iter() + .find(|l| l.record.task_id == created_a.task_id) + .cloned() + .expect("A recovered"); + assert!(lease_a2.fence > lease_a.fence, "recovery bumps fence"); + assert!( + matches!( + lease_a2.record.state, + TaskState::Working | TaskState::Queued + ), + "A state working/queued" + ); + + assert!( + !recovered + .iter() + .any(|l| l.record.task_id == created_b.task_id), + "B must not be redispatched" + ); + + let record_b = store.read(&access(AGENT), created_b.task_id).await.unwrap(); + assert_eq!(record_b.state, TaskState::OutcomeUnknown); + + // A prior attempt resolved as uncertain. + let attempts_a = store + .attempts(&access(AGENT), created_a.task_id) + .await + .unwrap(); + assert!(!attempts_a.is_empty()); + assert_eq!( + attempts_a[0].outcome, + Some(AttemptOutcomeClass::OutcomeUnknown) + ); + + // Further scans do not hand B back. + assert!( + store + .claim_batch("worker_live", 4, 10_000) + .await + .unwrap() + .iter() + .all(|l| l.record.task_id != created_b.task_id) + ); + + database.cleanup().await; +} + +#[tokio::test] +async fn before_dispatch_failed_attempt_does_not_erase_recorded_uncertainty() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let mut req = request(&key("bd_1"), &digest("fbd"), AGENT, 30_000); + req.pins.retry = TaskRetryPolicy::new(2, 0); + let created = store.accept(req).await.unwrap(); + + let leases = store.claim_batch("worker", 1, 10_000).await.unwrap(); + let lease_a = leases.into_iter().next().expect("claim A"); + + let committed_a = store + .checkpoint_dispatch(BeginAttempt { + task_id: lease_a.record.task_id, + owner: "worker".to_owned(), + fence: lease_a.fence, + expected_version: lease_a.record.version, + attempt_id: "attempt-a".to_owned(), + input_fingerprint: lease_a.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: true, + }) + .await + .expect("checkpoint A"); + + // Attempt A resolves to a bounded retry (uncertain outcome). + let retry = store + .resolve_attempt( + &committed_a, + AttemptOutcome::Uncertain { + error: TaskEnvelope::try_redacted( + &json!({"code":"outcome_unknown","phase":"dispatched"}), + ) + .unwrap(), + repeatable: true, + phase: AttemptPhase::Dispatched, + }, + ) + .await + .expect("resolve A"); + let retry_task = match retry { + AttemptResolution::RetryAt { task, .. } => task, + other => panic!("expected RetryAt, got {other:?}"), + }; + assert_eq!(retry_task.state, TaskState::Working); + assert!(retry_task.current_attempt_id.is_none()); + let retry_version = retry_task.version; + + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // The Task keeps its lease across the pinned backoff: the next attempt checkpoints under the + // same owner/fence and the version the RetryAt record reported. + let committed_b_result = store + .checkpoint_dispatch(BeginAttempt { + task_id: retry_task.task_id, + owner: "worker".to_owned(), + fence: lease_a.fence, + expected_version: retry_version, + attempt_id: "attempt-b".to_owned(), + input_fingerprint: retry_task.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) + .await; + let committed_b = committed_b_result.expect("checkpoint B"); + + // Attempt B never reached upstream: a before_dispatch failure must NOT overwrite + // the recorded uncertainty of the previously dispatched attempt. + let resolution = store + .resolve_attempt( + &committed_b, + AttemptOutcome::Failed { + error: TaskEnvelope::try_redacted( + &json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}), + ) + .unwrap(), + retryable: false, + phase: AttemptPhase::BeforeDispatch, + }, + ) + .await + .expect("resolve B"); + match resolution { + AttemptResolution::Unknown(record) => assert_eq!(record.state, TaskState::OutcomeUnknown), + other => panic!("expected Unknown, got {other:?}"), + } + + let record = store.read(&access(AGENT), created.task_id).await.unwrap(); + assert_eq!(record.state, TaskState::OutcomeUnknown); + assert!(record.result.is_none()); + let error = record.error.expect("error envelope present"); + assert_eq!(error.value()["code"], json!("shutdown_before_dispatch")); + assert_eq!(error.value()["phase"], json!("dispatched")); + assert_eq!(record.attempts, 2); + + let attempts = store + .attempts(&access(AGENT), created.task_id) + .await + .unwrap(); + assert_eq!(attempts.len(), 2); + assert_eq!( + attempts[0].outcome, + Some(AttemptOutcomeClass::RetryScheduled) + ); + assert_eq!( + attempts[0].certainty, + Some(crank_core::AttemptCertainty::Unknown) + ); + assert_eq!(attempts[0].phase, Some(AttemptPhase::Dispatched)); + assert_eq!(attempts[1].outcome, Some(AttemptOutcomeClass::Failed)); + assert_eq!( + attempts[1].certainty, + Some(crank_core::AttemptCertainty::Known) + ); + assert_eq!(attempts[1].phase, Some(AttemptPhase::BeforeDispatch)); + + // Repeated read/cancel must not turn the state terminal. + let again = store.read(&access(AGENT), created.task_id).await.unwrap(); + assert_eq!(again.state, TaskState::OutcomeUnknown); + + database.cleanup().await; +} + +#[tokio::test] +async fn before_dispatch_failed_single_attempt_stays_failed() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let created = store + .accept(request(&key("bd_single"), &digest("fbf"), AGENT, 30_000)) + .await + .unwrap(); + + let leases = store.claim_batch("worker", 1, 10_000).await.unwrap(); + let lease = leases.into_iter().next().expect("claim"); + let committed = store + .checkpoint_dispatch(BeginAttempt { + task_id: lease.record.task_id, + owner: "worker".to_owned(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: "attempt-single".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) + .await + .expect("checkpoint"); + + let resolution = store + .resolve_attempt( + &committed, + AttemptOutcome::Failed { + error: TaskEnvelope::try_redacted( + &json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}), + ) + .unwrap(), + retryable: false, + phase: AttemptPhase::BeforeDispatch, + }, + ) + .await + .expect("resolve"); + match resolution { + AttemptResolution::Finished(record) => assert_eq!(record.state, TaskState::Failed), + other => panic!("expected Finished, got {other:?}"), + } + + let record = store.read(&access(AGENT), created.task_id).await.unwrap(); + assert_eq!(record.state, TaskState::Failed); + let error = record.error.expect("error envelope present"); + assert_eq!(error.value()["phase"], json!("before_dispatch")); + + database.cleanup().await; +} diff --git a/crates/crank-registry/tests/integration/tasks/task_worker.rs b/crates/crank-registry/tests/integration/tasks/task_worker.rs index 0f2a261..514a714 100644 --- a/crates/crank-registry/tests/integration/tasks/task_worker.rs +++ b/crates/crank-registry/tests/integration/tasks/task_worker.rs @@ -87,12 +87,16 @@ async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_withou let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap(); let lease = leases.into_iter().next().expect("claim"); store - .checkpoint_dispatch( - "worker_dead", - lease.record.task_id, - lease.fence, - "attempt-uncertain".to_owned(), - ) + .checkpoint_dispatch(BeginAttempt { + task_id: lease.record.task_id, + owner: "worker_dead".to_owned(), + fence: lease.fence, + expected_version: lease.record.version, + attempt_id: "attempt-uncertain".to_owned(), + input_fingerprint: lease.record.input_fingerprint.clone(), + upstream_key_digest: None, + retry_allowed: false, + }) .await .expect("durable dispatch checkpoint"); diff --git a/crates/crank-runtime/src/executor.rs b/crates/crank-runtime/src/executor.rs index 7462636..42671d9 100644 --- a/crates/crank-runtime/src/executor.rs +++ b/crates/crank-runtime/src/executor.rs @@ -24,9 +24,13 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tracing::{Instrument, Span, debug, warn}; use uuid::Uuid; +use crate::task_attempt::{ + TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan, + TaskAttemptRefusal, +}; use crate::task_worker::TaskDispatchPermit; use crate::{ - AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError, + AdapterResponse, ExecutionAuthorization, ExecutionRequestError, PreparedRequest, RuntimeError, RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext, auth::apply_resolved_auth, request_preparation::{adapter_prepared_request, validate_selected_body_variant}, @@ -36,6 +40,18 @@ use crate::{ }, }; +/// How many outbound sends one prepared request may perform. +/// +/// `Legacy` keeps the existing synchronous `tools/call` retry behaviour inside the transport loop. +/// `Single` performs exactly one send, because a durable Task attempt already has its own committed +/// checkpoint and resolution around every send; it carries the pinned attempt ordinal and budget so +/// the one `adapter_attempt` span still names the real durable attempt. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SendMode { + Legacy, + Single { attempt: u32, max_attempts: u32 }, +} + #[derive(Clone)] pub struct RuntimeExecutor { adapters: AdapterRegistry, @@ -181,57 +197,273 @@ impl RuntimeExecutor { &self, request: RuntimeExecutionRequest<'_>, ) -> Result<(Value, Value), RuntimeError> { - self.execute_request_impl(request, None).await + self.execute_request_impl(request, None, SendMode::Legacy) + .await } + /// Drives the bounded, durable retry budget of one leased background Task (Story 4.4). + /// + /// Each outbound send is preceded by its own committed attempt checkpoint and followed by one + /// atomic resolution; the pinned budget, the safe-retry decision and the accepted absolute + /// deadline are enforced by PostgreSQL, so a restart or a second worker cannot extend them. pub async fn execute_task_outcome( &self, - request: RuntimeExecutionRequest<'_>, - permit: TaskDispatchPermit, - ) -> Result { - let operation = request.operation().clone(); - let operation_id = operation.operation_id.clone(); - let operation_version = operation.operation_version; - let origin = request.origin(); - let correlation = crank_core::CorrelationContext::new( - request.request_context().request_id.clone(), - request.request_context().trace_context.clone(), + context: &TaskAttemptContext, + authority: &dyn TaskAttemptAuthority, + ) -> TaskAttemptOutcome { + let store = context.store().clone(); + let lease = context.lease().clone(); + let mut record = lease.record.clone(); + loop { + if context.is_cancelled() || context.is_stopping() { + return TaskAttemptOutcome::Stopped; + } + let current_lease = crank_core::TaskLease { + record: record.clone(), + owner: lease.owner.clone(), + fence: lease.fence, + lease_until: lease.lease_until, + }; + let plan = match authority.prepare_attempt(¤t_lease).await { + Ok(plan) => plan, + Err(TaskAttemptRefusal::Stopped) => return TaskAttemptOutcome::Stopped, + Err(TaskAttemptRefusal::Failed { code }) => { + // A refused attempt never erases a proven ambiguity: if an earlier attempt of this + // Task may already have reached the upstream, the store keeps the Task in the + // non-terminal `outcome_unknown` instead of downgrading it to a plain failure. + return match store.refuse_attempt(¤t_lease, &code).await { + Ok(record) if record.state == crank_core::TaskState::OutcomeUnknown => { + TaskAttemptOutcome::OutcomeUnknown + } + _ => TaskAttemptOutcome::Failed { code }, + }; + } + }; + let attempt_id = format!("attempt-{}", Uuid::now_v7()); + let begin = crank_core::BeginAttempt { + task_id: record.task_id, + owner: lease.owner.clone(), + fence: lease.fence, + expected_version: record.version, + attempt_id: attempt_id.clone(), + input_fingerprint: record.input_fingerprint.clone(), + upstream_key_digest: plan.upstream_key_digest.clone(), + retry_allowed: plan.retry_allowed, + }; + let attempt = match store.checkpoint_dispatch(begin).await { + Ok(attempt) => attempt, + // Cancellation, lease loss or a concurrent lifecycle change: nothing was dispatched. + Err(crank_core::TaskError::StateConflict) + | Err(crank_core::TaskError::NotFound) + | Err(crank_core::TaskError::FingerprintConflict) => { + return TaskAttemptOutcome::Stopped; + } + // An unproven commit is never retried by guess: no outbound call without a proven + // durable checkpoint. + Err(error) => { + warn!(error = %error, "durable dispatch checkpoint failed; no outbound call"); + return TaskAttemptOutcome::Stopped; + } + }; + if context.is_stopping() { + let code = "shutdown_before_dispatch".to_owned(); + // A stop that lands before the send is a proven failure of *this* attempt only: when + // an earlier attempt of the same Task already lost its confirmation, the store keeps + // the non-terminal `outcome_unknown` instead of downgrading it to a plain failure. + return match store + .resolve_attempt( + &attempt, + crank_core::AttemptOutcome::Failed { + error: bounded_attempt_error( + &code, + crank_core::AttemptPhase::BeforeDispatch, + ), + retryable: false, + phase: crank_core::AttemptPhase::BeforeDispatch, + }, + ) + .await + { + Ok(crank_core::AttemptResolution::Unknown(_)) => { + TaskAttemptOutcome::OutcomeUnknown + } + _ => TaskAttemptOutcome::Failed { code }, + }; + } + if context.is_cancelled() { + let _ = store + .resolve_attempt( + &attempt, + crank_core::AttemptOutcome::Uncertain { + error: bounded_attempt_error( + "outcome_unknown", + crank_core::AttemptPhase::BeforeDispatch, + ), + repeatable: false, + phase: crank_core::AttemptPhase::BeforeDispatch, + }, + ) + .await; + return TaskAttemptOutcome::OutcomeUnknown; + } + // Each attempt owns exactly one `adapter_attempt` span, created around the single send + // inside the transport loop so the accepted trace shape (durable attempt → runtime + // execution → adapter attempt) is preserved. + let result = self.execute_task_send(&plan, &attempt).await; + let (outcome, code) = self.classify_task_attempt(&plan, result); + match store.resolve_attempt(&attempt, outcome).await { + Ok(crank_core::AttemptResolution::Finished(task)) => { + return if task.state == crank_core::TaskState::Completed { + TaskAttemptOutcome::Completed + } else { + TaskAttemptOutcome::Failed { + code: code.unwrap_or_else(|| "task_failed".to_owned()), + } + }; + } + Ok(crank_core::AttemptResolution::Unknown(_)) => { + return TaskAttemptOutcome::OutcomeUnknown; + } + Ok(crank_core::AttemptResolution::RetryAt { + task, + next_attempt_at_unix_ms, + }) => { + record = task; + let remaining = + next_attempt_at_unix_ms.saturating_sub(crank_core::current_unix_ms()); + let delay = Duration::from_millis(u64::try_from(remaining).unwrap_or(0)); + if !self.wait_before_attempt(context, delay).await { + return TaskAttemptOutcome::Stopped; + } + } + // A stale fence or version lost the resolution CAS: another writer owns the Task now. + Err(_) => return TaskAttemptOutcome::Stopped, + } + } + } + + /// One outbound send of one committed durable attempt. + async fn execute_task_send( + &self, + plan: &TaskAttemptPlan, + attempt: &crank_core::AttemptLease, + ) -> Result { + let permit = TaskDispatchPermit::new( + attempt.attempt.task_id, + plan.operation.operation_id.as_str().to_owned(), + plan.operation.operation_version, + plan.workspace_id.as_str().to_owned(), + plan.agent_id.as_ref().map(|id| id.as_str().to_owned()), + attempt.attempt.owner.clone(), + attempt.attempt.fence, + attempt.attempt.attempt_id.clone(), ); + let request = match RuntimeExecutionRequest::try_new( + &plan.workspace_id, + plan.origin, + plan.agent_id.as_ref(), + &plan.operation, + &plan.input, + plan.authorization, + plan.resolved_auth.as_ref(), + &plan.request_context, + plan.deadline, + ) { + Ok(request) => request, + Err(ExecutionRequestError::DeadlineElapsed) => { + return Err(RuntimeError::ExecutionDeadlineElapsed { + may_have_dispatched: false, + }); + } + Err(_) => return Err(RuntimeError::InvalidExecutionRequest), + }; + let (output, _preview) = self + .execute_request_impl( + request, + Some(permit), + SendMode::Single { + attempt: attempt.attempt.ordinal, + max_attempts: attempt.task.pins.retry.max_attempts, + }, + ) + .await?; + Ok(output) + } - // Validate permit binding before any adapter dispatch. - if permit.operation_id() != operation_id.as_str() - || permit.operation_version() != operation_version - || permit.workspace_id() != request.workspace_id().as_str() - || permit.agent_id() != request.agent_id().map(|value| value.as_str()) - { - return Err(crate::normalize_runtime_error_for_operation( - &RuntimeError::InvalidExecutionRequest, - &correlation, - &operation, - )); - } - if !permit.consume() { - return Err(crate::normalize_runtime_error_for_operation( - &RuntimeError::InvalidExecutionRequest, - &correlation, - &operation, - )); + /// Maps one outbound result of a durable attempt onto its persisted attempt outcome. + fn classify_task_attempt( + &self, + plan: &TaskAttemptPlan, + result: Result, + ) -> (crank_core::AttemptOutcome, Option) { + match result { + Ok(output) => match crank_core::TaskEnvelope::try_redacted(&output) { + Ok(envelope) => (crank_core::AttemptOutcome::Completed(envelope), None), + Err(_) => { + let code = "result_not_redactable".to_owned(); + ( + crank_core::AttemptOutcome::Failed { + error: bounded_attempt_error( + &code, + crank_core::AttemptPhase::Dispatched, + ), + retryable: false, + phase: crank_core::AttemptPhase::Dispatched, + }, + Some(code), + ) + } + }, + Err(error) => { + let phase = if runtime_error_may_have_dispatched(&error) { + crank_core::AttemptPhase::Dispatched + } else { + crank_core::AttemptPhase::BeforeDispatch + }; + let correlation = crank_core::CorrelationContext::new( + plan.request_context.request_id.clone(), + plan.request_context.trace_context.clone(), + ); + let failure = crate::normalize_runtime_error_for_operation( + &error, + &correlation, + &plan.operation, + ); + let code = format!("{:?}", failure.error_code()); + let safe_to_repeat = retry_is_allowed(&plan.operation, &error, plan.retry_allowed); + let envelope = bounded_attempt_error(&code, phase); + let outcome = match failure.outcome_certainty() { + crank_core::OutcomeCertainty::OutcomeUnknown => { + crank_core::AttemptOutcome::Uncertain { + error: envelope, + repeatable: safe_to_repeat, + phase, + } + } + _ => crank_core::AttemptOutcome::Failed { + error: envelope, + retryable: safe_to_repeat, + phase, + }, + }; + (outcome, Some(code)) + } } + } - match self.execute_request_impl(request, Some(permit)).await { - Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess { - operation_id: operation_id.clone(), - operation_version, - origin, - correlation, - request_preview, - output, - }), - Err(error) => Err(crate::normalize_runtime_error_for_operation( - &error, - &correlation, - &operation, - )), + /// Bounded cooperative wait inside the pinned backoff window. The lease keeps being renewed by + /// the worker, so a durable cancellation or lease loss still stops the local work here. + async fn wait_before_attempt(&self, context: &TaskAttemptContext, delay: Duration) -> bool { + let deadline = Instant::now() + delay; + loop { + if context.is_cancelled() || context.is_stopping() { + return false; + } + let now = Instant::now(); + if now >= deadline { + return true; + } + tokio::time::sleep((deadline - now).min(Duration::from_millis(50))).await; } } @@ -239,6 +471,7 @@ impl RuntimeExecutor { &self, request: RuntimeExecutionRequest<'_>, permit: Option, + send_mode: SendMode, ) -> Result<(Value, Value), RuntimeError> { let operation = request.operation(); // Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission. @@ -320,6 +553,7 @@ impl RuntimeExecutor { Some(request_context), Arc::clone(&dispatch_started), durable_attempt_id.as_deref(), + send_mode, ) .await; if let Some(lease) = lease.as_mut() { @@ -415,6 +649,7 @@ impl RuntimeExecutor { Ok(prepared) } + #[allow(clippy::too_many_arguments)] async fn execute_prepared( &self, operation: &RuntimeOperation, @@ -423,6 +658,7 @@ impl RuntimeExecutor { request_context: Option<&RuntimeRequestContext>, dispatch_started: Arc, durable_attempt_id: Option<&str>, + send_mode: SendMode, ) -> Result { let mut prepared_request = prepared_request; let idempotency_applicable = crate::idempotency::policy(operation).is_some(); @@ -472,7 +708,11 @@ impl RuntimeExecutor { approval_result?; } - let idempotency = if idempotency_applicable { + // The synchronous coordination guard is the replay authority of one `tools/call`. A durable + // Task attempt already has a PostgreSQL attempt authority around every send, so the guard is + // skipped there: an ambiguous first response must not block the bounded durable retry. The + // stable idempotency key is still delivered to the upstream by `prepare_idempotency` above. + let idempotency = if idempotency_applicable && matches!(send_mode, SendMode::Legacy) { let idempotency_span = Stage::RuntimeIdempotency.span(); let result = crate::idempotency::begin( self.coordination_store.as_deref(), @@ -540,6 +780,7 @@ impl RuntimeExecutor { idempotency_key.as_deref(), ), durable_attempt_id, + send_mode, ) .await; if let Ok(response) = &adapter_response { @@ -673,6 +914,7 @@ impl RuntimeExecutor { .map_err(|error| map_protocol_adapter_error(operation, error)) } + #[allow(clippy::too_many_arguments)] async fn execute_adapter_with_retry( &self, operation: &RuntimeOperation, @@ -681,20 +923,32 @@ impl RuntimeExecutor { dispatch_started: Arc, verified_idempotency: bool, durable_attempt_id: Option<&str>, + send_mode: SendMode, ) -> Result { - let max_attempts = retry_attempts(operation); - let mut attempt = 1; + // `max_attempts` bounds the transport loop; `reported_max_attempts` is the pinned Task budget + // the attempt span names. A durable attempt performs exactly one send, so its transport loop + // always has a single iteration even when the Task may retry later. + let (max_attempts, first_attempt, reported_max_attempts) = match send_mode { + SendMode::Single { + attempt, + max_attempts, + } => (1, attempt, max_attempts), + SendMode::Legacy => { + let attempts = retry_attempts(operation); + (attempts, 1, attempts) + } + }; + let mut attempt = first_attempt; loop { - // Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint; the - // retry index discriminates the attempts inside it. Only paths without a durable attempt - // (synchronous tools/call) mint a synthetic per-attempt id. + // Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint. Only + // paths without a durable attempt (synchronous tools/call) mint a synthetic identity. let attempt_id = durable_attempt_id .map(str::to_owned) .unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7())); let attempt_span = tracing::info_span!( "adapter_attempt", attempt, - max_attempts, + max_attempts = reported_max_attempts, attempt_id = %attempt_id ); let result = self @@ -846,6 +1100,22 @@ fn sanitized_request_preview(prepared: &PreparedRequest) -> Value { }) } +/// Bounded persisted error envelope of one durable attempt: a closed code and the attempt phase. +/// It never carries an upstream body, status text, URL or credential. +fn bounded_attempt_error(code: &str, phase: crank_core::AttemptPhase) -> crank_core::TaskEnvelope { + crank_core::TaskEnvelope::try_redacted(&json!({ + "code": code, + "phase": phase.as_str(), + })) + .unwrap_or_else(|_| { + crank_core::TaskEnvelope::try_redacted(&json!({ + "code": "task_error_not_recordable", + "phase": phase.as_str(), + })) + .expect("bounded attempt error envelope is always valid") + }) +} + fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool { matches!( error, diff --git a/crates/crank-runtime/src/lib.rs b/crates/crank-runtime/src/lib.rs index e6e47ea..09c1fab 100644 --- a/crates/crank-runtime/src/lib.rs +++ b/crates/crank-runtime/src/lib.rs @@ -15,6 +15,7 @@ mod rate_limit; mod request_context; mod request_preparation; mod response_cache; +mod task_attempt; mod task_worker; mod tasks; pub fn validate_resource_template_parameter( @@ -116,8 +117,11 @@ pub use rate_limit::{ }; pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext}; pub use secret_crypto::SecretCrypto; +pub use task_attempt::{ + TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan, + TaskAttemptRefusal, +}; pub use task_worker::{ - TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, - TaskWorkerError, + TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService}; diff --git a/crates/crank-runtime/src/task_attempt.rs b/crates/crank-runtime/src/task_attempt.rs new file mode 100644 index 0000000..37549c8 --- /dev/null +++ b/crates/crank-runtime/src/task_attempt.rs @@ -0,0 +1,113 @@ +//! Durable per-outbound attempt execution of one background Task (Story 4.4). +//! +//! The authoritative state of a Task stays in PostgreSQL. This module only carries the owned +//! execution plan of one attempt, the durable control (store + lease + cooperative stop) and the +//! bounded seam the application layer implements to revalidate authority before each attempt. +//! The single retry loop lives in [`crate::RuntimeExecutor::execute_task_outcome`]; there is no second +//! loop in the worker and no retry inside the transport adapter. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::time::Instant; + +use async_trait::async_trait; +use crank_core::{AgentId, ExecutionOrigin, TaskLease, TaskStore, WorkspaceId}; + +use crate::{ExecutionAuthorization, ResolvedAuth, RuntimeOperation, RuntimeRequestContext}; + +/// Owned execution plan of one durable attempt. +/// +/// It is rebuilt by the application through [`TaskAttemptAuthority::prepare_attempt`] before every +/// attempt, so a republished catalog, a revoked key or an elapsed deadline is revalidated between two +/// attempts instead of being trusted from the first one. +pub struct TaskAttemptPlan { + pub workspace_id: WorkspaceId, + pub origin: ExecutionOrigin, + pub agent_id: Option, + pub operation: RuntimeOperation, + pub input: serde_json::Value, + pub authorization: ExecutionAuthorization, + pub resolved_auth: Option, + pub request_context: RuntimeRequestContext, + /// Absolute deadline pinned at acceptance; never re-derived from a later publication. + pub deadline: Instant, + /// Safe-to-retry decision of this attempt: a read, or a write whose upstream idempotency is + /// backed by server-owned evidence. A client-supplied key or header never makes this true. + pub retry_allowed: bool, + /// Digest of the stable upstream idempotency key (never the raw key or a header value). + pub upstream_key_digest: Option, +} + +/// Application-owned seam: revalidate authority and prepare exactly one durable attempt. +#[async_trait] +pub trait TaskAttemptAuthority: Send + Sync { + async fn prepare_attempt( + &self, + lease: &TaskLease, + ) -> Result; +} + +/// Typed refusal of one attempt preparation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TaskAttemptRefusal { + /// Proven refusal before any dispatch of this attempt; the bounded code is persisted. + Failed { code: String }, + /// Durable control changed (cancellation, lease loss, shutdown): write nothing new from here. + Stopped, +} + +/// Durable control of one leased Task: the store, the leased record and cooperative stop signals. +#[derive(Clone)] +pub struct TaskAttemptContext { + pub(crate) store: Arc, + pub(crate) lease: TaskLease, + pub(crate) cancelled: Arc, + pub(crate) stopping: Arc, +} + +impl TaskAttemptContext { + pub(crate) fn new( + store: Arc, + lease: TaskLease, + cancelled: Arc, + stopping: Arc, + ) -> Self { + Self { + store, + lease, + cancelled, + stopping, + } + } + + pub fn store(&self) -> &Arc { + &self.store + } + + pub fn lease(&self) -> &TaskLease { + &self.lease + } + + /// Cooperative stop: the lease renewal or an observed durable cancellation set this flag. + pub fn is_cancelled(&self) -> bool { + self.cancelled.load(std::sync::atomic::Ordering::Acquire) + } + + /// Process shutdown was requested: no new outbound send may start after the current checkpoint. + pub fn is_stopping(&self) -> bool { + self.stopping.load(std::sync::atomic::Ordering::Acquire) + } +} + +/// Result of driving the whole pinned retry budget of one leased Task. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskAttemptOutcome { + /// The Task committed `completed` (exactly one terminal write). + Completed, + /// The Task committed `failed` without dispatch or with a proven failure. + Failed { code: String }, + /// The Task is non-terminal `outcome_unknown`; it is never retried or cancelled generically. + OutcomeUnknown, + /// Nothing new was written by this worker: cancellation, lease loss or an unavailable store. + Stopped, +} diff --git a/crates/crank-runtime/src/task_worker.rs b/crates/crank-runtime/src/task_worker.rs index d412070..4bc495e 100644 --- a/crates/crank-runtime/src/task_worker.rs +++ b/crates/crank-runtime/src/task_worker.rs @@ -11,12 +11,14 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use async_trait::async_trait; -use crank_core::{TaskEnvelope, TaskError, TaskFinishRequest, TaskLease, TaskState, TaskStore}; +use crank_core::{TaskError, TaskLease, TaskStore}; use tokio::sync::watch; use tokio::task::JoinSet; use tracing::{Instrument, debug, warn}; use uuid::Uuid; +use crate::task_attempt::{TaskAttemptContext, TaskAttemptOutcome}; + /// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets. #[derive(Clone, Debug)] pub struct TaskWorkerConfig { @@ -50,29 +52,16 @@ impl Default for TaskWorkerConfig { } } -/// Result of one durable Task attempt produced by the application layer. -#[derive(Clone, Debug)] -pub enum TaskAttemptOutcome { - Completed(serde_json::Value), - Failed(serde_json::Value), - OutcomeUnknown, -} - /// Application-owned seam for pre-dispatch revalidation and execution. /// -/// The worker owns lease renewal/CAS/fencing and the dispatch checkpoint. The handler only -/// receives the leased record and attempt identity; it must not trust `TaskRecord.input` as +/// The worker owns lease renewal/CAS/fencing and passes a durable [`TaskAttemptContext`]. The +/// application implements [`crate::TaskAttemptAuthority`] and drives the pinned retry budget through +/// [`crate::RuntimeExecutor::execute_task_outcome`]; it must not trust `TaskRecord.input` as /// executable payload. #[async_trait] pub trait TaskHandler: Send + Sync { async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>; - async fn execute( - &self, - lease: &TaskLease, - attempt_id: &str, - cancelled: Arc, - permit: TaskDispatchPermit, - ) -> TaskAttemptOutcome; + async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome; } /// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and @@ -91,6 +80,7 @@ pub struct TaskDispatchPermit { } impl TaskDispatchPermit { + #[allow(clippy::too_many_arguments)] pub(crate) fn new( task_id: crank_core::TaskId, operation_id: String, @@ -244,10 +234,10 @@ impl TaskWorker { if self.stopping.load(Ordering::Acquire) { break; } - if let Some(rx) = shutdown.as_mut() { - if *rx.borrow() { - break; - } + if let Some(rx) = shutdown.as_mut() + && *rx.borrow() + { + break; } // Reap finished attempts before claiming more. @@ -344,15 +334,7 @@ async fn process_lease( return; } debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch"); - finish_failed( - &store, - &owner, - task_id, - fence, - &lease, - "revalidation_failed", - ) - .await; + finish_failed(&store, &lease, "revalidation_failed").await; return; } if stopping.load(Ordering::Acquire) { @@ -364,49 +346,8 @@ async fn process_lease( let attempt_id = format!("attempt-{}", Uuid::now_v7()); - // Durable checkpoint before any network use. - if store - .checkpoint_dispatch(&owner, task_id, fence, attempt_id.clone()) - .await - .is_err() - { - // Lease lost or checkpoint rejected: do not dispatch. - cancelled.store(true, Ordering::Release); - return; - } - if stopping.load(Ordering::Acquire) { - // The checkpoint is durable, but we know the outbound callback has not started. - finish_failed( - &store, - &owner, - task_id, - fence, - &lease, - "shutdown_before_dispatch", - ) - .await; - return; - } - - // Sealed single-use permit for the trusted runtime dispatch bridge. It is created only after - // the current lease and durable checkpoint above. - let agent_id = if lease.record.owner_kind == crank_core::TaskOwnerKind::Agent { - Some(lease.record.owner_id.clone()) - } else { - None - }; - let permit = TaskDispatchPermit::new( - lease.record.task_id, - lease.record.operation_id.clone(), - lease.record.pins.operation_version, - lease.record.workspace_id.clone(), - agent_id, - owner.clone(), - fence, - attempt_id.clone(), - ); - - // Background lease renewal while the attempt runs. + // Background lease renewal covers the whole bounded retry budget, including the pinned backoff. + // A durable cancellation clears the lease, so the next renewal fails and the local work stops. let renewal_store = store.clone(); let renewal_owner = owner.clone(); let renewal_cancelled = cancelled.clone(); @@ -435,139 +376,43 @@ async fn process_lease( } }); - // Each dispatch gets a child span carrying the durable attempt identity, rooted in the parent - // trace persisted with the accepted Task (never a mutable queue/cache projection). - let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, attempt_id = %attempt_id, fence); + // The parent span names the leased Task generation; every individual outbound attempt adds its own + // `adapter_attempt` child span with the durable attempt ID committed by its checkpoint. + let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, lease_ref = %attempt_id, fence); if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) { crank_trace::set_parent_from_trace_context(&span, &parent); } - let outcome = handler - .execute(&lease, &attempt_id, cancelled.clone(), permit) - .instrument(span) - .await; + // Every durable write of this Task (checkpoint, attempt resolution, terminal state) happens inside + // the executor's single retry loop under this lease/fence; the worker only renews and stops. + let context = TaskAttemptContext::new( + store.clone(), + lease.clone(), + cancelled.clone(), + stopping.clone(), + ); + let outcome = handler.execute(context).instrument(span).await; cancelled.store(true, Ordering::Release); renewal_handle.abort(); - // Terminal CAS under valid lease. Only one writer wins; stale token is rejected by store. - let access = crank_core::TaskAccess { - workspace_id: lease.record.workspace_id.clone(), - owner_kind: lease.record.owner_kind, - owner_id: lease.record.owner_id.clone(), - authorization_scope: lease.record.authorization_scope.clone(), - }; - match outcome { - TaskAttemptOutcome::Completed(value) => { - let Ok(result) = TaskEnvelope::try_redacted(&value) else { - finish_failed( - &store, - &owner, - task_id, - fence, - &lease, - "result_not_redactable", - ) - .await; - return; - }; - let _ = store - .finish_leased( - TaskFinishRequest { - access, - task_id, - expected_state: leased_expected_state(&lease), - expected_version: lease.record.version, - target_state: TaskState::Completed, - result: Some(result), - error: None, - }, - &owner, - fence, - ) - .await; + TaskAttemptOutcome::Completed => { + debug!(task_id = %task_id, "durable task completed"); } - TaskAttemptOutcome::Failed(value) => { - let error = TaskEnvelope::try_redacted(&value).or_else(|_| { - TaskEnvelope::try_redacted(&serde_json::json!({ - "code": "task_error_not_redactable" - })) - }); - let Ok(error) = error else { - return; - }; - let _ = store - .finish_leased( - TaskFinishRequest { - access, - task_id, - expected_state: leased_expected_state(&lease), - expected_version: lease.record.version, - target_state: TaskState::Failed, - result: None, - error: Some(error), - }, - &owner, - fence, - ) - .await; + TaskAttemptOutcome::Failed { code } => { + debug!(task_id = %task_id, code = %code, "durable task failed"); } TaskAttemptOutcome::OutcomeUnknown => { - let _ = store - .finish_leased( - TaskFinishRequest { - access, - task_id, - expected_state: leased_expected_state(&lease), - expected_version: lease.record.version, - target_state: TaskState::OutcomeUnknown, - result: None, - error: None, - }, - &owner, - fence, - ) - .await; + warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry"); + } + TaskAttemptOutcome::Stopped => { + debug!(task_id = %task_id, "durable task attempt stopped without a new outcome"); } } } -fn leased_expected_state(lease: &TaskLease) -> TaskState { - // Claimed tasks are expected to be in a non-terminal processing state. - if lease.record.state == TaskState::Queued { - TaskState::Queued - } else { - TaskState::Working - } -} - -async fn finish_failed( - store: &Arc, - owner: &str, - task_id: crank_core::TaskId, - fence: u64, - lease: &TaskLease, - code: &str, -) { - let access = crank_core::TaskAccess { - workspace_id: lease.record.workspace_id.clone(), - owner_kind: lease.record.owner_kind, - owner_id: lease.record.owner_id.clone(), - authorization_scope: lease.record.authorization_scope.clone(), - }; - let _ = store - .finish_leased( - TaskFinishRequest { - access, - task_id, - expected_state: leased_expected_state(lease), - expected_version: lease.record.version, - target_state: TaskState::Failed, - result: None, - error: TaskEnvelope::try_redacted(&serde_json::json!({ "code": code })).ok(), - }, - owner, - fence, - ) - .await; +async fn finish_failed(store: &Arc, lease: &TaskLease, code: &str) { + // The store owns the fate of a refused pre-dispatch attempt: a Task that already carries a proven + // ambiguity still ends `outcome_unknown`, never a plain failure (Story 4.4). + let _ = store.refuse_attempt(lease, code).await; } diff --git a/crates/crank-runtime/src/tasks.rs b/crates/crank-runtime/src/tasks.rs index 038d6f8..8d90908 100644 --- a/crates/crank-runtime/src/tasks.rs +++ b/crates/crank-runtime/src/tasks.rs @@ -47,6 +47,8 @@ pub struct TaskAcceptDraft { /// Retention after the execution deadline. pub retention_ms: i64, pub approval_reference: Option, + /// Bounded retry snapshot pinned from the exact published operation at acceptance. + pub retry: crank_core::TaskRetryPolicy, } /// Projection of the durable status/result read. `result`/`error` are only present once a terminal @@ -161,6 +163,7 @@ impl TaskService { execution_deadline_unix_ms, retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms), approval_reference: draft.approval_reference, + retry: draft.retry, }, scoped_key, input_fingerprint, @@ -196,6 +199,35 @@ impl TaskService { )) } + /// Owner-scoped cancellation of one accepted Task. The store decides under a row lock whether the + /// Task can still be cancelled or whether an already checkpointed attempt makes the outcome + /// uncertain; the service only carries the request and a bounded reason. + pub async fn cancel( + &self, + access: &TaskAccess, + task_id: TaskId, + expected_version: Option, + reason: &str, + ) -> Result { + let request = crank_core::TaskCancelRequest { + access: access.clone(), + task_id, + expected_version, + reason: reason.to_owned(), + }; + request.validate()?; + self.store.cancel(request).await + } + + /// Bounded owner-scoped attempt history of one Task. + pub async fn attempts( + &self, + access: &TaskAccess, + task_id: TaskId, + ) -> Result, TaskError> { + self.store.attempts(access, task_id).await + } + /// Decrypts and validates the executable input bound to a valid current lease. pub async fn executable_input( &self, @@ -383,6 +415,7 @@ mod tests { deadline: Instant::now() + std::time::Duration::from_secs(30), retention_ms: TASK_DEFAULT_RETENTION_MS, approval_reference: None, + retry: crank_core::TaskRetryPolicy::single_attempt(), } } diff --git a/docs/execution-boundary.md b/docs/execution-boundary.md index d186a6f..ea003f7 100644 --- a/docs/execution-boundary.md +++ b/docs/execution-boundary.md @@ -88,3 +88,23 @@ checkpoint-ятся/дренируются в пределах 30s, а ошиб не отправляя запрос; это доказано регрессией, которая детерминированно задерживает checkpoint в его собственной транзакции, проверяет отсутствие вызова adapter и сохранность ещё queued работы. + +## Безопасные повторы, отмена и outcome_unknown (Story 4.4) + +До каждого outbound — отдельная durable-попытка в таблице `task_attempts` (attempt_id, ordinal, owner/fence, input fingerprint, digest upstream-ключа, pinned retry snapshot) в одной транзакции с инкрементом бюджета и версии Task; несостоявшийся commit = ноль вызовов. + +После каждой попытки — одна атомарная resolution; только текущая попытка/владелец/версия, иначе StateConflict; история попыток после закрытия неизменна. + +Bounded retry только для read и для write с server-owned verified-idempotency evidence; клиентский key/header сам по себе не даёт write-retry; бюджет/backoff/абсолютный deadline пинятся при приёме и не меняются новой публикацией/рестартом. + +Стабильный upstream-ключ доставляется тем же `prepare_idempotency`, но синхронный coordination guard одного `tools/call` на durable-пути не применяется: авторитет повтора — PG-попытка Task, иначе неоднозначный первый ответ заблокировал бы разрешённый bounded retry. В историю попытки пишется только digest ключа, не значение и не заголовки. + +Cancel до checkpoint -> cancelled и ноль outbound (сериализуется с claim и checkpoint на одной строке Task); после checkpoint (в т.ч. verified write) -> non-terminal outcome_unknown, без retry и без обещания rollback; подтверждённый результат, выигравший CAS раньше, неизменяем. + +Отмена в окне backoff между попытками и отказ перед следующей попыткой (истёкший deadline, отклонённая authority, shutdown/детерминированная ошибка до отправки) не стирают уже записанную неопределённость: если последняя закрытая **отправленная** (`phase=dispatched`) попытка Task имеет certainty=unknown, Task закрывается non-terminal `outcome_unknown` (cancel -> код `cancelled_after_dispatch`, отказ/резолюция -> код отказа и phase прошлой попытки), а `cancelled`/`failed` выдаются только когда ни одна попытка не отправлялась. Закрытая known-попытка с `phase=before_dispatch` (request вообще не уходил) такую неопределённость не перекрывает — аудит-строка остаётся known/before_dispatch, но Task не становится терминальным `failed`. + +Deadline/lease loss: read/safe-retry -> одна bounded new attempt (fence растёт, stale worker fenced), иначе outcome_unknown; сохранённый outcome_unknown обычными finish/cancel/retry не выводится (reconciliation — Story 4.6, вне scope). + +Миграция v29 additive; legacy Task получает max_attempts=1. + +Отмена после dispatch для read тоже даёт non-terminal `outcome_unknown`: в pinned snapshot Task'а нет read/write-класса, а выводить его из текущей публикации означало бы менять уже принятый snapshot. ARC-различие «read -> failed(`cancellation_after_dispatch`)» поэтому осознанно отложено до reconciliation Story 4.6 — это консервативное отклонение в безопасную сторону, а не забытый сценарий. diff --git a/docs/schemas/migration-sequence.json b/docs/schemas/migration-sequence.json index 2a9d103..4c5f659 100644 --- a/docs/schemas/migration-sequence.json +++ b/docs/schemas/migration-sequence.json @@ -1 +1 @@ -{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23},{"backfill":{"kind":"none"},"checksum":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-admission-v24","owner":"crank-registry","phase":"expand","readable_schema_max":24,"readable_schema_min":23,"source_digest":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","transactional":true,"version":24},{"backfill":{"kind":"none"},"checksum":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v25","owner":"crank-registry","phase":"expand","readable_schema_max":25,"readable_schema_min":24,"source_digest":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","transactional":true,"version":25},{"backfill":{"kind":"none"},"checksum":"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v26","owner":"crank-registry","phase":"expand","readable_schema_max":26,"readable_schema_min":25,"source_digest":"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089","transactional":true,"version":26},{"backfill":{"kind":"none"},"checksum":"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v27","owner":"crank-registry","phase":"expand","readable_schema_max":27,"readable_schema_min":26,"source_digest":"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8","transactional":true,"version":27},{"backfill":{"kind":"none"},"checksum":"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v28","owner":"crank-registry","phase":"expand","readable_schema_max":28,"readable_schema_min":27,"source_digest":"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676","transactional":true,"version":28}]} +{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23},{"backfill":{"kind":"none"},"checksum":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-admission-v24","owner":"crank-registry","phase":"expand","readable_schema_max":24,"readable_schema_min":23,"source_digest":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","transactional":true,"version":24},{"backfill":{"kind":"none"},"checksum":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v25","owner":"crank-registry","phase":"expand","readable_schema_max":25,"readable_schema_min":24,"source_digest":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","transactional":true,"version":25},{"backfill":{"kind":"none"},"checksum":"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v26","owner":"crank-registry","phase":"expand","readable_schema_max":26,"readable_schema_min":25,"source_digest":"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089","transactional":true,"version":26},{"backfill":{"kind":"none"},"checksum":"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v27","owner":"crank-registry","phase":"expand","readable_schema_max":27,"readable_schema_min":26,"source_digest":"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8","transactional":true,"version":27},{"backfill":{"kind":"none"},"checksum":"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v28","owner":"crank-registry","phase":"expand","readable_schema_max":28,"readable_schema_min":27,"source_digest":"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676","transactional":true,"version":28},{"backfill":{"kind":"none"},"checksum":"0666c7d2aa440e6aae2170ceaf2fce2549118612e1505ef78817be60436a75dd","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v29","owner":"crank-registry","phase":"expand","readable_schema_max":29,"readable_schema_min":28,"source_digest":"0666c7d2aa440e6aae2170ceaf2fce2549118612e1505ef78817be60436a75dd","transactional":true,"version":29}]} diff --git a/scripts/ci-story-checks.sh b/scripts/ci-story-checks.sh index b868bf3..940e387 100755 --- a/scripts/ci-story-checks.sh +++ b/scripts/ci-story-checks.sh @@ -85,3 +85,10 @@ run_atdd "durable task migration" -p crank-registry --test integration migration # claims before a slow HTTP drain, not after it) and one child attempt span per adapter retry. run_atdd "task worker process shutdown" -p mcp-server --bin mcp-server -- --test-threads=1 run_atdd "durable task retry attempt spans" -p crank-runtime --test stages retry -- --test-threads=1 + +# Story 4.4 (FR-22, FR-23): safe retries, cancellation and outcome_unknown. No new selector is added +# here on purpose: the Story 4.2 lines above already run `crank-core --lib task::` (attempt/retry +# bounds), `crank-registry --test integration tasks::` (checkpoint, resolution, cancel and recovery +# CAS) and `mcp-server --test integration durable_tasks::` (application → PG → worker attempts, +# cancellation through the application entry and the composition oracles). Duplicating one of them +# here would only run the same bounded set twice.