feat(epic4): продолжение ожидающей Task после ввода (Story 4.5)
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped

Durable worker приостанавливает Task как input_required до dispatch, если
закреплённая схема требует отсутствующее поле; authorized schema-validated
update атомарно и с аудитом возобновляет тот же Task ровно один раз.
Replay/conflict/TTL безопасны, ожидание удерживает admission permit.
Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
This commit is contained in:
bsodfather committed 2026-10-30 16:24:12 +00:00
1 parent d80eda7329
commit 90e5357a08
35 files changed
+3243 -91

No files matched your search

@@ -0,0 +1,965 @@
//! ATDD Story 4.5 (Beam): durable input continuation through TaskApplication -> PG -> worker -> upstream.
use super::common;
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
use crank_core::{
NoTasksFallback, PlatformApiKeyScope, TaskId, TaskState, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
};
use serde_json::json;
use sqlx::Row;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
const AGENT_SLUG: &str = "reports";
pub(crate) const TOOL_NAME: &str = "report_gen";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
pub(crate) async fn state_of(registry: &PostgresRegistry, task_id: TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
pub(crate) async fn wait_for_state(
registry: &PostgresRegistry,
task_id: TaskId,
desired: &str,
limit: Duration,
) -> bool {
let deadline = tokio::time::Instant::now() + limit;
while tokio::time::Instant::now() < deadline {
if state_of(registry, task_id).await.as_deref() == Some(desired) {
return true;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
state_of(registry, task_id).await.as_deref() == Some(desired)
}
pub(crate) async fn seed_tool(
registry: &PostgresRegistry,
upstream: &str,
policy: ToolExecutionPolicy,
) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![crank_core::AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
}
pub(crate) async fn principal(
registry: &PostgresRegistry,
name: &str,
) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(
registry,
AGENT_SLUG,
name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("active key");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
pub(crate) async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store, Arc::new(crypto)),
)
}
pub(crate) fn spawn_worker(
registry: &PostgresRegistry,
app: &TaskApplication,
) -> (
Arc<std::sync::atomic::AtomicBool>,
tokio::task::JoinHandle<()>,
) {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let pool = registry.pool().clone();
let store = Arc::new(
PostgresTaskStore::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
app,
registry.clone(),
runtime,
crypto,
));
let worker = TaskWorker::new(
store,
handler,
TaskWorkerConfig {
owner: "worker-input".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
renewal_interval: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(5),
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let jh = tokio::spawn(async move {
let _ = worker.run().await;
});
(stop_flag, jh)
}
#[tokio::test]
async fn input_continuation_happy_path_accept_update_resume_replay() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "input-happy").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-1",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
assert_eq!(acceptance.state, TaskState::Queued);
let task_id = acceptance.task_id;
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(
&registry,
task_id,
"input_required",
Duration::from_secs(10)
)
.await,
"worker must suspend for missing email"
);
// positive control: before update no outbound request
assert_eq!(calls.load(Ordering::SeqCst), 0);
// persisted request fields
let row = sqlx::query(
"select wait_id, expected_version, schema_snapshot, projected_schema, prompt_code, expires_at from task_input_requests where task_id = $1",
)
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
let wait_id: String = row.get("wait_id");
let expected_version: i32 = row.get("expected_version");
let schema_snapshot: serde_json::Value = row.get("schema_snapshot");
let projected_schema: serde_json::Value = row.get("projected_schema");
let prompt_code: String = row.get("prompt_code");
let _expires_at: OffsetDateTime = row.get("expires_at");
assert!(!wait_id.is_empty());
assert!(expected_version > 0);
assert!(schema_snapshot.is_object());
assert!(projected_schema.is_object());
assert_eq!(prompt_code, "required_input_missing");
let audit_count_before: i64 = sqlx::query_scalar(
"select count(*)::bigint from task_input_audit where task_id = $1 and event = 'input_required'",
)
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(audit_count_before >= 1);
// stop worker before update to avoid races
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let outcome = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version as u32,
"update-key-1",
&json!({"email": "user@example.com"}),
)
.await
.expect("valid update");
assert_eq!(outcome.state, TaskState::Working);
assert!(!outcome.replayed);
assert_eq!(outcome.task_id, task_id);
assert_eq!(outcome.wait_id, wait_id);
let (stop2, worker_task2) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"worker must complete after update"
);
stop2.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
let attempts = app
.attempts(&caller, TOOL_NAME, task_id)
.await
.expect("attempts");
assert_eq!(attempts.len(), 1);
let effective_fp: String =
sqlx::query_scalar("select effective_input_fingerprint from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts[0].input_fingerprint, effective_fp);
let input_fp: String =
sqlx::query_scalar("select input_fingerprint from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_ne!(effective_fp, input_fp);
// An identical answer replay still returns the committed outcome after the Task completed:
// no second transition, audit row or outbound call.
let completed_replay = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version as u32,
"update-key-1",
&json!({"email": "user@example.com"}),
)
.await
.expect("identical replay after completion");
assert!(completed_replay.replayed);
assert_eq!(completed_replay.version, outcome.version);
assert_eq!(calls.load(Ordering::SeqCst), 1);
// replay with original scoped key/partial input: same Task, no fingerprint conflict
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-1",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("replay accepted");
assert_eq!(replay.task_id, task_id);
server.abort();
}
pub(crate) async fn start_and_suspend(
registry: &PostgresRegistry,
app: &TaskApplication,
caller: &crank_community_mcp::TaskPrincipal,
scoped_key: &str,
) -> (TaskId, String, u32) {
let acceptance = app
.accept_background(
caller,
TOOL_NAME,
None,
scoped_key,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
let (stop, worker_task) = spawn_worker(registry, app);
assert!(
wait_for_state(
registry,
acceptance.task_id,
"input_required",
Duration::from_secs(10)
)
.await,
"worker must suspend for missing email"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let (wait_id, expected_version) = wait_row(registry, acceptance.task_id).await;
(acceptance.task_id, wait_id, expected_version)
}
async fn wait_row(registry: &PostgresRegistry, task_id: TaskId) -> (String, u32) {
let row =
sqlx::query("select wait_id, expected_version from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
(
row.get::<String, _>("wait_id"),
u32::try_from(row.get::<i32, _>("expected_version")).unwrap_or_default(),
)
}
pub(crate) async fn audit_events(
registry: &PostgresRegistry,
task_id: TaskId,
) -> Vec<(String, Option<String>)> {
sqlx::query("select event, reason_code from task_input_audit where task_id = $1 order by id")
.bind(task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap()
.into_iter()
.map(|row| (row.get("event"), row.get("reason_code")))
.collect()
}
pub(crate) async fn admission_entries(
registry: &PostgresRegistry,
task_id: TaskId,
) -> Vec<(String, i64)> {
sqlx::query(
"select state, count(*)::bigint as count from execution_admission_entries
where task_id = $1 group by state order by state",
)
.bind(task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap()
.into_iter()
.map(|row| (row.get("state"), row.get("count")))
.collect()
}
fn key_id(principal: &crank_community_mcp::TaskPrincipal) -> crank_core::PlatformApiKeyId {
match principal {
crank_community_mcp::TaskPrincipal::Agent { credential, .. } => credential
.platform_api_key_id
.clone()
.expect("agent principal carries a key id"),
crank_community_mcp::TaskPrincipal::Admin { .. } => panic!("agent principal expected"),
}
}
#[tokio::test]
async fn invalid_schema_foreign_scope_and_revoked_key_never_transition_or_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-scope").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-scope").await;
// Invalid answer: server-side schema validation refuses it, nothing transitions and nothing leaks.
let invalid = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-invalid",
&json!({}),
)
.await
.expect_err("the pinned schema requires email");
assert!(
matches!(
invalid,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::Invalid { .. })
),
"unexpected error: {invalid:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(
audit_events(&registry, task_id)
.await
.iter()
.all(|(event, _)| event == "input_required"),
"a rejected answer must not be audited as accepted"
);
let stored_snapshot: serde_json::Value =
sqlx::query_scalar("select schema_snapshot from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(!stored_snapshot.to_string().contains("user@example.com"));
// A different key of the same agent carries a different authorization scope: opaque not-found.
let foreign = principal(&registry, "input-foreign").await;
let foreign_result = app
.update_input(
&foreign,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-foreign",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a foreign scope cannot continue the Task");
assert_eq!(
foreign_result,
crank_community_mcp::TaskApplicationError::NotFound
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
// Positive control: the accepting principal (same key/scope) completes the transition.
let accepted = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-valid",
&json!({"email": "user@example.com"}),
)
.await
.expect("valid answer for the accepting scope");
assert_eq!(accepted.state, TaskState::Working);
assert!(!accepted.replayed);
// Finish the positive-control Task through a real worker so the revocation case starts from a
// known dispatch count instead of leaving resumable work behind.
let (finish_stop, finish_worker) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"the positive-control Task completes"
);
finish_stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), finish_worker).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
// Revocation race: the accepting key is revoked after suspension. The same waiting scope must
// fail closed inside the update transaction and never revive or dispatch the Task.
let revocable = principal(&registry, "input-revoked").await;
let (revoked_task, revoked_wait, revoked_version) =
start_and_suspend(&registry, &app, &revocable, "input-key-revoked").await;
registry
.revoke_platform_api_key(
&common::test_workspace_id(),
&key_id(&revocable),
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
let revoked_result = app
.update_input(
&revocable,
TOOL_NAME,
revoked_task,
&revoked_wait,
revoked_version,
"update-revoked",
&json!({"email": "revoked@example.com"}),
)
.await
.expect_err("a revoked key cannot continue the Task");
assert_eq!(
revoked_result,
crank_community_mcp::TaskApplicationError::NotFound
);
assert_eq!(
state_of(&registry, revoked_task).await.as_deref(),
Some("input_required")
);
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"the revoked answer must not add an outbound call"
);
server.abort();
}
#[tokio::test]
async fn replay_conflict_and_stale_answers_have_one_typed_outcome() {
let registry = common::test_registry().await;
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, "http://127.0.0.1:9", policy).await;
let caller = principal(&registry, "input-replay").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-replay").await;
let first = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "user@example.com"}),
)
.await
.expect("first valid answer");
assert!(!first.replayed);
assert_eq!(first.state, TaskState::Working);
// Identical replay returns the stored outcome without a second CAS/audit.
let replay = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "user@example.com"}),
)
.await
.expect("identical replay");
assert!(replay.replayed);
assert_eq!(replay.version, first.version);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("working")
);
let accepted_audit = audit_events(&registry, task_id)
.await
.into_iter()
.filter(|(event, _)| event == "input_accepted")
.count();
assert_eq!(accepted_audit, 1);
// Same key with another payload is a fingerprint conflict.
let conflict = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "other@example.com"}),
)
.await
.expect_err("same key with another payload conflicts");
assert_eq!(
conflict,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::FingerprintConflict)
);
// Another key after a committed answer, and a foreign wait id, are state conflicts.
let other_key = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
first.version,
"update-key-2",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("another key after commit conflicts");
assert_eq!(
other_key,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
let foreign_wait = app
.update_input(
&caller,
TOOL_NAME,
task_id,
"wait-does-not-exist",
first.version,
"update-key-3",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a foreign wait id conflicts");
assert_eq!(
foreign_wait,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
// A stale generation on a fresh wait is refused; the valid generation still works.
let (stale_task, stale_wait, stale_version) =
start_and_suspend(&registry, &app, &caller, "input-key-stale").await;
let stale = app
.update_input(
&caller,
TOOL_NAME,
stale_task,
&stale_wait,
stale_version.saturating_sub(1).max(1),
"update-stale",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a stale generation cannot be continued");
assert_eq!(
stale,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
let repaired = app
.update_input(
&caller,
TOOL_NAME,
stale_task,
&stale_wait,
stale_version,
"update-stale",
&json!({"email": "user@example.com"}),
)
.await
.expect("the current generation still accepts the answer");
assert!(!repaired.replayed);
}
#[tokio::test]
async fn ttl_expiry_fails_with_audit_and_releases_capacity_without_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
// The wait TTL is the remaining pinned absolute execution deadline.
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 1_200,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-ttl").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-ttl",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
let task_id = acceptance.task_id;
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "input_required", Duration::from_secs(5)).await,
"worker must suspend"
);
assert!(
wait_for_state(&registry, task_id, "failed", Duration::from_secs(6)).await,
"the per-poll sweeper must fail the expired wait"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let (wait_id, expected_version) = wait_row(&registry, task_id).await;
let events = audit_events(&registry, task_id).await;
assert!(
events.iter().any(|(event, reason)| event == "input_expired"
&& reason.as_deref() == Some("input_ttl_expired")),
"expiry reason must be audited: {events:?}"
);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
assert_eq!(calls.load(Ordering::SeqCst), 0);
// A late answer cannot revive the failed Task.
let late = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-late",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("an expired wait never revives");
assert!(
matches!(
late,
crank_community_mcp::TaskApplicationError::Task(
crank_core::TaskError::InputExpired | crank_core::TaskError::StateConflict
)
),
"unexpected late answer error: {late:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("failed")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn competing_identical_updates_commit_once_and_keep_one_permit() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-race").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-race").await;
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)],
"the waiting Task keeps exactly one executing permit"
);
let payload = json!({"email": "user@example.com"});
let first = app.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-race",
&payload,
);
let second = app.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-race",
&payload,
);
let (first, second) = tokio::join!(first, second);
let mut outcomes = [
first.expect("competing answer"),
second.expect("competing answer"),
];
outcomes.sort_by_key(|outcome| outcome.replayed);
assert!(!outcomes[0].replayed);
assert!(outcomes[1].replayed);
assert_eq!(outcomes[0].version, outcomes[1].version);
assert_eq!(
audit_events(&registry, task_id)
.await
.into_iter()
.filter(|(event, _)| event == "input_accepted")
.count(),
1
);
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)],
"the update must not grant a second permit"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"the resumed Task completes"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
server.abort();
}
@@ -0,0 +1,457 @@
//! ATDD Story 4.5 QA1 guards (F1/F3): one pinned deadline for several waiting Tasks, cancellation
//! while the Task waits for input and refusal of a stale lease/fence around the continuation seam.
use super::common;
use super::input_continuation::{
TOOL_NAME, admission_entries, application, audit_events, principal, seed_tool, spawn_worker,
start_and_suspend, state_of, wait_for_state,
};
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_core::{
BeginAttempt, NoTasksFallback, TASK_INPUT_SCHEMA_FORMAT_VERSION, TaskInputRequest,
TaskInputStore, TaskState, TaskStore, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{AdmissionLimits, PostgresTaskStore};
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
#[tokio::test]
async fn two_waits_with_one_pinned_deadline_and_cancelled_leftover_do_not_block_suspension() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "deadline-cancel").await;
let app = application(&registry).await;
let acc_a = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-a",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept a");
let acc_b = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-b",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept b");
let task_a = acc_a.task_id;
let task_b = acc_b.task_id;
let pinned_ms = (OffsetDateTime::now_utc().unix_timestamp() as f64 * 1000.0) + 60_000.0;
for tid in [task_a, task_b] {
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
.bind(pinned_ms)
.bind(tid.as_uuid())
.execute(registry.pool())
.await
.unwrap();
}
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_a, "input_required", Duration::from_secs(10)).await,
"a must suspend"
);
assert!(
wait_for_state(&registry, task_b, "input_required", Duration::from_secs(10)).await,
"b must suspend"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(calls.load(Ordering::SeqCst), 0);
// Both active waits carry the same absolute TTL value: the per-Task uniqueness above is the only
// uniqueness, so this must not be a UNIQUE index on expires_at.
let same_deadline_wait: i64 = sqlx::query_scalar(
"select count(*)::bigint from task_input_requests a
join task_input_requests b on b.task_id = $2
where a.task_id = $1 and a.accepted_at is null and b.accepted_at is null
and a.expires_at = b.expires_at",
)
.bind(task_a.as_uuid())
.bind(task_b.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
same_deadline_wait, 1,
"both waits share the pinned deadline"
);
let outcome = app
.cancel(&caller, TOOL_NAME, task_a, None)
.await
.expect("cancel a");
assert!(matches!(
outcome,
crank_core::TaskCancelOutcome::Cancelled(_)
));
assert_eq!(
state_of(&registry, task_a).await.as_deref(),
Some("cancelled")
);
assert_eq!(admission_entries(&registry, task_a).await, Vec::new());
// accept C with same pinned deadline
let acc_c = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-c",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept c");
let task_c = acc_c.task_id;
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
.bind(pinned_ms)
.bind(task_c.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let (stop2, worker_task2) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_c, "input_required", Duration::from_secs(10)).await,
"c must suspend despite leftover a"
);
stop2.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
assert_eq!(
state_of(&registry, task_b).await.as_deref(),
Some("input_required")
);
let c_deadline_ms: i64 = sqlx::query_scalar(
"select (extract(epoch from expires_at) * 1000)::bigint
from task_input_requests where task_id = $1",
)
.bind(task_c.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
c_deadline_ms, pinned_ms as i64,
"the third wait reuses the same pinned deadline value"
);
let a_accepted: Option<OffsetDateTime> =
sqlx::query_scalar("select accepted_at from task_input_requests where task_id = $1")
.bind(task_a.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(
a_accepted.is_none(),
"cancellation leaves the wait row without an accepted receipt"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn cancel_while_input_required_never_revives_or_dispatches() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "cancel-wait").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-cancel").await;
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)]
);
let outcome = app
.cancel(&caller, TOOL_NAME, task_id, None)
.await
.expect("cancel");
assert!(matches!(
outcome,
crank_core::TaskCancelOutcome::Cancelled(_)
));
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("cancelled")
);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
let late = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-late",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("late answer must not revive");
assert!(
matches!(
late,
crank_community_mcp::TaskApplicationError::Task(
crank_core::TaskError::StateConflict | crank_core::TaskError::InputExpired
)
),
"unexpected: {late:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("cancelled")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let events = audit_events(&registry, task_id).await;
assert!(events.iter().any(|(e, _)| e == "input_required"));
assert!(!events.iter().any(|(e, _)| e == "input_accepted"));
let attempts = app
.attempts(&caller, TOOL_NAME, task_id)
.await
.expect("attempts");
assert!(attempts.is_empty());
server.abort();
}
#[tokio::test]
async fn stale_lease_fence_cannot_suspend_or_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "stale-fence").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"stale-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let task_id = acceptance.task_id;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let mut old_claims = store
.claim_batch("worker-old", 1, 50)
.await
.expect("old claim");
assert!(!old_claims.is_empty());
let old = old_claims.remove(0);
tokio::time::sleep(Duration::from_millis(120)).await;
let mut new_claims = store
.claim_batch("worker-new", 1, 10_000)
.await
.expect("new claim");
assert!(!new_claims.is_empty());
let new = new_claims.remove(0);
assert!(new.fence > old.fence);
assert_eq!(new.record.task_id, task_id);
let request = TaskInputRequest {
wait_id: "stale-wait".to_string(),
expected_version: old.record.version + 1,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({}),
projected_schema: json!({}),
prompt_code: "required_input_missing".to_string(),
missing_fields: vec!["$.email".to_string()],
expires_at_unix_ms: OffsetDateTime::now_utc().unix_timestamp() * 1000 + 30_000,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
};
let suspended =
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &old, request).await;
assert!(
suspended.is_err(),
"stale fence must not suspend: {suspended:?}"
);
let attempt_id = "attempt-stale".to_string();
let checkpoint = store
.checkpoint_dispatch(BeginAttempt {
task_id,
owner: old.owner.clone(),
fence: old.fence,
expected_version: old.record.version,
attempt_id,
input_fingerprint: old.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(
checkpoint.is_err(),
"stale fence must not checkpoint: {checkpoint:?}"
);
assert_ne!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
let attempt_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempt_count, 0);
let req_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(req_count, 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
// Positive control: the same request under the current lease suspends the Task exactly once, so
// the refusal above is the stale fence and not an invalid request shape.
let request = TaskInputRequest {
wait_id: "current-wait".to_string(),
expected_version: new.record.version + 1,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({}),
projected_schema: json!({}),
prompt_code: "required_input_missing".to_string(),
missing_fields: vec!["$.email".to_string()],
expires_at_unix_ms: new.record.pins.execution_deadline_unix_ms,
};
let suspended =
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &new, request)
.await
.expect("the current lease suspends");
assert_eq!(suspended.state, TaskState::InputRequired);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)]
);
// The stale fence still cannot checkpoint a dispatch of this generation.
let checkpoint = store
.checkpoint_dispatch(BeginAttempt {
task_id,
owner: old.owner.clone(),
fence: old.fence,
expected_version: old.record.version,
attempt_id: "attempt-stale-after-suspend".to_string(),
input_fingerprint: old.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(
checkpoint.is_err(),
"stale fence must not checkpoint: {checkpoint:?}"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
@@ -653,5 +653,7 @@ pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::Attempt
mod attempt_guards;
mod attempt_trace;
mod cancel_chain;
mod input_continuation;
mod input_continuation_guards;
mod shutdown_window;
mod task_worker;
@@ -16,9 +16,9 @@ use crank_core::{
use crank_registry::PostgresRegistry;
use crank_registry::PublishedAgentTool;
use crank_runtime::{
ExecutionAuthorization, RuntimeExecutor, RuntimeOperation, SecretCrypto, TaskAttemptAuthority,
TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan, TaskAttemptRefusal, TaskHandler,
TaskService, TaskWorkerError,
ExecutionAuthorization, PrepareAttempt, RuntimeExecutor, RuntimeOperation, SecretCrypto,
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal, TaskHandler, TaskService, TaskWorkerError,
};
use sha2::{Digest, Sha256};
@@ -233,6 +233,39 @@ impl TaskWorkerHandler {
(true, Some(format!("{:x}", digest.finalize())))
}
/// Story 4.5 producer: the pinned operation still lacks a required input item, so the Task must be
/// suspended before any dispatch. The schema snapshot and its presentation projection are frozen
/// here; the TTL never exceeds the accepted absolute execution deadline.
fn input_request(
&self,
tool: &PublishedAgentTool,
lease: &TaskLease,
missing_path: &str,
expires_at_unix_ms: i64,
) -> Result<crank_core::TaskInputRequest, TaskAttemptRefusal> {
let schema_snapshot = serde_json::to_value(&tool.operation.input_schema).map_err(|_| {
TaskAttemptRefusal::Failed {
code: "task_input_schema_unavailable".to_owned(),
}
})?;
let request = crank_core::TaskInputRequest {
wait_id: format!("wait-{}", uuid::Uuid::now_v7()),
expected_version: lease.record.version.saturating_add(1),
schema_revision: tool.operation.version,
schema_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot,
projected_schema: crate::manifest::schema_to_json_schema(&tool.operation.input_schema),
prompt_code: "required_input_missing".to_owned(),
missing_fields: vec![missing_path.to_owned()],
expires_at_unix_ms,
fingerprint_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
};
request.validate().map_err(|_| TaskAttemptRefusal::Failed {
code: "task_input_request_invalid".to_owned(),
})?;
Ok(request)
}
/// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store
/// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal.
fn refusal(error: TaskError) -> TaskAttemptRefusal {
@@ -264,7 +297,7 @@ impl TaskAttemptAuthority for TaskWorkerHandler {
async fn prepare_attempt(
&self,
lease: &TaskLease,
) -> Result<TaskAttemptPlan, TaskAttemptRefusal> {
) -> Result<PrepareAttempt, TaskAttemptRefusal> {
let tool = self.authorize(lease).await.map_err(Self::refusal)?;
let input = self
.service
@@ -281,6 +314,24 @@ impl TaskAttemptAuthority for TaskWorkerHandler {
code: "task_deadline_elapsed".to_owned(),
});
}
// Only a missing required item of the pinned schema becomes a typed input request. Any other
// schema violation keeps the existing pre-dispatch attempt validation, and a generation that
// already accepted a continuation can never open a second wait for the same Task.
if let Some(missing_path) =
crank_runtime::missing_required_input(&tool.operation.input_schema, &input)
{
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
return Err(TaskAttemptRefusal::Failed {
code: "task_input_still_incomplete".to_owned(),
});
}
return Ok(PrepareAttempt::InputRequired(self.input_request(
&tool,
lease,
&missing_path,
deadline_ms,
)?));
}
let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64);
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
@@ -313,7 +364,7 @@ impl TaskAttemptAuthority for TaskWorkerHandler {
);
let (retry_allowed, upstream_key_digest) =
self.retry_classification(&operation, &input, deadline_ms - now_ms);
Ok(TaskAttemptPlan {
Ok(PrepareAttempt::Ready(Box::new(TaskAttemptPlan {
workspace_id,
origin: ExecutionOrigin::AgentSnapshot,
agent_id: Some(agent_id),
@@ -325,6 +376,6 @@ impl TaskAttemptAuthority for TaskWorkerHandler {
deadline,
retry_allowed,
upstream_key_digest,
})
})))
}
}
@@ -299,6 +299,43 @@ impl TaskApplication {
}
}
/// Applies one additional validated input to a waiting Task (Story 4.5).
///
/// This is the server-internal application entry: the public Task MCP API stays Story 4.7. The
/// principal is already authenticated here, the owner scope is derived from the verified
/// credential and the durable transaction revalidates the same key before any receipt lookup.
/// An identical authorized replay returns the committed outcome; a conflicting or stale answer is
/// typed, and an elapsed TTL never revives the Task.
#[allow(clippy::too_many_arguments)]
pub async fn update_input(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
wait_id: &str,
expected_version: u32,
update_key: &str,
input: &Value,
) -> Result<crank_core::TaskInputUpdateOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self
.service
.update_input(
&access,
task_id,
wait_id,
expected_version,
update_key,
input,
)
.await
{
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped attempt history of one Task of the addressed published Tool.
pub async fn attempts(
&self,
+9 -7
View File
@@ -209,13 +209,15 @@ pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use task::{
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_ATTEMPTS,
TASK_MAX_BACKOFF_STEP_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt,
TaskCancelOutcome, TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId,
TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState,
TaskStore, TaskSupportIds, contains_sensitive_task_keys, redact_task_envelope,
sensitive_task_key, validate_task_envelope,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
TASK_INPUT_PROMPT_CODE_MAX_BYTES, TASK_INPUT_SCHEMA_FORMAT_VERSION,
TASK_INPUT_WAIT_ID_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest,
TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskRetryPolicy, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
redact_task_envelope, sensitive_task_key, validate_task_envelope,
};
pub use tool_catalog::{
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
+295
View File
@@ -0,0 +1,295 @@
//! Continuation of an accepted Task through a validated additional input (Story 4.5, FR-25).
//!
//! A Task that cannot run because the pinned operation still lacks a required input item is
//! suspended as `input_required` before any outbound dispatch. The durable request keeps the exact
//! immutable schema snapshot, its JSON projection, safe prompt metadata, the absolute TTL and the
//! update fingerprint contract. A valid update moves the Task back to `working` exactly once; an
//! identical replay returns the committed outcome and a conflicting or stale reply is typed.
//!
//! Deliberately absent: raw input in the request/receipt/audit read models, a second cipher store and
//! any public transport. The regular executable-input ciphertext carries the continuation binding.
use serde_json::Value;
use crate::task::{TaskAccess, TaskError, TaskId};
/// Version of the persisted input-continuation contract. A stored snapshot always declares the
/// version it was written with, so a later binary never reinterprets it silently.
pub const TASK_INPUT_SCHEMA_FORMAT_VERSION: u32 = 1;
/// Bound of a wait identity before it is persisted.
pub const TASK_INPUT_WAIT_ID_MAX_BYTES: usize = 64;
/// Bound of the safe prompt code (never the prompt text or an input value).
pub const TASK_INPUT_PROMPT_CODE_MAX_BYTES: usize = 64;
/// Bound of the missing-field path metadata.
pub const TASK_INPUT_MAX_MISSING_FIELDS: usize = 32;
/// Bound of one missing-field path.
pub const TASK_INPUT_FIELD_PATH_MAX_BYTES: usize = 160;
/// Durable request of additional input for one suspended Task.
///
/// `schema_snapshot` is the native serialized schema subset the server validates against; the JSON
/// projection is presentation metadata only and never the validation authority. `expected_version`
/// is the Task version a caller must present with the answer.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskInputRequest {
pub wait_id: String,
pub expected_version: u32,
pub schema_revision: u32,
pub schema_format_version: u32,
pub schema_snapshot: Value,
pub projected_schema: Value,
pub prompt_code: String,
pub missing_fields: Vec<String>,
pub expires_at_unix_ms: i64,
pub fingerprint_format_version: u32,
}
impl TaskInputRequest {
pub fn validate(&self) -> Result<(), TaskError> {
if self.wait_id.is_empty()
|| self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES
|| self.wait_id.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task input wait identity is invalid"));
}
if self.expected_version == 0 || self.schema_revision == 0 {
return Err(TaskError::invalid(
"task input request generation is invalid",
));
}
if self.schema_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
|| self.fingerprint_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
{
return Err(TaskError::invalid(
"task input request format version is invalid",
));
}
if self.schema_snapshot.as_object().is_none() || self.projected_schema.as_object().is_none()
{
return Err(TaskError::invalid("task input schema snapshot is invalid"));
}
if self.prompt_code.is_empty() || self.prompt_code.len() > TASK_INPUT_PROMPT_CODE_MAX_BYTES
{
return Err(TaskError::invalid("task input prompt code is invalid"));
}
if self.missing_fields.len() > TASK_INPUT_MAX_MISSING_FIELDS
|| self
.missing_fields
.iter()
.any(|field| field.is_empty() || field.len() > TASK_INPUT_FIELD_PATH_MAX_BYTES)
{
return Err(TaskError::invalid(
"task input missing-field metadata is invalid",
));
}
if self.expires_at_unix_ms <= 0 {
return Err(TaskError::invalid("task input TTL is invalid"));
}
Ok(())
}
}
/// Authoritative write of one validated answer. The transaction owns schema-generation, TTL,
/// fingerprint and CAS checks; this request only carries the committed values.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskInputUpdateRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub wait_id: String,
pub expected_version: u32,
/// Digest of the scoped update key; the caller key is never persisted.
pub update_key_digest: String,
/// Canonical digest of the validated full executable input.
pub canonical_fingerprint: String,
/// Effective execution fingerprint of the resumed payload (never the acceptance fingerprint).
pub effective_input_fingerprint: String,
pub executable_input_ciphertext: String,
pub executable_input_key_version: String,
pub executable_input_master_key_epoch: i64,
}
impl TaskInputUpdateRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if self.wait_id.is_empty() || self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES {
return Err(TaskError::invalid("task input wait identity is invalid"));
}
if self.expected_version == 0 {
return Err(TaskError::invalid("task input version is invalid"));
}
if !valid_digest(&self.update_key_digest)
|| !valid_digest(&self.canonical_fingerprint)
|| !valid_digest(&self.effective_input_fingerprint)
{
return Err(TaskError::invalid("task input digest is invalid"));
}
if self.executable_input_ciphertext.is_empty()
|| self.executable_input_ciphertext.len()
> crate::task::TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
{
return Err(TaskError::invalid(
"task continuation ciphertext is invalid",
));
}
if self.executable_input_key_version.is_empty()
|| self.executable_input_key_version.len() > 64
|| self.executable_input_master_key_epoch < 1
{
return Err(TaskError::invalid(
"task continuation ciphertext is invalid",
));
}
Ok(())
}
}
/// Committed outcome of one input update. An identical replay returns the same stored outcome with
/// `replayed = true` instead of repeating the CAS, audit or dispatch.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskInputUpdateOutcome {
pub task_id: TaskId,
pub wait_id: String,
pub state: crate::task::TaskState,
pub version: u32,
pub replayed: bool,
}
fn valid_digest(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
}
/// Durable input-continuation operations of the Task store.
///
/// Split out of the general [`crate::task::TaskStore`] port so the lifecycle store stays readable and
/// the continuation contract (suspend, answer, receipt, expiry) is implemented and audited as one
/// cohesive surface. [`crate::task::TaskStore`] extends this trait, so an owned `Arc<dyn TaskStore>`
/// exposes both halves through one object.
#[async_trait::async_trait]
pub trait TaskInputStore: Send + Sync {
/// Suspends a leased Task that still lacks a required input of its pinned operation. The
/// transaction persists the immutable request and its audit row, moves `working` to
/// `input_required` exactly once, releases the worker lease and keeps the existing admission
/// permit. No outbound dispatch may have happened in this generation.
async fn suspend_for_input(
&self,
lease: &crate::task::TaskLease,
request: TaskInputRequest,
) -> Result<crate::task::TaskRecord, TaskError>;
/// Owner-scoped read of the durable input request of one Task.
async fn input_request(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<TaskInputRequest, TaskError>;
/// Owner-scoped read of the executable-input binding that a committed continuation may extend.
/// It is not lease-authorized: it only serves the update transaction and never dispatch.
async fn read_executable_input_for_update(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<(String, String, i64), TaskError>;
/// Applies exactly one validated answer under a row lock: authority recheck, receipt lookup,
/// generation/TTL check, atomic ciphertext/effective-fingerprint/receipt/`working` transition and
/// audit. An identical replay returns the stored outcome without a second transition.
async fn apply_input(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError>;
/// Bounded sweep of `input_required` Tasks whose TTL elapsed: `failed` with an audit reason and
/// the release of the admission permit, without any dispatch.
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError>;
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn request() -> TaskInputRequest {
TaskInputRequest {
wait_id: "wait-1".to_owned(),
expected_version: 2,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({"type": "object", "fields": {}}),
projected_schema: json!({"type": "object", "properties": {}}),
prompt_code: "required_input_missing".to_owned(),
missing_fields: vec!["$.email".to_owned()],
expires_at_unix_ms: 1_900_000_000_000,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
}
}
fn update() -> TaskInputUpdateRequest {
TaskInputUpdateRequest {
access: crate::task::TaskAccess {
workspace_id: "ws_default".to_owned(),
owner_kind: crate::task::TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
},
task_id: crate::task::TaskId::generate(),
wait_id: "wait-1".to_owned(),
expected_version: 2,
update_key_digest: "a".repeat(64),
canonical_fingerprint: "b".repeat(64),
effective_input_fingerprint: "c".repeat(64),
executable_input_ciphertext: "sealed".to_owned(),
executable_input_key_version: "key-v1".to_owned(),
executable_input_master_key_epoch: 1,
}
}
#[test]
fn input_request_holds_generation_schema_ttl_and_bounds() {
assert!(request().validate().is_ok());
let mut no_wait = request();
no_wait.wait_id.clear();
assert!(no_wait.validate().is_err());
let mut no_generation = request();
no_generation.expected_version = 0;
assert!(no_generation.validate().is_err());
let mut bad_schema = request();
bad_schema.schema_snapshot = json!("not-an-object");
assert!(bad_schema.validate().is_err());
let mut no_ttl = request();
no_ttl.expires_at_unix_ms = 0;
assert!(no_ttl.validate().is_err());
let mut too_many_fields = request();
too_many_fields.missing_fields = vec!["$.x".to_owned(); TASK_INPUT_MAX_MISSING_FIELDS + 1];
assert!(too_many_fields.validate().is_err());
}
#[test]
fn update_request_rejects_invalid_digests_and_ciphertext_bounds() {
assert!(update().validate().is_ok());
let mut short_digest = update();
short_digest.canonical_fingerprint = "abc".to_owned();
assert!(short_digest.validate().is_err());
let mut upper_digest = update();
upper_digest.update_key_digest = "A".repeat(64);
assert!(upper_digest.validate().is_err());
let mut no_ciphertext = update();
no_ciphertext.executable_input_ciphertext.clear();
assert!(no_ciphertext.validate().is_err());
let mut no_epoch = update();
no_epoch.executable_input_master_key_epoch = 0;
assert!(no_epoch.validate().is_err());
}
}
+9 -1
View File
@@ -10,8 +10,10 @@
//! Task was executed. A terminal state is written by a conditional transition only.
mod attempt;
mod input;
use async_trait::async_trait;
pub use attempt::*;
pub use input::*;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
@@ -523,6 +525,10 @@ pub struct TaskRecord {
pub operation_id: String,
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
pub input_fingerprint: String,
/// Effective fingerprint of the payload the next attempt will execute. It equals the acceptance
/// fingerprint until a committed input continuation replaces the effective payload; the
/// acceptance fingerprint above is never overwritten, so an original accept replay stays replay.
pub effective_input_fingerprint: String,
pub pins: TaskPins,
pub support: TaskSupportIds,
pub input: TaskEnvelope,
@@ -607,6 +613,8 @@ pub enum TaskError {
FingerprintConflict,
#[error("task lifecycle transition was rejected")]
StateConflict,
#[error("task input request has expired")]
InputExpired,
#[error("background task capacity is exhausted")]
Overloaded { retry_after_ms: u64 },
}
@@ -641,7 +649,7 @@ pub struct TaskLease {
/// Authoritative durable Task port.
#[async_trait]
pub trait TaskStore: Send + Sync {
pub trait TaskStore: TaskInputStore + Send + Sync {
/// Proves the durable Task state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), TaskError>;
+2
View File
@@ -6,6 +6,7 @@ mod artifact_cleanup_indexes_v13;
mod artifact_metadata_v12;
mod authority;
mod baseline_v1;
mod durable_task_input_v30;
mod durable_tasks_v25;
mod durable_tasks_v26;
mod durable_tasks_v27;
@@ -47,6 +48,7 @@ mod schema_guard_v26;
mod schema_guard_v27;
mod schema_guard_v28;
mod schema_guard_v29;
mod schema_guard_v30;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -4,6 +4,7 @@ use super::agent_resource_bindings_v18;
use super::approval_side_effects_v10;
use super::artifact_cleanup_indexes_v13;
use super::artifact_metadata_v12;
use super::durable_task_input_v30;
use super::durable_tasks_v25;
use super::durable_tasks_v26;
use super::durable_tasks_v27;
@@ -45,10 +46,10 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 29;
const CURRENT_VERSION: i64 = 30;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29,
27, 28, 29, 30,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -373,6 +374,12 @@ impl MigrationAuthority {
durable_tasks_v29::SOURCE_SHA256,
28,
),
expand_descriptor(
30,
durable_task_input_v30::NAME,
durable_task_input_v30::SOURCE_SHA256,
29,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -550,6 +557,9 @@ impl MigrationAuthority {
if from < 29 {
durable_tasks_v29::apply(&mut transaction, &Self::sequence()[28]).await?;
}
if from < 30 {
durable_task_input_v30::apply(&mut transaction, &Self::sequence()[29]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-task-input-v30";
pub(super) const SOURCE: &str = include_str!("durable_task_input_v30.sql");
pub(super) const SOURCE_SHA256: &str =
"84a51f712fd0bc439acba2ab3849232777c29e3b181586bbb4b06fd129843a38";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_task_input_v30",
Some(30),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_task_input_v30",
Some(30),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,59 @@
-- V30: continuation through validated input (Story 4.5).
alter table tasks add column if not exists effective_input_fingerprint text;
update tasks set effective_input_fingerprint = input_fingerprint where effective_input_fingerprint is null;
alter table tasks alter column effective_input_fingerprint set not null;
alter table tasks add constraint tasks_effective_input_fingerprint_check check (
length(effective_input_fingerprint) = 64 and effective_input_fingerprint ~ '^[0-9a-f]{64}$'
);
create table if not exists task_input_requests (
task_id uuid primary key references tasks (task_id) on delete cascade,
wait_id text not null,
expected_version integer not null,
schema_revision integer not null,
schema_format_version integer not null,
fingerprint_format_version integer not null,
schema_snapshot jsonb not null,
projected_schema jsonb not null,
prompt_code text not null,
missing_fields jsonb not null default '[]'::jsonb,
expires_at timestamptz not null,
created_at timestamptz not null default now(),
accepted_at timestamptz,
update_key_digest text,
canonical_fingerprint text,
effective_input_fingerprint text,
committed_state text,
committed_version integer,
constraint task_input_requests_wait_unique unique (wait_id),
constraint task_input_requests_prompt_check check (octet_length(prompt_code) between 1 and 64),
constraint task_input_requests_receipt_check check (
(accepted_at is null and update_key_digest is null and canonical_fingerprint is null
and effective_input_fingerprint is null and committed_state is null and committed_version is null)
or (accepted_at is not null and update_key_digest is not null and canonical_fingerprint is not null
and effective_input_fingerprint is not null and committed_state is not null and committed_version is not null)
)
);
-- Not unique on purpose: several waiting Tasks may share one absolute TTL value (the same pinned
-- policy deadline), and an expired/cancelled wait keeps its row. Per-Task uniqueness comes from the
-- primary key and task_input_requests_wait_unique.
create index if not exists task_input_requests_active_ttl_idx
on task_input_requests (expires_at) where accepted_at is null;
create table if not exists task_input_audit (
id bigint generated always as identity primary key,
task_id uuid not null references tasks (task_id) on delete cascade,
wait_id text not null,
event text not null,
from_state text,
to_state text,
from_version integer,
to_version integer,
actor text,
reason_code text,
created_at timestamptz not null default now(),
constraint task_input_audit_event_check check (event in ('input_required','input_accepted','input_expired')),
constraint task_input_audit_actor_check check (actor is null or octet_length(actor) between 1 and 160),
constraint task_input_audit_reason_check check (reason_code is null or octet_length(reason_code) between 1 and 64)
);
create index if not exists task_input_audit_task_idx on task_input_audit (task_id, id);
@@ -492,6 +492,9 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 26 {
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
@@ -503,6 +506,9 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 27 {
super::schema_guard_v26::validate_v26_durable_tasks(connection).await?;
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
@@ -514,6 +520,9 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 28 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v27::validate_v27_durable_tasks(connection).await?;
@@ -523,15 +532,27 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 29 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v28::validate_v28_durable_tasks(connection).await?;
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 30 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v29::validate_v29_durable_attempts(connection).await?;
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v30::validate_v30_input_continuation(connection).await?;
}
Ok(())
}
@@ -0,0 +1,194 @@
//! V30 schema guard: continuation through validated input (Story 4.5).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::{column_exists, relation_exists};
pub(super) async fn validate_v30_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, "tasks").await?
&& column_exists(connection, "tasks", "effective_input_fingerprint").await?
{
return Ok(false);
}
if relation_exists(connection, "task_input_requests").await? {
return Ok(false);
}
if relation_exists(connection, "task_input_audit").await? {
return Ok(false);
}
Ok(true)
}
pub(super) async fn validate_v30_input_continuation(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
// V30 builds on the applied V29 shape. The exact V29 column count (43) no longer holds once V30
// appends its column (44), so the V29-specific columns are re-checked here by name instead of
// calling the exact-count V29 guard, which is only valid before V30 is applied.
for column in [
"platform_api_key_id",
"max_attempts",
"retry_backoff_step_ms",
"next_attempt_at",
"cancel_requested_at",
"cancel_reason",
] {
if !column_exists(connection, "tasks", column).await? {
return Err(invalid(30, "v30.columns"));
}
}
// V30 appends one column to tasks.
if !column_exists(connection, "tasks", "effective_input_fingerprint").await? {
return Err(invalid(30, "v30.columns"));
}
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind("tasks")
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != 44_i64 {
return Err(invalid(30, "v30.columns"));
}
// Tables.
for table in ["task_input_requests", "task_input_audit"] {
if !relation_exists(connection, table).await? {
return Err(invalid(30, "v30.tables"));
}
}
// task_input_requests columns.
for column in [
"task_id",
"wait_id",
"expected_version",
"schema_revision",
"schema_format_version",
"fingerprint_format_version",
"schema_snapshot",
"projected_schema",
"prompt_code",
"missing_fields",
"expires_at",
"created_at",
"accepted_at",
"update_key_digest",
"canonical_fingerprint",
"effective_input_fingerprint",
"committed_state",
"committed_version",
] {
if !column_exists(connection, "task_input_requests", column).await? {
return Err(invalid(30, "v30.task_input_requests"));
}
}
// task_input_audit columns.
for column in [
"id",
"task_id",
"wait_id",
"event",
"from_state",
"to_state",
"from_version",
"to_version",
"actor",
"reason_code",
"created_at",
] {
if !column_exists(connection, "task_input_audit", column).await? {
return Err(invalid(30, "v30.task_input_audit"));
}
}
// Constraints.
for constraint in [
"task_input_requests_pkey",
"task_input_requests_wait_unique",
"task_input_requests_prompt_check",
"task_input_requests_receipt_check",
"task_input_audit_event_check",
"task_input_audit_actor_check",
"task_input_audit_reason_check",
"tasks_effective_input_fingerprint_check",
] {
let table = if constraint.starts_with("task_input_audit")
|| constraint.starts_with("task_input_audit_")
{
"task_input_audit"
} else if constraint == "tasks_effective_input_fingerprint_check" {
"tasks"
} else {
"task_input_requests"
};
if !named_constraint_exists(connection, table, constraint).await? {
return Err(invalid(30, "v30.constraints"));
}
}
// Indexes.
for index in [
"task_input_requests_active_ttl_idx",
"task_input_audit_task_idx",
] {
let present = query(
"select exists (select 1 from pg_catalog.pg_indexes
where schemaname = current_schema() and indexname = $1) as present",
)
.bind(index)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if !present {
return Err(invalid(30, "v30.indexes"));
}
}
Ok(())
}
fn invalid(version: i64, stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(version),
"restore_known_good_backup",
)
}
// helper
async fn named_constraint_exists(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema()
and t.relname = $1
and c.conname = $2
) as present",
)
.bind(table)
.bind(constraint)
.fetch_one(connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
@@ -57,12 +57,13 @@ impl PostgresTaskStore {
let created_ms: i64 = sqlx::query(
"insert into tasks (
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
operation_version, scoped_key, input_fingerprint, effective_input_fingerprint,
request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch, platform_api_key_id, max_attempts, retry_backoff_step_ms
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
$1, $2, $3, $4, $5, $6, $7, $8, $9, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22, $23, $24
)
@@ -119,7 +119,9 @@ impl PostgresTaskStore {
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
// The checkpoint must bind the payload this attempt actually executes: after a committed
// input continuation that is the effective fingerprint, not the unchanged acceptance one.
if record.effective_input_fingerprint != request.input_fingerprint {
transaction.rollback().await.ok();
return Err(TaskError::FingerprintConflict);
}
@@ -0,0 +1,659 @@
//! Input-continuation half of the PostgreSQL Task store (Story 4.5, FR-25).
//!
//! One transaction owns each transition: suspend (`working` -> `input_required`), the committed
//! answer (`input_required` -> `working`) and the TTL expiry (`input_required` -> `failed`). Authority,
//! generation, TTL and receipt checks run under the same Task row lock, so a revocation, a competing
//! answer, a sweeper or a cancellation has exactly one winner and never a duplicated side effect.
//!
//! Raw input never enters a request, receipt, audit row or error text: the validated payload is
//! encrypted by the runtime service before this module sees it.
use crank_core::{
TaskAccess, TaskError, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
TaskInputUpdateRequest, TaskRecord, TaskState,
};
use sqlx::{Postgres, Row, Transaction};
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row};
#[async_trait::async_trait]
impl TaskInputStore for PostgresTaskStore {
async fn suspend_for_input(
&self,
lease: &crank_core::TaskLease,
request: TaskInputRequest,
) -> Result<TaskRecord, TaskError> {
self.suspend_input_locked(lease, request).await
}
async fn input_request(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<TaskInputRequest, TaskError> {
self.read_input_request(access, task_id).await
}
async fn read_executable_input_for_update(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<(String, String, i64), TaskError> {
self.read_binding_for_update(access, task_id).await
}
async fn apply_input(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError> {
self.apply_input_locked(request).await
}
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError> {
self.expire_input_waits_locked(batch).await
}
}
impl PostgresTaskStore {
/// One transactional suspend of a leased Task before any dispatch checkpoint.
pub(super) async fn suspend_input_locked(
&self,
lease: &crank_core::TaskLease,
request: TaskInputRequest,
) -> Result<TaskRecord, TaskError> {
request.validate()?;
if request.expected_version != lease.record.version.saturating_add(1) {
return Err(TaskError::StateConflict);
}
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(lease.record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
let current = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
&& record.fence == lease.fence
&& record.version == lease.record.version
&& record.current_attempt_id.is_none()
&& record.attempts == 0
&& record.cancel_requested_at_unix_ms.is_none()
&& record
.lease_until_unix_ms
.is_some_and(|until| until > crank_core::current_unix_ms())
&& record.pins.execution_deadline_unix_ms > crank_core::current_unix_ms();
// A generation that already dispatched may not be suspended: that would hide a possible side
// effect behind an apparent "waiting for input".
if !current {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if request.expires_at_unix_ms > record.pins.execution_deadline_unix_ms {
transaction.rollback().await.ok();
return Err(TaskError::invalid(
"task input TTL exceeds the execution deadline",
));
}
sqlx::query(
"insert into task_input_requests (
task_id, wait_id, expected_version, schema_revision, schema_format_version,
fingerprint_format_version, schema_snapshot, projected_schema, prompt_code,
missing_fields, expires_at
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
to_timestamp($11::double precision / 1000.0)
)",
)
.bind(lease.record.task_id.as_uuid())
.bind(&request.wait_id)
.bind(i32::try_from(request.expected_version).unwrap_or(i32::MAX))
.bind(i32::try_from(request.schema_revision).unwrap_or(i32::MAX))
.bind(i32::try_from(request.schema_format_version).unwrap_or(i32::MAX))
.bind(i32::try_from(request.fingerprint_format_version).unwrap_or(i32::MAX))
.bind(request.schema_snapshot.clone())
.bind(request.projected_schema.clone())
.bind(&request.prompt_code)
.bind(serde_json::Value::Array(
request
.missing_fields
.iter()
.cloned()
.map(serde_json::Value::String)
.collect(),
))
.bind(request.expires_at_unix_ms)
.execute(&mut *transaction)
.await
.map_err(|error| {
match error
.as_database_error()
.and_then(|db| db.code())
.as_deref()
{
Some("23505") => TaskError::StateConflict,
_ => Self::unavailable(error),
}
})?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'input_required',
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working' and version = $2
and lease_owner = $3 and fence = $4 and current_attempt_id is null
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(lease.record.task_id.as_uuid())
.bind(i64::from(record.version))
.bind(&lease.owner)
.bind(i64::try_from(lease.fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let suspended = record_from_row(&row)?;
insert_audit(
&mut transaction,
lease.record.task_id,
&request.wait_id,
"input_required",
TaskState::Working,
TaskState::InputRequired,
record.version,
suspended.version,
None,
None,
)
.await?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(suspended)
}
/// Owner-scoped read of the durable request; a foreign Task is one opaque not-found.
pub(super) async fn read_input_request(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<TaskInputRequest, TaskError> {
access.validate()?;
let row = sqlx::query(
"select r.wait_id, r.expected_version, r.schema_revision, r.schema_format_version,
r.fingerprint_format_version, r.schema_snapshot, r.projected_schema,
r.prompt_code, r.missing_fields,
(extract(epoch from r.expires_at) * 1000)::bigint as expires_at_ms
from task_input_requests r
join tasks t on t.task_id = r.task_id
where r.task_id = $1 and t.workspace_id = $2 and t.owner_kind = $3
and t.owner_id = $4 and t.authorization_scope = $5",
)
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
let missing_fields: Vec<String> = serde_json::from_value(
row.try_get::<serde_json::Value, _>("missing_fields")
.map_err(Self::unavailable)?,
)
.map_err(|_| TaskError::unavailable("stored input request metadata is invalid"))?;
Ok(TaskInputRequest {
wait_id: row.try_get("wait_id").map_err(Self::unavailable)?,
expected_version: u32::try_from(
row.try_get::<i32, _>("expected_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_revision: u32::try_from(
row.try_get::<i32, _>("schema_revision")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_format_version: u32::try_from(
row.try_get::<i32, _>("schema_format_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_snapshot: row.try_get("schema_snapshot").map_err(Self::unavailable)?,
projected_schema: row.try_get("projected_schema").map_err(Self::unavailable)?,
prompt_code: row.try_get("prompt_code").map_err(Self::unavailable)?,
missing_fields,
expires_at_unix_ms: row.try_get("expires_at_ms").map_err(Self::unavailable)?,
fingerprint_format_version: u32::try_from(
row.try_get::<i32, _>("fingerprint_format_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
})
}
/// Owner-scoped read of the binding a continuation may extend. Never lease-authorized.
///
/// Deliberately not filtered by lifecycle state: an identical replay of an already committed
/// answer (including after completion or TTL expiry) must reach the receipt lookup, which is the
/// single authority for that outcome.
pub(super) async fn read_binding_for_update(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<(String, String, i64), TaskError> {
access.validate()?;
let row = sqlx::query(
"select executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch
from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
let ciphertext: Option<String> = row
.try_get("executable_input_ciphertext")
.map_err(Self::unavailable)?;
let key_version: Option<String> = row
.try_get("executable_input_key_version")
.map_err(Self::unavailable)?;
let epoch: Option<i64> = row
.try_get("executable_input_master_key_epoch")
.map_err(Self::unavailable)?;
match (ciphertext, key_version, epoch) {
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
Ok((ciphertext, key_version, epoch))
}
_ => Err(TaskError::NotFound),
}
}
/// One transactional answer application: authority, receipt, generation, TTL and CAS.
pub(super) async fn apply_input_locked(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError> {
request.validate()?;
let access = &request.access;
let mut transaction = self.start().await?;
super::super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
super::super::master_key::ensure_active_master_key_epoch(
&mut transaction,
request.executable_input_master_key_epoch,
)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
// Authority is revalidated inside this transaction, before any receipt lookup and under the
// same lock order: master key check, platform key share lock, Task row lock.
let key_row = sqlx::query(
"select k.id from platform_api_keys k
join tasks t on t.platform_api_key_id = k.id
where t.task_id = $1 and t.owner_kind = 'agent'
and k.workspace_id = $2 and k.agent_id = $3
and k.key_kind = 'mcp_client' and k.status = 'active'
and (k.expires_at is null or k.expires_at > clock_timestamp())
and (k.scopes_json ? 'write' or k.scopes_json ? 'deploy')
for share of k",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(&access.owner_id)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if key_row.is_none() {
transaction.rollback().await.ok();
// A revoked key, a foreign scope and an absent Task share one opaque answer.
return Err(TaskError::NotFound);
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5
for update"
)))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
// Receipt lookup precedes the generation and TTL checks: an authorized identical replay keeps
// returning the committed outcome even after completion, failure or TTL expiry.
let receipt = sqlx::query(
"select wait_id, update_key_digest, canonical_fingerprint, committed_state,
committed_version
from task_input_requests where task_id = $1 and accepted_at is not null",
)
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if let Some(receipt) = receipt {
let wait_id: String = receipt.try_get("wait_id").map_err(Self::unavailable)?;
if wait_id != request.wait_id {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let stored_key: String = receipt
.try_get("update_key_digest")
.map_err(Self::unavailable)?;
if stored_key != request.update_key_digest {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let stored_fingerprint: String = receipt
.try_get("canonical_fingerprint")
.map_err(Self::unavailable)?;
if stored_fingerprint != request.canonical_fingerprint {
transaction.rollback().await.ok();
return Err(TaskError::FingerprintConflict);
}
let state: String = receipt
.try_get("committed_state")
.map_err(Self::unavailable)?;
let version: i32 = receipt
.try_get("committed_version")
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
return Ok(TaskInputUpdateOutcome {
task_id: request.task_id,
wait_id,
state: TaskState::parse(&state)?,
version: u32::try_from(version).unwrap_or_default(),
replayed: true,
});
}
let pending = sqlx::query(
"select wait_id, expected_version,
(extract(epoch from expires_at) * 1000)::bigint as expires_at_ms,
(extract(epoch from clock_timestamp()) * 1000)::bigint as now_ms
from task_input_requests where task_id = $1 and accepted_at is null",
)
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(pending) = pending else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let wait_id: String = pending.try_get("wait_id").map_err(Self::unavailable)?;
let expected_version: i32 = pending
.try_get("expected_version")
.map_err(Self::unavailable)?;
let expires_at_ms: i64 = pending
.try_get("expires_at_ms")
.map_err(Self::unavailable)?;
// PostgreSQL clock, read after the row lock was taken: never a client timestamp.
let now_ms: i64 = pending.try_get("now_ms").map_err(Self::unavailable)?;
if wait_id != request.wait_id
|| u32::try_from(expected_version).unwrap_or_default() != request.expected_version
|| record.version != request.expected_version
|| record.state != TaskState::InputRequired
{
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if record.cancel_requested_at_unix_ms.is_some() {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if now_ms >= expires_at_ms {
// The same transactional helper the sweeper uses: the Task ends failed with an audit
// reason and no dispatch. The update never revives it.
expire_locked(&mut transaction, &record, &wait_id, now_ms).await?;
transaction.commit().await.map_err(Self::unavailable)?;
return Err(TaskError::InputExpired);
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'working',
effective_input_fingerprint = $1,
executable_input_ciphertext = $2,
executable_input_key_version = $3,
executable_input_master_key_epoch = $4,
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $5 and state = 'input_required' and version = $6
returning {TASK_PROJECTION}"
)))
.bind(&request.effective_input_fingerprint)
.bind(&request.executable_input_ciphertext)
.bind(&request.executable_input_key_version)
.bind(request.executable_input_master_key_epoch)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let accepted = record_from_row(&row)?;
let updated = sqlx::query(
"update task_input_requests
set accepted_at = now(),
update_key_digest = $1,
canonical_fingerprint = $2,
effective_input_fingerprint = $3,
committed_state = 'working',
committed_version = $4
where task_id = $5 and wait_id = $6 and accepted_at is null",
)
.bind(&request.update_key_digest)
.bind(&request.canonical_fingerprint)
.bind(&request.effective_input_fingerprint)
.bind(i64::from(accepted.version))
.bind(request.task_id.as_uuid())
.bind(&request.wait_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if updated.rows_affected() != 1 {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
insert_audit(
&mut transaction,
request.task_id,
&request.wait_id,
"input_accepted",
TaskState::InputRequired,
TaskState::Working,
record.version,
accepted.version,
Some(access.owner_id.as_str()),
None,
)
.await?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskInputUpdateOutcome {
task_id: request.task_id,
wait_id: request.wait_id,
state: TaskState::Working,
version: accepted.version,
replayed: false,
})
}
/// Bounded TTL sweep, safe to call on every poll while all worker slots are busy.
pub(super) async fn expire_input_waits_locked(&self, batch: u32) -> Result<u64, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks t
where t.state = 'input_required'
and exists (
select 1 from task_input_requests r
where r.task_id = t.task_id and r.accepted_at is null
and r.expires_at <= clock_timestamp()
)
order by t.task_id
limit $1
for update of t skip locked"
)))
.bind(batch)
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let mut expired = 0u64;
for row in rows {
let record = record_from_row(&row)?;
let Some(wait_id) = sqlx::query_scalar::<_, String>(
"select wait_id from task_input_requests
where task_id = $1 and accepted_at is null and expires_at <= clock_timestamp()",
)
.bind(record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?
else {
continue;
};
if expire_locked(
&mut transaction,
&record,
&wait_id,
crank_core::current_unix_ms(),
)
.await?
{
expired += 1;
}
}
transaction.commit().await.map_err(Self::unavailable)?;
Ok(expired)
}
}
/// Marks one locked `input_required` Task failed, releases its admission permit and writes the audit
/// reason. Returns whether this call performed the transition.
async fn expire_locked(
transaction: &mut Transaction<'_, Postgres>,
record: &TaskRecord,
wait_id: &str,
_now_ms: i64,
) -> Result<bool, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'input_ttl_expired'),
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'input_required' and version = $2
returning {TASK_PROJECTION}"
)))
.bind(record.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
let Some(row) = row else {
return Ok(false);
};
let failed = record_from_row(&row)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(record.task_id.as_uuid())
.execute(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
insert_audit(
transaction,
record.task_id,
wait_id,
"input_expired",
TaskState::InputRequired,
TaskState::Failed,
record.version,
failed.version,
None,
Some("input_ttl_expired"),
)
.await?;
Ok(true)
}
#[allow(clippy::too_many_arguments)]
async fn insert_audit(
transaction: &mut Transaction<'_, Postgres>,
task_id: crank_core::TaskId,
wait_id: &str,
event: &str,
from_state: TaskState,
to_state: TaskState,
from_version: u32,
to_version: u32,
actor: Option<&str>,
reason_code: Option<&str>,
) -> Result<(), TaskError> {
sqlx::query(
"insert into task_input_audit (
task_id, wait_id, event, from_state, to_state, from_version, to_version, actor,
reason_code
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
)
.bind(task_id.as_uuid())
.bind(wait_id)
.bind(event)
.bind(from_state.as_str())
.bind(to_state.as_str())
.bind(i32::try_from(from_version).unwrap_or(i32::MAX))
.bind(i32::try_from(to_version).unwrap_or(i32::MAX))
.bind(actor)
.bind(reason_code)
.execute(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}
@@ -26,9 +26,11 @@ use super::admission::{AdmissionLimits, AttemptError};
mod acceptance;
mod attempts;
mod input;
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version,
authorization_scope, platform_api_key_id, operation_id, input_fingerprint,
effective_input_fingerprint, operation_version,
policy_mode,
approval_reference,
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
@@ -521,7 +523,49 @@ and lease_owner is null
transaction.commit().await.map_err(Self::unavailable)?;
let mut leases = Vec::new();
for _ in 0..batch {
// Continuation claim: after a committed answer the Task is `working` without a lease and still
// owns its executing admission permit. It is handed to exactly one new fence here, without a
// second admission grant, so waiting capacity is never double-counted.
let mut transaction = self.start().await?;
let resumed = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set lease_owner = $1,
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
fence = fence + 1,
updated_at = now()
where task_id in (
select t.task_id
from tasks t
join task_input_requests r on r.task_id = t.task_id and r.accepted_at is not null
join execution_admission_entries e
on e.task_id = t.task_id and e.state = 'executing'
where t.state = 'working' and t.lease_owner is null
and t.current_attempt_id is null
order by t.task_id
limit $3
for update of t skip locked
)
returning {TASK_PROJECTION}"
)))
.bind(owner)
.bind(lease_ms)
.bind(i64::try_from(batch.clamp(1, 64)).unwrap_or(64))
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
for row in resumed {
let record = record_from_row(&row).map_err(Self::unavailable)?;
leases.push(TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
});
}
let remaining = batch.saturating_sub(leases.len());
for _ in 0..remaining {
let mut transaction = self.start().await?;
let workspace_cursor =
match super::admission::lock_admission_in_tx(&mut transaction, "*").await {
@@ -840,6 +884,9 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError>
.map(PlatformApiKeyId::new),
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
effective_input_fingerprint: row
.try_get("effective_input_fingerprint")
.map_err(unavailable_row)?,
pins: TaskPins {
operation_version: u32::try_from(
row.try_get::<i32, _>("operation_version")
@@ -77,6 +77,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
(27, "durable-tasks-v27"),
(28, "durable-tasks-v28"),
(29, "durable-tasks-v29"),
(30, "durable-task-input-v30"),
];
assert_eq!(rows.len(), expected.len());
for (row, (version, name)) in rows.iter().zip(expected) {
@@ -187,7 +188,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
);
assert_eq!(
MigrationAuthority::preflight(first.pool()).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
#[tokio::test]
@@ -235,7 +236,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
.expect("the V18-local guard must be restorable");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"restoring the exact V18 contract must return the current verdict"
);
}
@@ -361,7 +362,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -474,13 +475,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 2,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
let trace_column: bool = sqlx::query_scalar(
"select exists (
@@ -523,7 +524,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 3,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -583,7 +584,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 4,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -837,6 +838,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 29,
target: 30,
}
);
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
.unwrap();
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
for (drift, restore) in [
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"restore: {restore}"
);
}
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"a rejected primary-key nullability change must leave the schema current"
);
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"the baseline before the drift matrix must be the current V16 schema"
);
let mut observations: Vec<String> = Vec::new();
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
.await
.expect("restoring the exact definition must apply");
let restored = MigrationAuthority::preflight(&pool).await;
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 29 }));
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 30 }));
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
// мнением того же guard-а, который принимает неправильные определения (F4).
let restored_exact = catalog_definition_snapshot(&pool).await == before;
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"baseline before inserting the shadow schema"
);
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
// Тень не меняет вердикт guard-а по текущей схеме.
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"a same-named object in another schema must not change the verdict for the current schema"
);
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 },
MigrationPreflight::Current { version: 30 },
"the exact restoration must return the current V16 schema"
);
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
assert!(
@@ -22,16 +22,16 @@ async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 24,
target: 29,
target: 30,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 24, to: 29 }
MigrationApplyResult::Applied { from: 24, to: 30 }
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
let tables = sqlx::query(
"select table_name
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 13,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
assert_eq!(operation_count, 1);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
assert!(supports_elicitation);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 29,
target: 30,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
.fetch_one(&pool)
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 29,
target: 30,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 18, to: 29 }
MigrationApplyResult::Applied { from: 18, to: 30 }
);
let after: Vec<(i64, String, String)> =
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 29 }
MigrationPreflight::Current { version: 30 }
);
}
@@ -161,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 29,
target: 30,
}
);
}
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 29,
target: 30,
}
);
}
@@ -331,8 +331,12 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
// V25 created `tasks`; V26–V28 only altered it, so dropping the table removes their schema too.
// V30 adds tables that reference `tasks`; they are dropped explicitly so the legacy preflight
// sees an actually absent V30 shape instead of orphan continuation tables.
sqlx::raw_sql(
"drop table if exists tasks cascade;
"drop table if exists task_input_audit cascade;
drop table if exists task_input_requests cascade;
drop table if exists tasks cascade;
delete from __crank_migrations where version >= 25;",
)
.execute(pool)
+19 -3
View File
@@ -25,7 +25,7 @@ use tracing::{Instrument, Span, debug, warn};
use uuid::Uuid;
use crate::task_attempt::{
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
PrepareAttempt, TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal,
};
use crate::task_worker::TaskDispatchPermit;
@@ -225,7 +225,23 @@ impl RuntimeExecutor {
lease_until: lease.lease_until,
};
let plan = match authority.prepare_attempt(&current_lease).await {
Ok(plan) => plan,
Ok(PrepareAttempt::Ready(plan)) => *plan,
Ok(PrepareAttempt::InputRequired(request)) => {
// A missing required input is suspended before the dispatch checkpoint, so this
// generation provably produced no outbound side effect. A CAS/store failure stops
// without writing anything new; the lease expiry recovery makes it retryable.
return match store.suspend_for_input(&current_lease, request).await {
Ok(_) => TaskAttemptOutcome::WaitingForInput,
Err(crank_core::TaskError::StateConflict)
| Err(crank_core::TaskError::NotFound)
| Err(crank_core::TaskError::FingerprintConflict)
| Err(crank_core::TaskError::InputExpired) => TaskAttemptOutcome::Stopped,
Err(error) => {
warn!(error = %error, "input suspension failed; no outbound call");
TaskAttemptOutcome::Stopped
}
};
}
Err(TaskAttemptRefusal::Stopped) => return TaskAttemptOutcome::Stopped,
Err(TaskAttemptRefusal::Failed { code }) => {
// A refused attempt never erases a proven ambiguity: if an earlier attempt of this
@@ -246,7 +262,7 @@ impl RuntimeExecutor {
fence: lease.fence,
expected_version: record.version,
attempt_id: attempt_id.clone(),
input_fingerprint: record.input_fingerprint.clone(),
input_fingerprint: record.effective_input_fingerprint.clone(),
upstream_key_digest: plan.upstream_key_digest.clone(),
retry_allowed: plan.retry_allowed,
};
+2 -2
View File
@@ -118,8 +118,8 @@ pub use rate_limit::{
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
pub use secret_crypto::SecretCrypto;
pub use task_attempt::{
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal,
PrepareAttempt, TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal, missing_required_input,
};
pub use task_worker::{
TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
+28 -1
View File
@@ -44,7 +44,31 @@ pub trait TaskAttemptAuthority: Send + Sync {
async fn prepare_attempt(
&self,
lease: &TaskLease,
) -> Result<TaskAttemptPlan, TaskAttemptRefusal>;
) -> Result<PrepareAttempt, TaskAttemptRefusal>;
}
/// Typed preparation of one attempt: either it is ready to be checkpointed and dispatched, or the
/// pinned operation still lacks a required input and the Task must be suspended before any side
/// effect. The two are explicit so a real authority failure is never turned into a user question.
pub enum PrepareAttempt {
Ready(Box<TaskAttemptPlan>),
InputRequired(crank_core::TaskInputRequest),
}
/// Path of a required item the pinned schema still misses before dispatch.
///
/// This is the runtime-owned decision the application calls before checkpointing an attempt: only a
/// typed missing required field becomes a user question, every other schema violation keeps the
/// existing pre-dispatch attempt validation. Exposed as a plain value so the application layer never
/// maps execution validation itself (execution boundary).
pub fn missing_required_input(
schema: &crank_schema::Schema,
input: &serde_json::Value,
) -> Option<String> {
match schema.validate_shape(input) {
Err(crank_schema::SchemaError::MissingField { path }) => Some(path),
_ => None,
}
}
/// Typed refusal of one attempt preparation.
@@ -108,6 +132,9 @@ pub enum TaskAttemptOutcome {
Failed { code: String },
/// The Task is non-terminal `outcome_unknown`; it is never retried or cancelled generically.
OutcomeUnknown,
/// The Task was suspended as `input_required` before its first dispatch: no side effect happened
/// and the worker future must stop; the answer is applied through the application entry.
WaitingForInput,
/// Nothing new was written by this worker: cancellation, lease loss or an unavailable store.
Stopped,
}
+15
View File
@@ -243,6 +243,18 @@ impl TaskWorker {
// Reap finished attempts before claiming more.
while active.try_join_next().is_some() {}
// The bounded TTL sweep runs on every poll, including when all local slots are busy: an
// expired input wait must end `failed` with an audit reason even without free capacity.
match self.store.expire_input_waits(self.config.batch_limit).await {
Ok(expired) if expired > 0 => {
debug!(expired, "expired input waits were failed")
}
Ok(_) => {}
Err(error) => {
warn!(error = %error, "input TTL sweep failed; retrying on next scan")
}
}
let available = self.config.max_inflight.saturating_sub(active.len());
if available > 0 {
let batch_limit = self
@@ -405,6 +417,9 @@ async fn process_lease(
TaskAttemptOutcome::OutcomeUnknown => {
warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry");
}
TaskAttemptOutcome::WaitingForInput => {
debug!(task_id = %task_id, "durable task suspended until validated input arrives");
}
TaskAttemptOutcome::Stopped => {
debug!(task_id = %task_id, "durable task attempt stopped without a new outcome");
}
+193 -20
View File
@@ -17,9 +17,10 @@ use crank_core::PlatformApiKeyId;
use crank_core::{
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskLease,
TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds, ToolExecutionMode,
contains_sensitive_task_keys, current_unix_ms, validate_task_envelope,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskId,
TaskInputUpdateOutcome, TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskRecord,
TaskStore, TaskSupportIds, ToolExecutionMode, contains_sensitive_task_keys, current_unix_ms,
validate_task_envelope,
};
use sha2::{Digest, Sha256};
@@ -295,28 +296,46 @@ impl TaskService {
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
}
check_string("input_fingerprint", &lease.record.input_fingerprint)?;
let input = object
let base_input = object
.get("input")
.cloned()
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
let canonical = canonical_input_fingerprint(&input)?;
if canonical != lease.record.input_fingerprint {
if canonical_input_fingerprint(&base_input)? != lease.record.input_fingerprint {
return Err(TaskError::invalid("task executable input binding mismatch"));
}
validate_task_envelope(&input)?;
let encoded = serde_json::to_vec(&input)
.map_err(|_| TaskError::invalid("task executable input is not serializable"))?;
if encoded.len() > TASK_EXECUTABLE_INPUT_MAX_BYTES {
return Err(TaskError::invalid(
"task executable input exceeds its byte budget",
));
}
if contains_sensitive_task_keys(&input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
Ok(input)
// A committed continuation replaces only the effective payload. The acceptance fingerprint and
// the original input stay in the binding, so an original accept replay is still a replay.
let effective_input = match object.get("continuation") {
Some(continuation) => {
let continuation = continuation
.as_object()
.ok_or_else(|| TaskError::invalid("task input continuation is invalid"))?;
let effective = continuation
.get("effective_input_fingerprint")
.and_then(|value| value.as_str())
.ok_or_else(|| TaskError::invalid("task input continuation is invalid"))?;
if effective != lease.record.effective_input_fingerprint {
return Err(TaskError::invalid(
"task input continuation fingerprint mismatch",
));
}
let input = continuation.get("input").cloned().ok_or_else(|| {
TaskError::invalid("task input continuation is missing input")
})?;
if canonical_input_fingerprint(&input)? != effective {
return Err(TaskError::invalid("task input continuation is invalid"));
}
input
}
None => {
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
return Err(TaskError::invalid("task input continuation is missing"));
}
base_input
}
};
validate_executable_payload(&effective_input)?;
Ok(effective_input)
}
/// Default retention pinned by the application layer when it does not choose one explicitly.
@@ -344,6 +363,105 @@ impl TaskService {
pub fn input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
canonical_input_fingerprint(input)
}
/// Applies one additional validated input to a suspended Task (Story 4.5).
///
/// The stored native schema snapshot is the only validation authority; the canonical digest binds
/// the effective payload, and the existing encrypted executable-input binding is extended with a
/// continuation segment instead of a second cipher store. The store transaction owns authority,
/// receipt, generation, TTL and CAS, so an identical replay returns the committed outcome.
pub async fn update_input(
&self,
access: &TaskAccess,
task_id: TaskId,
wait_id: &str,
expected_version: u32,
update_key: &str,
input: &serde_json::Value,
) -> Result<TaskInputUpdateOutcome, TaskError> {
access.validate()?;
validate_task_envelope(input)?;
if contains_sensitive_task_keys(input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
let canonical_fingerprint = canonical_input_fingerprint(input)?;
let pending = self.store.input_request(access, task_id).await?;
if pending.wait_id != wait_id {
return Err(TaskError::StateConflict);
}
// The immutable snapshot, never the current publication, is the schema authority.
let schema: crank_schema::Schema = serde_json::from_value(pending.schema_snapshot.clone())
.map_err(|_| TaskError::unavailable("stored task input schema is invalid"))?;
schema
.validate_shape(input)
.map_err(|_| TaskError::invalid("task input does not match the pinned input schema"))?;
let update_key_digest = input_update_key_digest(access, task_id, wait_id, update_key)?;
let crypto = self.crypto.as_ref().ok_or_else(|| {
TaskError::invalid("secret crypto is required for durable Task continuation")
})?;
let (ciphertext, key_version, epoch) = self
.store
.read_executable_input_for_update(access, task_id)
.await?;
let binding = crypto
.decrypt_for_epoch(&key_version, epoch, &ciphertext)
.map_err(|error| TaskError::invalid(error.to_string()))?;
let mut object = binding
.as_object()
.cloned()
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
check_binding_string(&object, "task_id", &task_id.to_string())?;
check_binding_string(&object, "workspace_id", &access.workspace_id)?;
check_binding_string(&object, "owner_id", &access.owner_id)?;
check_binding_string(&object, "authorization_scope", &access.authorization_scope)?;
let base_input = object
.get("input")
.cloned()
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
let base_fingerprint = object
.get("input_fingerprint")
.and_then(|value| value.as_str())
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
if canonical_input_fingerprint(&base_input)? != base_fingerprint {
return Err(TaskError::invalid("task executable input binding mismatch"));
}
object.insert(
"binding_format_version".to_owned(),
serde_json::json!(2_u32),
);
object.insert(
"continuation".to_owned(),
serde_json::json!({
"wait_id": wait_id,
"schema_revision": pending.schema_revision,
"effective_input_fingerprint": canonical_fingerprint,
"input": input,
}),
);
let continuation = crypto
.encrypt(&serde_json::Value::Object(object))
.map_err(|error| TaskError::invalid(error.to_string()))?;
if continuation.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
return Err(TaskError::invalid(
"encrypted task input exceeds its byte budget",
));
}
let request = TaskInputUpdateRequest {
access: access.clone(),
task_id,
wait_id: wait_id.to_owned(),
expected_version,
update_key_digest,
effective_input_fingerprint: canonical_fingerprint.clone(),
canonical_fingerprint,
executable_input_ciphertext: continuation,
executable_input_key_version: crypto.key_version().to_owned(),
executable_input_master_key_epoch: crypto.master_key_epoch(),
};
self.store.apply_input(request).await
}
}
fn deadline_unix_ms(deadline: Instant) -> i64 {
@@ -394,6 +512,61 @@ fn hex_digest(bytes: &[u8]) -> String {
.collect()
}
fn check_binding_string(
object: &serde_json::Map<String, serde_json::Value>,
field: &str,
expected: &str,
) -> Result<(), TaskError> {
match object.get(field).and_then(|value| value.as_str()) {
Some(actual) if actual == expected => Ok(()),
_ => Err(TaskError::invalid("task executable input binding mismatch")),
}
}
fn validate_executable_payload(input: &serde_json::Value) -> Result<(), TaskError> {
validate_task_envelope(input)?;
let encoded = serde_json::to_vec(input)
.map_err(|_| TaskError::invalid("task executable input is not serializable"))?;
if encoded.len() > TASK_EXECUTABLE_INPUT_MAX_BYTES {
return Err(TaskError::invalid(
"task executable input exceeds its byte budget",
));
}
if contains_sensitive_task_keys(input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
Ok(())
}
/// Digest of the scoped update key. The key itself never leaves the caller, and the scope covers the
/// exact Task generation so one key cannot be replayed across Tasks or waits.
fn input_update_key_digest(
access: &TaskAccess,
task_id: TaskId,
wait_id: &str,
key: &str,
) -> Result<String, TaskError> {
if key.is_empty()
|| key.len() > TASK_SCOPED_KEY_MAX_BYTES
|| key.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task input update key is invalid"));
}
let material = format!(
"{}\n{}\n{}\n{}\n{}\n{}\n{}",
access.workspace_id,
access.owner_kind.as_str(),
access.owner_id,
access.authorization_scope,
task_id,
wait_id,
key
);
Ok(hex_digest(material.as_bytes()))
}
#[cfg(test)]
mod tests {
use super::*;
+52
View File
@@ -108,3 +108,55 @@ Deadline/lease loss: read/safe-retry -> одна bounded new attempt (fence ра
Миграция v29 additive; legacy Task получает max_attempts=1.
Отмена после dispatch для read тоже даёт non-terminal `outcome_unknown`: в pinned snapshot Task'а нет read/write-класса, а выводить его из текущей публикации означало бы менять уже принятый snapshot. ARC-различие «read -> failed(`cancellation_after_dispatch`)» поэтому осознанно отложено до reconciliation Story 4.6 — это консервативное отклонение в безопасную сторону, а не забытый сценарий.
## Продолжение Task через валидированный input (Story 4.5)
Отсутствующий обязательный input pinned-операции больше не превращается в ошибку после
checkpoint: `TaskWorkerHandler::prepare_attempt` сверяет исполняемый input с native
схемой pinned-версии и при `MissingField` возвращает типизированный
`PrepareAttempt::InputRequired`. Само решение о недостающем обязательном поле
принимает `crank_runtime::missing_required_input` (рядом с `TaskAttemptAuthority`), а
application-слой только использует результат: runtime остаётся единственным местом
execution-валидации (`scripts/check-rust-boundaries.sh`). Executor вызывает `TaskStore::suspend_for_input` до
`checkpoint_dispatch`, поэтому в этой генерации гарантированно нет outbound side effect.
Транзакция одной строкой Task проверяет owner/fence/version/attempts/lease/deadline,
пишет `task_input_requests` (wait_id, ожидаемая версия, schema revision/format и
fingerprint format version,
immutable native snapshot, JSON-проекция, безопасный prompt code, missing-field paths,
TTL), append-only `task_input_audit` и переводит `working → input_required`, снимая
lease. Executing admission permit сохраняется: ожидание ограничено pinned absolute
deadline и занимает уже выданную concurrency capacity, без второй reservation.
Ответ принимается внутренним `TaskApplication::update_input` (публичный MCP wire
остаётся Story 4.7): principal выводит owner/scope, сервер валидирует input той же
native snapshot, а PG-транзакция под row lock повторно проверяет key/workspace/agent/
status/expiry/scopes, wait_id, версию, generation и TTL по `clock_timestamp()`.
Успех атомарно пишет continuation в существующий encrypted executable input,
`tasks.effective_input_fingerprint`, receipt и audit и переводит
`input_required → working` ровно один раз; acceptance fingerprint не меняется, поэтому
исходный accept replay остаётся replay. Identical authorized replay возвращает
сохранённый outcome без нового CAS/audit/dispatch, другой payload с тем же
scoped update key — `FingerprintConflict`, другой key/устаревшая версия/чужой wait —
`StateConflict`.
Продолжение подхватывает bounded claim по committed receipt и существующему executing
permit (новый fence, без второго admission grant) и исполняет ту же pinned Task;
следующий checkpoint пишет effective fingerprint, а исходный accept ID/version/scope/
support IDs сохраняются. Sweeper `expire_input_waits` вызывается на каждом poll, в том
числе при заполненном local `max_inflight`, и под тем же row lock переводит
просроченную Task в `failed` (`input_ttl_expired`) с audit, удаляя admission entry, без
dispatch. Cancel ожидающей Task освобождает permit; update не оживляет уже
failed/cancelled/expired Task. Secret input не попадает в request/receipt/audit/log:
audit хранит только коды/версии/состояния, payload — только в шифрованном binding.
Ограничения этой генерации (осознанные, без обхода AC):
- Продолжение возможно ровно один раз на генерацию. Если принятый update всё ещё не
покрывает другое обязательное поле pinned-операции, Task завершается
`task_input_still_incomplete` до dispatch, а не задаёт второй вопрос — консервативно
и без side effect; повторный prompt после принятого ответа не входит в Story 4.5.
- v30 выполняет one-shot backfill (`update tasks set effective_input_fingerprint =
input_fingerprint`, затем `set not null`) и `set not null` под ACCESS EXCLUSIVE на
`tasks`, хотя descriptor объявляет `backfill: none`. На текущем объёме это короткая
блокировка; при деплое на большой `tasks` её длительность нужно учитывать в окне
миграции (bounded/backfill-политика для таких таблиц — отдельное решение, не эта story).
+1 -1
View File
@@ -1 +1 @@
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
+7
View File
@@ -92,3 +92,10 @@ run_atdd "durable task retry attempt spans" -p crank-runtime --test stages retry
# CAS) and `mcp-server --test integration durable_tasks::` (application → PG → worker attempts,
# cancellation through the application entry and the composition oracles). Duplicating one of them
# here would only run the same bounded set twice.
# Story 4.5 (FR-25): input continuation. No new selector is needed: the Story 4.2 lines
# above run `crank-core --lib task::` (input request bounds), `crank-registry --test
# integration tasks::` (store/tasks suites) and `mcp-server --test integration
# durable_tasks::` (the full application -> worker -> PG -> counted upstream input
# continuation ATDD, including TTL/replay/conflict/scope races). Duplicating them here
# would only run the same bounded set twice.