From f9f8c625d19f4fa2609a6f01556eb1f58c9289d0 Mon Sep 17 00:00:00 2001 From: bsodfather Date: Thu, 29 Oct 2026 22:26:47 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(epic4):=20=D0=B8=D1=81=D0=BF=D0=BE?= =?UTF-8?q?=D0=BB=D0=BD=D1=8F=D1=82=D1=8C=20durable=20Task=20=D1=87=D0=B5?= =?UTF-8?q?=D1=80=D0=B5=D0=B7=20PG=20lease/fence=20=D0=B8=20=D0=B2=D0=BE?= =?UTF-8?q?=D1=81=D1=81=D1=82=D0=B0=D0=BD=D0=B0=D0=B2=D0=BB=D0=B8=D0=B2?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D0=BF=D0=BE=D1=81=D0=BB=D0=B5=20=D1=81?= =?UTF-8?q?=D0=B1=D0=BE=D0=B5=D0=B2=20(Story=204.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с fence-токеном, зашифрованный input и повторная проверка доступа до сетевого вызова, checkpoint с AttemptID до dispatch и восстановление только по PG. Устаревший worker не может зафиксировать результат. Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса: стоп в окне после checkpoint и до permit завершает задачу как failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи остаются доступными. Retry-попытки получают отдельные дочерние spans с durable current_attempt_id. Shared admission/fairness из Story 4.1 сохранены. Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47, bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA и исходный runtime-прогон), точный exact-head CI остаётся обязательным. --- apps/admin-api/src/bin/crank-migrate.rs | 186 ++-- .../src/bin/crank_migrate/reporting.rs | 98 +++ apps/mcp-server/Cargo.toml | 3 +- apps/mcp-server/src/main.rs | 162 +++- .../durable_tasks/attempt_trace.rs | 413 +++++++++ .../mod.rs} | 47 +- .../durable_tasks/shutdown_window.rs | 190 ++++ .../integration/durable_tasks/task_worker.rs | 822 ++++++++++++++++++ crates/crank-community-mcp/src/access.rs | 5 +- crates/crank-community-mcp/src/app.rs | 4 +- .../src/app/auth_resolution.rs | 27 +- .../src/app/task_worker.rs | 282 ++++++ crates/crank-community-mcp/src/app/tasks.rs | 31 +- crates/crank-community-mcp/src/lib.rs | 6 +- crates/crank-core/src/lib.rs | 19 +- crates/crank-core/src/task.rs | 150 +++- crates/crank-registry/src/migrations.rs | 6 + .../src/migrations/authority.rs | 35 +- .../src/migrations/durable_tasks_v26.rs | 45 + .../src/migrations/durable_tasks_v26.sql | 6 + .../src/migrations/durable_tasks_v27.rs | 45 + .../src/migrations/durable_tasks_v27.sql | 19 + .../src/migrations/durable_tasks_v28.rs | 45 + .../src/migrations/durable_tasks_v28.sql | 8 + .../src/migrations/schema_guard.rs | 34 +- .../src/migrations/schema_guard_v26.rs | 120 +++ .../src/migrations/schema_guard_v27.rs | 85 ++ .../src/migrations/schema_guard_v28.rs | 89 ++ .../crank-registry/src/postgres/admission.rs | 139 +++ crates/crank-registry/src/postgres/api_key.rs | 63 ++ .../crank-registry/src/postgres/master_key.rs | 63 +- .../src/postgres/master_key_tasks.rs | 143 +++ crates/crank-registry/src/postgres/mod.rs | 1 + crates/crank-registry/src/postgres/tasks.rs | 418 ++++++++- .../tests/integration/migrations.rs | 20 +- .../migrations/artifact_metadata.rs | 12 +- .../artifact_metadata/exhaustive_v12.rs | 16 +- .../migrations/durable_tasks_guard.rs | 83 ++ .../integration/migrations/import_plan.rs | 4 +- .../integration/migrations/legacy_adoption.rs | 16 +- .../resource_security_revocations.rs | 20 +- .../tests/integration/migrations/rollback.rs | 2 +- .../integration/migrations/schema_removal.rs | 3 +- .../integration/{tasks.rs => tasks/mod.rs} | 233 ++++- .../tests/integration/tasks/task_worker.rs | 163 ++++ crates/crank-runtime/src/executor.rs | 108 ++- crates/crank-runtime/src/lib.rs | 5 + crates/crank-runtime/src/task_worker.rs | 573 ++++++++++++ crates/crank-runtime/src/tasks.rs | 166 +++- crates/crank-runtime/tests/stages.rs | 91 +- docs/execution-boundary.md | 43 +- scripts/ci-story-checks.sh | 7 + 52 files changed, 5142 insertions(+), 232 deletions(-) create mode 100644 apps/admin-api/src/bin/crank_migrate/reporting.rs create mode 100644 apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs rename apps/mcp-server/tests/integration/{durable_tasks.rs => durable_tasks/mod.rs} (92%) create mode 100644 apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs create mode 100644 apps/mcp-server/tests/integration/durable_tasks/task_worker.rs create mode 100644 crates/crank-community-mcp/src/app/task_worker.rs create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v26.rs create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v26.sql create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v27.rs create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v27.sql create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v28.rs create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v28.sql create mode 100644 crates/crank-registry/src/migrations/schema_guard_v26.rs create mode 100644 crates/crank-registry/src/migrations/schema_guard_v27.rs create mode 100644 crates/crank-registry/src/migrations/schema_guard_v28.rs create mode 100644 crates/crank-registry/src/postgres/master_key_tasks.rs create mode 100644 crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs rename crates/crank-registry/tests/integration/{tasks.rs => tasks/mod.rs} (75%) create mode 100644 crates/crank-registry/tests/integration/tasks/task_worker.rs create mode 100644 crates/crank-runtime/src/task_worker.rs diff --git a/apps/admin-api/src/bin/crank-migrate.rs b/apps/admin-api/src/bin/crank-migrate.rs index 8ace5e4..844509d 100644 --- a/apps/admin-api/src/bin/crank-migrate.rs +++ b/apps/admin-api/src/bin/crank-migrate.rs @@ -2,6 +2,10 @@ mod admin_auth_command; #[path = "crank_migrate/db_connect.rs"] mod db_connect; +#[path = "crank_migrate/reporting.rs"] +mod reporting; + +use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json}; use crank_config::{ConfigSource, parse_migrator}; use crank_registry::{ @@ -11,7 +15,7 @@ use crank_registry::{ }; use crank_runtime::SecretCrypto; use serde_json::json; -use std::{path::Path, process::ExitCode}; +use std::process::ExitCode; use time::OffsetDateTime; const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000; #[tokio::main] @@ -765,6 +769,59 @@ async fn process_rotation_batch( processed += 1; } } + let mut after_task_id: Option = None; + loop { + let tasks = registry + .list_tasks_for_master_key_epoch_page( + rotation.source_epoch, + after_task_id.as_deref(), + MASTER_KEY_ROTATION_PAGE_SIZE, + ) + .await + .map_err(CliError::from_registry)?; + if tasks.is_empty() { + break; + } + for task in tasks { + after_task_id = Some(task.task_id.clone()); + if task.target_epoch == Some(rotation.target_epoch) { + continue; + } + if max_versions.is_some_and(|limit| processed as usize >= limit) { + return Ok(processed); + } + let plaintext = current_crypto + .decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext) + .map_err(|_| { + CliError::new( + "master_key_rotation_verification_failed", + "master_key.rotation", + "rerun_rotation_or_abort", + ) + })?; + let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| { + CliError::new( + "master_key_rotation_verification_failed", + "master_key.rotation", + "rerun_rotation_or_abort", + ) + })?; + let now = OffsetDateTime::now_utc(); + registry + .stage_master_key_rotation_task_ciphertext( + &rotation.id, + &task.task_id, + rotation.source_epoch, + &target_ciphertext, + target_crypto.key_version(), + rotation.target_epoch, + &now, + ) + .await + .map_err(CliError::from_registry)?; + processed += 1; + } + } Ok(processed) } @@ -818,6 +875,47 @@ async fn verify_staged_targets( verified += 1; } } + let mut after_task_id: Option = None; + loop { + let tasks = registry + .list_target_tasks_for_master_key_rotation_page( + rotation.target_epoch, + after_task_id.as_deref(), + MASTER_KEY_ROTATION_PAGE_SIZE, + ) + .await + .map_err(CliError::from_registry)?; + if tasks.is_empty() { + break; + } + for task in tasks { + after_task_id = Some(task.task_id.clone()); + let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| { + CliError::new( + "master_key_rotation_verification_failed", + "master_key.rotation", + "rerun_rotation_or_abort", + ) + })?; + let key_version = task.target_key_version.as_deref().ok_or_else(|| { + CliError::new( + "master_key_rotation_verification_failed", + "master_key.rotation", + "rerun_rotation_or_abort", + ) + })?; + target_crypto + .decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext) + .map_err(|_| { + CliError::new( + "master_key_rotation_verification_failed", + "master_key.rotation", + "rerun_rotation_or_abort", + ) + })?; + verified += 1; + } + } if verified != rotation.total_secret_versions { return Err(CliError::new( "master_key_rotation_verification_failed", @@ -877,89 +975,3 @@ fn decrypt_current( ) }) } - -async fn active_rotation( - registry: &PostgresRegistry, - allowed_states: &[&str], -) -> Result, CliError> { - let status = registry - .master_key_rotation_status() - .await - .map_err(CliError::from_registry)?; - Ok(status - .rotations - .into_iter() - .find(|rotation| allowed_states.contains(&rotation.state.as_str()))) -} - -fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value { - json!({ - "id": rotation.id, - "source_epoch": rotation.source_epoch, - "target_epoch": rotation.target_epoch, - "state": rotation.state, - "backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"), - "checkpoint_secret_id": rotation.checkpoint_secret_id, - "total_secret_versions": rotation.total_secret_versions, - "processed_secret_versions": rotation.processed_secret_versions, - "verified_secret_versions": rotation.verified_secret_versions, - "failure_code": rotation.failure_code, - }) -} - -fn read_required_key_file(path: Option<&str>) -> Result { - let path = path.ok_or_else(|| { - CliError::new("invalid_command", "cli.arguments", "run_master_key_status") - })?; - if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) { - return Err(CliError::new( - "master_key_input_invalid", - "master_key.input", - "verify_operator_input", - )); - } - let metadata = std::fs::metadata(Path::new(path)).map_err(|_| { - CliError::new( - "master_key_input_invalid", - "master_key.input", - "verify_operator_input", - ) - })?; - if !metadata.is_file() || metadata.len() > 16_384 { - return Err(CliError::new( - "master_key_input_invalid", - "master_key.input", - "verify_operator_input", - )); - } - let value = std::fs::read_to_string(Path::new(path)).map_err(|_| { - CliError::new( - "master_key_input_invalid", - "master_key.input", - "verify_operator_input", - ) - })?; - if value.trim().is_empty() { - return Err(CliError::new( - "master_key_input_invalid", - "master_key.input", - "verify_operator_input", - )); - } - Ok(value) -} -fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> { - if options.current_key_file.is_some() - || options.target_key_file.is_some() - || options.backup_ref.is_some() - || options.rotation_id.is_some() - || options.max_versions.is_some() - { - return Err(CliError::new( - "invalid_command", - "cli.arguments", - "run_master_key_status", - )); - } - Ok(()) -} diff --git a/apps/admin-api/src/bin/crank_migrate/reporting.rs b/apps/admin-api/src/bin/crank_migrate/reporting.rs new file mode 100644 index 0000000..59f2f13 --- /dev/null +++ b/apps/admin-api/src/bin/crank_migrate/reporting.rs @@ -0,0 +1,98 @@ +//! Leaf/reporting helpers of the migrator CLI. +//! +//! Extracted from `crank-migrate.rs` without behaviour changes so the binary entry stays within +//! the code-health line limit. + +use std::path::Path; + +use crank_registry::{MasterKeyRotationRecord, PostgresRegistry}; +use serde_json::json; + +use super::{CliError, MasterKeyOptions}; + +pub(super) async fn active_rotation( + registry: &PostgresRegistry, + allowed_states: &[&str], +) -> Result, CliError> { + let status = registry + .master_key_rotation_status() + .await + .map_err(CliError::from_registry)?; + Ok(status + .rotations + .into_iter() + .find(|rotation| allowed_states.contains(&rotation.state.as_str()))) +} + +pub(super) fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value { + json!({ + "id": rotation.id, + "source_epoch": rotation.source_epoch, + "target_epoch": rotation.target_epoch, + "state": rotation.state, + "backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"), + "checkpoint_secret_id": rotation.checkpoint_secret_id, + "total_secret_versions": rotation.total_secret_versions, + "processed_secret_versions": rotation.processed_secret_versions, + "verified_secret_versions": rotation.verified_secret_versions, + "failure_code": rotation.failure_code, + }) +} + +pub(super) fn read_required_key_file(path: Option<&str>) -> Result { + let path = path.ok_or_else(|| { + CliError::new("invalid_command", "cli.arguments", "run_master_key_status") + })?; + if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) { + return Err(CliError::new( + "master_key_input_invalid", + "master_key.input", + "verify_operator_input", + )); + } + let metadata = std::fs::metadata(Path::new(path)).map_err(|_| { + CliError::new( + "master_key_input_invalid", + "master_key.input", + "verify_operator_input", + ) + })?; + if !metadata.is_file() || metadata.len() > 16_384 { + return Err(CliError::new( + "master_key_input_invalid", + "master_key.input", + "verify_operator_input", + )); + } + let value = std::fs::read_to_string(Path::new(path)).map_err(|_| { + CliError::new( + "master_key_input_invalid", + "master_key.input", + "verify_operator_input", + ) + })?; + if value.trim().is_empty() { + return Err(CliError::new( + "master_key_input_invalid", + "master_key.input", + "verify_operator_input", + )); + } + Ok(value) +} + +pub(super) fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> { + if options.current_key_file.is_some() + || options.target_key_file.is_some() + || options.backup_ref.is_some() + || options.rotation_id.is_some() + || options.max_versions.is_some() + { + return Err(CliError::new( + "invalid_command", + "cli.arguments", + "run_master_key_status", + )); + } + Ok(()) +} diff --git a/apps/mcp-server/Cargo.toml b/apps/mcp-server/Cargo.toml index 5747550..0b38ece 100644 --- a/apps/mcp-server/Cargo.toml +++ b/apps/mcp-server/Cargo.toml @@ -29,7 +29,7 @@ sha2.workspace = true sqlx.workspace = true thiserror.workspace = true time.workspace = true -tokio = { workspace = true, features = ["sync"] } +tokio = { workspace = true, features = ["sync", "signal"] } tracing.workspace = true tracing-subscriber.workspace = true uuid.workspace = true @@ -45,5 +45,6 @@ opentelemetry_sdk.workspace = true prost.workspace = true reqwest.workspace = true tower.workspace = true +tokio = { workspace = true, features = ["net", "time"] } tracing-opentelemetry.workspace = true metrics-util = "0.20.4" diff --git a/apps/mcp-server/src/main.rs b/apps/mcp-server/src/main.rs index 8a0e72a..6d4ae9d 100644 --- a/apps/mcp-server/src/main.rs +++ b/apps/mcp-server/src/main.rs @@ -201,20 +201,45 @@ async fn run( // admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root // fail-closed per read instead of preventing startup; nothing is ever written from this process. let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone()); - // Story 4.2: the durable Task application entry shares the registry pool, the published catalog - // refresh interval and the exact admission limits of the runtime; the public wire stays - // synchronous/unsupported and no worker is started. - let task_store = PostgresTaskStore::new( + // Story 4.2/4.3: the durable Task application entry shares the registry pool, the published + // catalog refresh interval and the exact admission limits of the runtime; the public wire stays + // synchronous/unsupported while a fenced in-process worker executes accepted Tasks. + let task_store = std::sync::Arc::new(PostgresTaskStore::new( registry.pool().clone(), admission_limits(runtime_limits), crank_core::AdmissionSchedulerConfig::default(), - )?; + )?); let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new( registry.clone(), Duration::from_millis(config.refresh_ms), cache_stores.coordination.clone(), - crank_runtime::TaskService::new(std::sync::Arc::new(task_store)), + crank_runtime::TaskService::with_secret_crypto( + task_store.clone(), + std::sync::Arc::new(secret_crypto.clone()), + ), )); + // Story 4.3: the durable Task worker runs in this process over the same PostgreSQL store, + // published catalog, executor and encrypted-input authority as the accepting application. + let task_handler = + std::sync::Arc::new(crank_community_mcp::TaskWorkerHandler::from_application( + &tasks, + registry.clone(), + runtime.clone(), + secret_crypto.clone(), + )); + let task_worker = std::sync::Arc::new(crank_runtime::TaskWorker::new( + task_store, + task_handler, + crank_runtime::TaskWorkerConfig { + owner: format!("mcp-server-{}", std::process::id()), + ..crank_runtime::TaskWorkerConfig::default() + }, + )?); + let (task_shutdown_tx, task_shutdown_rx) = tokio::sync::watch::channel(false); + let task_worker_handle = tokio::spawn({ + let worker = task_worker.clone(); + async move { worker.run_until_shutdown(task_shutdown_rx).await } + }); let app = build_app_with_artifacts_and_tasks( registry, Duration::from_millis(config.refresh_ms), @@ -252,15 +277,77 @@ async fn run( info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening"); *startup_completed = true; - if let Some(metrics_server) = metrics_server { - tokio::select! { - result = axum::serve(listener, app) => result?, - result = metrics_server.serve() => result?, + let serve_result: Result<(), Box> = async { + if let Some(metrics_server) = metrics_server { + tokio::select! { + result = serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) => result?, + result = metrics_server.serve() => result?, + } + } else { + serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) + .await?; } - } else { - axum::serve(listener, app).await?; + Ok(()) + } + .await; + + // Stop new claims; active attempts checkpoint/drain or are cancelled inside the bounded worker + // shutdown timeout (<=30s), and queued PostgreSQL work is left for recovery. + task_shutdown_tx.send(true).ok(); + if tokio::time::timeout(Duration::from_secs(30), task_worker_handle) + .await + .is_err() + { + tracing::warn!( + name: "mcp.task_worker.shutdown_timeout", + "durable Task worker did not stop within the story shutdown budget" + ); + } + serve_result +} + +/// Gracefully stop the durable Task worker as soon as shutdown is requested, while still +/// draining in-flight HTTP connections before returning. +async fn serve_with_worker_shutdown( + app: axum::Router, + listener: tokio::net::TcpListener, + signal: S, + worker_stop: tokio::sync::watch::Sender, +) -> std::io::Result<()> +where + S: std::future::Future + Send + 'static, +{ + let worker_stop_graceful = worker_stop.clone(); + let graceful = async move { + signal.await; + let _ = worker_stop_graceful.send(true); + }; + let result = axum::serve(listener, app) + .with_graceful_shutdown(graceful) + .await; + let _ = worker_stop.send(true); + result +} + +/// Process shutdown trigger for the HTTP surface and the durable Task worker. +async fn shutdown_signal() { + let ctrl_c = async { + let _ = tokio::signal::ctrl_c().await; + }; + #[cfg(unix)] + let terminate = async { + if let Ok(mut signal) = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + { + signal.recv().await; + } + }; + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + tokio::select! { + _ = ctrl_c => {} + _ = terminate => {} } - Ok(()) } fn init_observability( @@ -506,3 +593,52 @@ fn runtime_cache_config( .map(|value| value.expose_secret().to_owned()), ) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::{Router, routing::get}; + use std::time::Duration; + use tokio::sync::{oneshot, watch}; + + async fn slow_handler() -> &'static str { + tokio::time::sleep(Duration::from_millis(600)).await; + "ok" + } + + #[tokio::test] + async fn worker_stops_before_http_drain_completes() { + let app = Router::new().route("/slow", get(slow_handler)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (stop_tx, mut stop_rx) = watch::channel(false); + let (signal_tx, signal_rx) = oneshot::channel::<()>(); + + let handle = tokio::spawn(serve_with_worker_shutdown( + app, + listener, + async { + let _ = signal_rx.await; + }, + stop_tx, + )); + + let client = reqwest::Client::new(); + let request = client.get(format!("http://{}/slow", addr)); + let in_flight = tokio::spawn(async move { request.send().await }); + + tokio::time::sleep(Duration::from_millis(150)).await; + let _ = signal_tx.send(()); + + tokio::time::timeout(Duration::from_millis(250), stop_rx.changed()) + .await + .expect("worker stop signal should arrive before HTTP drain") + .expect("watch changed"); + assert!(*stop_rx.borrow()); + + let response = in_flight.await.unwrap().unwrap(); + assert!(response.status().is_success()); + let serve_result = handle.await.unwrap(); + assert!(serve_result.is_ok()); + } +} diff --git a/apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs b/apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs new file mode 100644 index 0000000..8be2e7d --- /dev/null +++ b/apps/mcp-server/tests/integration/durable_tasks/attempt_trace.rs @@ -0,0 +1,413 @@ +//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1). +//! +//! End-to-end: a real durable Task worker dispatches a retrying GET through a real local +//! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is +//! persisted in `tasks.current_attempt_id` on both attempts. + +use super::super::common; + +use axum::Router; +use axum::http::StatusCode; +use axum::routing::get; +use crank_community_mcp::TaskWorkerHandler; +use crank_core::{ + AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy, +}; +use crank_runtime::{ + OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use time::OffsetDateTime; +use tracing::{Id, Subscriber, field::Visit}; +use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan}; + +const AGENT_SLUG: &str = "reports"; +const TOOL_NAME: &str = "attempt_trace"; +const KEY: &str = "attempt-trace-key-1"; + +async fn publish( + registry: &crank_registry::PostgresRegistry, + operation_id: &crank_core::OperationId, + version: u32, +) { + registry + .publish_operation(crank_registry::PublishRequest { + workspace_id: &common::test_workspace_id(), + operation_id, + version, + published_at: &OffsetDateTime::parse( + "2026-03-26T10:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap(), + published_by: Some("alice"), + }) + .await + .unwrap(); +} + +async fn application_with_store( + registry: &crank_registry::PostgresRegistry, +) -> ( + crank_community_mcp::TaskApplication, + std::sync::Arc, + SecretCrypto, +) { + let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(); + let observed_at = OffsetDateTime::now_utc(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &observed_at, + }) + .await + .expect("test master-key identity registered"); + let store = std::sync::Arc::new( + crank_registry::PostgresTaskStore::new( + registry.pool().clone(), + crank_registry::AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let tasks = crank_community_mcp::TaskApplication::new( + registry.clone(), + std::time::Duration::ZERO, + std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()), + crank_runtime::TaskService::with_secret_crypto( + store.clone(), + std::sync::Arc::new(crypto.clone()), + ), + ); + (tasks, store, crypto) +} + +async fn principal( + registry: &crank_registry::PostgresRegistry, + name: &str, + scopes: &[crank_core::PlatformApiKeyScope], +) -> crank_community_mcp::TaskPrincipal { + let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await; + let secret_hash = common::hash_access_secret(&token); + let api_key = registry + .get_platform_api_key_by_secret_for_agent_slug( + common::test_workspace_slug(), + AGENT_SLUG, + &secret_hash, + ) + .await + .unwrap() + .expect("an active platform key resolves"); + registry + .touch_platform_api_key( + &api_key.api_key.workspace_id, + &api_key.api_key.id, + &OffsetDateTime::now_utc(), + ) + .await + .unwrap(); + crank_community_mcp::TaskPrincipal::Agent { + workspace_slug: common::test_workspace_slug().to_owned(), + agent_slug: AGENT_SLUG.to_owned(), + credential: crank_core::VerifiedMachineCredential { + machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey, + max_security_level: crank_core::OperationSecurityLevel::Standard, + scopes: api_key.api_key.scopes.clone(), + platform_api_key_id: Some(api_key.api_key.id.clone()), + }, + } +} +#[tokio::test] +async fn adapter_attempt_span_carries_durable_attempt_id() { + let registry = common::test_registry().await; + let calls = Arc::new(AtomicUsize::new(0)); + let upstream_app = Router::new().route( + "/crm/leads", + get({ + let calls = Arc::clone(&calls); + move || { + let calls = Arc::clone(&calls); + async move { + let n = calls.fetch_add(1, Ordering::SeqCst); + if n == 0 { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({"error":"boom"})), + ); + } + ( + StatusCode::OK, + axum::Json(serde_json::json!({"id":"lead-1"})), + ) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let mut operation = common::test_operation(&upstream, TOOL_NAME); + let Target::Rest(target) = &mut operation.target; + target.method = crank_core::HttpMethod::Get; + operation.execution_config.execution_policy = Some(ToolExecutionPolicy { + mode: crank_core::ToolExecutionMode::Background, + deadline_ms: 60_000, + no_tasks_fallback: crank_core::NoTasksFallback::Unsupported, + }); + operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 }); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish(®istry, &operation.id, 1).await; + common::publish_agent_with_bindings( + ®istry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; + + let caller = principal( + ®istry, + "attempt-trace", + &[ + crank_core::PlatformApiKeyScope::Read, + crank_core::PlatformApiKeyScope::Write, + ], + ) + .await; + + let (tasks, store, crypto) = application_with_store(®istry).await; + let acceptance = tasks + .accept_background( + &caller, + TOOL_NAME, + None, + KEY, + &serde_json::json!({"email": "user@example.com"}), + &CorrelationContext::generate(), + ) + .await + .expect("durable acceptance"); + + let runtime = + community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build(); + let handler = Arc::new(TaskWorkerHandler::from_application( + &tasks, + registry.clone(), + runtime, + crypto, + )); + let mut config = TaskWorkerConfig::default(); + config.owner = "runtime-worker-attempt-trace".to_owned(); + config.batch_limit = 1; + config.lease_ms = 10_000; + config.poll_interval = std::time::Duration::from_millis(50); + config.max_inflight = 1; + config.shutdown_timeout = std::time::Duration::from_secs(5); + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + + let capture = TraceCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _guard = tracing::subscriber::set_default(subscriber); + + let worker_task = tokio::spawn(async move { worker.run().await }); + + let mut state = None; + for _ in 0..200 { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let current: Option = + sqlx::query_scalar("select state from tasks where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + state = current; + if state.as_deref() == Some("completed") { + break; + } + } + stop_flag.store(true, Ordering::SeqCst); + tokio::time::timeout(std::time::Duration::from_secs(10), worker_task) + .await + .expect("worker stops inside the budget") + .expect("worker exits cleanly") + .expect("worker reports no error"); + server.abort(); + + let durable_attempt_id: Option = sqlx::query_scalar::<_, Option>( + "select current_attempt_id from tasks where task_id = $1", + ) + .bind(acceptance.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + + assert_eq!(state.as_deref(), Some("completed")); + + let spans = capture.snapshot(); + let attempt_spans: Vec<_> = spans + .iter() + .filter(|s| s.name == "adapter_attempt") + .collect(); + assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans"); + for span in &attempt_spans { + let aid = span.fields.get("attempt_id").cloned().unwrap_or_default(); + assert!(!aid.is_empty(), "attempt_id must not be empty"); + assert!( + !aid.contains("traceparent"), + "attempt_id must not contain traceparent" + ); + assert_eq!( + Some(aid.as_str()), + durable_attempt_id.as_deref(), + "span attempt_id must equal durable attempt id" + ); + } + let ids: Vec = attempt_spans + .iter() + .map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default()) + .collect(); + assert_eq!(ids[0], ids[1], "same durable id on both retries"); + assert_eq!( + spans + .iter() + .filter(|span| span.name == "durable_task_attempt") + .count(), + 1, + "one durable attempt span per dispatch" + ); + let runtime_span = spans + .iter() + .find(|span| span.name == "runtime.execute") + .expect("runtime.execute span"); + assert_eq!( + runtime_span.parent_name, + Some("durable_task_attempt"), + "the execution continues the durable attempt trace" + ); + for span in &attempt_spans { + assert_eq!( + span.parent_name, + Some("runtime.execute"), + "each retry is a child attempt span under the durable execution" + ); + } + + let attempts: Vec = attempt_spans + .iter() + .map(|s| s.fields.get("attempt").cloned().unwrap_or_default()) + .collect(); + assert!(attempts.contains(&"1".to_string())); + assert!(attempts.contains(&"2".to_string())); + for span in &attempt_spans { + assert_eq!( + span.fields.get("max_attempts").cloned().unwrap_or_default(), + "2" + ); + } +} + +// TraceCapture minimal port from crates/crank-runtime/tests/stages.rs +#[derive(Clone, Default)] +struct TraceCapture { + spans: Arc>>, +} + +impl TraceCapture { + fn snapshot(&self) -> Vec { + self.spans.lock().expect("span lock").clone() + } +} + +#[derive(Clone, Debug)] +struct CapturedSpan { + name: &'static str, + parent_name: Option<&'static str>, + fields: std::collections::BTreeMap, +} + +impl Layer for TraceCapture +where + S: Subscriber + for<'lookup> LookupSpan<'lookup>, +{ + fn on_new_span( + &self, + attributes: &tracing::span::Attributes<'_>, + id: &Id, + context: tracing_subscriber::layer::Context<'_, S>, + ) { + let parent = attributes + .parent() + .and_then(|parent| context.span(parent)) + .or_else(|| { + attributes + .is_contextual() + .then(|| context.lookup_current()) + .flatten() + }); + let mut visitor = FieldVisitor::default(); + attributes.record(&mut visitor); + let mut spans = self.spans.lock().expect("span lock"); + let index = spans.len(); + spans.push(CapturedSpan { + name: attributes.metadata().name(), + parent_name: parent.map(|span| span.metadata().name()), + fields: visitor.fields, + }); + context + .span(id) + .expect("span exists") + .extensions_mut() + .insert(index); + } + + fn on_record( + &self, + id: &Id, + values: &tracing::span::Record<'_>, + context: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = FieldVisitor::default(); + values.record(&mut visitor); + let span = context.span(id).expect("span exists"); + let index = *span.extensions().get::().expect("capture index"); + self.spans.lock().expect("span lock")[index] + .fields + .extend(visitor.fields); + } +} + +#[derive(Default)] +struct FieldVisitor { + fields: std::collections::BTreeMap, +} + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.fields + .insert(field.name().to_owned(), value.to_owned()); + } + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.fields + .insert(field.name().to_owned(), format!("{value:?}")); + } +} diff --git a/apps/mcp-server/tests/integration/durable_tasks.rs b/apps/mcp-server/tests/integration/durable_tasks/mod.rs similarity index 92% rename from apps/mcp-server/tests/integration/durable_tasks.rs rename to apps/mcp-server/tests/integration/durable_tasks/mod.rs index 9b4c867..c8ff9fb 100644 --- a/apps/mcp-server/tests/integration/durable_tasks.rs +++ b/apps/mcp-server/tests/integration/durable_tasks/mod.rs @@ -14,13 +14,14 @@ use std::time::Duration; use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal}; use crank_core::{ - AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState, - ToolExecutionMode, ToolExecutionPolicy, + AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId, + TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore, + TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy, }; use crank_registry::{ AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest, }; -use crank_runtime::{InMemoryCoordinationStateStore, TaskService}; +use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService}; use serde_json::json; use sqlx::{PgPool, Row}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -149,7 +150,9 @@ async fn publish_tool_version(registry: &PostgresRegistry, version: u32) { .unwrap(); } -fn task_service(pool: PgPool) -> TaskService { +const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000"; + +fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService { let store = PostgresTaskStore::new( pool, AdmissionLimits { @@ -161,15 +164,27 @@ fn task_service(pool: PgPool) -> TaskService { Default::default(), ) .expect("valid task store configuration"); - TaskService::new(std::sync::Arc::new(store)) + TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto)) } -fn application(registry: &PostgresRegistry) -> TaskApplication { +/// The application entry under the same encrypted-input authority the production process uses. +async fn application(registry: &PostgresRegistry) -> TaskApplication { + let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key"); + let observed_at = OffsetDateTime::now_utc(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &observed_at, + }) + .await + .expect("test master-key identity registered"); TaskApplication::new( registry.clone(), Duration::from_millis(0), std::sync::Arc::new(InMemoryCoordinationStateStore::default()), - task_service(registry.pool().clone()), + task_service(registry.pool().clone(), crypto), ) } @@ -243,7 +258,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() { ) .await; let correlation = unsampled_correlation(); - let app = application(®istry); + let app = application(®istry).await; let acceptance = app .accept_background( @@ -273,7 +288,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() { assert_eq!(counts(registry.pool()).await, (1, 1)); // A freshly constructed application + service reads the same durable identity and pins. - let reader = application(®istry); + let reader = application(®istry).await; let record = reader .status(&caller, TOOL_NAME, acceptance.task_id) .await @@ -317,7 +332,7 @@ async fn a_new_publication_never_rebinds_an_accepted_task() { &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; - let app = application(®istry); + let app = application(®istry).await; let accepted = app .accept_background( &caller, @@ -397,7 +412,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() { &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; - let app = application(®istry); + let app = application(®istry).await; let accepted = app .accept_background( &owner, @@ -433,7 +448,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() { )); // A synchronous publication is not accepted as background work at all. - let synchronous_app = application(®istry); + let synchronous_app = application(®istry).await; let refused = synchronous_app .accept_background( &owner, @@ -453,7 +468,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() { async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() { let registry = common::test_registry().await; seed_tool(®istry, background_policy()).await; - let app = application(®istry); + let app = application(®istry).await; let admin = TaskPrincipal::Admin { workspace_slug: common::test_workspace_slug().to_owned(), agent_slug: AGENT_SLUG.to_owned(), @@ -549,7 +564,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() { &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; - let app = application(®istry); + let app = application(®istry).await; assert!(app.health().await.is_ok()); // Fail closed: without the Task schema the entry accepts nothing and reports dependency failure. @@ -589,3 +604,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() { assert_eq!(body["checks"]["durable_tasks"], json!("not_ready")); server.abort(); } + +mod attempt_trace; +mod shutdown_window; +mod task_worker; diff --git a/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs b/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs new file mode 100644 index 0000000..37d794c --- /dev/null +++ b/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs @@ -0,0 +1,190 @@ +//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window. +//! +//! Split from `task_worker.rs` to keep both integration files inside the code-health limit. + +use super::*; + +/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit +/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and +/// does not consume work that is still queued. +/// +/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the +/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in +/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist). +#[tokio::test] +async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() { + use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; + use crank_runtime::{ + TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, + TaskWorkerError, + }; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let registry = common::test_registry().await; + // The test registry exposes a single pooled connection: delaying the checkpoint inside its own + // statement keeps the window deterministic without competing for a second connection. + sqlx::raw_sql(sqlx::AssertSqlSafe( + "create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$ + begin + if new.current_attempt_id is not null and old.current_attempt_id is null then + perform pg_sleep(1.5); + end if; + return new; + end $$; + create trigger delay_task_dispatch_checkpoint before update on tasks + for each row execute function delay_task_dispatch_checkpoint();", + )) + .execute(registry.pool()) + .await + .expect("checkpoint delay trigger"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let mut accepted = Vec::new(); + for (seed, scoped_key) in [("q", "a"), ("r", "c")] { + let request = TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_stop_window".to_owned(), + platform_api_key_id: None, + operation_id: "op_report".to_owned(), + pins: TaskPins { + operation_version: 1, + policy_mode: ToolExecutionMode::Background, + execution_deadline_unix_ms: 1_900_000_000_000, + retain_until_unix_ms: 1_900_000_000_000 + 60_000, + approval_reference: None, + }, + scoped_key: scoped_key.repeat(64), + input_fingerprint: scoped_key.repeat(64), + input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(), + support: TaskSupportIds::new( + RequestId::generate(), + &crank_core::TraceContext::parse( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", + ) + .unwrap(), + ), + executable_input_ciphertext: None, + executable_input_key_version: None, + executable_input_master_key_epoch: None, + }; + accepted.push(store.accept(request).await.expect("PG accepts")); + } + + // The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1). + let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0)); + let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0)); + let executed = Arc::new(AtomicUsize::new(0)); + struct GatedHandler { + entered: Arc, + release: Arc, + executed: Arc, + } + #[async_trait::async_trait] + impl TaskHandler for GatedHandler { + async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { + self.entered.add_permits(1); + let _ = self.release.acquire().await; + Ok(()) + } + async fn execute( + &self, + _lease: &TaskLease, + _attempt_id: &str, + _cancelled: Arc, + _permit: TaskDispatchPermit, + ) -> TaskAttemptOutcome { + self.executed.fetch_add(1, Ordering::SeqCst); + TaskAttemptOutcome::Completed(json!({ "ok": true })) + } + } + let handler: Arc = Arc::new(GatedHandler { + entered: revalidate_entered.clone(), + release: revalidate_release.clone(), + executed: executed.clone(), + }); + let mut config = TaskWorkerConfig::default(); + config.owner = "runtime-worker-stop-window".to_owned(); + config.batch_limit = 1; + config.lease_ms = 30_000; + config.poll_interval = Duration::from_millis(50); + config.renewal_interval = Duration::from_secs(20); + config.max_inflight = 1; + config.shutdown_timeout = Duration::from_secs(20); + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire()) + .await + .expect("worker must claim the first Task and enter revalidation") + .expect("revalidation gate is open") + .forget(); + + // Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s. + revalidate_release.add_permits(1); + tokio::time::sleep(Duration::from_millis(300)).await; + + // The stop now lands while the durable checkpoint statement is still in flight: the worker may + // finalize the Task, but no permit exists yet, so no outbound call can start. + let stop_requested = std::time::Instant::now(); + stop_flag.store(true, Ordering::SeqCst); + + tokio::time::timeout(Duration::from_secs(30), worker_task) + .await + .expect("worker stops inside the 30s budget") + .expect("worker exits cleanly") + .expect("worker reports no error"); + assert!( + stop_requested.elapsed() >= Duration::from_millis(500), + "the checkpoint must have still been in flight when the stop arrived" + ); + + assert_eq!( + executed.load(Ordering::SeqCst), + 0, + "a stop between checkpoint and permit must never reach the upstream adapter" + ); + let access = TaskAccess { + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_stop_window".to_owned(), + }; + let record = store.read(&access, accepted[0].task_id).await.unwrap(); + assert_eq!(record.state, TaskState::Failed); + assert_eq!(record.attempts, 1, "the durable checkpoint did commit"); + assert!(record.current_attempt_id.is_some()); + assert_eq!( + record + .error + .as_ref() + .and_then(|error| error.value().get("code")) + .and_then(|code| code.as_str()), + Some("shutdown_before_dispatch") + ); + + // Work that was still queued is not consumed by the stopped worker. + let queued = store.read(&access, accepted[1].task_id).await.unwrap(); + assert_eq!(queued.state, TaskState::Queued); + let recovered = store + .claim_batch("worker_after_stop", 4, 10_000) + .await + .expect("queued work stays claimable"); + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].record.task_id, accepted[1].task_id); +} diff --git a/apps/mcp-server/tests/integration/durable_tasks/task_worker.rs b/apps/mcp-server/tests/integration/durable_tasks/task_worker.rs new file mode 100644 index 0000000..cc8b85c --- /dev/null +++ b/apps/mcp-server/tests/integration/durable_tasks/task_worker.rs @@ -0,0 +1,822 @@ +//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch. +//! +//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit. + +use super::*; + +/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a +/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan, +/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented. +#[tokio::test] +async fn task_worker_store_lease_recovery_contract() { + let registry = common::test_registry().await; + let store = PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"); + + // Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and + // leases durable work, preserving the same admission-backed intent. + let request = TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_1".to_owned(), + platform_api_key_id: None, + operation_id: "op_report".to_owned(), + pins: TaskPins { + operation_version: 1, + policy_mode: ToolExecutionMode::Background, + execution_deadline_unix_ms: 1_900_000_000_000, + retain_until_unix_ms: 1_900_000_000_000 + 60_000, + approval_reference: None, + }, + scoped_key: "a".repeat(64), + input_fingerprint: "b".repeat(64), + input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(), + support: TaskSupportIds::new( + RequestId::generate(), + &crank_core::TraceContext::parse( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", + ) + .unwrap(), + ), + // This store-level control exercises lease/fence behavior, not the encrypted acceptance path. + executable_input_ciphertext: None, + executable_input_key_version: None, + executable_input_master_key_epoch: None, + }; + let accepted = store.accept(request).await.expect("PG accepts"); + let leases = store + .claim_batch("worker_1", 4, 10_000) + .await + .expect("claim"); + assert_eq!(leases.len(), 1); + assert_eq!(leases[0].record.task_id, accepted.task_id); + let lease = leases[0].clone(); + + // A stale or competing owner cannot win CAS with the issued fencing token. + assert!( + store + .renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000) + .await + .is_err() + ); + assert!( + store + .write_progress( + "worker_1", + lease.record.task_id, + lease.fence + 1, + TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(), + ) + .await + .is_err() + ); + + // A stale fencing token can never checkpoint dispatch, even under the issuing owner. + assert!( + store + .checkpoint_dispatch( + "worker_1", + lease.record.task_id, + lease.fence + 1, + "attempt-stale".to_owned() + ) + .await + .is_err() + ); + // The valid owner still commits the dispatch checkpoint before any outbound work. + assert!( + store + .checkpoint_dispatch( + "worker_1", + lease.record.task_id, + lease.fence, + "attempt-1".to_owned(), + ) + .await + .unwrap() + .current_attempt_id + .as_deref() + == Some("attempt-1") + ); +} + +#[tokio::test] +async fn task_worker_runtime_dispatch_and_recovery_contract() { + use async_trait::async_trait; + use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; + use crank_runtime::{ + TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, + TaskWorkerError, + }; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let registry = common::test_registry().await; + let pool = registry.pool().clone(); + let store = Arc::new( + PostgresTaskStore::new( + pool.clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let observed = Arc::new(AtomicUsize::new(0)); + + // Positive control: PostgreSQL alone accepts the durable Task. + let request = TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_runtime".to_owned(), + platform_api_key_id: None, + operation_id: "op_report".to_owned(), + pins: TaskPins { + operation_version: 1, + policy_mode: ToolExecutionMode::Background, + execution_deadline_unix_ms: 1_900_000_000_000, + retain_until_unix_ms: 1_900_000_000_000 + 60_000, + approval_reference: None, + }, + scoped_key: "r".repeat(64), + input_fingerprint: "b".repeat(64), + input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(), + support: TaskSupportIds::new( + RequestId::generate(), + &crank_core::TraceContext::parse( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", + ) + .unwrap(), + ), + executable_input_ciphertext: None, + executable_input_key_version: None, + executable_input_master_key_epoch: None, + }; + let accepted = store.accept(request).await.expect("PG accepts"); + let task_id = accepted.task_id; + + struct RuntimeHandler { + observed: Arc, + } + #[async_trait] + impl TaskHandler for RuntimeHandler { + async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { + Ok(()) + } + async fn execute( + &self, + _lease: &TaskLease, + _attempt_id: &str, + _cancelled: Arc, + _permit: TaskDispatchPermit, + ) -> TaskAttemptOutcome { + self.observed.fetch_add(1, Ordering::SeqCst); + TaskAttemptOutcome::Completed(serde_json::json!({"ok": true})) + } + } + let handler: Arc = Arc::new(RuntimeHandler { + observed: observed.clone(), + }); + + // PG-only authoritative dispatch: no Valkey or upstream process is involved. + let mut config = TaskWorkerConfig::default(); + config.owner = "runtime-worker-1".to_owned(); + config.batch_limit = 1; + config.lease_ms = 10_000; + config.poll_interval = Duration::from_millis(100); + config.renewal_interval = Duration::from_millis(1_000); + config.max_inflight = 1; + config.shutdown_timeout = Duration::from_secs(5); + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + let access = TaskAccess { + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_runtime".to_owned(), + }; + let mut state = None; + for _ in 0..100 { + tokio::time::sleep(Duration::from_millis(100)).await; + if let Ok(record) = store.read(&access, task_id).await { + state = Some(record.state); + if record.state == TaskState::Completed { + break; + } + } + } + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + + assert_eq!(state, Some(TaskState::Completed)); + assert_eq!( + observed.load(Ordering::SeqCst), + 1, + "exactly one outbound attempt" + ); + // Terminal state is not duplicated. + let record = store.read(&access, task_id).await.expect("terminal read"); + assert_eq!(record.state, TaskState::Completed); + + // Bounded shutdown with no fresh work returns promptly. + let mut idle_config = TaskWorkerConfig::default(); + idle_config.owner = "runtime-worker-idle".to_owned(); + idle_config.poll_interval = Duration::from_millis(50); + idle_config.shutdown_timeout = Duration::from_secs(2); + let idle_worker = TaskWorker::new( + store.clone(), + handler_for_no_work(store.clone()), + idle_config, + ) + .expect("valid idle worker"); + idle_worker.stop(); + let started = std::time::Instant::now(); + idle_worker.run().await.expect("idle worker stops"); + assert!(started.elapsed() <= Duration::from_secs(30)); +} + +/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call. +#[tokio::test] +async fn task_worker_revalidation_denial_never_dispatches() { + use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind}; + use crank_runtime::{ + TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, + TaskWorkerError, + }; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let registry = common::test_registry().await; + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let request = TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_denied".to_owned(), + platform_api_key_id: None, + operation_id: "op_report".to_owned(), + pins: TaskPins { + operation_version: 1, + policy_mode: ToolExecutionMode::Background, + execution_deadline_unix_ms: 1_900_000_000_000, + retain_until_unix_ms: 1_900_000_000_000 + 60_000, + approval_reference: None, + }, + scoped_key: "d".repeat(64), + input_fingerprint: "b".repeat(64), + input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(), + support: TaskSupportIds::new( + RequestId::generate(), + &crank_core::TraceContext::parse( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + ) + .unwrap(), + ), + executable_input_ciphertext: None, + executable_input_key_version: None, + executable_input_master_key_epoch: None, + }; + let accepted = store.accept(request).await.expect("PG accepts"); + + struct DeniedHandler { + observed: Arc, + } + #[async_trait::async_trait] + impl TaskHandler for DeniedHandler { + async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { + Err(TaskWorkerError::Store(TaskError::invalid( + "publication or credential revoked", + ))) + } + async fn execute( + &self, + _lease: &TaskLease, + _attempt_id: &str, + _cancelled: Arc, + _permit: TaskDispatchPermit, + ) -> TaskAttemptOutcome { + self.observed.fetch_add(1, Ordering::SeqCst); + TaskAttemptOutcome::Completed(json!({"ok": true})) + } + } + let observed = Arc::new(AtomicUsize::new(0)); + let handler: Arc = Arc::new(DeniedHandler { + observed: observed.clone(), + }); + let mut config = TaskWorkerConfig::default(); + config.owner = "runtime-worker-denied".to_owned(); + config.batch_limit = 1; + config.lease_ms = 10_000; + config.poll_interval = Duration::from_millis(50); + config.max_inflight = 1; + config.shutdown_timeout = Duration::from_secs(5); + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + let access = TaskAccess { + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_denied".to_owned(), + }; + let mut state = None; + for _ in 0..100 { + tokio::time::sleep(Duration::from_millis(50)).await; + if let Ok(record) = store.read(&access, accepted.task_id).await { + state = Some(record.state); + if record.state == TaskState::Failed { + break; + } + } + } + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + + assert_eq!(state, Some(TaskState::Failed)); + assert_eq!( + observed.load(Ordering::SeqCst), + 0, + "invalidated work never reaches the upstream" + ); +} + +fn handler_for_no_work( + _store: std::sync::Arc, +) -> std::sync::Arc { + struct NoWorkHandler; + #[async_trait::async_trait] + impl crank_runtime::TaskHandler for NoWorkHandler { + async fn revalidate( + &self, + _lease: &crank_core::TaskLease, + ) -> Result<(), crank_runtime::TaskWorkerError> { + Ok(()) + } + async fn execute( + &self, + _lease: &crank_core::TaskLease, + _attempt_id: &str, + _cancelled: std::sync::Arc, + _permit: crank_runtime::TaskDispatchPermit, + ) -> crank_runtime::TaskAttemptOutcome { + crank_runtime::TaskAttemptOutcome::Completed(serde_json::json!({"ok": true})) + } + } + std::sync::Arc::new(NoWorkHandler) +} + +/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a +/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the +/// accepted parent trace is continued on the wire. +#[tokio::test] +async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() { + use axum::Router; + use axum::http::{HeaderMap, StatusCode}; + use axum::routing::post; + use crank_community_mcp::TaskWorkerHandler; + use crank_core::{AgentOperationBinding, PlatformApiKeyScope}; + use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + + let registry = common::test_registry().await; + let calls = Arc::new(AtomicUsize::new(0)); + let seen_traceparent = Arc::new(Mutex::new(None::)); + + let app = Router::new().route( + "/crm/leads", + post({ + let calls = Arc::clone(&calls); + let seen = Arc::clone(&seen_traceparent); + move |headers: HeaderMap| { + let calls = Arc::clone(&calls); + let seen = Arc::clone(&seen); + async move { + calls.fetch_add(1, Ordering::SeqCst); + if let Some(value) = headers + .get("traceparent") + .and_then(|value| value.to_str().ok()) + { + *seen.lock().unwrap() = Some(value.to_owned()); + } + (StatusCode::OK, axum::Json(json!({"id": "lead-1"}))) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let mut operation = common::test_operation(&upstream, TOOL_NAME); + operation.execution_config.execution_policy = Some(background_policy()); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish(®istry, &operation.id, 1).await; + common::publish_agent_with_bindings( + ®istry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; + + let caller = principal( + ®istry, + "task-worker-e2e", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let correlation = unsampled_correlation(); + let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(); + let observed_at = OffsetDateTime::now_utc(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &observed_at, + }) + .await + .expect("test master-key identity registered"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let tasks = TaskApplication::new( + registry.clone(), + Duration::ZERO, + Arc::new(InMemoryCoordinationStateStore::default()), + TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())), + ); + let acceptance = tasks + .accept_background( + &caller, + TOOL_NAME, + None, + "worker-e2e-key", + &json!({"email": "user@example.com"}), + &correlation, + ) + .await + .expect("durable acceptance with executable input"); + + let runtime = crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .build(); + let handler = Arc::new(TaskWorkerHandler::from_application( + &tasks, + registry.clone(), + runtime, + crypto, + )); + let mut config = TaskWorkerConfig::default(); + config.owner = "runtime-worker-e2e".to_owned(); + config.batch_limit = 1; + config.lease_ms = 10_000; + config.poll_interval = Duration::from_millis(50); + config.max_inflight = 1; + config.shutdown_timeout = Duration::from_secs(5); + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + let mut state = None; + for _ in 0..200 { + tokio::time::sleep(Duration::from_millis(50)).await; + let current: Option = + sqlx::query_scalar("select state from tasks where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + state = current; + if state.as_deref() == Some("completed") { + break; + } + } + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + server.abort(); + + assert_eq!(state.as_deref(), Some("completed")); + assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call"); + assert_eq!( + seen_traceparent.lock().unwrap().as_deref(), + Some(correlation.trace_context().traceparent()), + "the durable attempt continues the accepted parent trace" + ); +} + +/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as +/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically. +#[tokio::test] +async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() { + use axum::Router; + use axum::http::StatusCode; + use axum::routing::post; + use crank_community_mcp::TaskWorkerHandler; + use crank_core::{AgentOperationBinding, PlatformApiKeyScope}; + use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig}; + use std::sync::Arc; + use std::sync::atomic::Ordering; + + let registry = common::test_registry().await; + let upstream_app = Router::new().route( + "/crm/leads", + post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); + let upstream = format!("http://{address}"); + + let mut operation = common::test_operation(&upstream, TOOL_NAME); + operation.execution_config.execution_policy = Some(background_policy()); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish(®istry, &operation.id, 1).await; + common::publish_agent_with_bindings( + ®istry, + AGENT_SLUG, + vec![AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + ) + .await; + + let caller = principal( + ®istry, + "task-worker-uncertain", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let correlation = unsampled_correlation(); + let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(); + let observed_at = OffsetDateTime::now_utc(); + registry + .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { + epoch: crypto.master_key_epoch(), + fingerprint: crypto.master_key_fingerprint(), + cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, + observed_at: &observed_at, + }) + .await + .expect("test master-key identity registered"); + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let tasks = TaskApplication::new( + registry.clone(), + Duration::ZERO, + Arc::new(InMemoryCoordinationStateStore::default()), + TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())), + ); + let acceptance = tasks + .accept_background( + &caller, + TOOL_NAME, + None, + "worker-uncertain-key", + &json!({"email": "user@example.com"}), + &correlation, + ) + .await + .expect("durable acceptance with executable input"); + + let runtime = crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .build(); + let handler = Arc::new(TaskWorkerHandler::from_application( + &tasks, + registry.clone(), + runtime, + crypto, + )); + let mut config = TaskWorkerConfig::default(); + config.owner = "runtime-worker-uncertain".to_owned(); + config.batch_limit = 1; + config.lease_ms = 10_000; + config.poll_interval = Duration::from_millis(50); + config.max_inflight = 1; + config.shutdown_timeout = Duration::from_secs(5); + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + let mut state = None; + for _ in 0..200 { + tokio::time::sleep(Duration::from_millis(50)).await; + let current: Option = + sqlx::query_scalar("select state from tasks where task_id = $1") + .bind(acceptance.task_id.as_uuid()) + .fetch_optional(registry.pool()) + .await + .unwrap(); + state = current; + if state.as_deref() == Some("outcome_unknown") { + break; + } + } + stop_flag.store(true, Ordering::SeqCst); + let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; + server.abort(); + + assert_eq!(state.as_deref(), Some("outcome_unknown")); + // No automatic retry: another authoritative scan never redispaches the uncertain attempt. + assert!( + store + .claim_batch("worker_after", 4, 10_000) + .await + .unwrap() + .is_empty() + ); +} + +/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced +/// terminal commit within the 30s budget. +#[tokio::test] +async fn task_worker_graceful_stop_drains_active_attempt() { + use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; + use crank_runtime::{ + TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, + TaskWorkerError, + }; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let registry = common::test_registry().await; + let store = Arc::new( + PostgresTaskStore::new( + registry.pool().clone(), + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"), + ); + let request = TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_drain".to_owned(), + platform_api_key_id: None, + operation_id: "op_report".to_owned(), + pins: TaskPins { + operation_version: 1, + policy_mode: ToolExecutionMode::Background, + execution_deadline_unix_ms: 1_900_000_000_000, + retain_until_unix_ms: 1_900_000_000_000 + 60_000, + approval_reference: None, + }, + scoped_key: "s".repeat(64), + input_fingerprint: "d".repeat(64), + input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(), + support: TaskSupportIds::new( + RequestId::generate(), + &crank_core::TraceContext::parse( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", + ) + .unwrap(), + ), + executable_input_ciphertext: None, + executable_input_key_version: None, + executable_input_master_key_epoch: None, + }; + let accepted = store.accept(request).await.expect("PG accepts"); + + struct SlowHandler { + started: Arc, + } + #[async_trait::async_trait] + impl TaskHandler for SlowHandler { + async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { + Ok(()) + } + async fn execute( + &self, + _lease: &TaskLease, + _attempt_id: &str, + _cancelled: Arc, + _permit: TaskDispatchPermit, + ) -> TaskAttemptOutcome { + self.started.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(500)).await; + TaskAttemptOutcome::Completed(json!({"ok": true})) + } + } + let started = Arc::new(AtomicUsize::new(0)); + let handler: Arc = Arc::new(SlowHandler { + started: started.clone(), + }); + let mut config = TaskWorkerConfig::default(); + config.owner = "runtime-worker-drain".to_owned(); + config.batch_limit = 1; + config.lease_ms = 10_000; + config.poll_interval = Duration::from_millis(50); + config.max_inflight = 1; + config.shutdown_timeout = Duration::from_secs(20); + let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); + let stop_flag = worker.stop_flag(); + let worker_task = tokio::spawn(async move { worker.run().await }); + + for _ in 0..100 { + if started.load(Ordering::SeqCst) > 0 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started"); + let stop_requested = std::time::Instant::now(); + stop_flag.store(true, Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(30), worker_task) + .await + .expect("worker stops inside the 30s budget") + .expect("worker exits cleanly") + .expect("worker reports no error"); + assert!(stop_requested.elapsed() <= Duration::from_secs(30)); + + let access = TaskAccess { + workspace_id: common::test_workspace_id().to_string(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_drain".to_owned(), + }; + let record = store.read(&access, accepted.task_id).await.unwrap(); + assert_eq!(record.state, TaskState::Completed); +} diff --git a/crates/crank-community-mcp/src/access.rs b/crates/crank-community-mcp/src/access.rs index 1eb1f32..90c93f6 100644 --- a/crates/crank-community-mcp/src/access.rs +++ b/crates/crank-community-mcp/src/access.rs @@ -272,7 +272,10 @@ fn record_unknown_approval_access_denial(reason: &'static str) { ); } -fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool { +pub(super) fn allows_scope( + scopes: &[PlatformApiKeyScope], + required_scope: PlatformApiKeyScope, +) -> bool { match required_scope { PlatformApiKeyScope::Read => scopes.iter().any(|scope| { matches!( diff --git a/crates/crank-community-mcp/src/app.rs b/crates/crank-community-mcp/src/app.rs index 2be7fad..96e3c74 100644 --- a/crates/crank-community-mcp/src/app.rs +++ b/crates/crank-community-mcp/src/app.rs @@ -51,15 +51,15 @@ mod auth_resolution; mod dispatch; mod endpoint; mod legacy_dispatch; +mod task_worker; mod tasks; - use self::approvals::{ approve_request, deny_request, get_approval_request, list_pending_approvals, }; pub(super) use self::auth_resolution::resolve_operation_auth; use self::endpoint::{mcp_delete, mcp_get, mcp_post}; +pub use self::task_worker::TaskWorkerHandler; pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal}; - // Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two // through `super`, and the plain lib build does not use them (same pattern as // `#[cfg(test)] use invocation_history::observe_invocation_history_outcome;` above). diff --git a/crates/crank-community-mcp/src/app/auth_resolution.rs b/crates/crank-community-mcp/src/app/auth_resolution.rs index 9bef8e4..f97f483 100644 --- a/crates/crank-community-mcp/src/app/auth_resolution.rs +++ b/crates/crank-community-mcp/src/app/auth_resolution.rs @@ -19,18 +19,31 @@ pub(crate) async fn resolve_operation_auth( workspace_id: &crank_core::WorkspaceId, execution_config: &crank_core::ExecutionConfig, ) -> Result, RuntimeError> { - if execution_config.auth_profile_ref.is_none() { - return Ok(None); - } - let span = Stage::AuthResolve.span(); - let result = resolve_runtime_auth_for_task( + resolve_operation_auth_with( &state.registry, &state.secret_crypto, workspace_id, execution_config, ) - .instrument(span.clone()) - .await; + .await +} + +/// Same resolution for callers that hold the authoritative registry/secret-crypto pair without the +/// HTTP application state (the durable Task worker). +pub(crate) async fn resolve_operation_auth_with( + registry: &PostgresRegistry, + secret_crypto: &SecretCrypto, + workspace_id: &crank_core::WorkspaceId, + execution_config: &crank_core::ExecutionConfig, +) -> Result, RuntimeError> { + if execution_config.auth_profile_ref.is_none() { + return Ok(None); + } + let span = Stage::AuthResolve.span(); + let result = + resolve_runtime_auth_for_task(registry, secret_crypto, workspace_id, execution_config) + .instrument(span.clone()) + .await; match &result { Ok(_) => StageOutcome::Success.record(&span), Err(_) => { diff --git a/crates/crank-community-mcp/src/app/task_worker.rs b/crates/crank-community-mcp/src/app/task_worker.rs new file mode 100644 index 0000000..e4fefc7 --- /dev/null +++ b/crates/crank-community-mcp/src/app/task_worker.rs @@ -0,0 +1,282 @@ +//! Application-level durable Task handler for Story 4.3. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use crank_core::{ + AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope, + TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId, + current_unix_ms, +}; +use crank_registry::PostgresRegistry; +use crank_registry::PublishedAgentTool; +use crank_runtime::{ + ExecutionAuthorization, ExecutionRequestError, RuntimeExecutionRequest, RuntimeExecutor, + SecretCrypto, TaskDispatchPermit, TaskHandler, TaskService, TaskWorkerError, +}; + +use super::TaskApplication; +use super::auth_resolution::resolve_operation_auth_with; +use super::tool_resolution::runtime_operation; +use crate::access::{allows_scope, credential_allows_security_level}; +use crate::catalog::PublishedToolCatalog; + +pub struct TaskWorkerHandler { + registry: PostgresRegistry, + catalog: PublishedToolCatalog, + runtime: RuntimeExecutor, + secret_crypto: SecretCrypto, + service: TaskService, +} + +impl TaskWorkerHandler { + /// Production construction from the same application entry, registry and runtime the MCP + /// process serves requests with. The worker needs no HTTP state, only these authoritative + /// dependencies. + pub fn from_application( + tasks: &TaskApplication, + registry: PostgresRegistry, + runtime: RuntimeExecutor, + secret_crypto: SecretCrypto, + ) -> Self { + Self { + registry, + catalog: tasks.catalog(), + runtime, + secret_crypto, + service: tasks.task_service(), + } + } + + /// Authoritative pre-dispatch authorization of one leased Task (Story 4.3). + /// + /// Resolves the exact pinned Published Version in the current publication, repeats the approval + /// policy and revalidates the typed persisted Platform API key identity against the registry: + /// the key must still be active, unexpired and bound to this workspace/agent, and its exact + /// scopes/security level must still authorize the operation. Any denial is a typed error, so the + /// worker finishes the attempt without dispatch (fail closed before any network use). + /// + /// An administrator Task has no authoritative Task revalidation API in the registry, so it stays + /// fail-closed here instead of inventing authority. + async fn authorize(&self, lease: &TaskLease) -> Result { + if lease.record.owner_kind != TaskOwnerKind::Agent { + return Err(TaskError::invalid( + "admin task authority revalidation is not implemented", + )); + } + let Some(key_id) = lease.record.platform_api_key_id.as_ref() else { + return Err(TaskError::invalid("task authority reference is missing")); + }; + + let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str()); + let agent_id = AgentId::new(lease.record.owner_id.as_str()); + let workspace = self + .registry + .get_workspace(&workspace_id) + .await + .map_err(|_| TaskError::unavailable("published catalog unavailable"))? + .ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?; + let agents = self + .registry + .list_agents(&workspace_id) + .await + .map_err(|_| TaskError::unavailable("published catalog unavailable"))?; + let agent = agents + .into_iter() + .find(|agent| agent.id.as_str() == lease.record.owner_id.as_str()) + .ok_or_else(|| TaskError::invalid("task agent no longer exists"))?; + + let catalog = self + .catalog + .get_catalog(&workspace.workspace.slug, &agent.slug) + .await + .map_err(|_| TaskError::unavailable("published catalog unavailable"))?; + let tool = catalog + .tools + .into_iter() + .find(|tool| { + tool.operation.id.as_str() == lease.record.operation_id.as_str() + && tool.operation.version == lease.record.pins.operation_version + }) + .ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?; + + if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() { + return Err(TaskError::invalid("task workspace binding mismatch")); + } + if tool.agent_id.as_str() != lease.record.owner_id.as_str() { + return Err(TaskError::invalid("task agent binding mismatch")); + } + if tool + .operation + .execution_config + .approval_policy + .as_ref() + .is_some_and(|policy| policy.required) + { + return Err(TaskError::invalid( + "missing approval proof for task operation", + )); + } + + // Authoritative revalidation of the exact immutable key identity. A revoked, expired, + // foreign-workspace or foreign-agent key is absent here and never reaches the network. + let key = self + .registry + .revalidate_platform_api_key(&workspace_id, &agent_id, key_id) + .await + .map_err(|_| TaskError::unavailable("task authority store unavailable"))? + .ok_or_else(|| TaskError::invalid("task authority is no longer active"))?; + let credential = VerifiedMachineCredential { + machine_access_mode: MachineAccessMode::StaticAgentKey, + max_security_level: OperationSecurityLevel::Standard, + scopes: key.api_key.scopes.clone(), + platform_api_key_id: Some(key.api_key.id.clone()), + }; + if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) { + return Err(TaskError::invalid("task authority scope is insufficient")); + } + if !credential_allows_security_level(&credential, tool.operation.security_level) { + return Err(TaskError::invalid( + "task authority security level is insufficient", + )); + } + Ok(tool) + } +} + +#[async_trait] +impl TaskHandler for TaskWorkerHandler { + async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> { + self.authorize(lease) + .await + .map_err(TaskWorkerError::Store)?; + Ok(()) + } + + async fn execute( + &self, + lease: &TaskLease, + _attempt_id: &str, + cancelled: Arc, + permit: TaskDispatchPermit, + ) -> crank_runtime::TaskAttemptOutcome { + use crank_runtime::TaskAttemptOutcome; + use std::sync::atomic::Ordering; + + // The lease may have been lost between the checkpoint and the outbound call; never dispatch + // on behalf of a worker that no longer owns the Task. + if cancelled.load(Ordering::Acquire) { + return TaskAttemptOutcome::OutcomeUnknown; + } + + let tool = match self.authorize(lease).await { + Ok(tool) => tool, + Err(error) => { + return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()})); + } + }; + + let input = match self.service.executable_input(lease).await { + Ok(input) => input, + Err(error) => { + return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()})); + } + }; + + let operation = runtime_operation(&tool); + let now_ms = current_unix_ms(); + let deadline_ms = lease.record.pins.execution_deadline_unix_ms; + if deadline_ms <= now_ms { + return TaskAttemptOutcome::Failed( + serde_json::json!({"code": "task deadline elapsed"}), + ); + } + let deadline = Instant::now() + + Duration::from_millis((deadline_ms - now_ms).min(i64::MAX as i64) as u64); + let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str()); + let agent_id = AgentId::new(lease.record.owner_id.as_str()); + let resolved_auth = match resolve_operation_auth_with( + &self.registry, + &self.secret_crypto, + &workspace_id, + &operation.execution_config, + ) + .await + { + Ok(auth) => auth, + Err(error) => { + return TaskAttemptOutcome::Failed( + serde_json::json!({ "code": error.to_string() }), + ); + } + }; + + // Continue the parent trace persisted with the accepted Task; the worker's attempt span is a + // child of it, and the executor records the attempt identity. Trace data stays out of labels. + let trace_context = match TraceContext::parse(&lease.record.support.traceparent) { + Ok(context) => context, + Err(_) => { + return TaskAttemptOutcome::Failed( + serde_json::json!({ "code": "task trace parent is invalid" }), + ); + } + }; + let request_context = crank_runtime::RuntimeRequestContext::new( + lease.record.support.request_id.clone(), + trace_context, + ) + .with_metering_context( + workspace_id.clone(), + Some(agent_id.clone()), + crank_core::InvocationSource::AgentToolCall, + ); + + let request = match RuntimeExecutionRequest::try_new( + &workspace_id, + ExecutionOrigin::AgentSnapshot, + Some(&agent_id), + &operation, + &input, + ExecutionAuthorization::Authorized, + resolved_auth.as_ref(), + &request_context, + deadline, + ) { + Ok(request) => request, + Err(error) => { + return TaskAttemptOutcome::Failed(match error { + ExecutionRequestError::InvalidOrigin => { + serde_json::json!({"code": "invalid execution origin"}) + } + ExecutionRequestError::InvalidOperationVersion => { + serde_json::json!({"code": "invalid operation version"}) + } + ExecutionRequestError::DeadlineElapsed => { + serde_json::json!({"code": "task deadline elapsed"}) + } + ExecutionRequestError::InvalidMeteringScope => { + serde_json::json!({"code": "invalid metering scope"}) + } + }); + } + }; + + if cancelled.load(Ordering::Acquire) { + return TaskAttemptOutcome::OutcomeUnknown; + } + match self.runtime.execute_task_outcome(request, permit).await { + Ok(success) => TaskAttemptOutcome::Completed(success.output), + Err(failure) => { + // An ambiguous outbound failure is recorded as uncertainty, never as a retryable + // failure: the durable checkpoint already exists, so Story 4.4 may reconcile it. + if failure.outcome_certainty() == crank_core::OutcomeCertainty::OutcomeUnknown { + return TaskAttemptOutcome::OutcomeUnknown; + } + TaskAttemptOutcome::Failed( + serde_json::json!({ "code": format!("{:?}", failure.error_code()) }), + ) + } + } + } +} diff --git a/crates/crank-community-mcp/src/app/tasks.rs b/crates/crank-community-mcp/src/app/tasks.rs index c25f85d..697709c 100644 --- a/crates/crank-community-mcp/src/app/tasks.rs +++ b/crates/crank-community-mcp/src/app/tasks.rs @@ -14,8 +14,8 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use crank_core::{ - CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord, - ToolExecutionMode, + CorrelationContext, PlatformApiKeyId, TaskAcceptance, TaskAccess, TaskError, TaskId, + TaskOwnerKind, TaskRecord, ToolExecutionMode, }; use crank_registry::{PostgresRegistry, PublishedAgentTool}; use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService}; @@ -64,6 +64,20 @@ impl TaskPrincipal { } } + /// Typed immutable Platform API key identity of this principal. A verified agent credential + /// without a key ID cannot be revalidated by a worker, so it is refused instead of accepted; + /// an administrator has no machine key and is stored as `None` (the worker fails closed). + fn platform_api_key_id(&self) -> Result, TaskApplicationError> { + match self { + Self::Agent { credential, .. } => credential + .platform_api_key_id + .clone() + .map(Some) + .ok_or(TaskApplicationError::Denied), + Self::Admin { .. } => Ok(None), + } + } + /// Bounded server-derived authorization scope identity of this principal. fn authorization_scope(&self) -> String { let identity = match self { @@ -131,6 +145,17 @@ impl TaskApplication { } } + /// Runtime task service carrying encrypted executable input. + pub fn task_service(&self) -> TaskService { + self.service.clone() + } + + /// Published catalog of the same application entry, for the durable Task worker's pre-dispatch + /// revalidation. + pub(super) fn catalog(&self) -> PublishedToolCatalog { + self.catalog.clone() + } + /// Readiness of the durable Task store this entry depends on. pub async fn health(&self) -> Result<(), TaskError> { self.service.health().await @@ -170,6 +195,7 @@ impl TaskApplication { // of accepting work a worker could never safely execute. return Err(TaskApplicationError::ApprovalRequired); } + let platform_api_key_id = principal.platform_api_key_id()?; let draft = |operation_version: u32| { let (owner_kind, owner_id) = principal.owner(&tool); TaskAcceptDraft { @@ -177,6 +203,7 @@ impl TaskApplication { owner_kind, owner_id, authorization_scope: principal.authorization_scope(), + platform_api_key_id: platform_api_key_id.clone(), operation_id: tool.operation.id.as_str().to_owned(), operation_version, policy_mode: policy.mode, diff --git a/crates/crank-community-mcp/src/lib.rs b/crates/crank-community-mcp/src/lib.rs index e2c4aca..217d031 100644 --- a/crates/crank-community-mcp/src/lib.rs +++ b/crates/crank-community-mcp/src/lib.rs @@ -16,7 +16,7 @@ mod tool_search; mod transport; pub use app::{ - TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts, - build_app_with_artifacts_and_tasks, build_app_with_background_workers, - build_app_with_background_workers_and_limits, + TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler, build_app, + build_app_with_artifacts, build_app_with_artifacts_and_tasks, + build_app_with_background_workers, build_app_with_background_workers_and_limits, }; diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index 54ecc1d..b688ca6 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -76,10 +76,12 @@ pub mod domain { pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion}; pub use crate::task::{ TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, - TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES, - TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, - TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, - TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds, + TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, + TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES, + TASK_IDENTITY_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, + TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, + TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, + TaskState, TaskSupportIds, }; pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget}; pub use crate::tool_quality::{ @@ -202,11 +204,12 @@ pub use product_event::{ pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion}; pub use task::{ - TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, + TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, + TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, - TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, - TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key, - validate_task_envelope, + TaskError, TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, + TaskRecord, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys, + redact_task_envelope, sensitive_task_key, validate_task_envelope, }; pub use tool_catalog::{ RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError, diff --git a/crates/crank-core/src/task.rs b/crates/crank-core/src/task.rs index 997da77..813514c 100644 --- a/crates/crank-core/src/task.rs +++ b/crates/crank-core/src/task.rs @@ -15,10 +15,15 @@ use serde_json::Value; use uuid::Uuid; use crate::correlation::{RequestId, TraceContext, TraceId}; +use crate::ids::PlatformApiKeyId; use crate::operation::ToolExecutionMode; /// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata). pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768; +/// Maximum byte budget of the decrypted executable Task input. +pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES; +/// Maximum byte budget of the stored encrypted executable Task input ciphertext. +pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304; /// Maximum nesting depth of a persisted Task envelope. pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8; /// Maximum number of JSON members/nodes of a persisted Task envelope. @@ -351,6 +356,16 @@ pub fn redact_task_envelope(value: &Value) -> Value { } } +/// Recursive check used to fail closed on sensitive fields before encryption. +pub fn contains_sensitive_task_keys(value: &Value) -> bool { + match value { + Value::Object(object) => object + .iter() + .any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)), + Value::Array(values) => values.iter().any(contains_sensitive_task_keys), + _ => false, + } +} /// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and /// queue capacity commit together, and no acknowledgement exists before that commit. #[derive(Clone, Debug, PartialEq, Eq)] @@ -360,6 +375,10 @@ pub struct TaskAcceptRequest { pub owner_kind: TaskOwnerKind, pub owner_id: String, pub authorization_scope: String, + /// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the + /// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively. + /// `None` means the accepting principal has no machine key (administrator) and fail-closes. + pub platform_api_key_id: Option, pub operation_id: String, pub pins: TaskPins, /// Digest of the caller key over the full owner/authorization/Tool scope. @@ -368,6 +387,10 @@ pub struct TaskAcceptRequest { pub input_fingerprint: String, pub input: TaskEnvelope, pub support: TaskSupportIds, + /// Bounded encrypted executable input, persisted alongside the redacted input projection. + pub executable_input_ciphertext: Option, + pub executable_input_key_version: Option, + pub executable_input_master_key_epoch: Option, } impl TaskAcceptRequest { @@ -382,6 +405,13 @@ impl TaskAcceptRequest { return Err(TaskError::invalid(format!("task {field} is invalid"))); } } + if let Some(key_id) = self.platform_api_key_id.as_ref() + && (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES) + { + return Err(TaskError::invalid( + "task platform api key reference is invalid", + )); + } if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES { return Err(TaskError::invalid("task scoped key digest is invalid")); } @@ -394,7 +424,40 @@ impl TaskAcceptRequest { return Err(TaskError::invalid("task input fingerprint is invalid")); } self.pins.validate()?; - self.support.validate() + self.support.validate()?; + let has_ciphertext = self.executable_input_ciphertext.is_some(); + let has_key = self.executable_input_key_version.is_some(); + let has_epoch = self.executable_input_master_key_epoch.is_some(); + match (has_ciphertext, has_key, has_epoch) { + (false, false, false) => {} + (true, true, true) => { + let ciphertext = self.executable_input_ciphertext.as_deref().unwrap(); + if ciphertext.is_empty() + || ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES + { + return Err(TaskError::invalid( + "task executable input ciphertext is invalid", + )); + } + let key_version = self.executable_input_key_version.as_deref().unwrap(); + if key_version.is_empty() || key_version.len() > 64 { + return Err(TaskError::invalid( + "task executable input ciphertext is invalid", + )); + } + if self.executable_input_master_key_epoch.unwrap_or_default() < 1 { + return Err(TaskError::invalid( + "task executable input ciphertext is invalid", + )); + } + } + _ => { + return Err(TaskError::invalid( + "task executable input ciphertext is invalid", + )); + } + } + Ok(()) } } @@ -448,6 +511,9 @@ pub struct TaskRecord { pub owner_kind: TaskOwnerKind, pub owner_id: String, pub authorization_scope: String, + /// Typed immutable identity of the Platform API key that authorized the acceptance; the worker + /// revalidates this exact key against the registry before any outbound dispatch. + pub platform_api_key_id: Option, pub operation_id: String, /// Canonical digest of the accepted executable input (compare-and-conflict identity). pub input_fingerprint: String, @@ -458,6 +524,16 @@ pub struct TaskRecord { pub result: Option, pub error: Option, pub attempts: u32, + /// Monotonic lease/fencing token. It never resets and protects Task state mutations. + pub fence: u64, + /// Random process-instance lease owner, if a worker currently owns the Task. + pub lease_owner: Option, + /// Lease expiry epoch milliseconds, if the Task is claimed. + pub lease_until_unix_ms: Option, + /// Current dispatch attempt identity, committed before any outbound adapter call. + pub current_attempt_id: Option, + /// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed. + pub dispatch_started_at_unix_ms: Option, /// Monotonic lifecycle version used by the terminal CAS. pub version: u32, pub created_at_unix_ms: i64, @@ -542,6 +618,15 @@ impl TaskError { } } +/// Leased view of a claimed Task: its durable record plus lease fencing identity. +#[derive(Clone, Debug, PartialEq)] +pub struct TaskLease { + pub record: TaskRecord, + pub owner: String, + pub fence: u64, + pub lease_until: i64, +} + /// Authoritative durable Task port. #[async_trait] pub trait TaskStore: Send + Sync { @@ -566,9 +651,59 @@ pub trait TaskStore: Send + Sync { /// Applies exactly one conditional terminal transition and releases its intent atomically. async fn finish(&self, request: TaskFinishRequest) -> Result; - /// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes - /// expired terminal records. Live and non-terminal work is never deleted. + /// Reads encrypted executable input only under a valid, unexpired current lease. + async fn read_executable_input( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + ) -> Result<(String, String, i64), TaskError>; + + /// Bounded retention... async fn purge_expired(&self, batch: u32) -> Result; + + /// Claims a bounded batch of queued or safely recoverable Tasks under leases. + async fn claim_batch( + &self, + owner: &str, + batch_limit: u32, + lease_ms: i64, + ) -> Result, TaskError>; + + /// Renews a lease owned by `owner` when fence and expiry still match. + async fn renew_lease( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + lease_ms: i64, + ) -> Result; + + /// Persists redacted progress under a valid lease. + async fn write_progress( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + progress: TaskEnvelope, + ) -> Result; + + /// Commits an outbound attempt identity and dispatch timestamp before network use. + async fn checkpoint_dispatch( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + attempt_id: String, + ) -> Result; + + /// Applies a terminal transition only under a valid lease and releases intent atomically. + async fn finish_leased( + &self, + request: TaskFinishRequest, + owner: &str, + fence: u64, + ) -> Result; } #[cfg(test)] @@ -583,6 +718,7 @@ mod tests { owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_1".to_owned(), + platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")), operation_id: "op_report".to_owned(), pins: TaskPins { operation_version: 3, @@ -599,6 +735,9 @@ mod tests { &TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00") .unwrap(), ), + executable_input_ciphertext: None, + executable_input_key_version: None, + executable_input_master_key_epoch: None, } } @@ -694,6 +833,11 @@ mod tests { let mut bad_retention = request(); bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1; assert!(bad_retention.validate().is_err()); + + // The persisted authority reference is bounded and never empty. + let mut empty_key_reference = request(); + empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new("")); + assert!(empty_key_reference.validate().is_err()); } #[test] diff --git a/crates/crank-registry/src/migrations.rs b/crates/crank-registry/src/migrations.rs index bb8e989..8e7ad1b 100644 --- a/crates/crank-registry/src/migrations.rs +++ b/crates/crank-registry/src/migrations.rs @@ -7,6 +7,9 @@ mod artifact_metadata_v12; mod authority; mod baseline_v1; mod durable_tasks_v25; +mod durable_tasks_v26; +mod durable_tasks_v27; +mod durable_tasks_v28; mod execution_admission_v24; mod execution_outcome_v5; mod import_plan_v14; @@ -39,6 +42,9 @@ mod schema_guard_v22; mod schema_guard_v23; mod schema_guard_v24; mod schema_guard_v25; +mod schema_guard_v26; +mod schema_guard_v27; +mod schema_guard_v28; mod schema_guard_v7; mod schema_guard_v8; mod schema_guard_v9; diff --git a/crates/crank-registry/src/migrations/authority.rs b/crates/crank-registry/src/migrations/authority.rs index eeb5354..abb4594 100644 --- a/crates/crank-registry/src/migrations/authority.rs +++ b/crates/crank-registry/src/migrations/authority.rs @@ -5,6 +5,9 @@ use super::approval_side_effects_v10; use super::artifact_cleanup_indexes_v13; use super::artifact_metadata_v12; use super::durable_tasks_v25; +use super::durable_tasks_v26; +use super::durable_tasks_v27; +use super::durable_tasks_v28; use super::execution_admission_v24; use super::execution_outcome_v5; use super::import_plan_v14; @@ -41,9 +44,10 @@ use preflight::*; use contract::baseline_source_digest; use contract::validate_descriptors; const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947; -const CURRENT_VERSION: i64 = 25; +const CURRENT_VERSION: i64 = 28; const IMPLEMENTED_VERSIONS: &[i64] = &[ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, ]; const BASELINE_SOURCE_SHA256: &str = "eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675"; @@ -344,6 +348,24 @@ impl MigrationAuthority { durable_tasks_v25::SOURCE_SHA256, 24, ), + expand_descriptor( + 26, + durable_tasks_v26::NAME, + durable_tasks_v26::SOURCE_SHA256, + 25, + ), + expand_descriptor( + 27, + durable_tasks_v27::NAME, + durable_tasks_v27::SOURCE_SHA256, + 26, + ), + expand_descriptor( + 28, + durable_tasks_v28::NAME, + durable_tasks_v28::SOURCE_SHA256, + 27, + ), ] } pub fn validate_sequence() -> Result<(), MigrationError> { @@ -509,6 +531,15 @@ impl MigrationAuthority { if from < 25 { durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?; } + if from < 26 { + durable_tasks_v26::apply(&mut transaction, &Self::sequence()[25]).await?; + } + if from < 27 { + durable_tasks_v27::apply(&mut transaction, &Self::sequence()[26]).await?; + } + if from < 28 { + durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?; + } match inspect(&mut transaction).await? { MigrationPreflight::Current { version: CURRENT_VERSION, diff --git a/crates/crank-registry/src/migrations/durable_tasks_v26.rs b/crates/crank-registry/src/migrations/durable_tasks_v26.rs new file mode 100644 index 0000000..4611f7b --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v26.rs @@ -0,0 +1,45 @@ +use sqlx::{Postgres, Transaction, query}; + +use super::authority::{MigrationDescriptor, MigrationError}; + +pub(super) const NAME: &str = "durable-tasks-v26"; +pub(super) const SOURCE: &str = include_str!("durable_tasks_v26.sql"); +pub(super) const SOURCE_SHA256: &str = + "6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089"; + +pub(super) async fn apply( + transaction: &mut Transaction<'_, Postgres>, + descriptor: &MigrationDescriptor, +) -> Result<(), MigrationError> { + sqlx::raw_sql(SOURCE) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v26", + Some(26), + "restore_known_good_backup", + ) + })?; + query( + "insert into __crank_migrations (version, name, checksum, phase, compatibility) + values ($1, $2, $3, $4, $5)", + ) + .bind(descriptor.version) + .bind(descriptor.name) + .bind(&descriptor.checksum) + .bind(descriptor.phase) + .bind(descriptor.compatibility) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v26", + Some(26), + "restore_known_good_backup", + ) + })?; + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/durable_tasks_v26.sql b/crates/crank-registry/src/migrations/durable_tasks_v26.sql new file mode 100644 index 0000000..411fbe7 --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v26.sql @@ -0,0 +1,6 @@ +-- V26: PostgreSQL lease/fencing/CAS additions for durable background Tasks (Story 4.3). +alter table tasks add column if not exists fence bigint not null default 0; +alter table tasks add column if not exists lease_owner text; +alter table tasks add column if not exists lease_until timestamptz; +alter table tasks add column if not exists current_attempt_id text; +alter table tasks add column if not exists dispatch_started_at timestamptz; diff --git a/crates/crank-registry/src/migrations/durable_tasks_v27.rs b/crates/crank-registry/src/migrations/durable_tasks_v27.rs new file mode 100644 index 0000000..780a11a --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v27.rs @@ -0,0 +1,45 @@ +use sqlx::{Postgres, Transaction, query}; + +use super::authority::{MigrationDescriptor, MigrationError}; + +pub(super) const NAME: &str = "durable-tasks-v27"; +pub(super) const SOURCE: &str = include_str!("durable_tasks_v27.sql"); +pub(super) const SOURCE_SHA256: &str = + "a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8"; + +pub(super) async fn apply( + transaction: &mut Transaction<'_, Postgres>, + descriptor: &MigrationDescriptor, +) -> Result<(), MigrationError> { + sqlx::raw_sql(SOURCE) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v27", + Some(27), + "restore_known_good_backup", + ) + })?; + query( + "insert into __crank_migrations (version, name, checksum, phase, compatibility) + values ($1, $2, $3, $4, $5)", + ) + .bind(descriptor.version) + .bind(descriptor.name) + .bind(&descriptor.checksum) + .bind(descriptor.phase) + .bind(descriptor.compatibility) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v27", + Some(27), + "restore_known_good_backup", + ) + })?; + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/durable_tasks_v27.sql b/crates/crank-registry/src/migrations/durable_tasks_v27.sql new file mode 100644 index 0000000..ed6d904 --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v27.sql @@ -0,0 +1,19 @@ +-- V27: encrypted executable Task input (Story 4.3). +alter table tasks add column if not exists executable_input_ciphertext text; +alter table tasks add column if not exists executable_input_key_version text; +alter table tasks add column if not exists executable_input_master_key_epoch bigint; + +alter table tasks add constraint tasks_executable_input_key_version_check check ( + executable_input_ciphertext is null + or (executable_input_key_version is not null and octet_length(executable_input_key_version) between 1 and 64) +); + +alter table tasks add column if not exists target_executable_input_ciphertext text; +alter table tasks add column if not exists target_executable_input_key_version text; +alter table tasks add column if not exists target_executable_input_master_key_epoch bigint; + +alter table tasks add constraint tasks_executable_input_epoch_check check ( + executable_input_ciphertext is null + or (executable_input_master_key_epoch is not null + and executable_input_master_key_epoch >= 1) +); \ No newline at end of file diff --git a/crates/crank-registry/src/migrations/durable_tasks_v28.rs b/crates/crank-registry/src/migrations/durable_tasks_v28.rs new file mode 100644 index 0000000..30e2bca --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v28.rs @@ -0,0 +1,45 @@ +use sqlx::{Postgres, Transaction, query}; + +use super::authority::{MigrationDescriptor, MigrationError}; + +pub(super) const NAME: &str = "durable-tasks-v28"; +pub(super) const SOURCE: &str = include_str!("durable_tasks_v28.sql"); +pub(super) const SOURCE_SHA256: &str = + "4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676"; + +pub(super) async fn apply( + transaction: &mut Transaction<'_, Postgres>, + descriptor: &MigrationDescriptor, +) -> Result<(), MigrationError> { + sqlx::raw_sql(SOURCE) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v28", + Some(28), + "restore_known_good_backup", + ) + })?; + query( + "insert into __crank_migrations (version, name, checksum, phase, compatibility) + values ($1, $2, $3, $4, $5)", + ) + .bind(descriptor.version) + .bind(descriptor.name) + .bind(&descriptor.checksum) + .bind(descriptor.phase) + .bind(descriptor.compatibility) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v28", + Some(28), + "restore_known_good_backup", + ) + })?; + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/durable_tasks_v28.sql b/crates/crank-registry/src/migrations/durable_tasks_v28.sql new file mode 100644 index 0000000..9260a4f --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v28.sql @@ -0,0 +1,8 @@ +-- V28: typed Platform API key authority reference of a durable Task (Story 4.3). +-- Only the immutable key identifier is stored, never the bearer secret. +alter table tasks add column if not exists platform_api_key_id text; + +alter table tasks add constraint tasks_platform_api_key_id_check check ( + platform_api_key_id is null + or octet_length(platform_api_key_id) between 1 and 160 +); diff --git a/crates/crank-registry/src/migrations/schema_guard.rs b/crates/crank-registry/src/migrations/schema_guard.rs index 7eaa520..d5df30e 100644 --- a/crates/crank-registry/src/migrations/schema_guard.rs +++ b/crates/crank-registry/src/migrations/schema_guard.rs @@ -480,8 +480,40 @@ pub(super) async fn validate_schema_fingerprint( if !super::schema_guard_v25::validate_v25_absent(connection).await? { return Err(schema_error(current_version)); } - } else { + if !super::schema_guard_v26::validate_v26_absent(connection).await? { + return Err(schema_error(current_version)); + } + if !super::schema_guard_v27::validate_v27_absent(connection).await? { + return Err(schema_error(current_version)); + } + if !super::schema_guard_v28::validate_v28_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else if current_version < 26 { super::schema_guard_v25::validate_v25_durable_tasks(connection).await?; + if !super::schema_guard_v26::validate_v26_absent(connection).await? { + return Err(schema_error(current_version)); + } + if !super::schema_guard_v28::validate_v28_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else if current_version < 27 { + super::schema_guard_v26::validate_v26_durable_tasks(connection).await?; + if !super::schema_guard_v27::validate_v27_absent(connection).await? { + return Err(schema_error(current_version)); + } + if !super::schema_guard_v28::validate_v28_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else if current_version < 28 { + super::schema_guard_v26::validate_v26_base_columns(connection).await?; + super::schema_guard_v27::validate_v27_durable_tasks(connection).await?; + if !super::schema_guard_v28::validate_v28_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else { + super::schema_guard_v26::validate_v26_base_columns(connection).await?; + super::schema_guard_v28::validate_v28_durable_tasks(connection).await?; } Ok(()) } diff --git a/crates/crank-registry/src/migrations/schema_guard_v26.rs b/crates/crank-registry/src/migrations/schema_guard_v26.rs new file mode 100644 index 0000000..8fc1731 --- /dev/null +++ b/crates/crank-registry/src/migrations/schema_guard_v26.rs @@ -0,0 +1,120 @@ +//! V26 schema guard: lease/fencing columns for durable background Tasks (Story 4.3). + +use sqlx::{PgConnection, Row, query}; + +use super::authority::MigrationError; +use super::schema_guard::relation_exists; + +const TASKS: &str = "tasks"; + +const TASK_COLUMNS: &[(&str, &str, bool)] = &[ + ("task_id", "uuid", false), + ("workspace_id", "text", false), + ("owner_kind", "text", false), + ("owner_id", "text", false), + ("authorization_scope", "text", false), + ("operation_id", "text", false), + ("operation_version", "integer", false), + ("scoped_key", "text", false), + ("input_fingerprint", "text", false), + ("request_id", "text", false), + ("trace_id", "text", false), + ("traceparent", "text", false), + ("trace_sampled", "boolean", false), + ("policy_mode", "text", false), + ("approval_reference", "text", true), + ("state", "text", false), + ("version", "integer", false), + ("attempts", "integer", false), + ("input_envelope", "jsonb", false), + ("progress_envelope", "jsonb", true), + ("result_envelope", "jsonb", true), + ("error_envelope", "jsonb", true), + ("execution_deadline", "timestamp with time zone", false), + ("retain_until", "timestamp with time zone", false), + ("created_at", "timestamp with time zone", false), + ("updated_at", "timestamp with time zone", false), + ("fence", "bigint", false), + ("lease_owner", "text", true), + ("lease_until", "timestamp with time zone", true), + ("current_attempt_id", "text", true), + ("dispatch_started_at", "timestamp with time zone", true), +]; + +fn v26_error(stage: &'static str) -> MigrationError { + MigrationError::new( + "partial_sequence", + stage, + Some(26), + "restore_known_good_backup", + ) +} + +pub(super) async fn validate_v26_absent( + connection: &mut PgConnection, +) -> Result { + if relation_exists(connection, TASKS).await? { + // delegate to v25 absent for relation mostly; V26 columns must not exist. + for column in [ + "fence", + "lease_owner", + "lease_until", + "current_attempt_id", + "dispatch_started_at", + ] { + if super::schema_guard::column_exists(connection, TASKS, column).await? { + return Ok(false); + } + } + } + Ok(true) +} + +pub(super) async fn validate_v26_base_columns( + connection: &mut PgConnection, +) -> Result<(), MigrationError> { + if !relation_exists(connection, TASKS).await? { + return Err(v26_error("v26.relations")); + } + for (column, data_type, nullable) in TASK_COLUMNS { + let row = query( + "select data_type, is_nullable from information_schema.columns + where table_schema = current_schema() and table_name = $1 and column_name = $2", + ) + .bind(TASKS) + .bind(column) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + let valid = row.is_some_and(|row| { + row.try_get::("data_type").ok().as_deref() == Some(*data_type) + && row.try_get::("is_nullable").ok().as_deref() + == Some(if *nullable { "YES" } else { "NO" }) + }); + if !valid { + return Err(v26_error("v26.columns")); + } + } + Ok(()) +} + +pub(super) async fn validate_v26_durable_tasks( + connection: &mut PgConnection, +) -> Result<(), MigrationError> { + validate_v26_base_columns(connection).await?; + let count: i64 = query( + "select count(*)::bigint as count from information_schema.columns + where table_schema = current_schema() and table_name = $1", + ) + .bind(TASKS) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get("count") + .map_err(|_| MigrationError::storage("preflight.schema"))?; + if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() { + return Err(v26_error("v26.columns")); + } + // Constraints/index checks inherited conceptually from V25; V26 adds columns only. + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/schema_guard_v27.rs b/crates/crank-registry/src/migrations/schema_guard_v27.rs new file mode 100644 index 0000000..04638af --- /dev/null +++ b/crates/crank-registry/src/migrations/schema_guard_v27.rs @@ -0,0 +1,85 @@ +//! V27 schema guard: encrypted executable Task input columns (Story 4.3). + +use sqlx::{PgConnection, Row, query}; + +use super::authority::MigrationError; +use super::schema_guard::relation_exists; +use super::schema_guard_v26::validate_v26_base_columns; + +const TASKS: &str = "tasks"; + +pub(super) async fn validate_v27_absent( + connection: &mut PgConnection, +) -> Result { + if relation_exists(connection, TASKS).await? { + for column in [ + "executable_input_ciphertext", + "executable_input_key_version", + "executable_input_master_key_epoch", + ] { + if super::schema_guard::column_exists(connection, TASKS, column).await? { + return Ok(false); + } + } + } + Ok(true) +} + +pub(super) async fn validate_v27_durable_tasks( + connection: &mut PgConnection, +) -> Result<(), MigrationError> { + validate_v26_base_columns(connection).await?; + // Additional V27 columns. + let extra = [ + ("executable_input_ciphertext", "text", true), + ("executable_input_key_version", "text", true), + ("executable_input_master_key_epoch", "bigint", true), + ("target_executable_input_ciphertext", "text", true), + ("target_executable_input_key_version", "text", true), + ("target_executable_input_master_key_epoch", "bigint", true), + ]; + for (column, data_type, nullable) in extra { + let row = query( + "select data_type, is_nullable from information_schema.columns + where table_schema = current_schema() and table_name = $1 and column_name = $2", + ) + .bind(TASKS) + .bind(column) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + let valid = row.is_some_and(|row| { + row.try_get::("data_type").ok().as_deref() == Some(data_type) + && row.try_get::("is_nullable").ok().as_deref() + == Some(if nullable { "YES" } else { "NO" }) + }); + if !valid { + return Err(MigrationError::new( + "partial_sequence", + "v27.columns", + Some(27), + "restore_known_good_backup", + )); + } + } + // Exact total column count for version 27: 43 V26 columns + 3 V27 columns. + let count: i64 = query( + "select count(*)::bigint as count from information_schema.columns + where table_schema = current_schema() and table_name = $1", + ) + .bind(TASKS) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get("count") + .map_err(|_| MigrationError::storage("preflight.schema"))?; + if count != i64::try_from(37).unwrap_or_default() { + return Err(MigrationError::new( + "partial_sequence", + "v27.columns", + Some(27), + "restore_known_good_backup", + )); + } + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/schema_guard_v28.rs b/crates/crank-registry/src/migrations/schema_guard_v28.rs new file mode 100644 index 0000000..bd529fd --- /dev/null +++ b/crates/crank-registry/src/migrations/schema_guard_v28.rs @@ -0,0 +1,89 @@ +//! V28 schema guard: typed Platform API key authority reference of a durable Task (Story 4.3). + +use sqlx::{PgConnection, Row, query}; + +use super::authority::MigrationError; +use super::schema_guard::relation_exists; + +const TASKS: &str = "tasks"; + +pub(super) async fn validate_v28_absent( + connection: &mut PgConnection, +) -> Result { + if relation_exists(connection, TASKS).await? { + if super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await? { + return Ok(false); + } + } + Ok(true) +} + +pub(super) async fn validate_v28_durable_tasks( + connection: &mut PgConnection, +) -> Result<(), MigrationError> { + // The V27 column set is still required and unchanged; V28 only appends one column. + validate_v27_base_columns(connection).await?; + let row = query( + "select data_type, is_nullable from information_schema.columns + where table_schema = current_schema() and table_name = $1 and column_name = $2", + ) + .bind(TASKS) + .bind("platform_api_key_id") + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + let valid = row.is_some_and(|row| { + row.try_get::("data_type").ok().as_deref() == Some("text") + && row.try_get::("is_nullable").ok().as_deref() == Some("YES") + }); + if !valid { + return Err(MigrationError::new( + "partial_sequence", + "v28.columns", + Some(28), + "restore_known_good_backup", + )); + } + // Exact total column count for version 28: the 37 V27 columns + 1 V28 column. + let count: i64 = query( + "select count(*)::bigint as count from information_schema.columns + where table_schema = current_schema() and table_name = $1", + ) + .bind(TASKS) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get("count") + .map_err(|_| MigrationError::storage("preflight.schema"))?; + if count != i64::try_from(38).unwrap_or_default() { + return Err(MigrationError::new( + "partial_sequence", + "v28.columns", + Some(28), + "restore_known_good_backup", + )); + } + Ok(()) +} + +/// V27 columns must still be exactly present before V28 is considered applied. +async fn validate_v27_base_columns(connection: &mut PgConnection) -> Result<(), MigrationError> { + for column in [ + "executable_input_ciphertext", + "executable_input_key_version", + "executable_input_master_key_epoch", + "target_executable_input_ciphertext", + "target_executable_input_key_version", + "target_executable_input_master_key_epoch", + ] { + if !super::schema_guard::column_exists(connection, TASKS, column).await? { + return Err(MigrationError::new( + "partial_sequence", + "v28.columns", + Some(28), + "restore_known_good_backup", + )); + } + } + Ok(()) +} diff --git a/crates/crank-registry/src/postgres/admission.rs b/crates/crank-registry/src/postgres/admission.rs index 5807e64..4889e6e 100644 --- a/crates/crank-registry/src/postgres/admission.rs +++ b/crates/crank-registry/src/postgres/admission.rs @@ -577,6 +577,145 @@ pub(super) async fn insert_admission_entry( Ok(u64::try_from(ticket_id).unwrap_or_default()) } +/// Shared grant step used by the durable Task store: grants one reserved Task intent if capacity allows, +/// marks it `executing`, persists the weighted round-robin cursor, and returns its Task id. It reuses +/// the same selector/capacity logic as synchronous admission so no second policy exists. +pub(super) async fn grant_next_task( + transaction: &mut Transaction<'_, Postgres>, + workspace_id: &str, + cursor: &AdmissionCursor, + limits: &AdmissionLimits, + scheduler: AdmissionSchedulerConfig, +) -> Result<(Option, AdmissionCursor), AttemptError> { + let rows = sqlx::query( + "select ticket_id, agent_key, operation_id, task_id + from execution_admission_entries + where state = 'reserved' and task_id is not null + order by ticket_id + limit $1 + for update skip locked", + ) + .bind(QUEUE_READ_LIMIT) + .fetch_all(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + if rows.is_empty() { + return Ok((None, cursor.clone())); + } + struct Candidate { + entry: AdmissionQueueEntry, + task_id: Uuid, + } + let mut candidates = Vec::with_capacity(rows.len()); + for row in rows { + let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?; + let task_id: Uuid = row.try_get("task_id").map_err(AttemptError::from_sql)?; + candidates.push(Candidate { + entry: AdmissionQueueEntry { + ticket_id: u64::try_from(ticket_id).unwrap_or_default(), + sequence: 0, + agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?, + operation_id: row + .try_get("operation_id") + .map_err(AttemptError::from_sql)?, + }, + task_id, + }); + } + let queue: Vec = candidates.iter().map(|c| c.entry.clone()).collect(); + let Some(selection) = select_next_grant(&queue, cursor, scheduler) else { + return Ok((None, cursor.clone())); + }; + let head_index = selection.index; + let head = &candidates[head_index]; + + let global_executing: i64 = sqlx::query( + "select count(*)::bigint as count from execution_admission_entries where state = 'executing'", + ) + .fetch_one(&mut **transaction) + .await + .map_err(AttemptError::from_sql)? + .try_get("count") + .map_err(AttemptError::from_sql)?; + let agent_counts = sqlx::query( + "select agent_key as key, count(*)::bigint as count + from execution_admission_entries where state = 'executing' group by agent_key", + ) + .fetch_all(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + let mut agent_map = std::collections::BTreeMap::new(); + for row in agent_counts { + let key: String = row.try_get("key").map_err(AttemptError::from_sql)?; + let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?; + agent_map.insert(key, u64::try_from(count).unwrap_or_default()); + } + let tool_counts = sqlx::query( + "select operation_id as key, count(*)::bigint as count + from execution_admission_entries where state = 'executing' group by operation_id", + ) + .fetch_all(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + let mut tool_map = std::collections::BTreeMap::new(); + for row in tool_counts { + let key: String = row.try_get("key").map_err(AttemptError::from_sql)?; + let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?; + tool_map.insert(key, u64::try_from(count).unwrap_or_default()); + } + + if u64::try_from(global_executing).unwrap_or(u64::MAX) + >= u64::try_from(limits.global_limit).unwrap_or(u64::MAX) + { + return Ok((None, cursor.clone())); + } + let agent_executing = agent_map + .get(head.entry.agent_key.as_str()) + .copied() + .unwrap_or_default(); + if agent_executing >= u64::try_from(limits.agent_limit).unwrap_or(u64::MAX) { + return Ok((None, cursor.clone())); + } + let tool_executing = tool_map + .get(head.entry.operation_id.as_str()) + .copied() + .unwrap_or_default(); + if tool_executing >= u64::try_from(limits.tool_limit).unwrap_or(u64::MAX) { + return Ok((None, cursor.clone())); + } + + sqlx::query( + "update execution_admission_entries + set state = 'executing', granted_at = clock_timestamp() + where ticket_id = $1 and state = 'reserved'", + ) + .bind(i64::try_from(head.entry.ticket_id).unwrap_or(i64::MAX)) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + + sqlx::query( + "update execution_admission_controllers + set cursor_agent = $2, + cursor_agent_credit = $3, + cursor_tool = $4, + cursor_tool_credit = $5, + updated_at = now() + where workspace_id = $1", + ) + .bind(workspace_id) + .bind(selection.cursor.agent.as_deref()) + .bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default()) + .bind(selection.cursor.tool.as_deref()) + .bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default()) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + + Ok((Some(head.task_id), selection.cursor)) +} + +#[derive(Debug)] pub(super) enum AttemptError { /// Lock or statement timeout: the transaction made no decision and the waiter retries. Contended, diff --git a/crates/crank-registry/src/postgres/api_key.rs b/crates/crank-registry/src/postgres/api_key.rs index fafecc0..3fe1a3a 100644 --- a/crates/crank-registry/src/postgres/api_key.rs +++ b/crates/crank-registry/src/postgres/api_key.rs @@ -170,6 +170,69 @@ impl PostgresRegistry { .transpose() } + /// Authoritative revalidation of the Platform API key that authorized a durable Task (Story 4.3). + /// + /// The key is returned only when the exact immutable ID is still active, unexpired and bound to + /// the same workspace and agent; otherwise `None`, so a worker fails closed. Only the immutable + /// identity is supplied here — the bearer secret is never stored or re-read. + pub async fn revalidate_platform_api_key( + &self, + workspace_id: &WorkspaceId, + agent_id: &AgentId, + key_id: &PlatformApiKeyId, + ) -> Result, RegistryError> { + let row = sqlx::query( + "select + k.id, + k.workspace_id, + k.agent_id, + k.key_kind, + k.name, + k.prefix, + k.scopes_json, + k.status, + k.created_at, + k.last_used_at, + k.expires_at, + k.allowed_origins_json + from platform_api_keys k + where k.id = $1 + and k.workspace_id = $2 + and k.agent_id = $3 + and k.key_kind = 'mcp_client' + and k.status = 'active' + and (k.expires_at is null or k.expires_at > now()) + limit 1", + ) + .bind(key_id.as_str()) + .bind(workspace_id.as_str()) + .bind(agent_id.as_str()) + .fetch_optional(&self.pool) + .await?; + + row.map(|row| { + Ok(PlatformApiKeyRecord { + api_key: PlatformApiKey { + id: PlatformApiKeyId::new(row.get::("id")), + workspace_id: WorkspaceId::new(row.get::("workspace_id")), + agent_id: row.get::, _>("agent_id").map(AgentId::new), + key_kind: deserialize_enum_text(&row.get::("key_kind"), "key_kind")?, + name: row.get("name"), + prefix: row.get("prefix"), + scopes: deserialize_json_value(row.get::("scopes_json"))?, + status: deserialize_enum_text(&row.get::("status"), "status")?, + created_at: row.get("created_at"), + last_used_at: row.get("last_used_at"), + expires_at: row.get("expires_at"), + allowed_origins: deserialize_json_value( + row.get::("allowed_origins_json"), + )?, + }, + }) + }) + .transpose() + } + pub async fn get_approval_api_key_by_secret_for_agent_slug( &self, workspace_slug: &str, diff --git a/crates/crank-registry/src/postgres/master_key.rs b/crates/crank-registry/src/postgres/master_key.rs index 496fffe..a943fbb 100644 --- a/crates/crank-registry/src/postgres/master_key.rs +++ b/crates/crank-registry/src/postgres/master_key.rs @@ -1,6 +1,6 @@ use super::*; -const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000; +pub(super) const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000; impl PostgresRegistry { pub async fn verify_or_register_master_key_identity( @@ -356,12 +356,19 @@ impl PostgresRegistry { return Err(RegistryError::MasterKeyRotationConflict); } - let total = sqlx::query_scalar::<_, i64>( + let secret_total = sqlx::query_scalar::<_, i64>( "select count(*) from secret_versions where master_key_epoch = $1", ) .bind(source_epoch) .fetch_one(&mut *tx) .await?; + let task_total = sqlx::query_scalar::<_, i64>( + "select count(*) from tasks where executable_input_master_key_epoch = $1", + ) + .bind(source_epoch) + .fetch_one(&mut *tx) + .await?; + let total = secret_total + task_total; let rotation_id = format!("master-key-e{source_epoch}-to-e{target_epoch}"); let aborted = sqlx::query( "select @@ -403,6 +410,16 @@ impl PostgresRegistry { .bind(target_epoch) .execute(&mut *tx) .await?; + sqlx::query( + "update tasks + set target_executable_input_ciphertext = null, + target_executable_input_key_version = null, + target_executable_input_master_key_epoch = null + where target_executable_input_master_key_epoch = $1", + ) + .bind(target_epoch) + .execute(&mut *tx) + .await?; sqlx::query( "update master_key_rotations set state = 'running', @@ -662,9 +679,24 @@ impl PostgresRegistry { .bind(rotation.target_epoch) .execute(&mut *tx) .await?; + sqlx::query( + "update tasks + set executable_input_ciphertext = target_executable_input_ciphertext, + executable_input_key_version = target_executable_input_key_version, + executable_input_master_key_epoch = target_executable_input_master_key_epoch, + target_executable_input_ciphertext = null, + target_executable_input_key_version = null, + target_executable_input_master_key_epoch = null + where executable_input_master_key_epoch = $1 + and target_executable_input_master_key_epoch = $2", + ) + .bind(rotation.source_epoch) + .bind(rotation.target_epoch) + .execute(&mut *tx) + .await?; sqlx::query( "update master_key_rotations - set state = 'promoted', + set state = 'promoted', updated_at = $2::timestamptz where id = $1", ) @@ -697,6 +729,16 @@ impl PostgresRegistry { .bind(rotation.target_epoch) .execute(&mut *tx) .await?; + sqlx::query( + "update tasks + set target_executable_input_ciphertext = null, + target_executable_input_key_version = null, + target_executable_input_master_key_epoch = null + where target_executable_input_master_key_epoch = $1", + ) + .bind(rotation.target_epoch) + .execute(&mut *tx) + .await?; sqlx::query( "update master_key_rotations set state = 'aborted', @@ -883,7 +925,7 @@ async fn select_active_master_key_identity_for_update( Ok(row.map(map_master_key_identity)) } -async fn select_master_key_rotation_for_update( +pub(super) async fn select_master_key_rotation_for_update( transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, rotation_id: &str, ) -> Result { @@ -915,16 +957,19 @@ async fn select_master_key_rotation_for_update( }) } -async fn count_staged_rotation_targets( +pub(super) async fn count_staged_rotation_targets( transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, source_epoch: i64, target_epoch: i64, ) -> Result { let count = sqlx::query_scalar::<_, i64>( - "select count(*) - from secret_versions - where master_key_epoch = $1 - and target_master_key_epoch = $2", + "select + (select count(*) from secret_versions + where master_key_epoch = $1 + and target_master_key_epoch = $2) + + (select count(*) from tasks + where executable_input_master_key_epoch = $1 + and target_executable_input_master_key_epoch = $2)", ) .bind(source_epoch) .bind(target_epoch) diff --git a/crates/crank-registry/src/postgres/master_key_tasks.rs b/crates/crank-registry/src/postgres/master_key_tasks.rs new file mode 100644 index 0000000..81ba671 --- /dev/null +++ b/crates/crank-registry/src/postgres/master_key_tasks.rs @@ -0,0 +1,143 @@ +//! Task ciphertext paging/staging of the master-key rotation (Story 4.3). +//! +//! Extracted from `master_key.rs` without logic changes so that module stays within the +//! code-health line limit; the operations and their SQL are unchanged. + +use super::master_key::{ + MASTER_KEY_PAGE_LIMIT_MAX, count_staged_rotation_targets, select_master_key_rotation_for_update, +}; +use super::*; + +fn map_master_key_task_version( + row: sqlx::postgres::PgRow, +) -> Result { + Ok(MasterKeyTaskVersion { + task_id: row.get::("task_id"), + ciphertext: row + .get::, _>("executable_input_ciphertext") + .unwrap_or_default(), + key_version: row + .get::, _>("executable_input_key_version") + .unwrap_or_default(), + epoch: row + .get::, _>("executable_input_master_key_epoch") + .unwrap_or_default(), + target_ciphertext: row.get::, _>("target_executable_input_ciphertext"), + target_key_version: row.get::, _>("target_executable_input_key_version"), + target_epoch: row.get::, _>("target_executable_input_master_key_epoch"), + }) +} + +impl PostgresRegistry { + pub async fn list_tasks_for_master_key_epoch_page( + &self, + epoch: i64, + after_task_id: Option<&str>, + limit: i64, + ) -> Result, RegistryError> { + if epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) { + return Err(RegistryError::InvalidMasterKeyIdentity); + } + let rows = sqlx::query( + "select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3", + ) + .bind(epoch) + .bind(after_task_id) + .bind(limit) + .fetch_all(&self.pool) + .await?; + rows.into_iter().map(map_master_key_task_version).collect() + } + + pub async fn list_target_tasks_for_master_key_rotation_page( + &self, + target_epoch: i64, + after_task_id: Option<&str>, + limit: i64, + ) -> Result, RegistryError> { + if target_epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) { + return Err(RegistryError::InvalidMasterKeyIdentity); + } + let rows = sqlx::query( + "select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where target_executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3", + ) + .bind(target_epoch) + .bind(after_task_id) + .bind(limit) + .fetch_all(&self.pool) + .await?; + rows.into_iter().map(map_master_key_task_version).collect() + } + + pub async fn stage_master_key_rotation_task_ciphertext( + &self, + rotation_id: &str, + task_id: &str, + source_epoch: i64, + target_ciphertext: &str, + target_key_version: &str, + target_epoch: i64, + now: &OffsetDateTime, + ) -> Result { + let mut tx = self.pool.begin().await?; + let rotation = select_master_key_rotation_for_update(&mut tx, rotation_id).await?; + if rotation.state != "running" + || rotation.source_epoch != source_epoch + || rotation.target_epoch != target_epoch + { + return Err(RegistryError::MasterKeyRotationConflict); + } + let result = sqlx::query( + "update tasks + set target_executable_input_ciphertext = $3, + target_executable_input_key_version = $4, + target_executable_input_master_key_epoch = $5 + where task_id::text = $1 + and executable_input_master_key_epoch = $2 + and ( + target_executable_input_master_key_epoch is null + or target_executable_input_master_key_epoch = $5 + ) + and ( + target_executable_input_ciphertext is null + or target_executable_input_ciphertext = $3 + )", + ) + .bind(task_id) + .bind(source_epoch) + .bind(target_ciphertext) + .bind(target_key_version) + .bind(target_epoch) + .execute(&mut *tx) + .await?; + if result.rows_affected() != 1 { + return Err(RegistryError::MasterKeyRotationConflict); + } + let processed = count_staged_rotation_targets(&mut tx, source_epoch, target_epoch).await?; + sqlx::query( + "update master_key_rotations + set checkpoint_secret_id = checkpoint_secret_id, + processed_secret_versions = $2, + updated_at = $3::timestamptz + where id = $1", + ) + .bind(rotation_id) + .bind(processed) + .bind(now) + .execute(&mut *tx) + .await?; + let updated = select_master_key_rotation_for_update(&mut tx, rotation_id).await?; + tx.commit().await?; + Ok(updated) + } +} + +pub struct MasterKeyTaskVersion { + pub task_id: String, + pub ciphertext: String, + pub key_version: String, + pub epoch: i64, + pub target_ciphertext: Option, + pub target_key_version: Option, + pub target_epoch: Option, +} diff --git a/crates/crank-registry/src/postgres/mod.rs b/crates/crank-registry/src/postgres/mod.rs index 78ae840..3d049c0 100644 --- a/crates/crank-registry/src/postgres/mod.rs +++ b/crates/crank-registry/src/postgres/mod.rs @@ -10,6 +10,7 @@ mod auth; mod connection; mod import_job; mod master_key; +mod master_key_tasks; mod observability; mod onboarding; mod onboarding_import; diff --git a/crates/crank-registry/src/postgres/tasks.rs b/crates/crank-registry/src/postgres/tasks.rs index 473c512..63dd71f 100644 --- a/crates/crank-registry/src/postgres/tasks.rs +++ b/crates/crank-registry/src/postgres/tasks.rs @@ -13,9 +13,9 @@ use async_trait::async_trait; use crank_core::{ AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback, - TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, - TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore, - TaskSupportIds, ToolExecutionMode, + PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, + TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, + TaskState, TaskStore, TaskSupportIds, ToolExecutionMode, }; use serde_json::Value; use sqlx::{PgPool, Postgres, Row, Transaction}; @@ -27,10 +27,14 @@ use super::admission::{ }; const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id, - authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode, + authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version, + policy_mode, approval_reference, request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope, - result_envelope, error_envelope, attempts, version, + result_envelope, error_envelope, attempts, fence, lease_owner, lease_until, + (extract(epoch from lease_until) * 1000)::bigint as lease_until_ms, + current_attempt_id, dispatch_started_at, + (extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version, (extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms, (extract(epoch from retain_until) * 1000)::bigint as retain_until_ms, (extract(epoch from created_at) * 1000)::bigint as created_ms, @@ -40,6 +44,7 @@ const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_i pub struct PostgresTaskStore { pool: PgPool, limits: AdmissionLimits, + scheduler: crank_core::AdmissionSchedulerConfig, } impl PostgresTaskStore { @@ -54,7 +59,11 @@ impl PostgresTaskStore { scheduler .validate() .map_err(|error| TaskError::invalid(error.to_string()))?; - Ok(Self { pool, limits }) + Ok(Self { + pool, + limits, + scheduler, + }) } fn unavailable(error: impl std::fmt::Display) -> TaskError { @@ -151,6 +160,20 @@ impl TaskStore for PostgresTaskStore { async fn accept(&self, request: TaskAcceptRequest) -> Result { request.validate()?; let mut transaction = self.start().await?; + super::master_key::ensure_no_active_master_key_rotation(&mut transaction) + .await + .map_err(|error| match error { + crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()), + _ => TaskError::StateConflict, + })?; + if let Some(epoch) = request.executable_input_master_key_epoch { + super::master_key::ensure_active_master_key_epoch(&mut transaction, epoch) + .await + .map_err(|error| match error { + crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()), + _ => TaskError::StateConflict, + })?; + } let outcome = self.accept_locked(&mut transaction, &request).await; match outcome { Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => { @@ -252,6 +275,7 @@ impl TaskStore for PostgresTaskStore { where task_id = $4 and workspace_id = $5 and owner_kind = $6 and owner_id = $7 and authorization_scope = $8 and state = $9 and version = $10 + and (lease_owner is null or fence = 0) returning {TASK_PROJECTION}" ))) .bind(request.target_state.as_str()) @@ -285,6 +309,12 @@ impl TaskStore for PostgresTaskStore { async fn purge_expired(&self, batch: u32) -> Result { let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT)); let mut transaction = self.start().await?; + super::master_key::ensure_no_active_master_key_rotation(&mut transaction) + .await + .map_err(|error| match error { + crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()), + _ => TaskError::StateConflict, + })?; let expired = sqlx::query( "with expired as ( update tasks @@ -295,6 +325,7 @@ impl TaskStore for PostgresTaskStore { where task_id in ( select task_id from tasks where state = 'queued' and execution_deadline <= now() +and lease_owner is null order by task_id limit $1 ) @@ -332,6 +363,351 @@ impl TaskStore for PostgresTaskStore { expired_terminal: purged.rows_affected(), }) } + + async fn claim_batch( + &self, + owner: &str, + batch_limit: u32, + lease_ms: i64, + ) -> Result, TaskError> { + let batch = (batch_limit.clamp(1, 64)) as usize; + // A checkpointed attempt is never redispatched: once its lease expires the outbound outcome + // is uncertain, so it resolves to outcome_unknown and releases its admission slot. Story 4.4 + // owns any later retry/cancellation of that uncertain outcome. + let mut transaction = self.start().await?; + sqlx::query( + "with candidates as ( + select task_id from tasks + where state = 'working' and lease_owner is not null + and current_attempt_id is not null + and lease_until < clock_timestamp() + order by task_id + limit 64 + ), resolved as ( + update tasks + set state = 'outcome_unknown', + lease_owner = null, + lease_until = null, + version = version + 1, + updated_at = now() + where task_id in (select task_id from candidates) + and state = 'working' + returning task_id + ) + delete from execution_admission_entries + where task_id in (select task_id from resolved)", + ) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + + // A crash before the durable dispatch checkpoint left the Task queued for execution: its + // admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is + // bounded by the <=30s lease expiry, inside the 60s story budget. + sqlx::query( + "with candidates as ( + select task_id from tasks + where state = 'working' and lease_owner is not null + and current_attempt_id is null + and lease_until < clock_timestamp() + order by task_id + limit 64 + ), requeued as ( + update tasks + set state = 'queued', + lease_owner = null, + lease_until = null, + version = version + 1, + updated_at = now() + where task_id in (select task_id from candidates) + and state = 'working' + returning task_id + ) + update execution_admission_entries + set state = 'reserved', granted_at = null + where task_id in (select task_id from requeued) and state = 'executing'", + ) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + + let mut leases = Vec::new(); + for _ in 0..batch { + let mut transaction = self.start().await?; + let workspace_cursor = + match super::admission::lock_admission_in_tx(&mut transaction, "*").await { + Ok(cursor) => cursor, + Err(error) => { + transaction.rollback().await.ok(); + return Err(TaskError::unavailable(format!("{error:?}"))); + } + }; + let (task_id, _updated_cursor) = match super::admission::grant_next_task( + &mut transaction, + "*", + &workspace_cursor, + &self.limits, + self.scheduler, + ) + .await + { + Ok(value) => value, + Err(error) => { + transaction.rollback().await.ok(); + return Err(TaskError::unavailable(format!("{error:?}"))); + } + }; + // Persist the updated controller cursor even when no task is granted. + let Some(task_id) = task_id else { + transaction.commit().await.map_err(Self::unavailable)?; + break; + }; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set state = 'working', + lease_owner = $1, + lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval, + fence = fence + 1, + updated_at = now() + where task_id = $3 + and ( + (state = 'queued' and lease_owner is null) + or (state = 'working' and lease_owner is not null + and lease_until < clock_timestamp() and current_attempt_id is null) + ) + returning {TASK_PROJECTION}" + ))) + .bind(owner) + .bind(lease_ms) + .bind(task_id) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + // Persist cursor update (grant_next_task already did it) regardless. + let Some(row) = row else { + transaction.rollback().await.ok(); + break; + }; + let record = record_from_row(&row).map_err(Self::unavailable)?; + leases.push(TaskLease { + owner: owner.to_owned(), + fence: record.fence, + lease_until: record.lease_until_unix_ms.unwrap_or_default(), + record, + }); + transaction.commit().await.map_err(Self::unavailable)?; + } + Ok(leases) + } + + async fn renew_lease( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + lease_ms: i64, + ) -> Result { + let mut transaction = self.start().await?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set lease_until = clock_timestamp() + ($1::bigint || ' milliseconds')::interval, + updated_at = now() + where task_id = $2 and lease_owner = $3 and fence = $4 + and lease_until > clock_timestamp() + returning {TASK_PROJECTION}" + ))) + .bind(lease_ms) + .bind(task_id.as_uuid()) + .bind(owner) + .bind(i64::try_from(fence).unwrap_or_default()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let record = record_from_row(&row).map_err(Self::unavailable)?; + let lease = TaskLease { + owner: owner.to_owned(), + fence: record.fence, + lease_until: record.lease_until_unix_ms.unwrap_or_default(), + record, + }; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(lease) + } + + async fn write_progress( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + progress: TaskEnvelope, + ) -> Result { + let mut transaction = self.start().await?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set progress_envelope = $1, updated_at = now() + where task_id = $2 and lease_owner = $3 and fence = $4 + and lease_until > clock_timestamp() + returning {TASK_PROJECTION}" + ))) + .bind(progress.value().clone()) + .bind(task_id.as_uuid()) + .bind(owner) + .bind(i64::try_from(fence).unwrap_or_default()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let record = record_from_row(&row).map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(record) + } + + async fn read_executable_input( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + ) -> Result<(String, String, i64), TaskError> { + let row = sqlx::query( + "select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch + from tasks + where task_id = $1 and state = 'working' and lease_owner = $2 and fence = $3 + and lease_until > clock_timestamp()", + ) + .bind(task_id.as_uuid()) + .bind(owner) + .bind(i64::try_from(fence).unwrap_or_default()) + .fetch_optional(&self.pool) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + return Err(TaskError::StateConflict); + }; + let ciphertext: Option = row + .try_get("executable_input_ciphertext") + .map_err(Self::unavailable)?; + let key_version: Option = row + .try_get("executable_input_key_version") + .map_err(Self::unavailable)?; + let epoch: Option = row + .try_get("executable_input_master_key_epoch") + .map_err(Self::unavailable)?; + if let Some(epoch) = epoch { + let active = sqlx::query_scalar::<_, i64>( + "select epoch from master_key_identities where status = 'active' order by epoch desc limit 1", + ) + .fetch_optional(&self.pool) + .await + .map_err(Self::unavailable)?; + if let Some(active) = active { + if active != epoch { + return Err(TaskError::StateConflict); + } + } + } + match (ciphertext, key_version, epoch) { + (Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => { + Ok((ciphertext, key_version, epoch)) + } + _ => Err(TaskError::NotFound), + } + } + + async fn checkpoint_dispatch( + &self, + owner: &str, + task_id: TaskId, + fence: u64, + attempt_id: String, + ) -> Result { + let mut transaction = self.start().await?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set current_attempt_id = $1, + dispatch_started_at = now(), + attempts = attempts + 1, + updated_at = now() + where task_id = $2 and lease_owner = $3 and fence = $4 + and lease_until > clock_timestamp() and current_attempt_id is null + returning {TASK_PROJECTION}" + ))) + .bind(&attempt_id) + .bind(task_id.as_uuid()) + .bind(owner) + .bind(i64::try_from(fence).unwrap_or_default()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let record = record_from_row(&row).map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(record) + } + + async fn finish_leased( + &self, + request: TaskFinishRequest, + owner: &str, + fence: u64, + ) -> Result { + request.validate()?; + let mut transaction = self.start().await?; + let access = &request.access; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set state = $1, + result_envelope = $2, + error_envelope = $3, + version = version + 1, + updated_at = now(), + lease_owner = null, + lease_until = null + where task_id = $4 and workspace_id = $5 and owner_kind = $6 + and owner_id = $7 and authorization_scope = $8 + and state = $9 and version = $10 + and lease_owner = $11 and fence = $12 + and lease_until > clock_timestamp() + returning {TASK_PROJECTION}" + ))) + .bind(request.target_state.as_str()) + .bind(request.result.as_ref().map(|value| value.value().clone())) + .bind(request.error.as_ref().map(|value| value.value().clone())) + .bind(request.task_id.as_uuid()) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .bind(request.expected_state.as_str()) + .bind(i64::from(request.expected_version)) + .bind(owner) + .bind(i64::try_from(fence).unwrap_or_default()) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + transaction.rollback().await.ok(); + return Err(TaskError::StateConflict); + }; + let record = record_from_row(&row).map_err(Self::unavailable)?; + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(request.task_id.as_uuid()) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(record) + } } impl PostgresTaskStore { @@ -377,11 +753,12 @@ impl PostgresTaskStore { task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id, operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent, trace_sampled, policy_mode, approval_reference, state, input_envelope, - execution_deadline, retain_until + execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version, + executable_input_master_key_epoch, platform_api_key_id ) values ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', - $16, to_timestamp($17::double precision / 1000.0), - to_timestamp($18::double precision / 1000.0) + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16, + to_timestamp($17::double precision / 1000.0), + to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22 ) returning (extract(epoch from created_at) * 1000)::bigint as created_ms", ) @@ -403,6 +780,15 @@ impl PostgresTaskStore { .bind(request.input.value().clone()) .bind(request.pins.execution_deadline_unix_ms) .bind(request.pins.retain_until_unix_ms) + .bind(request.executable_input_ciphertext.as_deref()) + .bind(request.executable_input_key_version.as_deref()) + .bind(request.executable_input_master_key_epoch) + .bind( + request + .platform_api_key_id + .as_ref() + .map(PlatformApiKeyId::as_str), + ) .fetch_one(&mut **transaction) .await .map_err(task_sql_error)? @@ -540,6 +926,10 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result authorization_scope: row .try_get("authorization_scope") .map_err(unavailable_row)?, + platform_api_key_id: row + .try_get::, _>("platform_api_key_id") + .map_err(unavailable_row)? + .map(PlatformApiKeyId::new), operation_id: row.try_get("operation_id").map_err(unavailable_row)?, input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?, pins: TaskPins { @@ -574,6 +964,14 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result error: envelope("error_envelope")?, attempts: u32::try_from(row.try_get::("attempts").map_err(unavailable_row)?) .unwrap_or_default(), + fence: u64::try_from(row.try_get::("fence").map_err(unavailable_row)?) + .unwrap_or_default(), + lease_owner: row.try_get("lease_owner").map_err(unavailable_row)?, + lease_until_unix_ms: row.try_get("lease_until_ms").map_err(unavailable_row)?, + current_attempt_id: row.try_get("current_attempt_id").map_err(unavailable_row)?, + dispatch_started_at_unix_ms: row + .try_get("dispatch_started_at_ms") + .map_err(unavailable_row)?, version: u32::try_from(row.try_get::("version").map_err(unavailable_row)?) .unwrap_or_default(), created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?, diff --git a/crates/crank-registry/tests/integration/migrations.rs b/crates/crank-registry/tests/integration/migrations.rs index 720e85b..886c0e6 100644 --- a/crates/crank-registry/tests/integration/migrations.rs +++ b/crates/crank-registry/tests/integration/migrations.rs @@ -2,6 +2,7 @@ use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry}; use sqlx::Row; mod artifact_metadata; +mod durable_tasks_guard; mod import_plan; mod legacy_adoption; mod preflight; @@ -72,6 +73,9 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() { (23, "prompt-dependencies-v23"), (24, "execution-admission-v24"), (25, "durable-tasks-v25"), + (26, "durable-tasks-v26"), + (27, "durable-tasks-v27"), + (28, "durable-tasks-v28"), ]; assert_eq!(rows.len(), expected.len()); for (row, (version, name)) in rows.iter().zip(expected) { @@ -182,7 +186,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() { ); assert_eq!( MigrationAuthority::preflight(first.pool()).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } #[tokio::test] @@ -230,7 +234,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() { .expect("the V18-local guard must be restorable"); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "restoring the exact V18 contract must return the current verdict" ); } @@ -356,7 +360,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -469,13 +473,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 2, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); let trace_column: bool = sqlx::query_scalar( "select exists ( @@ -518,7 +522,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 3, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -578,7 +582,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 4, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -832,6 +836,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } diff --git a/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs b/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs index d2a102d..548beb5 100644 --- a/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs +++ b/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs @@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 11, - target: 25, + target: 28, } ); @@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "restore: {restore}" ); } @@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "restore: {restore}" ); } @@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) { .unwrap(); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "restore: {restore}" ); } @@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); for (drift, restore) in [ @@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "restore: {restore}" ); } diff --git a/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs b/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs index 430d931..42d8886 100644 --- a/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs +++ b/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs @@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column ); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "a rejected primary-key nullability change must leave the schema current" ); @@ -473,7 +473,7 @@ async fn assert_definition_only_drift( assert_eq!(super::catalog_constraint_definitions(pool).await, before); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } @@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "the baseline before the drift matrix must be the current V16 schema" ); let mut observations: Vec = Vec::new(); @@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift .await .expect("restoring the exact definition must apply"); let restored = MigrationAuthority::preflight(&pool).await; - let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 })); + let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 28 })); // Точное восстановление доказывается сравнением снятого снимка с baseline, а не // мнением того же guard-а, который принимает неправильные определения (F4). let restored_exact = catalog_definition_snapshot(&pool).await == before; @@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "baseline before inserting the shadow schema" ); @@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() // Тень не меняет вердикт guard-а по текущей схеме. assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "a same-named object in another schema must not change the verdict for the current schema" ); @@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() ); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 }, + MigrationPreflight::Current { version: 28 }, "the exact restoration must return the current V16 schema" ); @@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() .unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); assert!( diff --git a/crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs b/crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs new file mode 100644 index 0000000..1b92dd2 --- /dev/null +++ b/crates/crank-registry/tests/integration/migrations/durable_tasks_guard.rs @@ -0,0 +1,83 @@ +use super::*; +use crank_registry::MigrationApplyResult; + +#[tokio::test] +async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() { + let database_url = crank_test_support::postgres_schema_url("test_v43_clean_legacy").await; + let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); + MigrationAuthority::apply(&pool).await.unwrap(); + super::remove_v25_schema(&pool).await; + // remove_v25_schema drops `tasks` and ledger >= 25, but V25 also added the + // nullable dispatch-intent column/index/FK to execution_admission_entries; + // restore those to the V24 shape so the legacy ledger is self-consistent. + sqlx::raw_sql( + "alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey; + drop index if exists execution_admission_entries_task_uidx; + alter table execution_admission_entries drop column if exists task_id;", + ) + .execute(&pool) + .await + .unwrap(); + assert_eq!( + MigrationAuthority::preflight(&pool).await.unwrap(), + MigrationPreflight::MigrationRequired { + current: 24, + target: 28, + } + ); + assert_eq!( + MigrationAuthority::apply(&pool).await.unwrap(), + MigrationApplyResult::Applied { from: 24, to: 28 } + ); + assert_eq!( + MigrationAuthority::preflight(&pool).await.unwrap(), + MigrationPreflight::Current { version: 28 } + ); +} + +#[tokio::test] +async fn partial_v27_on_legacy_ledger_is_rejected_by_preflight() { + let database_url = crank_test_support::postgres_schema_url("test_v43_partial_v27").await; + let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); + MigrationAuthority::apply(&pool).await.unwrap(); + super::remove_v25_schema(&pool).await; + sqlx::raw_sql( + "alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey; + drop index if exists execution_admission_entries_task_uidx; + alter table execution_admission_entries drop column if exists task_id;", + ) + .execute(&pool) + .await + .unwrap(); + // Reintroduce a partial V27-shaped `tasks` table on top of the legacy (v24) ledger. + sqlx::raw_sql(sqlx::AssertSqlSafe( + "create table tasks ( + task_id uuid primary key, + executable_input_ciphertext text, + executable_input_key_version text, + executable_input_master_key_epoch bigint +);", + )) + .execute(&pool) + .await + .unwrap(); + + let error = MigrationAuthority::preflight(&pool).await.unwrap_err(); + assert_eq!(error.code(), "partial_sequence"); + // The rejection today is attributed to V25's absent-schema guard because the + // `tasks` relation already exists, so a behavioral RED pin for the missing + // V27 guard line cannot be reached; this test fixes the rejection contract + // instead of claiming new behavior. + assert_eq!(error.version(), Some(24)); + assert_eq!(error.stage(), "preflight.schema"); + + let count: i64 = + sqlx::query_scalar("select count(*)::bigint from __crank_migrations where version >= 25") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + count, 0, + "nothing at or above V25 should have been applied over the partial fixture" + ); +} diff --git a/crates/crank-registry/tests/integration/migrations/import_plan.rs b/crates/crank-registry/tests/integration/migrations/import_plan.rs index 2dcf092..1a101b2 100644 --- a/crates/crank-registry/tests/integration/migrations/import_plan.rs +++ b/crates/crank-registry/tests/integration/migrations/import_plan.rs @@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); let tables = sqlx::query( "select table_name @@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 13, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); diff --git a/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs b/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs index 2bc519c..5beb44c 100644 --- a/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs +++ b/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs @@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() { assert_eq!(operation_count, 1); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } @@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los ); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } @@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() { assert!(supports_elicitation); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } @@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 25, + target: 28, } ); MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } diff --git a/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs b/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs index 7db1994..8745973 100644 --- a/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs +++ b/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs @@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations") .fetch_one(&pool) @@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 18, - target: 25, + target: 28, } ); assert_eq!( MigrationAuthority::apply(&pool).await.unwrap(), - MigrationApplyResult::Applied { from: 18, to: 25 } + MigrationApplyResult::Applied { from: 18, to: 28 } ); let after: Vec<(i64, String, String)> = sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version") @@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() { assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten"); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 25 } + MigrationPreflight::Current { version: 28 } ); } @@ -86,16 +86,22 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() { MigrationAuthority::apply(&pool).await.unwrap(); // The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer // database; a synthetic version above the current one exercises it without a second binary. + let current: i64 = sqlx::query_scalar("select max(version) from __crank_migrations") + .fetch_one(&pool) + .await + .unwrap(); + let future = current + 1; sqlx::query( "insert into __crank_migrations (version, name, checksum, phase, compatibility) - values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')", + values ($1, 'future-synthetic', repeat('a', 64), 'expand', 'n-minus-one-readable')", ) + .bind(future) .execute(&pool) .await .unwrap(); let error = MigrationAuthority::preflight(&pool).await.unwrap_err(); assert_eq!(error.code(), "future_version"); - assert_eq!(error.version(), Some(26)); + assert_eq!(error.version(), Some(future)); assert_eq!(error.recovery(), "install_matching_application"); assert!( MigrationAuthority::require_current(&pool).await.is_err(), @@ -155,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 18, - target: 25, + target: 28, } ); } diff --git a/crates/crank-registry/tests/integration/migrations/rollback.rs b/crates/crank-registry/tests/integration/migrations/rollback.rs index add4333..329ab07 100644 --- a/crates/crank-registry/tests/integration/migrations/rollback.rs +++ b/crates/crank-registry/tests/integration/migrations/rollback.rs @@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 11, - target: 25, + target: 28, } ); } diff --git a/crates/crank-registry/tests/integration/migrations/schema_removal.rs b/crates/crank-registry/tests/integration/migrations/schema_removal.rs index 1bfa76a..a22fa86 100644 --- a/crates/crank-registry/tests/integration/migrations/schema_removal.rs +++ b/crates/crank-registry/tests/integration/migrations/schema_removal.rs @@ -330,9 +330,10 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) { /// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25. pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) { + // V25 created `tasks`; V26–V28 only altered it, so dropping the table removes their schema too. sqlx::raw_sql( "drop table if exists tasks cascade; - delete from __crank_migrations where version = 25;", + delete from __crank_migrations where version >= 25;", ) .execute(pool) .await diff --git a/crates/crank-registry/tests/integration/tasks.rs b/crates/crank-registry/tests/integration/tasks/mod.rs similarity index 75% rename from crates/crank-registry/tests/integration/tasks.rs rename to crates/crank-registry/tests/integration/tasks/mod.rs index f169fff..a478a33 100644 --- a/crates/crank-registry/tests/integration/tasks.rs +++ b/crates/crank-registry/tests/integration/tasks/mod.rs @@ -8,14 +8,18 @@ use super::common; use crank_core::{ - AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence, - ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope, - TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore, - TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms, + AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, AgentId, ApprovalEvidence, + ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind, + PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest, TaskAccess, + TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, + TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId, current_unix_ms, +}; +use crank_registry::{ + AdmissionLimits, CreatePlatformApiKeyRequest, PostgresExecutionAdmission, PostgresTaskStore, }; -use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore}; use serde_json::json; use sqlx::{PgPool, Row}; +use time::{Duration as TimeDuration, OffsetDateTime}; const WORKSPACE: &str = "ws_default"; const AGENT: &str = "agent_a"; @@ -48,6 +52,7 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta owner_kind: TaskOwnerKind::Agent, owner_id: agent.to_owned(), authorization_scope: format!("credential:pk_{agent}"), + platform_api_key_id: Some(PlatformApiKeyId::new(format!("pk_{agent}"))), operation_id: "op_report".to_owned(), pins: TaskPins { operation_version: 3, @@ -60,6 +65,9 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta input_fingerprint: fingerprint.to_owned(), input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(), support: unsampled_support(), + executable_input_ciphertext: None, + executable_input_key_version: None, + executable_input_master_key_epoch: None, } } @@ -664,6 +672,70 @@ async fn invalid_bounded_envelope_is_rejected_before_persistence() { database.cleanup().await; } +#[tokio::test] +async fn task_worker_lease_and_fencing_contract() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let created = store + .accept(request(&key("lease1"), &digest("f1"), AGENT, 30_000)) + .await + .unwrap(); + let leases = store.claim_batch("worker_1", 4, 10_000).await.unwrap(); + assert_eq!(leases.len(), 1); + assert_eq!(leases[0].record.task_id, created.task_id); + assert_eq!(leases[0].record.state, TaskState::Working); + assert_eq!(leases[0].owner, "worker_1"); + + let lease = &leases[0]; + let fenced_progress = store + .write_progress( + "worker_1", + lease.record.task_id, + lease.fence, + TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(), + ) + .await + .unwrap(); + assert!(fenced_progress.progress.is_some()); + + // stale fence rejected + let bad = store + .renew_lease("worker_1", lease.record.task_id, lease.fence + 1, 10_000) + .await; + assert!(bad.is_err()); + + // checkpoint dispatch once + let checkpointed = store + .checkpoint_dispatch( + "worker_1", + lease.record.task_id, + lease.fence, + "attempt-1".to_owned(), + ) + .await + .unwrap(); + assert_eq!( + checkpointed.current_attempt_id.as_deref(), + Some("attempt-1") + ); + assert!(checkpointed.dispatch_started_at_unix_ms.is_some()); + + // second checkpoint rejected + let second = store + .checkpoint_dispatch( + "worker_1", + lease.record.task_id, + lease.fence, + "attempt-2".to_owned(), + ) + .await; + assert!(second.is_err()); + + database.cleanup().await; +} + #[tokio::test] async fn store_failure_fails_closed_without_records() { let database = common::TestDatabase::new().await; @@ -680,3 +752,154 @@ async fn store_failure_fails_closed_without_records() { database.cleanup().await; } + +/// Story 4.3 ATDD: the typed immutable Platform API key reference of an accepted Task is persisted, +/// read back through the lease, and only an active, unexpired, exactly bound key is authoritatively +/// revalidated. Revocation, expiry and a foreign workspace/agent binding all close; an administrator +/// Task stores no machine key identity at all, so a worker fails closed on it. +#[tokio::test] +async fn accepted_task_authority_reference_is_persisted_and_revalidated() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let workspace = WorkspaceId::new(WORKSPACE); + let now = OffsetDateTime::now_utc(); + // `ws_default` is seeded by the baseline migration; the keys only need that workspace row. + let agent = AgentId::new(AGENT); + + let active_id = PlatformApiKeyId::new("pk_task_active"); + for key in [ + authority_key( + "pk_task_active", + &workspace, + AGENT, + PlatformApiKeyStatus::Active, + None, + ), + authority_key( + "pk_task_revoked", + &workspace, + AGENT, + PlatformApiKeyStatus::Revoked, + None, + ), + authority_key( + "pk_task_expired", + &workspace, + AGENT, + PlatformApiKeyStatus::Active, + Some(now - TimeDuration::hours(1)), + ), + ] { + let secret_hash = format!("secret-hash-{}", key.id.as_str()); + registry + .create_platform_api_key(CreatePlatformApiKeyRequest { + secret_hash: &secret_hash, + api_key: &key, + }) + .await + .unwrap(); + } + + // Acceptance persists only the immutable typed reference, never the bearer secret. + let mut accepted = request(&key("k_authority"), &digest("fa"), AGENT, 300_000); + accepted.platform_api_key_id = Some(active_id.clone()); + let task_id = accepted.task_id; + store.accept(accepted).await.expect("acceptance commits"); + + let leases = store + .claim_batch("worker_authority", 8, 30_000) + .await + .expect("claim"); + let lease = leases + .into_iter() + .find(|lease| lease.record.task_id == task_id) + .expect("accepted task is leased"); + assert_eq!(lease.record.platform_api_key_id.as_ref(), Some(&active_id)); + + // Positive: the exact active key of this workspace/agent is revalidated authoritatively. + let revalidated = registry + .revalidate_platform_api_key(&workspace, &agent, &active_id) + .await + .expect("authoritative read") + .expect("active key is authoritative"); + assert!( + revalidated + .api_key + .scopes + .contains(&PlatformApiKeyScope::Write) + ); + + // Negatives: revocation, expiry and a foreign workspace/agent binding all fail closed. + let foreign_agent = AgentId::new("agent_b"); + let foreign_workspace = WorkspaceId::new("ws_elsewhere"); + for (other_workspace, other_agent, other_key) in [ + (&workspace, &agent, PlatformApiKeyId::new("pk_task_revoked")), + (&workspace, &agent, PlatformApiKeyId::new("pk_task_expired")), + (&workspace, &foreign_agent, active_id.clone()), + (&foreign_workspace, &agent, active_id.clone()), + ] { + let found = registry + .revalidate_platform_api_key(other_workspace, other_agent, &other_key) + .await + .unwrap(); + assert!( + found.is_none(), + "authority must fail closed for {}", + other_key.as_str() + ); + } + + // An administrator Task carries no machine key identity: the worker has nothing to revalidate. + let mut admin_request = request(&key("k_admin"), &digest("fb"), AGENT, 300_000); + admin_request.owner_kind = TaskOwnerKind::Admin; + admin_request.owner_id = "user_admin".to_owned(); + admin_request.authorization_scope = "admin:user_admin".to_owned(); + admin_request.platform_api_key_id = None; + let admin_task = admin_request.task_id; + store + .accept(admin_request) + .await + .expect("administrator acceptance commits"); + let admin_record = store + .read( + &TaskAccess { + workspace_id: WORKSPACE.to_owned(), + owner_kind: TaskOwnerKind::Admin, + owner_id: "user_admin".to_owned(), + authorization_scope: "admin:user_admin".to_owned(), + }, + admin_task, + ) + .await + .expect("owner read"); + assert_eq!(admin_record.platform_api_key_id, None); + + database.cleanup().await; +} + +fn authority_key( + id: &str, + workspace_id: &WorkspaceId, + agent: &str, + status: PlatformApiKeyStatus, + expires_at: Option, +) -> PlatformApiKey { + PlatformApiKey { + id: PlatformApiKeyId::new(id), + workspace_id: workspace_id.clone(), + agent_id: Some(AgentId::new(agent)), + key_kind: PlatformApiKeyKind::McpClient, + name: id.to_owned(), + prefix: "crk_live".to_owned(), + scopes: vec![PlatformApiKeyScope::Write], + status, + created_at: OffsetDateTime::now_utc(), + last_used_at: None, + expires_at, + allowed_origins: Vec::new(), + } +} + +mod task_worker; diff --git a/crates/crank-registry/tests/integration/tasks/task_worker.rs b/crates/crank-registry/tests/integration/tasks/task_worker.rs new file mode 100644 index 0000000..0f2a261 --- /dev/null +++ b/crates/crank-registry/tests/integration/tasks/task_worker.rs @@ -0,0 +1,163 @@ +//! Story 4.3 ATDD: durable Task leases, fencing, crash recovery and bounded claim behavior. +//! +//! Split from `tasks.rs` to keep both integration files inside the code-health limit. + +use super::*; +use serde_json::json; + +/// Story 4.3 ATDD: a worker crash before the durable dispatch checkpoint leaves only an expired +/// lease. PostgreSQL is authoritative, so another worker recovers the Task and the dead owner can +/// never commit a terminal outcome with its old fence. +#[tokio::test] +async fn task_worker_expired_pre_checkpoint_lease_is_recovered_and_stale_owner_cannot_commit() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let created = store + .accept(request(&key("recover1"), &digest("f2"), AGENT, 30_000)) + .await + .unwrap(); + let first = store.claim_batch("worker_dead", 1, 150).await.unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(first[0].record.task_id, created.task_id); + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + + // Recovery on the next bounded authoritative scan, with no Valkey wake-up. + let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap(); + let lease = recovered + .into_iter() + .find(|lease| lease.record.task_id == created.task_id) + .expect("expired pre-checkpoint Task is recovered"); + assert!(lease.fence > first[0].fence, "recovery bumps the fence"); + + // The previous owner's terminal CAS is rejected; the live owner wins exactly once. + let stale = store + .finish_leased( + TaskFinishRequest { + access: access(AGENT), + task_id: created.task_id, + expected_state: first[0].record.state, + expected_version: first[0].record.version, + target_state: TaskState::Completed, + result: Some(TaskEnvelope::try_redacted(&json!({"stale": true})).unwrap()), + error: None, + }, + "worker_dead", + first[0].fence, + ) + .await; + assert!(matches!(stale, Err(TaskError::StateConflict))); + + let finished = store + .finish_leased( + TaskFinishRequest { + access: access(AGENT), + task_id: created.task_id, + expected_state: lease.record.state, + expected_version: lease.record.version, + target_state: TaskState::Completed, + result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()), + error: None, + }, + "worker_live", + lease.fence, + ) + .await + .expect("current lease wins the terminal CAS"); + assert_eq!(finished.state, TaskState::Completed); + + database.cleanup().await; +} + +/// Story 4.3 ATDD: after the dispatch checkpoint the outbound outcome is uncertain. An expired +/// attempt resolves to `outcome_unknown`, is never redispatched automatically and releases the +/// admission slot it held. +#[tokio::test] +async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_without_redispatch() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = store_with(registry.pool().clone(), 8); + + let created = store + .accept(request(&key("uncertain1"), &digest("f3"), AGENT, 30_000)) + .await + .unwrap(); + let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap(); + let lease = leases.into_iter().next().expect("claim"); + store + .checkpoint_dispatch( + "worker_dead", + lease.record.task_id, + lease.fence, + "attempt-uncertain".to_owned(), + ) + .await + .expect("durable dispatch checkpoint"); + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + + // The authority scan resolves uncertainty instead of handing the attempt to another worker. + let after = store.claim_batch("worker_live", 4, 10_000).await.unwrap(); + assert!( + after.is_empty(), + "a checkpointed attempt must never be redispatched" + ); + let record = store.read(&access(AGENT), created.task_id).await.unwrap(); + assert_eq!(record.state, TaskState::OutcomeUnknown); + assert_eq!( + record.current_attempt_id.as_deref(), + Some("attempt-uncertain") + ); + let intents: i64 = sqlx::query_scalar( + "select count(*)::bigint from execution_admission_entries where task_id = $1", + ) + .bind(created.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(intents, 0, "uncertain terminal outcome releases capacity"); + + // Still no automatic retry on the following scans. + assert!( + store + .claim_batch("worker_live", 4, 10_000) + .await + .unwrap() + .is_empty() + ); + + database.cleanup().await; +} + +/// Story 4.3 ATDD: concurrent claimers never hold the same Task at once. +#[tokio::test] +async fn task_worker_concurrent_claimers_never_double_lease_one_task() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let store = std::sync::Arc::new(store_with(registry.pool().clone(), 8)); + + let created = store + .accept(request(&key("concurrent1"), &digest("f4"), AGENT, 30_000)) + .await + .unwrap(); + let mut handles = Vec::new(); + for index in 0..4 { + let store = store.clone(); + handles.push(tokio::spawn(async move { + store + .claim_batch(&format!("worker_{index}"), 1, 10_000) + .await + .unwrap() + })); + } + let mut leased = Vec::new(); + for handle in handles { + leased.extend(handle.await.unwrap()); + } + assert_eq!(leased.len(), 1, "exactly one owner wins the claim"); + assert_eq!(leased[0].record.task_id, created.task_id); + + database.cleanup().await; +} diff --git a/crates/crank-runtime/src/executor.rs b/crates/crank-runtime/src/executor.rs index e3e6cd5..7462636 100644 --- a/crates/crank-runtime/src/executor.rs +++ b/crates/crank-runtime/src/executor.rs @@ -22,7 +22,9 @@ use serde_json::{Map, Value, json}; use time::OffsetDateTime; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tracing::{Instrument, Span, debug, warn}; +use uuid::Uuid; +use crate::task_worker::TaskDispatchPermit; use crate::{ AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError, RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext, @@ -178,8 +180,72 @@ impl RuntimeExecutor { pub(crate) async fn execute_request( &self, request: RuntimeExecutionRequest<'_>, + ) -> Result<(Value, Value), RuntimeError> { + self.execute_request_impl(request, None).await + } + + pub async fn execute_task_outcome( + &self, + request: RuntimeExecutionRequest<'_>, + permit: TaskDispatchPermit, + ) -> Result { + let operation = request.operation().clone(); + let operation_id = operation.operation_id.clone(); + let operation_version = operation.operation_version; + let origin = request.origin(); + let correlation = crank_core::CorrelationContext::new( + request.request_context().request_id.clone(), + request.request_context().trace_context.clone(), + ); + + // Validate permit binding before any adapter dispatch. + if permit.operation_id() != operation_id.as_str() + || permit.operation_version() != operation_version + || permit.workspace_id() != request.workspace_id().as_str() + || permit.agent_id() != request.agent_id().map(|value| value.as_str()) + { + return Err(crate::normalize_runtime_error_for_operation( + &RuntimeError::InvalidExecutionRequest, + &correlation, + &operation, + )); + } + if !permit.consume() { + return Err(crate::normalize_runtime_error_for_operation( + &RuntimeError::InvalidExecutionRequest, + &correlation, + &operation, + )); + } + + match self.execute_request_impl(request, Some(permit)).await { + Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess { + operation_id: operation_id.clone(), + operation_version, + origin, + correlation, + request_preview, + output, + }), + Err(error) => Err(crate::normalize_runtime_error_for_operation( + &error, + &correlation, + &operation, + )), + } + } + + pub(crate) async fn execute_request_impl( + &self, + request: RuntimeExecutionRequest<'_>, + permit: Option, ) -> Result<(Value, Value), RuntimeError> { let operation = request.operation(); + // Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission. + let trusted_task_dispatch = permit.is_some(); + // The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the + // trace names the real persisted attempt instead of inventing one (Story 4.3 AC). + let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned()); let input = request.input(); let request_context = request.request_context(); debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized); @@ -205,7 +271,9 @@ impl RuntimeExecutor { let dispatch_started = Arc::new(AtomicBool::new(false)); // True until the admission gate answered. A deadline that fires while the request is still // waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout. - let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some())); + let admission_pending = Arc::new(AtomicBool::new( + !trusted_task_dispatch && self.admission.is_some(), + )); let invocation_metrics = ToolInvocationMetrics::start_with_exemplar( metric_invocation_source(Some(request_context)), request_context @@ -222,14 +290,19 @@ impl RuntimeExecutor { // One admission gate for every executable path (Story 4.1): the published policy decides // the effective mode, the absolute deadline includes queue waiting and the scoped permit // is held for the whole local execution future, including retries. - let mut lease = crate::admission::acquire( - self.admission.as_ref(), - self.coordination_store.as_deref(), - &request, - execution_deadline, - ) - .await?; - admission_pending.store(false, Ordering::Release); + let mut lease = if trusted_task_dispatch { + None + } else { + let result = crate::admission::acquire( + self.admission.as_ref(), + self.coordination_store.as_deref(), + &request, + execution_deadline, + ) + .await?; + admission_pending.store(false, Ordering::Release); + result + }; let _permit = self.acquire_unary_permit(operation)?; let _inflight = InFlightGuard::runtime(); let mapping_span = Stage::RuntimeArgumentsMap.span(); @@ -246,6 +319,7 @@ impl RuntimeExecutor { prepared_request, Some(request_context), Arc::clone(&dispatch_started), + durable_attempt_id.as_deref(), ) .await; if let Some(lease) = lease.as_mut() { @@ -348,6 +422,7 @@ impl RuntimeExecutor { prepared_request: PreparedRequest, request_context: Option<&RuntimeRequestContext>, dispatch_started: Arc, + durable_attempt_id: Option<&str>, ) -> Result { let mut prepared_request = prepared_request; let idempotency_applicable = crate::idempotency::policy(operation).is_some(); @@ -464,6 +539,7 @@ impl RuntimeExecutor { operation, idempotency_key.as_deref(), ), + durable_attempt_id, ) .await; if let Ok(response) = &adapter_response { @@ -604,10 +680,23 @@ impl RuntimeExecutor { request_context: Option<&RuntimeRequestContext>, dispatch_started: Arc, verified_idempotency: bool, + durable_attempt_id: Option<&str>, ) -> Result { let max_attempts = retry_attempts(operation); let mut attempt = 1; loop { + // Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint; the + // retry index discriminates the attempts inside it. Only paths without a durable attempt + // (synchronous tools/call) mint a synthetic per-attempt id. + let attempt_id = durable_attempt_id + .map(str::to_owned) + .unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7())); + let attempt_span = tracing::info_span!( + "adapter_attempt", + attempt, + max_attempts, + attempt_id = %attempt_id + ); let result = self .execute_adapter( operation, @@ -615,6 +704,7 @@ impl RuntimeExecutor { request_context, Arc::clone(&dispatch_started), ) + .instrument(attempt_span) .await; match result { Ok(response) => return Ok(response), diff --git a/crates/crank-runtime/src/lib.rs b/crates/crank-runtime/src/lib.rs index 41114ba..e6e47ea 100644 --- a/crates/crank-runtime/src/lib.rs +++ b/crates/crank-runtime/src/lib.rs @@ -15,6 +15,7 @@ mod rate_limit; mod request_context; mod request_preparation; mod response_cache; +mod task_worker; mod tasks; pub fn validate_resource_template_parameter( schema: &crank_schema::Schema, @@ -115,4 +116,8 @@ pub use rate_limit::{ }; pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext}; pub use secret_crypto::SecretCrypto; +pub use task_worker::{ + TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, + TaskWorkerError, +}; pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService}; diff --git a/crates/crank-runtime/src/task_worker.rs b/crates/crank-runtime/src/task_worker.rs new file mode 100644 index 0000000..d412070 --- /dev/null +++ b/crates/crank-runtime/src/task_worker.rs @@ -0,0 +1,573 @@ +//! Fenced durable Task worker loop for Story 4.3. +//! +//! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded, +//! Valkey may only wake it up (never represented here as correctness), and every lease-protected +//! write goes through CAS using owner + fencing token. The actual executable input and revalidation +//! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a +//! redacted projection and is never dispatched. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use crank_core::{TaskEnvelope, TaskError, TaskFinishRequest, TaskLease, TaskState, TaskStore}; +use tokio::sync::watch; +use tokio::task::JoinSet; +use tracing::{Instrument, debug, warn}; +use uuid::Uuid; + +/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets. +#[derive(Clone, Debug)] +pub struct TaskWorkerConfig { + /// Stable random-ish owner identity for this process instance. Required lease owner. + pub owner: String, + /// Maximum number of Tasks claimed in one bounded PG scan. + pub batch_limit: u32, + /// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used. + pub lease_ms: i64, + /// Authoritative PG polling interval when no wake-up hint is available. + pub poll_interval: Duration, + /// Renewal cadence while processing a lease. + pub renewal_interval: Duration, + /// Concurrent in-flight Tasks for this worker. + pub max_inflight: usize, + /// Maximum time to wait for active work during graceful shutdown. + pub shutdown_timeout: Duration, +} + +impl Default for TaskWorkerConfig { + fn default() -> Self { + Self { + owner: format!("task-worker-{}", Uuid::now_v7()), + batch_limit: 8, + lease_ms: 30_000, + poll_interval: Duration::from_millis(250), + renewal_interval: Duration::from_millis(5_000), + max_inflight: 4, + shutdown_timeout: Duration::from_secs(30), + } + } +} + +/// Result of one durable Task attempt produced by the application layer. +#[derive(Clone, Debug)] +pub enum TaskAttemptOutcome { + Completed(serde_json::Value), + Failed(serde_json::Value), + OutcomeUnknown, +} + +/// Application-owned seam for pre-dispatch revalidation and execution. +/// +/// The worker owns lease renewal/CAS/fencing and the dispatch checkpoint. The handler only +/// receives the leased record and attempt identity; it must not trust `TaskRecord.input` as +/// executable payload. +#[async_trait] +pub trait TaskHandler: Send + Sync { + async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>; + async fn execute( + &self, + lease: &TaskLease, + attempt_id: &str, + cancelled: Arc, + permit: TaskDispatchPermit, + ) -> TaskAttemptOutcome; +} + +/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and +/// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and +/// it binds one durable Task attempt to its execution request. +pub struct TaskDispatchPermit { + task_id: crank_core::TaskId, + operation_id: String, + operation_version: u32, + workspace_id: String, + agent_id: Option, + owner: String, + fence: u64, + attempt_id: String, + consumed: std::sync::atomic::AtomicBool, +} + +impl TaskDispatchPermit { + pub(crate) fn new( + task_id: crank_core::TaskId, + operation_id: String, + operation_version: u32, + workspace_id: String, + agent_id: Option, + owner: String, + fence: u64, + attempt_id: String, + ) -> Self { + Self { + task_id, + operation_id, + operation_version, + workspace_id, + agent_id, + owner, + fence, + attempt_id, + consumed: std::sync::atomic::AtomicBool::new(false), + } + } + + pub fn task_id(&self) -> crank_core::TaskId { + self.task_id + } + + pub fn operation_id(&self) -> &str { + &self.operation_id + } + + pub fn operation_version(&self) -> u32 { + self.operation_version + } + + pub fn workspace_id(&self) -> &str { + &self.workspace_id + } + + pub fn agent_id(&self) -> Option<&str> { + self.agent_id.as_deref() + } + + pub fn owner(&self) -> &str { + &self.owner + } + + pub fn fence(&self) -> u64 { + self.fence + } + + pub fn attempt_id(&self) -> &str { + &self.attempt_id + } + + pub fn is_consumed(&self) -> bool { + self.consumed.load(std::sync::atomic::Ordering::Acquire) + } + + pub fn consume(&self) -> bool { + !self + .consumed + .swap(true, std::sync::atomic::Ordering::AcqRel) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum TaskWorkerError { + #[error("task store error: {0}")] + Store(#[from] TaskError), + #[error("invalid worker configuration: {0}")] + Config(String), + #[error("task terminal transition CAS failed")] + CasRejected, +} + +pub struct TaskWorker { + store: Arc, + handler: Arc, + config: TaskWorkerConfig, + stopping: Arc, +} + +impl TaskWorker { + pub fn new( + store: Arc, + handler: Arc, + config: TaskWorkerConfig, + ) -> Result { + if config.owner.is_empty() { + return Err(TaskWorkerError::Config( + "owner must not be empty".to_owned(), + )); + } + if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 { + return Err(TaskWorkerError::Config( + "batch_limit must be 1..=64 and lease_ms must be positive".to_owned(), + )); + } + if config.lease_ms > 30_000 + || config.poll_interval.is_zero() + || config.poll_interval > Duration::from_secs(30) + || config.renewal_interval.is_zero() + || config.renewal_interval >= Duration::from_millis(config.lease_ms as u64) + || config.max_inflight == 0 + || config.shutdown_timeout.is_zero() + || config.shutdown_timeout > Duration::from_secs(30) + { + return Err(TaskWorkerError::Config( + "lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds" + .to_owned(), + )); + } + Ok(Self { + store, + handler, + config, + stopping: Arc::new(AtomicBool::new(false)), + }) + } + + /// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled. + pub fn stop(&self) { + self.stopping.store(true, Ordering::Release); + } + + /// Returns a handle whose drop does not stop the worker; call `stop()` on the worker. + pub fn stop_flag(&self) -> Arc { + self.stopping.clone() + } + + /// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan. + pub async fn run(&self) -> Result<(), TaskWorkerError> { + self.run_inner(None).await + } + + /// Run until either `stop()` or the provided shutdown receiver becomes true. + pub async fn run_until_shutdown( + &self, + mut shutdown: watch::Receiver, + ) -> Result<(), TaskWorkerError> { + self.run_inner(Some(&mut shutdown)).await + } + + async fn run_inner( + &self, + mut shutdown: Option<&mut watch::Receiver>, + ) -> Result<(), TaskWorkerError> { + let mut active: JoinSet<()> = JoinSet::new(); + loop { + if self.stopping.load(Ordering::Acquire) { + break; + } + if let Some(rx) = shutdown.as_mut() { + if *rx.borrow() { + break; + } + } + + // Reap finished attempts before claiming more. + while active.try_join_next().is_some() {} + + let available = self.config.max_inflight.saturating_sub(active.len()); + if available > 0 { + let batch_limit = self + .config + .batch_limit + .min(u32::try_from(available).unwrap_or(u32::MAX)); + match self + .store + .claim_batch(&self.config.owner, batch_limit, self.config.lease_ms) + .await + { + Ok(leases) => { + for lease in leases { + let store = self.store.clone(); + let handler = self.handler.clone(); + let owner = self.config.owner.clone(); + let lease_ms = self.config.lease_ms; + let renewal_interval = self.config.renewal_interval; + let stopping = self.stopping.clone(); + active.spawn(async move { + process_lease( + store, + handler, + owner, + lease, + lease_ms, + renewal_interval, + stopping, + ) + .await; + }); + } + } + Err(error) => { + warn!(error = %error, "bounded task claim failed; retrying on next scan"); + } + } + } + + tokio::select! { + _ = tokio::time::sleep(self.config.poll_interval) => {} + _ = async { + if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await } + } => {} + } + } + + // Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded. + // The same stop flag reaches cooperative cancellation and lease-renewal loops. + self.stopping.store(true, Ordering::Release); + let deadline = Instant::now() + self.config.shutdown_timeout; + while !active.is_empty() { + if Instant::now() >= deadline { + warn!("graceful shutdown timeout reached; aborting remaining task attempts"); + active.abort_all(); + break; + } + tokio::select! { + _ = active.join_next() => {} + _ = tokio::time::sleep(Duration::from_millis(50)) => {} + } + } + Ok(()) + } +} + +async fn process_lease( + store: Arc, + handler: Arc, + owner: String, + lease: TaskLease, + lease_ms: i64, + renewal_interval: Duration, + stopping: Arc, +) { + let task_id = lease.record.task_id; + let fence = lease.fence; + let cancelled = Arc::new(AtomicBool::new(false)); + + // Pre-dispatch revalidation under the fresh lease. + if let Err(error) = handler.revalidate(&lease).await { + // A transient authority-store outage is not evidence that the work is invalid: leave the + // lease to expire so the bounded recovery scan can retry it without dispatching now. + if matches!( + &error, + TaskWorkerError::Store(TaskError::Unavailable { .. }) + ) { + debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery"); + return; + } + debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch"); + finish_failed( + &store, + &owner, + task_id, + fence, + &lease, + "revalidation_failed", + ) + .await; + return; + } + if stopping.load(Ordering::Acquire) { + // Graceful shutdown before the durable dispatch checkpoint: no outbound call can have + // happened, so the Task is left leased. The bounded recovery scan requeues it after the + // lease expires instead of turning a never-started attempt into a user-visible failure. + return; + } + + let attempt_id = format!("attempt-{}", Uuid::now_v7()); + + // Durable checkpoint before any network use. + if store + .checkpoint_dispatch(&owner, task_id, fence, attempt_id.clone()) + .await + .is_err() + { + // Lease lost or checkpoint rejected: do not dispatch. + cancelled.store(true, Ordering::Release); + return; + } + if stopping.load(Ordering::Acquire) { + // The checkpoint is durable, but we know the outbound callback has not started. + finish_failed( + &store, + &owner, + task_id, + fence, + &lease, + "shutdown_before_dispatch", + ) + .await; + return; + } + + // Sealed single-use permit for the trusted runtime dispatch bridge. It is created only after + // the current lease and durable checkpoint above. + let agent_id = if lease.record.owner_kind == crank_core::TaskOwnerKind::Agent { + Some(lease.record.owner_id.clone()) + } else { + None + }; + let permit = TaskDispatchPermit::new( + lease.record.task_id, + lease.record.operation_id.clone(), + lease.record.pins.operation_version, + lease.record.workspace_id.clone(), + agent_id, + owner.clone(), + fence, + attempt_id.clone(), + ); + + // Background lease renewal while the attempt runs. + let renewal_store = store.clone(); + let renewal_owner = owner.clone(); + let renewal_cancelled = cancelled.clone(); + let renewal_stopping = stopping.clone(); + let renewal_task_id = task_id; + let renewal_handle = tokio::spawn(async move { + let mut interval = tokio::time::interval(renewal_interval); + interval.tick().await; + loop { + interval.tick().await; + if renewal_cancelled.load(Ordering::Acquire) { + break; + } + if renewal_stopping.load(Ordering::Acquire) { + renewal_cancelled.store(true, Ordering::Release); + break; + } + if renewal_store + .renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms) + .await + .is_err() + { + renewal_cancelled.store(true, Ordering::Release); + break; + } + } + }); + + // Each dispatch gets a child span carrying the durable attempt identity, rooted in the parent + // trace persisted with the accepted Task (never a mutable queue/cache projection). + let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, attempt_id = %attempt_id, fence); + if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) { + crank_trace::set_parent_from_trace_context(&span, &parent); + } + let outcome = handler + .execute(&lease, &attempt_id, cancelled.clone(), permit) + .instrument(span) + .await; + + cancelled.store(true, Ordering::Release); + renewal_handle.abort(); + + // Terminal CAS under valid lease. Only one writer wins; stale token is rejected by store. + let access = crank_core::TaskAccess { + workspace_id: lease.record.workspace_id.clone(), + owner_kind: lease.record.owner_kind, + owner_id: lease.record.owner_id.clone(), + authorization_scope: lease.record.authorization_scope.clone(), + }; + + match outcome { + TaskAttemptOutcome::Completed(value) => { + let Ok(result) = TaskEnvelope::try_redacted(&value) else { + finish_failed( + &store, + &owner, + task_id, + fence, + &lease, + "result_not_redactable", + ) + .await; + return; + }; + let _ = store + .finish_leased( + TaskFinishRequest { + access, + task_id, + expected_state: leased_expected_state(&lease), + expected_version: lease.record.version, + target_state: TaskState::Completed, + result: Some(result), + error: None, + }, + &owner, + fence, + ) + .await; + } + TaskAttemptOutcome::Failed(value) => { + let error = TaskEnvelope::try_redacted(&value).or_else(|_| { + TaskEnvelope::try_redacted(&serde_json::json!({ + "code": "task_error_not_redactable" + })) + }); + let Ok(error) = error else { + return; + }; + let _ = store + .finish_leased( + TaskFinishRequest { + access, + task_id, + expected_state: leased_expected_state(&lease), + expected_version: lease.record.version, + target_state: TaskState::Failed, + result: None, + error: Some(error), + }, + &owner, + fence, + ) + .await; + } + TaskAttemptOutcome::OutcomeUnknown => { + let _ = store + .finish_leased( + TaskFinishRequest { + access, + task_id, + expected_state: leased_expected_state(&lease), + expected_version: lease.record.version, + target_state: TaskState::OutcomeUnknown, + result: None, + error: None, + }, + &owner, + fence, + ) + .await; + } + } +} + +fn leased_expected_state(lease: &TaskLease) -> TaskState { + // Claimed tasks are expected to be in a non-terminal processing state. + if lease.record.state == TaskState::Queued { + TaskState::Queued + } else { + TaskState::Working + } +} + +async fn finish_failed( + store: &Arc, + owner: &str, + task_id: crank_core::TaskId, + fence: u64, + lease: &TaskLease, + code: &str, +) { + let access = crank_core::TaskAccess { + workspace_id: lease.record.workspace_id.clone(), + owner_kind: lease.record.owner_kind, + owner_id: lease.record.owner_id.clone(), + authorization_scope: lease.record.authorization_scope.clone(), + }; + let _ = store + .finish_leased( + TaskFinishRequest { + access, + task_id, + expected_state: leased_expected_state(lease), + expected_version: lease.record.version, + target_state: TaskState::Failed, + result: None, + error: TaskEnvelope::try_redacted(&serde_json::json!({ "code": code })).ok(), + }, + owner, + fence, + ) + .await; +} diff --git a/crates/crank-runtime/src/tasks.rs b/crates/crank-runtime/src/tasks.rs index 1acafa3..038d6f8 100644 --- a/crates/crank-runtime/src/tasks.rs +++ b/crates/crank-runtime/src/tasks.rs @@ -13,11 +13,13 @@ use std::sync::Arc; use std::time::Instant; +use crank_core::PlatformApiKeyId; use crank_core::{ - CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, - TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, - TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds, - ToolExecutionMode, current_unix_ms, validate_task_envelope, + CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, + TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_SCOPED_KEY_MAX_BYTES, + TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskLease, + TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds, ToolExecutionMode, + contains_sensitive_task_keys, current_unix_ms, validate_task_envelope, }; use sha2::{Digest, Sha256}; @@ -29,6 +31,9 @@ pub struct TaskAcceptDraft { pub owner_id: String, /// Server-derived authorization scope identity of the verified principal. pub authorization_scope: String, + /// Typed immutable Platform API key identity of the verified machine credential. It is sealed + /// into the executable-input binding and revalidated authoritatively by the worker. + pub platform_api_key_id: Option, pub operation_id: String, pub operation_version: u32, pub policy_mode: ToolExecutionMode, @@ -68,11 +73,22 @@ impl TaskOutcome { #[derive(Clone)] pub struct TaskService { store: Arc, + crypto: Option>, } impl TaskService { pub fn new(store: Arc) -> Self { - Self { store } + Self { + store, + crypto: None, + } + } + + pub fn with_secret_crypto(store: Arc, crypto: Arc) -> Self { + Self { + store, + crypto: Some(crypto), + } } /// Readiness of the mandatory durable Task state. @@ -83,18 +99,61 @@ impl TaskService { /// Durable acceptance: the returned identity exists only after the store committed it. pub async fn accept(&self, draft: TaskAcceptDraft) -> Result { validate_task_envelope(&draft.input)?; + if contains_sensitive_task_keys(&draft.input) { + return Err(TaskError::invalid( + "task executable input contains sensitive fields", + )); + } let (scoped_key, input_fingerprint) = ( scoped_key_digest(&draft)?, canonical_input_fingerprint(&draft.input)?, ); let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline); let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS); + let task_id = TaskId::generate(); + let crypto = self.crypto.as_ref().ok_or_else(|| { + TaskError::invalid("secret crypto is required for durable Task acceptance") + })?; + let binding = serde_json::json!({ + "task_id": task_id.to_string(), + "workspace_id": draft.workspace_id, + "owner_kind": draft.owner_kind.as_str(), + "owner_id": draft.owner_id, + "authorization_scope": draft.authorization_scope, + "platform_api_key_id": draft + .platform_api_key_id + .as_ref() + .map(PlatformApiKeyId::as_str), + "operation_id": draft.operation_id, + "operation_version": draft.operation_version, + "policy_mode": draft.policy_mode, + "input_fingerprint": input_fingerprint, + "input": draft.input, + }); + let ciphertext = crypto + .encrypt(&binding) + .map_err(|error| TaskError::invalid(error.to_string()))?; + if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES { + return Err(TaskError::invalid( + "encrypted task input exceeds its byte budget", + )); + } + let ( + executable_input_ciphertext, + executable_input_key_version, + executable_input_master_key_epoch, + ) = ( + Some(ciphertext), + Some(crypto.key_version().to_owned()), + Some(crypto.master_key_epoch()), + ); let request = TaskAcceptRequest { - task_id: TaskId::generate(), + task_id, workspace_id: draft.workspace_id, owner_kind: draft.owner_kind, owner_id: draft.owner_id, authorization_scope: draft.authorization_scope, + platform_api_key_id: draft.platform_api_key_id, operation_id: draft.operation_id, pins: TaskPins { operation_version: draft.operation_version, @@ -110,6 +169,9 @@ impl TaskService { draft.correlation.request_id().clone(), draft.correlation.trace_context(), ), + executable_input_ciphertext, + executable_input_key_version, + executable_input_master_key_epoch, }; self.store.accept(request).await } @@ -134,6 +196,97 @@ impl TaskService { )) } + /// Decrypts and validates the executable input bound to a valid current lease. + pub async fn executable_input( + &self, + lease: &TaskLease, + ) -> Result { + let crypto = self.crypto.as_ref().ok_or_else(|| { + TaskError::invalid("secret crypto is required for durable Task execution") + })?; + let (ciphertext, key_version, epoch) = self + .store + .read_executable_input(&lease.owner, lease.record.task_id, lease.fence) + .await?; + if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES { + return Err(TaskError::invalid( + "task executable input ciphertext is invalid", + )); + } + let binding = crypto + .decrypt_for_epoch(&key_version, epoch, &ciphertext) + .map_err(|error| TaskError::invalid(error.to_string()))?; + let object = binding + .as_object() + .ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?; + let check_string = |field: &str, expected: &str| -> Result<(), TaskError> { + match object.get(field).and_then(|value| value.as_str()) { + Some(actual) if actual == expected => Ok(()), + _ => Err(TaskError::invalid("task executable input binding mismatch")), + } + }; + check_string("task_id", &lease.record.task_id.to_string())?; + check_string("workspace_id", &lease.record.workspace_id)?; + check_string("owner_id", &lease.record.owner_id)?; + check_string("authorization_scope", &lease.record.authorization_scope)?; + let expected_key = lease + .record + .platform_api_key_id + .as_ref() + .map(PlatformApiKeyId::as_str); + match (object.get("platform_api_key_id"), expected_key) { + (None | Some(serde_json::Value::Null), None) => {} + (Some(serde_json::Value::String(actual)), Some(expected)) if actual == expected => {} + _ => { + return Err(TaskError::invalid("task executable input binding mismatch")); + } + } + check_string("operation_id", &lease.record.operation_id)?; + match object.get("owner_kind").and_then(|value| value.as_str()) { + Some(actual) if actual == lease.record.owner_kind.as_str() => {} + _ => return Err(TaskError::invalid("task executable input binding mismatch")), + } + match object + .get("operation_version") + .and_then(|value| value.as_u64()) + { + Some(actual) if actual == u64::from(lease.record.pins.operation_version) => {} + _ => return Err(TaskError::invalid("task executable input binding mismatch")), + } + let expected_policy = match lease.record.pins.policy_mode { + ToolExecutionMode::Synchronous => "synchronous", + ToolExecutionMode::Background => "background", + ToolExecutionMode::Adaptive => "adaptive", + }; + match object.get("policy_mode").and_then(|value| value.as_str()) { + Some(actual) if actual == expected_policy => {} + _ => return Err(TaskError::invalid("task executable input binding mismatch")), + } + check_string("input_fingerprint", &lease.record.input_fingerprint)?; + let input = object + .get("input") + .cloned() + .ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?; + let canonical = canonical_input_fingerprint(&input)?; + if canonical != lease.record.input_fingerprint { + return Err(TaskError::invalid("task executable input binding mismatch")); + } + validate_task_envelope(&input)?; + let encoded = serde_json::to_vec(&input) + .map_err(|_| TaskError::invalid("task executable input is not serializable"))?; + if encoded.len() > TASK_EXECUTABLE_INPUT_MAX_BYTES { + return Err(TaskError::invalid( + "task executable input exceeds its byte budget", + )); + } + if contains_sensitive_task_keys(&input) { + return Err(TaskError::invalid( + "task executable input contains sensitive fields", + )); + } + Ok(input) + } + /// Default retention pinned by the application layer when it does not choose one explicitly. pub fn default_retention_ms() -> i64 { TASK_DEFAULT_RETENTION_MS @@ -220,6 +373,7 @@ mod tests { owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_1".to_owned(), + platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")), operation_id: "op_report".to_owned(), operation_version: 2, policy_mode: ToolExecutionMode::Background, diff --git a/crates/crank-runtime/tests/stages.rs b/crates/crank-runtime/tests/stages.rs index ee4858d..b7a6452 100644 --- a/crates/crank-runtime/tests/stages.rs +++ b/crates/crank-runtime/tests/stages.rs @@ -106,6 +106,83 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() { assert_eq!(failure.stage(), ExecutionStage::Upstream); } +#[tokio::test] +async fn adapter_retry_child_spans_are_distinct_without_a_durable_attempt() { + let _tracing_test_guard = TRACING_TEST_LOCK.lock().await; + let capture = TraceCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + + let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(FlakyReadAdapter { + call_count: Arc::clone(&call_count), + })) + .build(); + let mut operation: crank_runtime::RuntimeOperation = operation().into(); + let Target::Rest(target) = &mut operation.target; + target.method = HttpMethod::Get; + operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 }); + + // The synchronous tools/call path has no durable Attempt: every adapter attempt gets its own + // synthetic id. A Task dispatch instead reuses the durable id (mcp-server `attempt_trace`). + let result = async { + let request_root = tracing::info_span!("request_root"); + executor + .execute(&operation, &json!({"name": "retry-read"})) + .instrument(request_root) + .await + } + .with_subscriber(subscriber) + .await + .expect("read retry should recover"); + + assert_eq!(result, json!({"accepted": true})); + + let spans = capture.snapshot(); + let runtime_span = spans + .iter() + .find(|span| span.name == "runtime.execute") + .expect("runtime.execute span"); + assert_eq!( + runtime_span.parent_name, + Some("request_root"), + "the execution continues the caller's trace root" + ); + let attempt_spans: Vec<_> = spans + .iter() + .filter(|span| span.name == "adapter_attempt") + .collect(); + assert_eq!(attempt_spans.len(), 2); + for span in &attempt_spans { + assert_eq!(span.parent_name, Some("runtime.execute")); + assert!( + !span + .fields + .get("attempt_id") + .unwrap_or(&"".to_string()) + .is_empty() + ); + assert!( + !span + .fields + .get("attempt_id") + .unwrap_or(&"".to_string()) + .contains("traceparent") + ); + } + let attempts: Vec = attempt_spans + .iter() + .map(|span| span.fields.get("attempt").cloned().unwrap_or_default()) + .collect(); + assert!(attempts.contains(&"1".to_string())); + assert!(attempts.contains(&"2".to_string())); + let ids: Vec = attempt_spans + .iter() + .map(|span| span.fields.get("attempt_id").cloned().unwrap_or_default()) + .collect(); + assert_ne!(ids[0], ids[1]); +} + #[tokio::test] async fn read_retry_revalidates_adapter_attempt_and_keeps_one_success_outcome() { let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -236,11 +313,7 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() { .find(|span| span.name == "runtime.execute") .expect("runtime span"); assert_eq!(runtime.parent_name, Some("mcp.request")); - for child in [ - "runtime.arguments.map", - "upstream.http", - "runtime.response.transform", - ] { + for child in ["runtime.arguments.map", "runtime.response.transform"] { assert_eq!( spans .iter() @@ -250,6 +323,14 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() { "{child} must be a runtime child" ); } + assert_eq!( + spans + .iter() + .find(|span| span.name == "upstream.http") + .and_then(|span| span.parent_name), + Some("adapter_attempt"), + "upstream.http must be an adapter child" + ); } #[tokio::test] diff --git a/docs/execution-boundary.md b/docs/execution-boundary.md index d7f4697..d186a6f 100644 --- a/docs/execution-boundary.md +++ b/docs/execution-boundary.md @@ -48,6 +48,43 @@ input fingerprint — типизированный конфликт; явная authorization scope) входит в каждый read; чужой Task неотличим от отсутствующего. Границы этой story: публичный MCP wire Tasks/capabilities/activation остаётся -Story 4.7, worker/leases/recovery — Story 4.3. Существующий `tools/call` сохраняет -synchronous/unsupported поведение; durable acceptance не запускает ephemeral -execution и не рекламирует клиенту background capability. +Story 4.7, worker/leases/recovery реализованы в Story 4.3 (ниже). Существующий +`tools/call` сохраняет synchronous/unsupported поведение; durable acceptance не +запускает ephemeral execution и не рекламирует клиенту background capability. + +## Исполнение durable Task (Story 4.3) + +`mcp-server` запускает fenced worker в том же процессе: bounded claim через +PostgreSQL `FOR UPDATE SKIP LOCKED` над reserved dispatch intent, lease owner/expiry +и monotonic fence; Valkey не является источником истины и может только ускорять +wake-up. Claim использует уже принятый shared admission из Story 4.1 +(`lock_admission_in_tx` по controller-строке `*` + `grant_next_task` с его +scoped-лимитами, bounded queue и weighted fairness cursor) и не добавляет второго +scheduler'а: 4.3-дельта лишь fencing/lease-поля и CAS-предикаты поверх того же +grant, поэтому принятые fairness-инварианты 4.1 сохраняются. Progress, renewal, +dispatch checkpoint и terminal outcome пишутся только через CAS по текущему +owner/fence/expiry. До сетевого вызова worker повторно проверяет публикацию и +точную версию capability, активный Platform API key/scope, approval policy и +deadline; невалидная работа завершается без dispatch. Attempt ID и dispatch +checkpoint коммитятся атомарно до outbound; после checkpoint потеря lease означает +uncertain outcome и Task переходит в `outcome_unknown` без автоматического retry. +Исполняемый input берётся только из зашифрованного Task payload через +`TaskService::executable_input` (redacted projection никогда не исполняется). +Parent TraceContext сохраняется с acceptance; durable attempt span продолжает его, +а каждая adapter-попытка (включая retry существующих GET/verified-idempotent write) +получает собственный child span `adapter_attempt` с тем же durable Attempt ID и +номером попытки, поэтому retry не сливается с первым attempt в трассе, а любой +такой span сопоставим с `tasks.current_attempt_id`; синтетический attempt id +остаётся только у синхронного tools/call, где durable попытки нет. Скан PostgreSQL авторитетен: +crash до checkpoint возвращает Task в queue по истечении lease (recovery ≤60s). + +Graceful shutdown является процессным, а не только worker-локальным: сигнал +SIGTERM/ctrl-c одновременно прекращает новые claims и запускает HTTP graceful +drain (worker не ждёт завершения долгого соединения), активные attempts +checkpoint-ятся/дренируются в пределах 30s, а ошибочное завершение HTTP-сервера +также останавливает worker. Уже поставленные в queue Tasks не теряются. Остаётся +узкое окно между durable checkpoint и созданием sealed dispatch permit: если stop +приходит именно туда, Task финализируется как failed(`shutdown_before_dispatch`), +не отправляя запрос; это доказано регрессией, которая детерминированно +задерживает checkpoint в его собственной транзакции, проверяет отсутствие вызова +adapter и сохранность ещё queued работы. diff --git a/scripts/ci-story-checks.sh b/scripts/ci-story-checks.sh index 6b276a7..b868bf3 100755 --- a/scripts/ci-story-checks.sh +++ b/scripts/ci-story-checks.sh @@ -78,3 +78,10 @@ run_atdd "runtime task identity" -p crank-runtime --lib tasks:: -- --test-thread run_atdd "durable task store" -p crank-registry --test integration tasks:: -- --test-threads=1 run_atdd "durable task application entry" -p mcp-server --test integration durable_tasks:: -- --test-threads=1 run_atdd "durable task migration" -p crank-registry --test integration migrations:: -- --test-threads=1 + +# Story 4.3 (FR-22, FR-23): fenced durable Task execution. The Story 4.2 selectors above already run +# the lease/fencing/recovery store, the Task application entry and the migration guards; these two +# oracles bound the rest of this story's AC: the process-level shutdown ordering (signal stops new +# claims before a slow HTTP drain, not after it) and one child attempt span per adapter retry. +run_atdd "task worker process shutdown" -p mcp-server --bin mcp-server -- --test-threads=1 +run_atdd "durable task retry attempt spans" -p crank-runtime --test stages retry -- --test-threads=1 -- 2.54.0 From 5488eb73ec6d26bfdcf06294e1dc70040d3013e0 Mon Sep 17 00:00:00 2001 From: bsodfather Date: Thu, 29 Oct 2026 22:43:05 +0000 Subject: [PATCH 2/2] =?UTF-8?q?chore(migrations):=20=D0=B7=D0=B0=D1=84?= =?UTF-8?q?=D0=B8=D0=BA=D1=81=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20?= =?UTF-8?q?=D0=BA=D0=B0=D0=BD=D0=BE=D0=BD=D0=B8=D1=87=D0=B5=D1=81=D0=BA?= =?UTF-8?q?=D1=83=D1=8E=20=D0=BF=D0=BE=D1=81=D0=BB=D0=B5=D0=B4=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D1=82=D0=B5=D0=BB=D1=8C=D0=BD=D0=BE=D1=81=D1=82=D1=8C=20?= =?UTF-8?q?v26=E2=80=93v28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Обязательный CI-гейт «Check canonical migration contract» сравнивает `crank-migrate plan` с docs/schemas/migration-sequence.json. Принятая дельта Story 4.3 добавила миграции v26–v28, но снимок контракта не был регенерирован, поэтому exact-head CI падал на точном PR head с contract_drift (plan.compare). Файл получен машинной регенерацией из того же дерева: `crank-migrate plan > docs/schemas/migration-sequence.json`; проверка `crank-migrate plan --check` возвращает {"status":"contract_current"}. Содержимое меняется только добавлением трёх записей 26/27/28; существующие записи 1–25 и их checksum не изменены, поведение продукта не меняется. --- docs/schemas/migration-sequence.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/schemas/migration-sequence.json b/docs/schemas/migration-sequence.json index 0facea5..2a9d103 100644 --- a/docs/schemas/migration-sequence.json +++ b/docs/schemas/migration-sequence.json @@ -1 +1 @@ -{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23},{"backfill":{"kind":"none"},"checksum":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-admission-v24","owner":"crank-registry","phase":"expand","readable_schema_max":24,"readable_schema_min":23,"source_digest":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","transactional":true,"version":24},{"backfill":{"kind":"none"},"checksum":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v25","owner":"crank-registry","phase":"expand","readable_schema_max":25,"readable_schema_min":24,"source_digest":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","transactional":true,"version":25}]} +{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23},{"backfill":{"kind":"none"},"checksum":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-admission-v24","owner":"crank-registry","phase":"expand","readable_schema_max":24,"readable_schema_min":23,"source_digest":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","transactional":true,"version":24},{"backfill":{"kind":"none"},"checksum":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v25","owner":"crank-registry","phase":"expand","readable_schema_max":25,"readable_schema_min":24,"source_digest":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","transactional":true,"version":25},{"backfill":{"kind":"none"},"checksum":"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v26","owner":"crank-registry","phase":"expand","readable_schema_max":26,"readable_schema_min":25,"source_digest":"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089","transactional":true,"version":26},{"backfill":{"kind":"none"},"checksum":"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v27","owner":"crank-registry","phase":"expand","readable_schema_max":27,"readable_schema_min":26,"source_digest":"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8","transactional":true,"version":27},{"backfill":{"kind":"none"},"checksum":"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v28","owner":"crank-registry","phase":"expand","readable_schema_max":28,"readable_schema_min":27,"source_digest":"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676","transactional":true,"version":28}]} -- 2.54.0