Compare commits

...
Author SHA1 Message Date
hermes_dev 3b5fcc51cc fix(ui): pin npm build stage to TLS 1.2 on the CI host uplink
CI / UI Checks (pull_request) Successful in 9s
CI / Rust Checks (pull_request) Successful in 20m41s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
The CI host uplink (192.168.1.108) drops TLS 1.3 ClientHello carrying the
hybrid X25519MLKEM768 key share, so registry.npmjs.org is unreachable from
the seeded node:22-alpine build stage while the same host answers over TLS
1.2. RUN npm ci hung until the build was cancelled.

Cap only the UI build stage at TLS 1.2 via NODE_OPTIONS=--tls-max-v1.2.
Certificates, SRI and the lockfile are untouched; no mirror, fallback or
cache-substituted packages. Remove the line once TLS 1.3 works from the CI
host again.
2026-10-30 09:56:19 +00:00
bsodfather 433987413e Merge pull request 'Story 4.3 — исполнять и восстанавливать durable Task через PG lease/fence' (#110) from delivery/epic4-story-4.3 into main
CI / UI Checks (push) Successful in 8s
CI / Rust Checks (push) Successful in 26m20s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 2s
CI / Deploy (push) Skipped
Community Release / release (push) Failing after 31m2s
2026-10-29 23:08:04 +00:00
bsodfather 5488eb73ec chore(migrations): зафиксировать каноническую последовательность v26–v28
CI / Rust Checks (pull_request) Successful in 22m6s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
CI / UI Checks (pull_request) Successful in 12s
Обязательный CI-гейт «Check canonical migration contract» сравнивает
`crank-migrate plan` с docs/schemas/migration-sequence.json. Принятая дельта
Story 4.3 добавила миграции v26–v28, но снимок контракта не был регенерирован,
поэтому exact-head CI падал на точном PR head с contract_drift (plan.compare).

Файл получен машинной регенерацией из того же дерева:
`crank-migrate plan > docs/schemas/migration-sequence.json`; проверка
`crank-migrate plan --check` возвращает {"status":"contract_current"}.
Содержимое меняется только добавлением трёх записей 26/27/28; существующие
записи 1–25 и их checksum не изменены, поведение продукта не меняется.
2026-10-29 22:43:05 +00:00
bsodfather f9f8c625d1 feat(epic4): исполнять durable Task через PG lease/fence и восстанавливать после сбоев (Story 4.3)
CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с
fence-токеном, зашифрованный input и повторная проверка доступа до сетевого
вызова, checkpoint с AttemptID до dispatch и восстановление только по PG.
Устаревший worker не может зафиксировать результат.

Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой
схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса:
стоп в окне после checkpoint и до permit завершает задачу как
failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи
остаются доступными. Retry-попытки получают отдельные дочерние spans с durable
current_attempt_id. Shared admission/fairness из Story 4.1 сохранены.

Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47,
bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA
и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
2026-10-29 22:26:52 +00:00
54 changed files with 5148 additions and 233 deletions

No files matched your search

+99 -87
View File
@@ -2,6 +2,10 @@
mod admin_auth_command;
#[path = "crank_migrate/db_connect.rs"]
mod db_connect;
#[path = "crank_migrate/reporting.rs"]
mod reporting;
use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json};
use crank_config::{ConfigSource, parse_migrator};
use crank_registry::{
@@ -11,7 +15,7 @@ use crank_registry::{
};
use crank_runtime::SecretCrypto;
use serde_json::json;
use std::{path::Path, process::ExitCode};
use std::process::ExitCode;
use time::OffsetDateTime;
const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000;
#[tokio::main]
@@ -765,6 +769,59 @@ async fn process_rotation_batch(
processed += 1;
}
}
let mut after_task_id: Option<String> = None;
loop {
let tasks = registry
.list_tasks_for_master_key_epoch_page(
rotation.source_epoch,
after_task_id.as_deref(),
MASTER_KEY_ROTATION_PAGE_SIZE,
)
.await
.map_err(CliError::from_registry)?;
if tasks.is_empty() {
break;
}
for task in tasks {
after_task_id = Some(task.task_id.clone());
if task.target_epoch == Some(rotation.target_epoch) {
continue;
}
if max_versions.is_some_and(|limit| processed as usize >= limit) {
return Ok(processed);
}
let plaintext = current_crypto
.decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext)
.map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let now = OffsetDateTime::now_utc();
registry
.stage_master_key_rotation_task_ciphertext(
&rotation.id,
&task.task_id,
rotation.source_epoch,
&target_ciphertext,
target_crypto.key_version(),
rotation.target_epoch,
&now,
)
.await
.map_err(CliError::from_registry)?;
processed += 1;
}
}
Ok(processed)
}
@@ -818,6 +875,47 @@ async fn verify_staged_targets(
verified += 1;
}
}
let mut after_task_id: Option<String> = None;
loop {
let tasks = registry
.list_target_tasks_for_master_key_rotation_page(
rotation.target_epoch,
after_task_id.as_deref(),
MASTER_KEY_ROTATION_PAGE_SIZE,
)
.await
.map_err(CliError::from_registry)?;
if tasks.is_empty() {
break;
}
for task in tasks {
after_task_id = Some(task.task_id.clone());
let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let key_version = task.target_key_version.as_deref().ok_or_else(|| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
target_crypto
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
.map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
verified += 1;
}
}
if verified != rotation.total_secret_versions {
return Err(CliError::new(
"master_key_rotation_verification_failed",
@@ -877,89 +975,3 @@ fn decrypt_current(
)
})
}
async fn active_rotation(
registry: &PostgresRegistry,
allowed_states: &[&str],
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
let status = registry
.master_key_rotation_status()
.await
.map_err(CliError::from_registry)?;
Ok(status
.rotations
.into_iter()
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
}
fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
json!({
"id": rotation.id,
"source_epoch": rotation.source_epoch,
"target_epoch": rotation.target_epoch,
"state": rotation.state,
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
"checkpoint_secret_id": rotation.checkpoint_secret_id,
"total_secret_versions": rotation.total_secret_versions,
"processed_secret_versions": rotation.processed_secret_versions,
"verified_secret_versions": rotation.verified_secret_versions,
"failure_code": rotation.failure_code,
})
}
fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
let path = path.ok_or_else(|| {
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
})?;
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if !metadata.is_file() || metadata.len() > 16_384 {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if value.trim().is_empty() {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
Ok(value)
}
fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
if options.current_key_file.is_some()
|| options.target_key_file.is_some()
|| options.backup_ref.is_some()
|| options.rotation_id.is_some()
|| options.max_versions.is_some()
{
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_master_key_status",
));
}
Ok(())
}
@@ -0,0 +1,98 @@
//! Leaf/reporting helpers of the migrator CLI.
//!
//! Extracted from `crank-migrate.rs` without behaviour changes so the binary entry stays within
//! the code-health line limit.
use std::path::Path;
use crank_registry::{MasterKeyRotationRecord, PostgresRegistry};
use serde_json::json;
use super::{CliError, MasterKeyOptions};
pub(super) async fn active_rotation(
registry: &PostgresRegistry,
allowed_states: &[&str],
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
let status = registry
.master_key_rotation_status()
.await
.map_err(CliError::from_registry)?;
Ok(status
.rotations
.into_iter()
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
}
pub(super) fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
json!({
"id": rotation.id,
"source_epoch": rotation.source_epoch,
"target_epoch": rotation.target_epoch,
"state": rotation.state,
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
"checkpoint_secret_id": rotation.checkpoint_secret_id,
"total_secret_versions": rotation.total_secret_versions,
"processed_secret_versions": rotation.processed_secret_versions,
"verified_secret_versions": rotation.verified_secret_versions,
"failure_code": rotation.failure_code,
})
}
pub(super) fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
let path = path.ok_or_else(|| {
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
})?;
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if !metadata.is_file() || metadata.len() > 16_384 {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if value.trim().is_empty() {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
Ok(value)
}
pub(super) fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
if options.current_key_file.is_some()
|| options.target_key_file.is_some()
|| options.backup_ref.is_some()
|| options.rotation_id.is_some()
|| options.max_versions.is_some()
{
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_master_key_status",
));
}
Ok(())
}
+2 -1
View File
@@ -29,7 +29,7 @@ sha2.workspace = true
sqlx.workspace = true
thiserror.workspace = true
time.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio = { workspace = true, features = ["sync", "signal"] }
tracing.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
@@ -45,5 +45,6 @@ opentelemetry_sdk.workspace = true
prost.workspace = true
reqwest.workspace = true
tower.workspace = true
tokio = { workspace = true, features = ["net", "time"] }
tracing-opentelemetry.workspace = true
metrics-util = "0.20.4"
+149 -13
View File
@@ -201,20 +201,45 @@ async fn run(
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
// Story 4.2: the durable Task application entry shares the registry pool, the published catalog
// refresh interval and the exact admission limits of the runtime; the public wire stays
// synchronous/unsupported and no worker is started.
let task_store = PostgresTaskStore::new(
// Story 4.2/4.3: the durable Task application entry shares the registry pool, the published
// catalog refresh interval and the exact admission limits of the runtime; the public wire stays
// synchronous/unsupported while a fenced in-process worker executes accepted Tasks.
let task_store = std::sync::Arc::new(PostgresTaskStore::new(
registry.pool().clone(),
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?;
)?);
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
registry.clone(),
Duration::from_millis(config.refresh_ms),
cache_stores.coordination.clone(),
crank_runtime::TaskService::new(std::sync::Arc::new(task_store)),
crank_runtime::TaskService::with_secret_crypto(
task_store.clone(),
std::sync::Arc::new(secret_crypto.clone()),
),
));
// Story 4.3: the durable Task worker runs in this process over the same PostgreSQL store,
// published catalog, executor and encrypted-input authority as the accepting application.
let task_handler =
std::sync::Arc::new(crank_community_mcp::TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime.clone(),
secret_crypto.clone(),
));
let task_worker = std::sync::Arc::new(crank_runtime::TaskWorker::new(
task_store,
task_handler,
crank_runtime::TaskWorkerConfig {
owner: format!("mcp-server-{}", std::process::id()),
..crank_runtime::TaskWorkerConfig::default()
},
)?);
let (task_shutdown_tx, task_shutdown_rx) = tokio::sync::watch::channel(false);
let task_worker_handle = tokio::spawn({
let worker = task_worker.clone();
async move { worker.run_until_shutdown(task_shutdown_rx).await }
});
let app = build_app_with_artifacts_and_tasks(
registry,
Duration::from_millis(config.refresh_ms),
@@ -252,15 +277,77 @@ async fn run(
info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening");
*startup_completed = true;
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = axum::serve(listener, app) => result?,
result = metrics_server.serve() => result?,
let serve_result: Result<(), Box<dyn std::error::Error>> = async {
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) => result?,
result = metrics_server.serve() => result?,
}
} else {
serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone())
.await?;
}
} else {
axum::serve(listener, app).await?;
Ok(())
}
.await;
// Stop new claims; active attempts checkpoint/drain or are cancelled inside the bounded worker
// shutdown timeout (<=30s), and queued PostgreSQL work is left for recovery.
task_shutdown_tx.send(true).ok();
if tokio::time::timeout(Duration::from_secs(30), task_worker_handle)
.await
.is_err()
{
tracing::warn!(
name: "mcp.task_worker.shutdown_timeout",
"durable Task worker did not stop within the story shutdown budget"
);
}
serve_result
}
/// Gracefully stop the durable Task worker as soon as shutdown is requested, while still
/// draining in-flight HTTP connections before returning.
async fn serve_with_worker_shutdown<S>(
app: axum::Router,
listener: tokio::net::TcpListener,
signal: S,
worker_stop: tokio::sync::watch::Sender<bool>,
) -> std::io::Result<()>
where
S: std::future::Future<Output = ()> + Send + 'static,
{
let worker_stop_graceful = worker_stop.clone();
let graceful = async move {
signal.await;
let _ = worker_stop_graceful.send(true);
};
let result = axum::serve(listener, app)
.with_graceful_shutdown(graceful)
.await;
let _ = worker_stop.send(true);
result
}
/// Process shutdown trigger for the HTTP surface and the durable Task worker.
async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let terminate = async {
if let Ok(mut signal) =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
{
signal.recv().await;
}
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {}
_ = terminate => {}
}
Ok(())
}
fn init_observability(
@@ -506,3 +593,52 @@ fn runtime_cache_config(
.map(|value| value.expose_secret().to_owned()),
)
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{Router, routing::get};
use std::time::Duration;
use tokio::sync::{oneshot, watch};
async fn slow_handler() -> &'static str {
tokio::time::sleep(Duration::from_millis(600)).await;
"ok"
}
#[tokio::test]
async fn worker_stops_before_http_drain_completes() {
let app = Router::new().route("/slow", get(slow_handler));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (stop_tx, mut stop_rx) = watch::channel(false);
let (signal_tx, signal_rx) = oneshot::channel::<()>();
let handle = tokio::spawn(serve_with_worker_shutdown(
app,
listener,
async {
let _ = signal_rx.await;
},
stop_tx,
));
let client = reqwest::Client::new();
let request = client.get(format!("http://{}/slow", addr));
let in_flight = tokio::spawn(async move { request.send().await });
tokio::time::sleep(Duration::from_millis(150)).await;
let _ = signal_tx.send(());
tokio::time::timeout(Duration::from_millis(250), stop_rx.changed())
.await
.expect("worker stop signal should arrive before HTTP drain")
.expect("watch changed");
assert!(*stop_rx.borrow());
let response = in_flight.await.unwrap().unwrap();
assert!(response.status().is_success());
let serve_result = handle.await.unwrap();
assert!(serve_result.is_ok());
}
}
@@ -0,0 +1,413 @@
//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1).
//!
//! End-to-end: a real durable Task worker dispatches a retrying GET through a real local
//! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is
//! persisted in `tasks.current_attempt_id` on both attempts.
use super::super::common;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::get;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{
AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy,
};
use crank_runtime::{
OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy,
};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use time::OffsetDateTime;
use tracing::{Id, Subscriber, field::Visit};
use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "attempt_trace";
const KEY: &str = "attempt-trace-key-1";
async fn publish(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
version: u32,
) {
registry
.publish_operation(crank_registry::PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id,
version,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
}
async fn application_with_store(
registry: &crank_registry::PostgresRegistry,
) -> (
crank_community_mcp::TaskApplication,
std::sync::Arc<crank_registry::PostgresTaskStore>,
SecretCrypto,
) {
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = std::sync::Arc::new(
crank_registry::PostgresTaskStore::new(
registry.pool().clone(),
crank_registry::AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = crank_community_mcp::TaskApplication::new(
registry.clone(),
std::time::Duration::ZERO,
std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()),
crank_runtime::TaskService::with_secret_crypto(
store.clone(),
std::sync::Arc::new(crypto.clone()),
),
);
(tasks, store, crypto)
}
async fn principal(
registry: &crank_registry::PostgresRegistry,
name: &str,
scopes: &[crank_core::PlatformApiKeyScope],
) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("an active platform key resolves");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
#[tokio::test]
async fn adapter_attempt_span_carries_durable_attempt_id() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
get({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
let n = calls.fetch_add(1, Ordering::SeqCst);
if n == 0 {
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(serde_json::json!({"error":"boom"})),
);
}
(
StatusCode::OK,
axum::Json(serde_json::json!({"id":"lead-1"})),
)
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
let Target::Rest(target) = &mut operation.target;
target.method = crank_core::HttpMethod::Get;
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"attempt-trace",
&[
crank_core::PlatformApiKeyScope::Read,
crank_core::PlatformApiKeyScope::Write,
],
)
.await;
let (tasks, store, crypto) = application_with_store(&registry).await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&serde_json::json!({"email": "user@example.com"}),
&CorrelationContext::generate(),
)
.await
.expect("durable acceptance");
let runtime =
community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-attempt-trace".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = std::time::Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = std::time::Duration::from_secs(5);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
let _guard = tracing::subscriber::set_default(subscriber);
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(std::time::Duration::from_secs(10), worker_task)
.await
.expect("worker stops inside the budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
server.abort();
let durable_attempt_id: Option<String> = sqlx::query_scalar::<_, Option<String>>(
"select current_attempt_id from tasks where task_id = $1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(state.as_deref(), Some("completed"));
let spans = capture.snapshot();
let attempt_spans: Vec<_> = spans
.iter()
.filter(|s| s.name == "adapter_attempt")
.collect();
assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans");
for span in &attempt_spans {
let aid = span.fields.get("attempt_id").cloned().unwrap_or_default();
assert!(!aid.is_empty(), "attempt_id must not be empty");
assert!(
!aid.contains("traceparent"),
"attempt_id must not contain traceparent"
);
assert_eq!(
Some(aid.as_str()),
durable_attempt_id.as_deref(),
"span attempt_id must equal durable attempt id"
);
}
let ids: Vec<String> = attempt_spans
.iter()
.map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default())
.collect();
assert_eq!(ids[0], ids[1], "same durable id on both retries");
assert_eq!(
spans
.iter()
.filter(|span| span.name == "durable_task_attempt")
.count(),
1,
"one durable attempt span per dispatch"
);
let runtime_span = spans
.iter()
.find(|span| span.name == "runtime.execute")
.expect("runtime.execute span");
assert_eq!(
runtime_span.parent_name,
Some("durable_task_attempt"),
"the execution continues the durable attempt trace"
);
for span in &attempt_spans {
assert_eq!(
span.parent_name,
Some("runtime.execute"),
"each retry is a child attempt span under the durable execution"
);
}
let attempts: Vec<String> = attempt_spans
.iter()
.map(|s| s.fields.get("attempt").cloned().unwrap_or_default())
.collect();
assert!(attempts.contains(&"1".to_string()));
assert!(attempts.contains(&"2".to_string()));
for span in &attempt_spans {
assert_eq!(
span.fields.get("max_attempts").cloned().unwrap_or_default(),
"2"
);
}
}
// TraceCapture minimal port from crates/crank-runtime/tests/stages.rs
#[derive(Clone, Default)]
struct TraceCapture {
spans: Arc<Mutex<Vec<CapturedSpan>>>,
}
impl TraceCapture {
fn snapshot(&self) -> Vec<CapturedSpan> {
self.spans.lock().expect("span lock").clone()
}
}
#[derive(Clone, Debug)]
struct CapturedSpan {
name: &'static str,
parent_name: Option<&'static str>,
fields: std::collections::BTreeMap<String, String>,
}
impl<S> Layer<S> for TraceCapture
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
{
fn on_new_span(
&self,
attributes: &tracing::span::Attributes<'_>,
id: &Id,
context: tracing_subscriber::layer::Context<'_, S>,
) {
let parent = attributes
.parent()
.and_then(|parent| context.span(parent))
.or_else(|| {
attributes
.is_contextual()
.then(|| context.lookup_current())
.flatten()
});
let mut visitor = FieldVisitor::default();
attributes.record(&mut visitor);
let mut spans = self.spans.lock().expect("span lock");
let index = spans.len();
spans.push(CapturedSpan {
name: attributes.metadata().name(),
parent_name: parent.map(|span| span.metadata().name()),
fields: visitor.fields,
});
context
.span(id)
.expect("span exists")
.extensions_mut()
.insert(index);
}
fn on_record(
&self,
id: &Id,
values: &tracing::span::Record<'_>,
context: tracing_subscriber::layer::Context<'_, S>,
) {
let mut visitor = FieldVisitor::default();
values.record(&mut visitor);
let span = context.span(id).expect("span exists");
let index = *span.extensions().get::<usize>().expect("capture index");
self.spans.lock().expect("span lock")[index]
.fields
.extend(visitor.fields);
}
}
#[derive(Default)]
struct FieldVisitor {
fields: std::collections::BTreeMap<String, String>,
}
impl Visit for FieldVisitor {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.fields
.insert(field.name().to_owned(), value.to_owned());
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.fields
.insert(field.name().to_owned(), format!("{value:?}"));
}
}
@@ -14,13 +14,14 @@ use std::time::Duration;
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
use crank_core::{
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState,
ToolExecutionMode, ToolExecutionPolicy,
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId,
TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
};
use crank_runtime::{InMemoryCoordinationStateStore, TaskService};
use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
@@ -149,7 +150,9 @@ async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
.unwrap();
}
fn task_service(pool: PgPool) -> TaskService {
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService {
let store = PostgresTaskStore::new(
pool,
AdmissionLimits {
@@ -161,15 +164,27 @@ fn task_service(pool: PgPool) -> TaskService {
Default::default(),
)
.expect("valid task store configuration");
TaskService::new(std::sync::Arc::new(store))
TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto))
}
fn application(registry: &PostgresRegistry) -> TaskApplication {
/// The application entry under the same encrypted-input authority the production process uses.
async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key");
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
TaskApplication::new(
registry.clone(),
Duration::from_millis(0),
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
task_service(registry.pool().clone()),
task_service(registry.pool().clone(), crypto),
)
}
@@ -243,7 +258,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
)
.await;
let correlation = unsampled_correlation();
let app = application(&registry);
let app = application(&registry).await;
let acceptance = app
.accept_background(
@@ -273,7 +288,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
assert_eq!(counts(registry.pool()).await, (1, 1));
// A freshly constructed application + service reads the same durable identity and pins.
let reader = application(&registry);
let reader = application(&registry).await;
let record = reader
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
@@ -317,7 +332,7 @@ async fn a_new_publication_never_rebinds_an_accepted_task() {
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let app = application(&registry).await;
let accepted = app
.accept_background(
&caller,
@@ -397,7 +412,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let app = application(&registry).await;
let accepted = app
.accept_background(
&owner,
@@ -433,7 +448,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
));
// A synchronous publication is not accepted as background work at all.
let synchronous_app = application(&registry);
let synchronous_app = application(&registry).await;
let refused = synchronous_app
.accept_background(
&owner,
@@ -453,7 +468,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let app = application(&registry);
let app = application(&registry).await;
let admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
@@ -549,7 +564,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let app = application(&registry).await;
assert!(app.health().await.is_ok());
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
@@ -589,3 +604,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
server.abort();
}
mod attempt_trace;
mod shutdown_window;
mod task_worker;
@@ -0,0 +1,190 @@
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
//!
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
use super::*;
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
/// does not consume work that is still queued.
///
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
#[tokio::test]
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
// statement keeps the window deterministic without competing for a second connection.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
begin
if new.current_attempt_id is not null and old.current_attempt_id is null then
perform pg_sleep(1.5);
end if;
return new;
end $$;
create trigger delay_task_dispatch_checkpoint before update on tasks
for each row execute function delay_task_dispatch_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint delay trigger");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let mut accepted = Vec::new();
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
},
scoped_key: scoped_key.repeat(64),
input_fingerprint: scoped_key.repeat(64),
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
accepted.push(store.accept(request).await.expect("PG accepts"));
}
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
let executed = Arc::new(AtomicUsize::new(0));
struct GatedHandler {
entered: Arc<tokio::sync::Semaphore>,
release: Arc<tokio::sync::Semaphore>,
executed: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for GatedHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
self.entered.add_permits(1);
let _ = self.release.acquire().await;
Ok(())
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
self.executed.fetch_add(1, Ordering::SeqCst);
TaskAttemptOutcome::Completed(json!({ "ok": true }))
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
entered: revalidate_entered.clone(),
release: revalidate_release.clone(),
executed: executed.clone(),
});
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-stop-window".to_owned();
config.batch_limit = 1;
config.lease_ms = 30_000;
config.poll_interval = Duration::from_millis(50);
config.renewal_interval = Duration::from_secs(20);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(20);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
.await
.expect("worker must claim the first Task and enter revalidation")
.expect("revalidation gate is open")
.forget();
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
revalidate_release.add_permits(1);
tokio::time::sleep(Duration::from_millis(300)).await;
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
// finalize the Task, but no permit exists yet, so no outbound call can start.
let stop_requested = std::time::Instant::now();
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(Duration::from_secs(30), worker_task)
.await
.expect("worker stops inside the 30s budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
assert!(
stop_requested.elapsed() >= Duration::from_millis(500),
"the checkpoint must have still been in flight when the stop arrived"
);
assert_eq!(
executed.load(Ordering::SeqCst),
0,
"a stop between checkpoint and permit must never reach the upstream adapter"
);
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
};
let record = store.read(&access, accepted[0].task_id).await.unwrap();
assert_eq!(record.state, TaskState::Failed);
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
assert!(record.current_attempt_id.is_some());
assert_eq!(
record
.error
.as_ref()
.and_then(|error| error.value().get("code"))
.and_then(|code| code.as_str()),
Some("shutdown_before_dispatch")
);
// Work that was still queued is not consumed by the stopped worker.
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
assert_eq!(queued.state, TaskState::Queued);
let recovered = store
.claim_batch("worker_after_stop", 4, 10_000)
.await
.expect("queued work stays claimable");
assert_eq!(recovered.len(), 1);
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
}
@@ -0,0 +1,822 @@
//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch.
//!
//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit.
use super::*;
/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a
/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan,
/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented.
#[tokio::test]
async fn task_worker_store_lease_recovery_contract() {
let registry = common::test_registry().await;
let store = PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration");
// Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and
// leases durable work, preserving the same admission-backed intent.
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
// This store-level control exercises lease/fence behavior, not the encrypted acceptance path.
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
let leases = store
.claim_batch("worker_1", 4, 10_000)
.await
.expect("claim");
assert_eq!(leases.len(), 1);
assert_eq!(leases[0].record.task_id, accepted.task_id);
let lease = leases[0].clone();
// A stale or competing owner cannot win CAS with the issued fencing token.
assert!(
store
.renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000)
.await
.is_err()
);
assert!(
store
.write_progress(
"worker_1",
lease.record.task_id,
lease.fence + 1,
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
)
.await
.is_err()
);
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
assert!(
store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence + 1,
"attempt-stale".to_owned()
)
.await
.is_err()
);
// The valid owner still commits the dispatch checkpoint before any outbound work.
assert!(
store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence,
"attempt-1".to_owned(),
)
.await
.unwrap()
.current_attempt_id
.as_deref()
== Some("attempt-1")
);
}
#[tokio::test]
async fn task_worker_runtime_dispatch_and_recovery_contract() {
use async_trait::async_trait;
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let pool = registry.pool().clone();
let store = Arc::new(
PostgresTaskStore::new(
pool.clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let observed = Arc::new(AtomicUsize::new(0));
// Positive control: PostgreSQL alone accepts the durable Task.
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_runtime".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
},
scoped_key: "r".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
let task_id = accepted.task_id;
struct RuntimeHandler {
observed: Arc<AtomicUsize>,
}
#[async_trait]
impl TaskHandler for RuntimeHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
observed: observed.clone(),
});
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-1".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(100);
config.renewal_interval = Duration::from_millis(1_000);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_runtime".to_owned(),
};
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(100)).await;
if let Ok(record) = store.read(&access, task_id).await {
state = Some(record.state);
if record.state == TaskState::Completed {
break;
}
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state, Some(TaskState::Completed));
assert_eq!(
observed.load(Ordering::SeqCst),
1,
"exactly one outbound attempt"
);
// Terminal state is not duplicated.
let record = store.read(&access, task_id).await.expect("terminal read");
assert_eq!(record.state, TaskState::Completed);
// Bounded shutdown with no fresh work returns promptly.
let mut idle_config = TaskWorkerConfig::default();
idle_config.owner = "runtime-worker-idle".to_owned();
idle_config.poll_interval = Duration::from_millis(50);
idle_config.shutdown_timeout = Duration::from_secs(2);
let idle_worker = TaskWorker::new(
store.clone(),
handler_for_no_work(store.clone()),
idle_config,
)
.expect("valid idle worker");
idle_worker.stop();
let started = std::time::Instant::now();
idle_worker.run().await.expect("idle worker stops");
assert!(started.elapsed() <= Duration::from_secs(30));
}
/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call.
#[tokio::test]
async fn task_worker_revalidation_denial_never_dispatches() {
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_denied".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
},
scoped_key: "d".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
struct DeniedHandler {
observed: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for DeniedHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Err(TaskWorkerError::Store(TaskError::invalid(
"publication or credential revoked",
)))
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
TaskAttemptOutcome::Completed(json!({"ok": true}))
}
}
let observed = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
observed: observed.clone(),
});
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-denied".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_denied".to_owned(),
};
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(50)).await;
if let Ok(record) = store.read(&access, accepted.task_id).await {
state = Some(record.state);
if record.state == TaskState::Failed {
break;
}
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state, Some(TaskState::Failed));
assert_eq!(
observed.load(Ordering::SeqCst),
0,
"invalidated work never reaches the upstream"
);
}
fn handler_for_no_work(
_store: std::sync::Arc<PostgresTaskStore>,
) -> std::sync::Arc<dyn crank_runtime::TaskHandler> {
struct NoWorkHandler;
#[async_trait::async_trait]
impl crank_runtime::TaskHandler for NoWorkHandler {
async fn revalidate(
&self,
_lease: &crank_core::TaskLease,
) -> Result<(), crank_runtime::TaskWorkerError> {
Ok(())
}
async fn execute(
&self,
_lease: &crank_core::TaskLease,
_attempt_id: &str,
_cancelled: std::sync::Arc<std::sync::atomic::AtomicBool>,
_permit: crank_runtime::TaskDispatchPermit,
) -> crank_runtime::TaskAttemptOutcome {
crank_runtime::TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
}
}
std::sync::Arc::new(NoWorkHandler)
}
/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a
/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the
/// accepted parent trace is continued on the wire.
#[tokio::test]
async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
use axum::Router;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let seen_traceparent = Arc::new(Mutex::new(None::<String>));
let app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
let seen = Arc::clone(&seen_traceparent);
move |headers: HeaderMap| {
let calls = Arc::clone(&calls);
let seen = Arc::clone(&seen);
async move {
calls.fetch_add(1, Ordering::SeqCst);
if let Some(value) = headers
.get("traceparent")
.and_then(|value| value.to_str().ok())
{
*seen.lock().unwrap() = Some(value.to_owned());
}
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"task-worker-e2e",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"worker-e2e-key",
&json!({"email": "user@example.com"}),
&correlation,
)
.await
.expect("durable acceptance with executable input");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-e2e".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("completed"));
assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call");
assert_eq!(
seen_traceparent.lock().unwrap().as_deref(),
Some(correlation.trace_context().traceparent()),
"the durable attempt continues the accepted parent trace"
);
}
/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as
/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically.
#[tokio::test]
async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::Arc;
use std::sync::atomic::Ordering;
let registry = common::test_registry().await;
let upstream_app = Router::new().route(
"/crm/leads",
post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"task-worker-uncertain",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"worker-uncertain-key",
&json!({"email": "user@example.com"}),
&correlation,
)
.await
.expect("durable acceptance with executable input");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-uncertain".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("outcome_unknown") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("outcome_unknown"));
// No automatic retry: another authoritative scan never redispaches the uncertain attempt.
assert!(
store
.claim_batch("worker_after", 4, 10_000)
.await
.unwrap()
.is_empty()
);
}
/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced
/// terminal commit within the 30s budget.
#[tokio::test]
async fn task_worker_graceful_stop_drains_active_attempt() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_drain".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
},
scoped_key: "s".repeat(64),
input_fingerprint: "d".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
struct SlowHandler {
started: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for SlowHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
self.started.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(500)).await;
TaskAttemptOutcome::Completed(json!({"ok": true}))
}
}
let started = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
started: started.clone(),
});
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-drain".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(20);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
for _ in 0..100 {
if started.load(Ordering::SeqCst) > 0 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started");
let stop_requested = std::time::Instant::now();
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(Duration::from_secs(30), worker_task)
.await
.expect("worker stops inside the 30s budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
assert!(stop_requested.elapsed() <= Duration::from_secs(30));
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_drain".to_owned(),
};
let record = store.read(&access, accepted.task_id).await.unwrap();
assert_eq!(record.state, TaskState::Completed);
}
+5
View File
@@ -1,5 +1,10 @@
FROM node:22-alpine AS build
# The CI host uplink drops TLS 1.3 ClientHello carrying the hybrid
# X25519MLKEM768 key share; pin the npm stage to TLS 1.2 until that network
# defect is fixed. Certificates, SRI and the lockfile are unaffected.
ENV NODE_OPTIONS=--tls-max-v1.2
WORKDIR /app
COPY apps/ui/package.json apps/ui/package-lock.json ./
+4 -1
View File
@@ -272,7 +272,10 @@ fn record_unknown_approval_access_denial(reason: &'static str) {
);
}
fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool {
pub(super) fn allows_scope(
scopes: &[PlatformApiKeyScope],
required_scope: PlatformApiKeyScope,
) -> bool {
match required_scope {
PlatformApiKeyScope::Read => scopes.iter().any(|scope| {
matches!(
+2 -2
View File
@@ -51,15 +51,15 @@ mod auth_resolution;
mod dispatch;
mod endpoint;
mod legacy_dispatch;
mod task_worker;
mod tasks;
use self::approvals::{
approve_request, deny_request, get_approval_request, list_pending_approvals,
};
pub(super) use self::auth_resolution::resolve_operation_auth;
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
pub use self::task_worker::TaskWorkerHandler;
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
// through `super`, and the plain lib build does not use them (same pattern as
// `#[cfg(test)] use invocation_history::observe_invocation_history_outcome;` above).
@@ -19,18 +19,31 @@ pub(crate) async fn resolve_operation_auth(
workspace_id: &crank_core::WorkspaceId,
execution_config: &crank_core::ExecutionConfig,
) -> Result<Option<ResolvedAuth>, RuntimeError> {
if execution_config.auth_profile_ref.is_none() {
return Ok(None);
}
let span = Stage::AuthResolve.span();
let result = resolve_runtime_auth_for_task(
resolve_operation_auth_with(
&state.registry,
&state.secret_crypto,
workspace_id,
execution_config,
)
.instrument(span.clone())
.await;
.await
}
/// Same resolution for callers that hold the authoritative registry/secret-crypto pair without the
/// HTTP application state (the durable Task worker).
pub(crate) async fn resolve_operation_auth_with(
registry: &PostgresRegistry,
secret_crypto: &SecretCrypto,
workspace_id: &crank_core::WorkspaceId,
execution_config: &crank_core::ExecutionConfig,
) -> Result<Option<ResolvedAuth>, RuntimeError> {
if execution_config.auth_profile_ref.is_none() {
return Ok(None);
}
let span = Stage::AuthResolve.span();
let result =
resolve_runtime_auth_for_task(registry, secret_crypto, workspace_id, execution_config)
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
@@ -0,0 +1,282 @@
//! Application-level durable Task handler for Story 4.3.
use std::sync::Arc;
use std::time::{Duration, Instant};
use async_trait::async_trait;
use crank_core::{
AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId,
current_unix_ms,
};
use crank_registry::PostgresRegistry;
use crank_registry::PublishedAgentTool;
use crank_runtime::{
ExecutionAuthorization, ExecutionRequestError, RuntimeExecutionRequest, RuntimeExecutor,
SecretCrypto, TaskDispatchPermit, TaskHandler, TaskService, TaskWorkerError,
};
use super::TaskApplication;
use super::auth_resolution::resolve_operation_auth_with;
use super::tool_resolution::runtime_operation;
use crate::access::{allows_scope, credential_allows_security_level};
use crate::catalog::PublishedToolCatalog;
pub struct TaskWorkerHandler {
registry: PostgresRegistry,
catalog: PublishedToolCatalog,
runtime: RuntimeExecutor,
secret_crypto: SecretCrypto,
service: TaskService,
}
impl TaskWorkerHandler {
/// Production construction from the same application entry, registry and runtime the MCP
/// process serves requests with. The worker needs no HTTP state, only these authoritative
/// dependencies.
pub fn from_application(
tasks: &TaskApplication,
registry: PostgresRegistry,
runtime: RuntimeExecutor,
secret_crypto: SecretCrypto,
) -> Self {
Self {
registry,
catalog: tasks.catalog(),
runtime,
secret_crypto,
service: tasks.task_service(),
}
}
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
///
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
/// policy and revalidates the typed persisted Platform API key identity against the registry:
/// the key must still be active, unexpired and bound to this workspace/agent, and its exact
/// scopes/security level must still authorize the operation. Any denial is a typed error, so the
/// worker finishes the attempt without dispatch (fail closed before any network use).
///
/// An administrator Task has no authoritative Task revalidation API in the registry, so it stays
/// fail-closed here instead of inventing authority.
async fn authorize(&self, lease: &TaskLease) -> Result<PublishedAgentTool, TaskError> {
if lease.record.owner_kind != TaskOwnerKind::Agent {
return Err(TaskError::invalid(
"admin task authority revalidation is not implemented",
));
}
let Some(key_id) = lease.record.platform_api_key_id.as_ref() else {
return Err(TaskError::invalid("task authority reference is missing"));
};
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
let workspace = self
.registry
.get_workspace(&workspace_id)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?
.ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?;
let agents = self
.registry
.list_agents(&workspace_id)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
let agent = agents
.into_iter()
.find(|agent| agent.id.as_str() == lease.record.owner_id.as_str())
.ok_or_else(|| TaskError::invalid("task agent no longer exists"))?;
let catalog = self
.catalog
.get_catalog(&workspace.workspace.slug, &agent.slug)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
let tool = catalog
.tools
.into_iter()
.find(|tool| {
tool.operation.id.as_str() == lease.record.operation_id.as_str()
&& tool.operation.version == lease.record.pins.operation_version
})
.ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?;
if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() {
return Err(TaskError::invalid("task workspace binding mismatch"));
}
if tool.agent_id.as_str() != lease.record.owner_id.as_str() {
return Err(TaskError::invalid("task agent binding mismatch"));
}
if tool
.operation
.execution_config
.approval_policy
.as_ref()
.is_some_and(|policy| policy.required)
{
return Err(TaskError::invalid(
"missing approval proof for task operation",
));
}
// Authoritative revalidation of the exact immutable key identity. A revoked, expired,
// foreign-workspace or foreign-agent key is absent here and never reaches the network.
let key = self
.registry
.revalidate_platform_api_key(&workspace_id, &agent_id, key_id)
.await
.map_err(|_| TaskError::unavailable("task authority store unavailable"))?
.ok_or_else(|| TaskError::invalid("task authority is no longer active"))?;
let credential = VerifiedMachineCredential {
machine_access_mode: MachineAccessMode::StaticAgentKey,
max_security_level: OperationSecurityLevel::Standard,
scopes: key.api_key.scopes.clone(),
platform_api_key_id: Some(key.api_key.id.clone()),
};
if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) {
return Err(TaskError::invalid("task authority scope is insufficient"));
}
if !credential_allows_security_level(&credential, tool.operation.security_level) {
return Err(TaskError::invalid(
"task authority security level is insufficient",
));
}
Ok(tool)
}
}
#[async_trait]
impl TaskHandler for TaskWorkerHandler {
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> {
self.authorize(lease)
.await
.map_err(TaskWorkerError::Store)?;
Ok(())
}
async fn execute(
&self,
lease: &TaskLease,
_attempt_id: &str,
cancelled: Arc<std::sync::atomic::AtomicBool>,
permit: TaskDispatchPermit,
) -> crank_runtime::TaskAttemptOutcome {
use crank_runtime::TaskAttemptOutcome;
use std::sync::atomic::Ordering;
// The lease may have been lost between the checkpoint and the outbound call; never dispatch
// on behalf of a worker that no longer owns the Task.
if cancelled.load(Ordering::Acquire) {
return TaskAttemptOutcome::OutcomeUnknown;
}
let tool = match self.authorize(lease).await {
Ok(tool) => tool,
Err(error) => {
return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()}));
}
};
let input = match self.service.executable_input(lease).await {
Ok(input) => input,
Err(error) => {
return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()}));
}
};
let operation = runtime_operation(&tool);
let now_ms = current_unix_ms();
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
if deadline_ms <= now_ms {
return TaskAttemptOutcome::Failed(
serde_json::json!({"code": "task deadline elapsed"}),
);
}
let deadline = Instant::now()
+ Duration::from_millis((deadline_ms - now_ms).min(i64::MAX as i64) as u64);
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
let resolved_auth = match resolve_operation_auth_with(
&self.registry,
&self.secret_crypto,
&workspace_id,
&operation.execution_config,
)
.await
{
Ok(auth) => auth,
Err(error) => {
return TaskAttemptOutcome::Failed(
serde_json::json!({ "code": error.to_string() }),
);
}
};
// Continue the parent trace persisted with the accepted Task; the worker's attempt span is a
// child of it, and the executor records the attempt identity. Trace data stays out of labels.
let trace_context = match TraceContext::parse(&lease.record.support.traceparent) {
Ok(context) => context,
Err(_) => {
return TaskAttemptOutcome::Failed(
serde_json::json!({ "code": "task trace parent is invalid" }),
);
}
};
let request_context = crank_runtime::RuntimeRequestContext::new(
lease.record.support.request_id.clone(),
trace_context,
)
.with_metering_context(
workspace_id.clone(),
Some(agent_id.clone()),
crank_core::InvocationSource::AgentToolCall,
);
let request = match RuntimeExecutionRequest::try_new(
&workspace_id,
ExecutionOrigin::AgentSnapshot,
Some(&agent_id),
&operation,
&input,
ExecutionAuthorization::Authorized,
resolved_auth.as_ref(),
&request_context,
deadline,
) {
Ok(request) => request,
Err(error) => {
return TaskAttemptOutcome::Failed(match error {
ExecutionRequestError::InvalidOrigin => {
serde_json::json!({"code": "invalid execution origin"})
}
ExecutionRequestError::InvalidOperationVersion => {
serde_json::json!({"code": "invalid operation version"})
}
ExecutionRequestError::DeadlineElapsed => {
serde_json::json!({"code": "task deadline elapsed"})
}
ExecutionRequestError::InvalidMeteringScope => {
serde_json::json!({"code": "invalid metering scope"})
}
});
}
};
if cancelled.load(Ordering::Acquire) {
return TaskAttemptOutcome::OutcomeUnknown;
}
match self.runtime.execute_task_outcome(request, permit).await {
Ok(success) => TaskAttemptOutcome::Completed(success.output),
Err(failure) => {
// An ambiguous outbound failure is recorded as uncertainty, never as a retryable
// failure: the durable checkpoint already exists, so Story 4.4 may reconcile it.
if failure.outcome_certainty() == crank_core::OutcomeCertainty::OutcomeUnknown {
return TaskAttemptOutcome::OutcomeUnknown;
}
TaskAttemptOutcome::Failed(
serde_json::json!({ "code": format!("{:?}", failure.error_code()) }),
)
}
}
}
}
+29 -2
View File
@@ -14,8 +14,8 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord,
ToolExecutionMode,
CorrelationContext, PlatformApiKeyId, TaskAcceptance, TaskAccess, TaskError, TaskId,
TaskOwnerKind, TaskRecord, ToolExecutionMode,
};
use crank_registry::{PostgresRegistry, PublishedAgentTool};
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
@@ -64,6 +64,20 @@ impl TaskPrincipal {
}
}
/// Typed immutable Platform API key identity of this principal. A verified agent credential
/// without a key ID cannot be revalidated by a worker, so it is refused instead of accepted;
/// an administrator has no machine key and is stored as `None` (the worker fails closed).
fn platform_api_key_id(&self) -> Result<Option<PlatformApiKeyId>, TaskApplicationError> {
match self {
Self::Agent { credential, .. } => credential
.platform_api_key_id
.clone()
.map(Some)
.ok_or(TaskApplicationError::Denied),
Self::Admin { .. } => Ok(None),
}
}
/// Bounded server-derived authorization scope identity of this principal.
fn authorization_scope(&self) -> String {
let identity = match self {
@@ -131,6 +145,17 @@ impl TaskApplication {
}
}
/// Runtime task service carrying encrypted executable input.
pub fn task_service(&self) -> TaskService {
self.service.clone()
}
/// Published catalog of the same application entry, for the durable Task worker's pre-dispatch
/// revalidation.
pub(super) fn catalog(&self) -> PublishedToolCatalog {
self.catalog.clone()
}
/// Readiness of the durable Task store this entry depends on.
pub async fn health(&self) -> Result<(), TaskError> {
self.service.health().await
@@ -170,6 +195,7 @@ impl TaskApplication {
// of accepting work a worker could never safely execute.
return Err(TaskApplicationError::ApprovalRequired);
}
let platform_api_key_id = principal.platform_api_key_id()?;
let draft = |operation_version: u32| {
let (owner_kind, owner_id) = principal.owner(&tool);
TaskAcceptDraft {
@@ -177,6 +203,7 @@ impl TaskApplication {
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
platform_api_key_id: platform_api_key_id.clone(),
operation_id: tool.operation.id.as_str().to_owned(),
operation_version,
policy_mode: policy.mode,
+3 -3
View File
@@ -16,7 +16,7 @@ mod tool_search;
mod transport;
pub use app::{
TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts,
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
build_app_with_background_workers_and_limits,
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler, build_app,
build_app_with_artifacts, build_app_with_artifacts_and_tasks,
build_app_with_background_workers, build_app_with_background_workers_and_limits,
};
+11 -8
View File
@@ -76,10 +76,12 @@ pub mod domain {
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use crate::task::{
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
TASK_IDENTITY_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskSupportIds,
};
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
pub use crate::tool_quality::{
@@ -202,11 +204,12 @@ pub use product_event::{
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use task::{
TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key,
validate_task_envelope,
TaskError, TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport,
TaskRecord, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
redact_task_envelope, sensitive_task_key, validate_task_envelope,
};
pub use tool_catalog::{
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
+147 -3
View File
@@ -15,10 +15,15 @@ use serde_json::Value;
use uuid::Uuid;
use crate::correlation::{RequestId, TraceContext, TraceId};
use crate::ids::PlatformApiKeyId;
use crate::operation::ToolExecutionMode;
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
/// Maximum byte budget of the decrypted executable Task input.
pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES;
/// Maximum byte budget of the stored encrypted executable Task input ciphertext.
pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304;
/// Maximum nesting depth of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
/// Maximum number of JSON members/nodes of a persisted Task envelope.
@@ -351,6 +356,16 @@ pub fn redact_task_envelope(value: &Value) -> Value {
}
}
/// Recursive check used to fail closed on sensitive fields before encryption.
pub fn contains_sensitive_task_keys(value: &Value) -> bool {
match value {
Value::Object(object) => object
.iter()
.any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)),
Value::Array(values) => values.iter().any(contains_sensitive_task_keys),
_ => false,
}
}
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
/// queue capacity commit together, and no acknowledgement exists before that commit.
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -360,6 +375,10 @@ pub struct TaskAcceptRequest {
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
/// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the
/// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively.
/// `None` means the accepting principal has no machine key (administrator) and fail-closes.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
pub pins: TaskPins,
/// Digest of the caller key over the full owner/authorization/Tool scope.
@@ -368,6 +387,10 @@ pub struct TaskAcceptRequest {
pub input_fingerprint: String,
pub input: TaskEnvelope,
pub support: TaskSupportIds,
/// Bounded encrypted executable input, persisted alongside the redacted input projection.
pub executable_input_ciphertext: Option<String>,
pub executable_input_key_version: Option<String>,
pub executable_input_master_key_epoch: Option<i64>,
}
impl TaskAcceptRequest {
@@ -382,6 +405,13 @@ impl TaskAcceptRequest {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
if let Some(key_id) = self.platform_api_key_id.as_ref()
&& (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES)
{
return Err(TaskError::invalid(
"task platform api key reference is invalid",
));
}
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
return Err(TaskError::invalid("task scoped key digest is invalid"));
}
@@ -394,7 +424,40 @@ impl TaskAcceptRequest {
return Err(TaskError::invalid("task input fingerprint is invalid"));
}
self.pins.validate()?;
self.support.validate()
self.support.validate()?;
let has_ciphertext = self.executable_input_ciphertext.is_some();
let has_key = self.executable_input_key_version.is_some();
let has_epoch = self.executable_input_master_key_epoch.is_some();
match (has_ciphertext, has_key, has_epoch) {
(false, false, false) => {}
(true, true, true) => {
let ciphertext = self.executable_input_ciphertext.as_deref().unwrap();
if ciphertext.is_empty()
|| ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
{
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
let key_version = self.executable_input_key_version.as_deref().unwrap();
if key_version.is_empty() || key_version.len() > 64 {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
if self.executable_input_master_key_epoch.unwrap_or_default() < 1 {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
}
_ => {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
}
Ok(())
}
}
@@ -448,6 +511,9 @@ pub struct TaskRecord {
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
/// Typed immutable identity of the Platform API key that authorized the acceptance; the worker
/// revalidates this exact key against the registry before any outbound dispatch.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
pub input_fingerprint: String,
@@ -458,6 +524,16 @@ pub struct TaskRecord {
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
pub attempts: u32,
/// Monotonic lease/fencing token. It never resets and protects Task state mutations.
pub fence: u64,
/// Random process-instance lease owner, if a worker currently owns the Task.
pub lease_owner: Option<String>,
/// Lease expiry epoch milliseconds, if the Task is claimed.
pub lease_until_unix_ms: Option<i64>,
/// Current dispatch attempt identity, committed before any outbound adapter call.
pub current_attempt_id: Option<String>,
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
pub dispatch_started_at_unix_ms: Option<i64>,
/// Monotonic lifecycle version used by the terminal CAS.
pub version: u32,
pub created_at_unix_ms: i64,
@@ -542,6 +618,15 @@ impl TaskError {
}
}
/// Leased view of a claimed Task: its durable record plus lease fencing identity.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskLease {
pub record: TaskRecord,
pub owner: String,
pub fence: u64,
pub lease_until: i64,
}
/// Authoritative durable Task port.
#[async_trait]
pub trait TaskStore: Send + Sync {
@@ -566,9 +651,59 @@ pub trait TaskStore: Send + Sync {
/// Applies exactly one conditional terminal transition and releases its intent atomically.
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
/// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes
/// expired terminal records. Live and non-terminal work is never deleted.
/// Reads encrypted executable input only under a valid, unexpired current lease.
async fn read_executable_input(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
) -> Result<(String, String, i64), TaskError>;
/// Bounded retention...
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
/// Claims a bounded batch of queued or safely recoverable Tasks under leases.
async fn claim_batch(
&self,
owner: &str,
batch_limit: u32,
lease_ms: i64,
) -> Result<Vec<TaskLease>, TaskError>;
/// Renews a lease owned by `owner` when fence and expiry still match.
async fn renew_lease(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
lease_ms: i64,
) -> Result<TaskLease, TaskError>;
/// Persists redacted progress under a valid lease.
async fn write_progress(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
progress: TaskEnvelope,
) -> Result<TaskRecord, TaskError>;
/// Commits an outbound attempt identity and dispatch timestamp before network use.
async fn checkpoint_dispatch(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
attempt_id: String,
) -> Result<TaskRecord, TaskError>;
/// Applies a terminal transition only under a valid lease and releases intent atomically.
async fn finish_leased(
&self,
request: TaskFinishRequest,
owner: &str,
fence: u64,
) -> Result<TaskRecord, TaskError>;
}
#[cfg(test)]
@@ -583,6 +718,7 @@ mod tests {
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
@@ -599,6 +735,9 @@ mod tests {
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
}
}
@@ -694,6 +833,11 @@ mod tests {
let mut bad_retention = request();
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
assert!(bad_retention.validate().is_err());
// The persisted authority reference is bounded and never empty.
let mut empty_key_reference = request();
empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new(""));
assert!(empty_key_reference.validate().is_err());
}
#[test]
+6
View File
@@ -7,6 +7,9 @@ mod artifact_metadata_v12;
mod authority;
mod baseline_v1;
mod durable_tasks_v25;
mod durable_tasks_v26;
mod durable_tasks_v27;
mod durable_tasks_v28;
mod execution_admission_v24;
mod execution_outcome_v5;
mod import_plan_v14;
@@ -39,6 +42,9 @@ mod schema_guard_v22;
mod schema_guard_v23;
mod schema_guard_v24;
mod schema_guard_v25;
mod schema_guard_v26;
mod schema_guard_v27;
mod schema_guard_v28;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -5,6 +5,9 @@ use super::approval_side_effects_v10;
use super::artifact_cleanup_indexes_v13;
use super::artifact_metadata_v12;
use super::durable_tasks_v25;
use super::durable_tasks_v26;
use super::durable_tasks_v27;
use super::durable_tasks_v28;
use super::execution_admission_v24;
use super::execution_outcome_v5;
use super::import_plan_v14;
@@ -41,9 +44,10 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 25;
const CURRENT_VERSION: i64 = 28;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -344,6 +348,24 @@ impl MigrationAuthority {
durable_tasks_v25::SOURCE_SHA256,
24,
),
expand_descriptor(
26,
durable_tasks_v26::NAME,
durable_tasks_v26::SOURCE_SHA256,
25,
),
expand_descriptor(
27,
durable_tasks_v27::NAME,
durable_tasks_v27::SOURCE_SHA256,
26,
),
expand_descriptor(
28,
durable_tasks_v28::NAME,
durable_tasks_v28::SOURCE_SHA256,
27,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -509,6 +531,15 @@ impl MigrationAuthority {
if from < 25 {
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
}
if from < 26 {
durable_tasks_v26::apply(&mut transaction, &Self::sequence()[25]).await?;
}
if from < 27 {
durable_tasks_v27::apply(&mut transaction, &Self::sequence()[26]).await?;
}
if from < 28 {
durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v26";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v26.sql");
pub(super) const SOURCE_SHA256: &str =
"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v26",
Some(26),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v26",
Some(26),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,6 @@
-- V26: PostgreSQL lease/fencing/CAS additions for durable background Tasks (Story 4.3).
alter table tasks add column if not exists fence bigint not null default 0;
alter table tasks add column if not exists lease_owner text;
alter table tasks add column if not exists lease_until timestamptz;
alter table tasks add column if not exists current_attempt_id text;
alter table tasks add column if not exists dispatch_started_at timestamptz;
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v27";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v27.sql");
pub(super) const SOURCE_SHA256: &str =
"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v27",
Some(27),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v27",
Some(27),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,19 @@
-- V27: encrypted executable Task input (Story 4.3).
alter table tasks add column if not exists executable_input_ciphertext text;
alter table tasks add column if not exists executable_input_key_version text;
alter table tasks add column if not exists executable_input_master_key_epoch bigint;
alter table tasks add constraint tasks_executable_input_key_version_check check (
executable_input_ciphertext is null
or (executable_input_key_version is not null and octet_length(executable_input_key_version) between 1 and 64)
);
alter table tasks add column if not exists target_executable_input_ciphertext text;
alter table tasks add column if not exists target_executable_input_key_version text;
alter table tasks add column if not exists target_executable_input_master_key_epoch bigint;
alter table tasks add constraint tasks_executable_input_epoch_check check (
executable_input_ciphertext is null
or (executable_input_master_key_epoch is not null
and executable_input_master_key_epoch >= 1)
);
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v28";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v28.sql");
pub(super) const SOURCE_SHA256: &str =
"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v28",
Some(28),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v28",
Some(28),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,8 @@
-- V28: typed Platform API key authority reference of a durable Task (Story 4.3).
-- Only the immutable key identifier is stored, never the bearer secret.
alter table tasks add column if not exists platform_api_key_id text;
alter table tasks add constraint tasks_platform_api_key_id_check check (
platform_api_key_id is null
or octet_length(platform_api_key_id) between 1 and 160
);
@@ -480,8 +480,40 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 26 {
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 27 {
super::schema_guard_v26::validate_v26_durable_tasks(connection).await?;
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 28 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v27::validate_v27_durable_tasks(connection).await?;
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v28::validate_v28_durable_tasks(connection).await?;
}
Ok(())
}
@@ -0,0 +1,120 @@
//! V26 schema guard: lease/fencing columns for durable background Tasks (Story 4.3).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const TASKS: &str = "tasks";
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
("task_id", "uuid", false),
("workspace_id", "text", false),
("owner_kind", "text", false),
("owner_id", "text", false),
("authorization_scope", "text", false),
("operation_id", "text", false),
("operation_version", "integer", false),
("scoped_key", "text", false),
("input_fingerprint", "text", false),
("request_id", "text", false),
("trace_id", "text", false),
("traceparent", "text", false),
("trace_sampled", "boolean", false),
("policy_mode", "text", false),
("approval_reference", "text", true),
("state", "text", false),
("version", "integer", false),
("attempts", "integer", false),
("input_envelope", "jsonb", false),
("progress_envelope", "jsonb", true),
("result_envelope", "jsonb", true),
("error_envelope", "jsonb", true),
("execution_deadline", "timestamp with time zone", false),
("retain_until", "timestamp with time zone", false),
("created_at", "timestamp with time zone", false),
("updated_at", "timestamp with time zone", false),
("fence", "bigint", false),
("lease_owner", "text", true),
("lease_until", "timestamp with time zone", true),
("current_attempt_id", "text", true),
("dispatch_started_at", "timestamp with time zone", true),
];
fn v26_error(stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(26),
"restore_known_good_backup",
)
}
pub(super) async fn validate_v26_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
// delegate to v25 absent for relation mostly; V26 columns must not exist.
for column in [
"fence",
"lease_owner",
"lease_until",
"current_attempt_id",
"dispatch_started_at",
] {
if super::schema_guard::column_exists(connection, TASKS, column).await? {
return Ok(false);
}
}
}
Ok(true)
}
pub(super) async fn validate_v26_base_columns(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
if !relation_exists(connection, TASKS).await? {
return Err(v26_error("v26.relations"));
}
for (column, data_type, nullable) in TASK_COLUMNS {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if *nullable { "YES" } else { "NO" })
});
if !valid {
return Err(v26_error("v26.columns"));
}
}
Ok(())
}
pub(super) async fn validate_v26_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
validate_v26_base_columns(connection).await?;
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
return Err(v26_error("v26.columns"));
}
// Constraints/index checks inherited conceptually from V25; V26 adds columns only.
Ok(())
}
@@ -0,0 +1,85 @@
//! V27 schema guard: encrypted executable Task input columns (Story 4.3).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
use super::schema_guard_v26::validate_v26_base_columns;
const TASKS: &str = "tasks";
pub(super) async fn validate_v27_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
for column in [
"executable_input_ciphertext",
"executable_input_key_version",
"executable_input_master_key_epoch",
] {
if super::schema_guard::column_exists(connection, TASKS, column).await? {
return Ok(false);
}
}
}
Ok(true)
}
pub(super) async fn validate_v27_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
validate_v26_base_columns(connection).await?;
// Additional V27 columns.
let extra = [
("executable_input_ciphertext", "text", true),
("executable_input_key_version", "text", true),
("executable_input_master_key_epoch", "bigint", true),
("target_executable_input_ciphertext", "text", true),
("target_executable_input_key_version", "text", true),
("target_executable_input_master_key_epoch", "bigint", true),
];
for (column, data_type, nullable) in extra {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if nullable { "YES" } else { "NO" })
});
if !valid {
return Err(MigrationError::new(
"partial_sequence",
"v27.columns",
Some(27),
"restore_known_good_backup",
));
}
}
// Exact total column count for version 27: 43 V26 columns + 3 V27 columns.
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(37).unwrap_or_default() {
return Err(MigrationError::new(
"partial_sequence",
"v27.columns",
Some(27),
"restore_known_good_backup",
));
}
Ok(())
}
@@ -0,0 +1,89 @@
//! V28 schema guard: typed Platform API key authority reference of a durable Task (Story 4.3).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const TASKS: &str = "tasks";
pub(super) async fn validate_v28_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
if super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await? {
return Ok(false);
}
}
Ok(true)
}
pub(super) async fn validate_v28_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
// The V27 column set is still required and unchanged; V28 only appends one column.
validate_v27_base_columns(connection).await?;
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind("platform_api_key_id")
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some("text")
&& row.try_get::<String, _>("is_nullable").ok().as_deref() == Some("YES")
});
if !valid {
return Err(MigrationError::new(
"partial_sequence",
"v28.columns",
Some(28),
"restore_known_good_backup",
));
}
// Exact total column count for version 28: the 37 V27 columns + 1 V28 column.
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(38).unwrap_or_default() {
return Err(MigrationError::new(
"partial_sequence",
"v28.columns",
Some(28),
"restore_known_good_backup",
));
}
Ok(())
}
/// V27 columns must still be exactly present before V28 is considered applied.
async fn validate_v27_base_columns(connection: &mut PgConnection) -> Result<(), MigrationError> {
for column in [
"executable_input_ciphertext",
"executable_input_key_version",
"executable_input_master_key_epoch",
"target_executable_input_ciphertext",
"target_executable_input_key_version",
"target_executable_input_master_key_epoch",
] {
if !super::schema_guard::column_exists(connection, TASKS, column).await? {
return Err(MigrationError::new(
"partial_sequence",
"v28.columns",
Some(28),
"restore_known_good_backup",
));
}
}
Ok(())
}
@@ -577,6 +577,145 @@ pub(super) async fn insert_admission_entry(
Ok(u64::try_from(ticket_id).unwrap_or_default())
}
/// Shared grant step used by the durable Task store: grants one reserved Task intent if capacity allows,
/// marks it `executing`, persists the weighted round-robin cursor, and returns its Task id. It reuses
/// the same selector/capacity logic as synchronous admission so no second policy exists.
pub(super) async fn grant_next_task(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
cursor: &AdmissionCursor,
limits: &AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> Result<(Option<Uuid>, AdmissionCursor), AttemptError> {
let rows = sqlx::query(
"select ticket_id, agent_key, operation_id, task_id
from execution_admission_entries
where state = 'reserved' and task_id is not null
order by ticket_id
limit $1
for update skip locked",
)
.bind(QUEUE_READ_LIMIT)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
if rows.is_empty() {
return Ok((None, cursor.clone()));
}
struct Candidate {
entry: AdmissionQueueEntry,
task_id: Uuid,
}
let mut candidates = Vec::with_capacity(rows.len());
for row in rows {
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
let task_id: Uuid = row.try_get("task_id").map_err(AttemptError::from_sql)?;
candidates.push(Candidate {
entry: AdmissionQueueEntry {
ticket_id: u64::try_from(ticket_id).unwrap_or_default(),
sequence: 0,
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
operation_id: row
.try_get("operation_id")
.map_err(AttemptError::from_sql)?,
},
task_id,
});
}
let queue: Vec<AdmissionQueueEntry> = candidates.iter().map(|c| c.entry.clone()).collect();
let Some(selection) = select_next_grant(&queue, cursor, scheduler) else {
return Ok((None, cursor.clone()));
};
let head_index = selection.index;
let head = &candidates[head_index];
let global_executing: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries where state = 'executing'",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
let agent_counts = sqlx::query(
"select agent_key as key, count(*)::bigint as count
from execution_admission_entries where state = 'executing' group by agent_key",
)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let mut agent_map = std::collections::BTreeMap::new();
for row in agent_counts {
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
agent_map.insert(key, u64::try_from(count).unwrap_or_default());
}
let tool_counts = sqlx::query(
"select operation_id as key, count(*)::bigint as count
from execution_admission_entries where state = 'executing' group by operation_id",
)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let mut tool_map = std::collections::BTreeMap::new();
for row in tool_counts {
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
tool_map.insert(key, u64::try_from(count).unwrap_or_default());
}
if u64::try_from(global_executing).unwrap_or(u64::MAX)
>= u64::try_from(limits.global_limit).unwrap_or(u64::MAX)
{
return Ok((None, cursor.clone()));
}
let agent_executing = agent_map
.get(head.entry.agent_key.as_str())
.copied()
.unwrap_or_default();
if agent_executing >= u64::try_from(limits.agent_limit).unwrap_or(u64::MAX) {
return Ok((None, cursor.clone()));
}
let tool_executing = tool_map
.get(head.entry.operation_id.as_str())
.copied()
.unwrap_or_default();
if tool_executing >= u64::try_from(limits.tool_limit).unwrap_or(u64::MAX) {
return Ok((None, cursor.clone()));
}
sqlx::query(
"update execution_admission_entries
set state = 'executing', granted_at = clock_timestamp()
where ticket_id = $1 and state = 'reserved'",
)
.bind(i64::try_from(head.entry.ticket_id).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
sqlx::query(
"update execution_admission_controllers
set cursor_agent = $2,
cursor_agent_credit = $3,
cursor_tool = $4,
cursor_tool_credit = $5,
updated_at = now()
where workspace_id = $1",
)
.bind(workspace_id)
.bind(selection.cursor.agent.as_deref())
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
.bind(selection.cursor.tool.as_deref())
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok((Some(head.task_id), selection.cursor))
}
#[derive(Debug)]
pub(super) enum AttemptError {
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
Contended,
@@ -170,6 +170,69 @@ impl PostgresRegistry {
.transpose()
}
/// Authoritative revalidation of the Platform API key that authorized a durable Task (Story 4.3).
///
/// The key is returned only when the exact immutable ID is still active, unexpired and bound to
/// the same workspace and agent; otherwise `None`, so a worker fails closed. Only the immutable
/// identity is supplied here — the bearer secret is never stored or re-read.
pub async fn revalidate_platform_api_key(
&self,
workspace_id: &WorkspaceId,
agent_id: &AgentId,
key_id: &PlatformApiKeyId,
) -> Result<Option<PlatformApiKeyRecord>, RegistryError> {
let row = sqlx::query(
"select
k.id,
k.workspace_id,
k.agent_id,
k.key_kind,
k.name,
k.prefix,
k.scopes_json,
k.status,
k.created_at,
k.last_used_at,
k.expires_at,
k.allowed_origins_json
from platform_api_keys k
where k.id = $1
and k.workspace_id = $2
and k.agent_id = $3
and k.key_kind = 'mcp_client'
and k.status = 'active'
and (k.expires_at is null or k.expires_at > now())
limit 1",
)
.bind(key_id.as_str())
.bind(workspace_id.as_str())
.bind(agent_id.as_str())
.fetch_optional(&self.pool)
.await?;
row.map(|row| {
Ok(PlatformApiKeyRecord {
api_key: PlatformApiKey {
id: PlatformApiKeyId::new(row.get::<String, _>("id")),
workspace_id: WorkspaceId::new(row.get::<String, _>("workspace_id")),
agent_id: row.get::<Option<String>, _>("agent_id").map(AgentId::new),
key_kind: deserialize_enum_text(&row.get::<String, _>("key_kind"), "key_kind")?,
name: row.get("name"),
prefix: row.get("prefix"),
scopes: deserialize_json_value(row.get::<Value, _>("scopes_json"))?,
status: deserialize_enum_text(&row.get::<String, _>("status"), "status")?,
created_at: row.get("created_at"),
last_used_at: row.get("last_used_at"),
expires_at: row.get("expires_at"),
allowed_origins: deserialize_json_value(
row.get::<Value, _>("allowed_origins_json"),
)?,
},
})
})
.transpose()
}
pub async fn get_approval_api_key_by_secret_for_agent_slug(
&self,
workspace_slug: &str,
@@ -1,6 +1,6 @@
use super::*;
const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
pub(super) const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
impl PostgresRegistry {
pub async fn verify_or_register_master_key_identity(
@@ -356,12 +356,19 @@ impl PostgresRegistry {
return Err(RegistryError::MasterKeyRotationConflict);
}
let total = sqlx::query_scalar::<_, i64>(
let secret_total = sqlx::query_scalar::<_, i64>(
"select count(*) from secret_versions where master_key_epoch = $1",
)
.bind(source_epoch)
.fetch_one(&mut *tx)
.await?;
let task_total = sqlx::query_scalar::<_, i64>(
"select count(*) from tasks where executable_input_master_key_epoch = $1",
)
.bind(source_epoch)
.fetch_one(&mut *tx)
.await?;
let total = secret_total + task_total;
let rotation_id = format!("master-key-e{source_epoch}-to-e{target_epoch}");
let aborted = sqlx::query(
"select
@@ -403,6 +410,16 @@ impl PostgresRegistry {
.bind(target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update tasks
set target_executable_input_ciphertext = null,
target_executable_input_key_version = null,
target_executable_input_master_key_epoch = null
where target_executable_input_master_key_epoch = $1",
)
.bind(target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update master_key_rotations
set state = 'running',
@@ -662,9 +679,24 @@ impl PostgresRegistry {
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update tasks
set executable_input_ciphertext = target_executable_input_ciphertext,
executable_input_key_version = target_executable_input_key_version,
executable_input_master_key_epoch = target_executable_input_master_key_epoch,
target_executable_input_ciphertext = null,
target_executable_input_key_version = null,
target_executable_input_master_key_epoch = null
where executable_input_master_key_epoch = $1
and target_executable_input_master_key_epoch = $2",
)
.bind(rotation.source_epoch)
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update master_key_rotations
set state = 'promoted',
set state = 'promoted',
updated_at = $2::timestamptz
where id = $1",
)
@@ -697,6 +729,16 @@ impl PostgresRegistry {
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update tasks
set target_executable_input_ciphertext = null,
target_executable_input_key_version = null,
target_executable_input_master_key_epoch = null
where target_executable_input_master_key_epoch = $1",
)
.bind(rotation.target_epoch)
.execute(&mut *tx)
.await?;
sqlx::query(
"update master_key_rotations
set state = 'aborted',
@@ -883,7 +925,7 @@ async fn select_active_master_key_identity_for_update(
Ok(row.map(map_master_key_identity))
}
async fn select_master_key_rotation_for_update(
pub(super) async fn select_master_key_rotation_for_update(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
rotation_id: &str,
) -> Result<MasterKeyRotationRecord, RegistryError> {
@@ -915,16 +957,19 @@ async fn select_master_key_rotation_for_update(
})
}
async fn count_staged_rotation_targets(
pub(super) async fn count_staged_rotation_targets(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
source_epoch: i64,
target_epoch: i64,
) -> Result<i64, RegistryError> {
let count = sqlx::query_scalar::<_, i64>(
"select count(*)
from secret_versions
where master_key_epoch = $1
and target_master_key_epoch = $2",
"select
(select count(*) from secret_versions
where master_key_epoch = $1
and target_master_key_epoch = $2) +
(select count(*) from tasks
where executable_input_master_key_epoch = $1
and target_executable_input_master_key_epoch = $2)",
)
.bind(source_epoch)
.bind(target_epoch)
@@ -0,0 +1,143 @@
//! Task ciphertext paging/staging of the master-key rotation (Story 4.3).
//!
//! Extracted from `master_key.rs` without logic changes so that module stays within the
//! code-health line limit; the operations and their SQL are unchanged.
use super::master_key::{
MASTER_KEY_PAGE_LIMIT_MAX, count_staged_rotation_targets, select_master_key_rotation_for_update,
};
use super::*;
fn map_master_key_task_version(
row: sqlx::postgres::PgRow,
) -> Result<MasterKeyTaskVersion, RegistryError> {
Ok(MasterKeyTaskVersion {
task_id: row.get::<String, _>("task_id"),
ciphertext: row
.get::<Option<String>, _>("executable_input_ciphertext")
.unwrap_or_default(),
key_version: row
.get::<Option<String>, _>("executable_input_key_version")
.unwrap_or_default(),
epoch: row
.get::<Option<i64>, _>("executable_input_master_key_epoch")
.unwrap_or_default(),
target_ciphertext: row.get::<Option<String>, _>("target_executable_input_ciphertext"),
target_key_version: row.get::<Option<String>, _>("target_executable_input_key_version"),
target_epoch: row.get::<Option<i64>, _>("target_executable_input_master_key_epoch"),
})
}
impl PostgresRegistry {
pub async fn list_tasks_for_master_key_epoch_page(
&self,
epoch: i64,
after_task_id: Option<&str>,
limit: i64,
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
if epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
return Err(RegistryError::InvalidMasterKeyIdentity);
}
let rows = sqlx::query(
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
)
.bind(epoch)
.bind(after_task_id)
.bind(limit)
.fetch_all(&self.pool)
.await?;
rows.into_iter().map(map_master_key_task_version).collect()
}
pub async fn list_target_tasks_for_master_key_rotation_page(
&self,
target_epoch: i64,
after_task_id: Option<&str>,
limit: i64,
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
if target_epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
return Err(RegistryError::InvalidMasterKeyIdentity);
}
let rows = sqlx::query(
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where target_executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
)
.bind(target_epoch)
.bind(after_task_id)
.bind(limit)
.fetch_all(&self.pool)
.await?;
rows.into_iter().map(map_master_key_task_version).collect()
}
pub async fn stage_master_key_rotation_task_ciphertext(
&self,
rotation_id: &str,
task_id: &str,
source_epoch: i64,
target_ciphertext: &str,
target_key_version: &str,
target_epoch: i64,
now: &OffsetDateTime,
) -> Result<MasterKeyRotationRecord, RegistryError> {
let mut tx = self.pool.begin().await?;
let rotation = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
if rotation.state != "running"
|| rotation.source_epoch != source_epoch
|| rotation.target_epoch != target_epoch
{
return Err(RegistryError::MasterKeyRotationConflict);
}
let result = sqlx::query(
"update tasks
set target_executable_input_ciphertext = $3,
target_executable_input_key_version = $4,
target_executable_input_master_key_epoch = $5
where task_id::text = $1
and executable_input_master_key_epoch = $2
and (
target_executable_input_master_key_epoch is null
or target_executable_input_master_key_epoch = $5
)
and (
target_executable_input_ciphertext is null
or target_executable_input_ciphertext = $3
)",
)
.bind(task_id)
.bind(source_epoch)
.bind(target_ciphertext)
.bind(target_key_version)
.bind(target_epoch)
.execute(&mut *tx)
.await?;
if result.rows_affected() != 1 {
return Err(RegistryError::MasterKeyRotationConflict);
}
let processed = count_staged_rotation_targets(&mut tx, source_epoch, target_epoch).await?;
sqlx::query(
"update master_key_rotations
set checkpoint_secret_id = checkpoint_secret_id,
processed_secret_versions = $2,
updated_at = $3::timestamptz
where id = $1",
)
.bind(rotation_id)
.bind(processed)
.bind(now)
.execute(&mut *tx)
.await?;
let updated = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
tx.commit().await?;
Ok(updated)
}
}
pub struct MasterKeyTaskVersion {
pub task_id: String,
pub ciphertext: String,
pub key_version: String,
pub epoch: i64,
pub target_ciphertext: Option<String>,
pub target_key_version: Option<String>,
pub target_epoch: Option<i64>,
}
@@ -10,6 +10,7 @@ mod auth;
mod connection;
mod import_job;
mod master_key;
mod master_key_tasks;
mod observability;
mod onboarding;
mod onboarding_import;
+408 -10
View File
@@ -13,9 +13,9 @@
use async_trait::async_trait;
use crank_core::{
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode,
PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError,
TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskStore, TaskSupportIds, ToolExecutionMode,
};
use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction};
@@ -27,10 +27,14 @@ use super::admission::{
};
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode,
authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version,
policy_mode,
approval_reference,
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
result_envelope, error_envelope, attempts, version,
result_envelope, error_envelope, attempts, fence, lease_owner, lease_until,
(extract(epoch from lease_until) * 1000)::bigint as lease_until_ms,
current_attempt_id, dispatch_started_at,
(extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version,
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
(extract(epoch from created_at) * 1000)::bigint as created_ms,
@@ -40,6 +44,7 @@ const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_i
pub struct PostgresTaskStore {
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
}
impl PostgresTaskStore {
@@ -54,7 +59,11 @@ impl PostgresTaskStore {
scheduler
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
Ok(Self { pool, limits })
Ok(Self {
pool,
limits,
scheduler,
})
}
fn unavailable(error: impl std::fmt::Display) -> TaskError {
@@ -151,6 +160,20 @@ impl TaskStore for PostgresTaskStore {
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
if let Some(epoch) = request.executable_input_master_key_epoch {
super::master_key::ensure_active_master_key_epoch(&mut transaction, epoch)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
}
let outcome = self.accept_locked(&mut transaction, &request).await;
match outcome {
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
@@ -252,6 +275,7 @@ impl TaskStore for PostgresTaskStore {
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
and (lease_owner is null or fence = 0)
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
@@ -285,6 +309,12 @@ impl TaskStore for PostgresTaskStore {
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
let expired = sqlx::query(
"with expired as (
update tasks
@@ -295,6 +325,7 @@ impl TaskStore for PostgresTaskStore {
where task_id in (
select task_id from tasks
where state = 'queued' and execution_deadline <= now()
and lease_owner is null
order by task_id
limit $1
)
@@ -332,6 +363,351 @@ impl TaskStore for PostgresTaskStore {
expired_terminal: purged.rows_affected(),
})
}
async fn claim_batch(
&self,
owner: &str,
batch_limit: u32,
lease_ms: i64,
) -> Result<Vec<TaskLease>, TaskError> {
let batch = (batch_limit.clamp(1, 64)) as usize;
// A checkpointed attempt is never redispatched: once its lease expires the outbound outcome
// is uncertain, so it resolves to outcome_unknown and releases its admission slot. Story 4.4
// owns any later retry/cancellation of that uncertain outcome.
let mut transaction = self.start().await?;
sqlx::query(
"with candidates as (
select task_id from tasks
where state = 'working' and lease_owner is not null
and current_attempt_id is not null
and lease_until < clock_timestamp()
order by task_id
limit 64
), resolved as (
update tasks
set state = 'outcome_unknown',
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id in (select task_id from candidates)
and state = 'working'
returning task_id
)
delete from execution_admission_entries
where task_id in (select task_id from resolved)",
)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
// A crash before the durable dispatch checkpoint left the Task queued for execution: its
// admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is
// bounded by the <=30s lease expiry, inside the 60s story budget.
sqlx::query(
"with candidates as (
select task_id from tasks
where state = 'working' and lease_owner is not null
and current_attempt_id is null
and lease_until < clock_timestamp()
order by task_id
limit 64
), requeued as (
update tasks
set state = 'queued',
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id in (select task_id from candidates)
and state = 'working'
returning task_id
)
update execution_admission_entries
set state = 'reserved', granted_at = null
where task_id in (select task_id from requeued) and state = 'executing'",
)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
let mut leases = Vec::new();
for _ in 0..batch {
let mut transaction = self.start().await?;
let workspace_cursor =
match super::admission::lock_admission_in_tx(&mut transaction, "*").await {
Ok(cursor) => cursor,
Err(error) => {
transaction.rollback().await.ok();
return Err(TaskError::unavailable(format!("{error:?}")));
}
};
let (task_id, _updated_cursor) = match super::admission::grant_next_task(
&mut transaction,
"*",
&workspace_cursor,
&self.limits,
self.scheduler,
)
.await
{
Ok(value) => value,
Err(error) => {
transaction.rollback().await.ok();
return Err(TaskError::unavailable(format!("{error:?}")));
}
};
// Persist the updated controller cursor even when no task is granted.
let Some(task_id) = task_id else {
transaction.commit().await.map_err(Self::unavailable)?;
break;
};
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'working',
lease_owner = $1,
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
fence = fence + 1,
updated_at = now()
where task_id = $3
and (
(state = 'queued' and lease_owner is null)
or (state = 'working' and lease_owner is not null
and lease_until < clock_timestamp() and current_attempt_id is null)
)
returning {TASK_PROJECTION}"
)))
.bind(owner)
.bind(lease_ms)
.bind(task_id)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
// Persist cursor update (grant_next_task already did it) regardless.
let Some(row) = row else {
transaction.rollback().await.ok();
break;
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
leases.push(TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
});
transaction.commit().await.map_err(Self::unavailable)?;
}
Ok(leases)
}
async fn renew_lease(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
lease_ms: i64,
) -> Result<TaskLease, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set lease_until = clock_timestamp() + ($1::bigint || ' milliseconds')::interval,
updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(lease_ms)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
let lease = TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
};
transaction.commit().await.map_err(Self::unavailable)?;
Ok(lease)
}
async fn write_progress(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
progress: TaskEnvelope,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set progress_envelope = $1, updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(progress.value().clone())
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn read_executable_input(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
) -> Result<(String, String, i64), TaskError> {
let row = sqlx::query(
"select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch
from tasks
where task_id = $1 and state = 'working' and lease_owner = $2 and fence = $3
and lease_until > clock_timestamp()",
)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::StateConflict);
};
let ciphertext: Option<String> = row
.try_get("executable_input_ciphertext")
.map_err(Self::unavailable)?;
let key_version: Option<String> = row
.try_get("executable_input_key_version")
.map_err(Self::unavailable)?;
let epoch: Option<i64> = row
.try_get("executable_input_master_key_epoch")
.map_err(Self::unavailable)?;
if let Some(epoch) = epoch {
let active = sqlx::query_scalar::<_, i64>(
"select epoch from master_key_identities where status = 'active' order by epoch desc limit 1",
)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
if let Some(active) = active {
if active != epoch {
return Err(TaskError::StateConflict);
}
}
}
match (ciphertext, key_version, epoch) {
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
Ok((ciphertext, key_version, epoch))
}
_ => Err(TaskError::NotFound),
}
}
async fn checkpoint_dispatch(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
attempt_id: String,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set current_attempt_id = $1,
dispatch_started_at = now(),
attempts = attempts + 1,
updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp() and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(&attempt_id)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn finish_leased(
&self,
request: TaskFinishRequest,
owner: &str,
fence: u64,
) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now(),
lease_owner = null,
lease_until = null
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
and lease_owner = $11 and fence = $12
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
}
impl PostgresTaskStore {
@@ -377,11 +753,12 @@ impl PostgresTaskStore {
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until
execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch, platform_api_key_id
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued',
$16, to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0)
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22
)
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
)
@@ -403,6 +780,15 @@ impl PostgresTaskStore {
.bind(request.input.value().clone())
.bind(request.pins.execution_deadline_unix_ms)
.bind(request.pins.retain_until_unix_ms)
.bind(request.executable_input_ciphertext.as_deref())
.bind(request.executable_input_key_version.as_deref())
.bind(request.executable_input_master_key_epoch)
.bind(
request
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str),
)
.fetch_one(&mut **transaction)
.await
.map_err(task_sql_error)?
@@ -540,6 +926,10 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError>
authorization_scope: row
.try_get("authorization_scope")
.map_err(unavailable_row)?,
platform_api_key_id: row
.try_get::<Option<String>, _>("platform_api_key_id")
.map_err(unavailable_row)?
.map(PlatformApiKeyId::new),
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
pins: TaskPins {
@@ -574,6 +964,14 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError>
error: envelope("error_envelope")?,
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
.unwrap_or_default(),
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
.unwrap_or_default(),
lease_owner: row.try_get("lease_owner").map_err(unavailable_row)?,
lease_until_unix_ms: row.try_get("lease_until_ms").map_err(unavailable_row)?,
current_attempt_id: row.try_get("current_attempt_id").map_err(unavailable_row)?,
dispatch_started_at_unix_ms: row
.try_get("dispatch_started_at_ms")
.map_err(unavailable_row)?,
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
.unwrap_or_default(),
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
@@ -2,6 +2,7 @@ use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry};
use sqlx::Row;
mod artifact_metadata;
mod durable_tasks_guard;
mod import_plan;
mod legacy_adoption;
mod preflight;
@@ -72,6 +73,9 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
(23, "prompt-dependencies-v23"),
(24, "execution-admission-v24"),
(25, "durable-tasks-v25"),
(26, "durable-tasks-v26"),
(27, "durable-tasks-v27"),
(28, "durable-tasks-v28"),
];
assert_eq!(rows.len(), expected.len());
for (row, (version, name)) in rows.iter().zip(expected) {
@@ -182,7 +186,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
);
assert_eq!(
MigrationAuthority::preflight(first.pool()).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
#[tokio::test]
@@ -230,7 +234,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
.expect("the V18-local guard must be restorable");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"restoring the exact V18 contract must return the current verdict"
);
}
@@ -356,7 +360,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -469,13 +473,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 2,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
let trace_column: bool = sqlx::query_scalar(
"select exists (
@@ -518,7 +522,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 3,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -578,7 +582,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 4,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -832,6 +836,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 25,
target: 28,
}
);
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"restore: {restore}"
);
}
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"restore: {restore}"
);
}
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
.unwrap();
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"restore: {restore}"
);
}
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
for (drift, restore) in [
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"restore: {restore}"
);
}
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"a rejected primary-key nullability change must leave the schema current"
);
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"the baseline before the drift matrix must be the current V16 schema"
);
let mut observations: Vec<String> = Vec::new();
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
.await
.expect("restoring the exact definition must apply");
let restored = MigrationAuthority::preflight(&pool).await;
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 }));
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 28 }));
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
// мнением того же guard-а, который принимает неправильные определения (F4).
let restored_exact = catalog_definition_snapshot(&pool).await == before;
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"baseline before inserting the shadow schema"
);
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
// Тень не меняет вердикт guard-а по текущей схеме.
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"a same-named object in another schema must not change the verdict for the current schema"
);
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 },
MigrationPreflight::Current { version: 28 },
"the exact restoration must return the current V16 schema"
);
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
assert!(
@@ -0,0 +1,83 @@
use super::*;
use crank_registry::MigrationApplyResult;
#[tokio::test]
async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() {
let database_url = crank_test_support::postgres_schema_url("test_v43_clean_legacy").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
super::remove_v25_schema(&pool).await;
// remove_v25_schema drops `tasks` and ledger >= 25, but V25 also added the
// nullable dispatch-intent column/index/FK to execution_admission_entries;
// restore those to the V24 shape so the legacy ledger is self-consistent.
sqlx::raw_sql(
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
drop index if exists execution_admission_entries_task_uidx;
alter table execution_admission_entries drop column if exists task_id;",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 24,
target: 28,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 24, to: 28 }
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
);
}
#[tokio::test]
async fn partial_v27_on_legacy_ledger_is_rejected_by_preflight() {
let database_url = crank_test_support::postgres_schema_url("test_v43_partial_v27").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
super::remove_v25_schema(&pool).await;
sqlx::raw_sql(
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
drop index if exists execution_admission_entries_task_uidx;
alter table execution_admission_entries drop column if exists task_id;",
)
.execute(&pool)
.await
.unwrap();
// Reintroduce a partial V27-shaped `tasks` table on top of the legacy (v24) ledger.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create table tasks (
task_id uuid primary key,
executable_input_ciphertext text,
executable_input_key_version text,
executable_input_master_key_epoch bigint
);",
))
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "partial_sequence");
// The rejection today is attributed to V25's absent-schema guard because the
// `tasks` relation already exists, so a behavioral RED pin for the missing
// V27 guard line cannot be reached; this test fixes the rejection contract
// instead of claiming new behavior.
assert_eq!(error.version(), Some(24));
assert_eq!(error.stage(), "preflight.schema");
let count: i64 =
sqlx::query_scalar("select count(*)::bigint from __crank_migrations where version >= 25")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
count, 0,
"nothing at or above V25 should have been applied over the partial fixture"
);
}
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
let tables = sqlx::query(
"select table_name
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 13,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
assert_eq!(operation_count, 1);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
assert!(supports_elicitation);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 25,
target: 28,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
.fetch_one(&pool)
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 25,
target: 28,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 18, to: 25 }
MigrationApplyResult::Applied { from: 18, to: 28 }
);
let after: Vec<(i64, String, String)> =
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 25 }
MigrationPreflight::Current { version: 28 }
);
}
@@ -86,16 +86,22 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() {
MigrationAuthority::apply(&pool).await.unwrap();
// The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer
// database; a synthetic version above the current one exercises it without a second binary.
let current: i64 = sqlx::query_scalar("select max(version) from __crank_migrations")
.fetch_one(&pool)
.await
.unwrap();
let future = current + 1;
sqlx::query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')",
values ($1, 'future-synthetic', repeat('a', 64), 'expand', 'n-minus-one-readable')",
)
.bind(future)
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "future_version");
assert_eq!(error.version(), Some(26));
assert_eq!(error.version(), Some(future));
assert_eq!(error.recovery(), "install_matching_application");
assert!(
MigrationAuthority::require_current(&pool).await.is_err(),
@@ -155,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 25,
target: 28,
}
);
}
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 25,
target: 28,
}
);
}
@@ -330,9 +330,10 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
// V25 created `tasks`; V26–V28 only altered it, so dropping the table removes their schema too.
sqlx::raw_sql(
"drop table if exists tasks cascade;
delete from __crank_migrations where version = 25;",
delete from __crank_migrations where version >= 25;",
)
.execute(pool)
.await
@@ -8,14 +8,18 @@
use super::common;
use crank_core::{
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms,
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, AgentId, ApprovalEvidence,
ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind,
PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest, TaskAccess,
TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState,
TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId, current_unix_ms,
};
use crank_registry::{
AdmissionLimits, CreatePlatformApiKeyRequest, PostgresExecutionAdmission, PostgresTaskStore,
};
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{Duration as TimeDuration, OffsetDateTime};
const WORKSPACE: &str = "ws_default";
const AGENT: &str = "agent_a";
@@ -48,6 +52,7 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
owner_kind: TaskOwnerKind::Agent,
owner_id: agent.to_owned(),
authorization_scope: format!("credential:pk_{agent}"),
platform_api_key_id: Some(PlatformApiKeyId::new(format!("pk_{agent}"))),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
@@ -60,6 +65,9 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
input_fingerprint: fingerprint.to_owned(),
input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(),
support: unsampled_support(),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
}
}
@@ -664,6 +672,70 @@ async fn invalid_bounded_envelope_is_rejected_before_persistence() {
database.cleanup().await;
}
#[tokio::test]
async fn task_worker_lease_and_fencing_contract() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("lease1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker_1", 4, 10_000).await.unwrap();
assert_eq!(leases.len(), 1);
assert_eq!(leases[0].record.task_id, created.task_id);
assert_eq!(leases[0].record.state, TaskState::Working);
assert_eq!(leases[0].owner, "worker_1");
let lease = &leases[0];
let fenced_progress = store
.write_progress(
"worker_1",
lease.record.task_id,
lease.fence,
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
)
.await
.unwrap();
assert!(fenced_progress.progress.is_some());
// stale fence rejected
let bad = store
.renew_lease("worker_1", lease.record.task_id, lease.fence + 1, 10_000)
.await;
assert!(bad.is_err());
// checkpoint dispatch once
let checkpointed = store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence,
"attempt-1".to_owned(),
)
.await
.unwrap();
assert_eq!(
checkpointed.current_attempt_id.as_deref(),
Some("attempt-1")
);
assert!(checkpointed.dispatch_started_at_unix_ms.is_some());
// second checkpoint rejected
let second = store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence,
"attempt-2".to_owned(),
)
.await;
assert!(second.is_err());
database.cleanup().await;
}
#[tokio::test]
async fn store_failure_fails_closed_without_records() {
let database = common::TestDatabase::new().await;
@@ -680,3 +752,154 @@ async fn store_failure_fails_closed_without_records() {
database.cleanup().await;
}
/// Story 4.3 ATDD: the typed immutable Platform API key reference of an accepted Task is persisted,
/// read back through the lease, and only an active, unexpired, exactly bound key is authoritatively
/// revalidated. Revocation, expiry and a foreign workspace/agent binding all close; an administrator
/// Task stores no machine key identity at all, so a worker fails closed on it.
#[tokio::test]
async fn accepted_task_authority_reference_is_persisted_and_revalidated() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let workspace = WorkspaceId::new(WORKSPACE);
let now = OffsetDateTime::now_utc();
// `ws_default` is seeded by the baseline migration; the keys only need that workspace row.
let agent = AgentId::new(AGENT);
let active_id = PlatformApiKeyId::new("pk_task_active");
for key in [
authority_key(
"pk_task_active",
&workspace,
AGENT,
PlatformApiKeyStatus::Active,
None,
),
authority_key(
"pk_task_revoked",
&workspace,
AGENT,
PlatformApiKeyStatus::Revoked,
None,
),
authority_key(
"pk_task_expired",
&workspace,
AGENT,
PlatformApiKeyStatus::Active,
Some(now - TimeDuration::hours(1)),
),
] {
let secret_hash = format!("secret-hash-{}", key.id.as_str());
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
secret_hash: &secret_hash,
api_key: &key,
})
.await
.unwrap();
}
// Acceptance persists only the immutable typed reference, never the bearer secret.
let mut accepted = request(&key("k_authority"), &digest("fa"), AGENT, 300_000);
accepted.platform_api_key_id = Some(active_id.clone());
let task_id = accepted.task_id;
store.accept(accepted).await.expect("acceptance commits");
let leases = store
.claim_batch("worker_authority", 8, 30_000)
.await
.expect("claim");
let lease = leases
.into_iter()
.find(|lease| lease.record.task_id == task_id)
.expect("accepted task is leased");
assert_eq!(lease.record.platform_api_key_id.as_ref(), Some(&active_id));
// Positive: the exact active key of this workspace/agent is revalidated authoritatively.
let revalidated = registry
.revalidate_platform_api_key(&workspace, &agent, &active_id)
.await
.expect("authoritative read")
.expect("active key is authoritative");
assert!(
revalidated
.api_key
.scopes
.contains(&PlatformApiKeyScope::Write)
);
// Negatives: revocation, expiry and a foreign workspace/agent binding all fail closed.
let foreign_agent = AgentId::new("agent_b");
let foreign_workspace = WorkspaceId::new("ws_elsewhere");
for (other_workspace, other_agent, other_key) in [
(&workspace, &agent, PlatformApiKeyId::new("pk_task_revoked")),
(&workspace, &agent, PlatformApiKeyId::new("pk_task_expired")),
(&workspace, &foreign_agent, active_id.clone()),
(&foreign_workspace, &agent, active_id.clone()),
] {
let found = registry
.revalidate_platform_api_key(other_workspace, other_agent, &other_key)
.await
.unwrap();
assert!(
found.is_none(),
"authority must fail closed for {}",
other_key.as_str()
);
}
// An administrator Task carries no machine key identity: the worker has nothing to revalidate.
let mut admin_request = request(&key("k_admin"), &digest("fb"), AGENT, 300_000);
admin_request.owner_kind = TaskOwnerKind::Admin;
admin_request.owner_id = "user_admin".to_owned();
admin_request.authorization_scope = "admin:user_admin".to_owned();
admin_request.platform_api_key_id = None;
let admin_task = admin_request.task_id;
store
.accept(admin_request)
.await
.expect("administrator acceptance commits");
let admin_record = store
.read(
&TaskAccess {
workspace_id: WORKSPACE.to_owned(),
owner_kind: TaskOwnerKind::Admin,
owner_id: "user_admin".to_owned(),
authorization_scope: "admin:user_admin".to_owned(),
},
admin_task,
)
.await
.expect("owner read");
assert_eq!(admin_record.platform_api_key_id, None);
database.cleanup().await;
}
fn authority_key(
id: &str,
workspace_id: &WorkspaceId,
agent: &str,
status: PlatformApiKeyStatus,
expires_at: Option<OffsetDateTime>,
) -> PlatformApiKey {
PlatformApiKey {
id: PlatformApiKeyId::new(id),
workspace_id: workspace_id.clone(),
agent_id: Some(AgentId::new(agent)),
key_kind: PlatformApiKeyKind::McpClient,
name: id.to_owned(),
prefix: "crk_live".to_owned(),
scopes: vec![PlatformApiKeyScope::Write],
status,
created_at: OffsetDateTime::now_utc(),
last_used_at: None,
expires_at,
allowed_origins: Vec::new(),
}
}
mod task_worker;
@@ -0,0 +1,163 @@
//! Story 4.3 ATDD: durable Task leases, fencing, crash recovery and bounded claim behavior.
//!
//! Split from `tasks.rs` to keep both integration files inside the code-health limit.
use super::*;
use serde_json::json;
/// Story 4.3 ATDD: a worker crash before the durable dispatch checkpoint leaves only an expired
/// lease. PostgreSQL is authoritative, so another worker recovers the Task and the dead owner can
/// never commit a terminal outcome with its old fence.
#[tokio::test]
async fn task_worker_expired_pre_checkpoint_lease_is_recovered_and_stale_owner_cannot_commit() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("recover1"), &digest("f2"), AGENT, 30_000))
.await
.unwrap();
let first = store.claim_batch("worker_dead", 1, 150).await.unwrap();
assert_eq!(first.len(), 1);
assert_eq!(first[0].record.task_id, created.task_id);
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
// Recovery on the next bounded authoritative scan, with no Valkey wake-up.
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
let lease = recovered
.into_iter()
.find(|lease| lease.record.task_id == created.task_id)
.expect("expired pre-checkpoint Task is recovered");
assert!(lease.fence > first[0].fence, "recovery bumps the fence");
// The previous owner's terminal CAS is rejected; the live owner wins exactly once.
let stale = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: first[0].record.state,
expected_version: first[0].record.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"stale": true})).unwrap()),
error: None,
},
"worker_dead",
first[0].fence,
)
.await;
assert!(matches!(stale, Err(TaskError::StateConflict)));
let finished = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: lease.record.state,
expected_version: lease.record.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
},
"worker_live",
lease.fence,
)
.await
.expect("current lease wins the terminal CAS");
assert_eq!(finished.state, TaskState::Completed);
database.cleanup().await;
}
/// Story 4.3 ATDD: after the dispatch checkpoint the outbound outcome is uncertain. An expired
/// attempt resolves to `outcome_unknown`, is never redispatched automatically and releases the
/// admission slot it held.
#[tokio::test]
async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_without_redispatch() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("uncertain1"), &digest("f3"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
store
.checkpoint_dispatch(
"worker_dead",
lease.record.task_id,
lease.fence,
"attempt-uncertain".to_owned(),
)
.await
.expect("durable dispatch checkpoint");
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
// The authority scan resolves uncertainty instead of handing the attempt to another worker.
let after = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
assert!(
after.is_empty(),
"a checkpointed attempt must never be redispatched"
);
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::OutcomeUnknown);
assert_eq!(
record.current_attempt_id.as_deref(),
Some("attempt-uncertain")
);
let intents: i64 = sqlx::query_scalar(
"select count(*)::bigint from execution_admission_entries where task_id = $1",
)
.bind(created.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(intents, 0, "uncertain terminal outcome releases capacity");
// Still no automatic retry on the following scans.
assert!(
store
.claim_batch("worker_live", 4, 10_000)
.await
.unwrap()
.is_empty()
);
database.cleanup().await;
}
/// Story 4.3 ATDD: concurrent claimers never hold the same Task at once.
#[tokio::test]
async fn task_worker_concurrent_claimers_never_double_lease_one_task() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = std::sync::Arc::new(store_with(registry.pool().clone(), 8));
let created = store
.accept(request(&key("concurrent1"), &digest("f4"), AGENT, 30_000))
.await
.unwrap();
let mut handles = Vec::new();
for index in 0..4 {
let store = store.clone();
handles.push(tokio::spawn(async move {
store
.claim_batch(&format!("worker_{index}"), 1, 10_000)
.await
.unwrap()
}));
}
let mut leased = Vec::new();
for handle in handles {
leased.extend(handle.await.unwrap());
}
assert_eq!(leased.len(), 1, "exactly one owner wins the claim");
assert_eq!(leased[0].record.task_id, created.task_id);
database.cleanup().await;
}
+99 -9
View File
@@ -22,7 +22,9 @@ use serde_json::{Map, Value, json};
use time::OffsetDateTime;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tracing::{Instrument, Span, debug, warn};
use uuid::Uuid;
use crate::task_worker::TaskDispatchPermit;
use crate::{
AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError,
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
@@ -178,8 +180,72 @@ impl RuntimeExecutor {
pub(crate) async fn execute_request(
&self,
request: RuntimeExecutionRequest<'_>,
) -> Result<(Value, Value), RuntimeError> {
self.execute_request_impl(request, None).await
}
pub async fn execute_task_outcome(
&self,
request: RuntimeExecutionRequest<'_>,
permit: TaskDispatchPermit,
) -> Result<crank_core::ExecutionSuccess, crank_core::ExecutionFailure> {
let operation = request.operation().clone();
let operation_id = operation.operation_id.clone();
let operation_version = operation.operation_version;
let origin = request.origin();
let correlation = crank_core::CorrelationContext::new(
request.request_context().request_id.clone(),
request.request_context().trace_context.clone(),
);
// Validate permit binding before any adapter dispatch.
if permit.operation_id() != operation_id.as_str()
|| permit.operation_version() != operation_version
|| permit.workspace_id() != request.workspace_id().as_str()
|| permit.agent_id() != request.agent_id().map(|value| value.as_str())
{
return Err(crate::normalize_runtime_error_for_operation(
&RuntimeError::InvalidExecutionRequest,
&correlation,
&operation,
));
}
if !permit.consume() {
return Err(crate::normalize_runtime_error_for_operation(
&RuntimeError::InvalidExecutionRequest,
&correlation,
&operation,
));
}
match self.execute_request_impl(request, Some(permit)).await {
Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess {
operation_id: operation_id.clone(),
operation_version,
origin,
correlation,
request_preview,
output,
}),
Err(error) => Err(crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&operation,
)),
}
}
pub(crate) async fn execute_request_impl(
&self,
request: RuntimeExecutionRequest<'_>,
permit: Option<TaskDispatchPermit>,
) -> Result<(Value, Value), RuntimeError> {
let operation = request.operation();
// Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission.
let trusted_task_dispatch = permit.is_some();
// The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the
// trace names the real persisted attempt instead of inventing one (Story 4.3 AC).
let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned());
let input = request.input();
let request_context = request.request_context();
debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized);
@@ -205,7 +271,9 @@ impl RuntimeExecutor {
let dispatch_started = Arc::new(AtomicBool::new(false));
// True until the admission gate answered. A deadline that fires while the request is still
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
let admission_pending = Arc::new(AtomicBool::new(
!trusted_task_dispatch && self.admission.is_some(),
));
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
metric_invocation_source(Some(request_context)),
request_context
@@ -222,14 +290,19 @@ impl RuntimeExecutor {
// One admission gate for every executable path (Story 4.1): the published policy decides
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
// is held for the whole local execution future, including retries.
let mut lease = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
let mut lease = if trusted_task_dispatch {
None
} else {
let result = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
result
};
let _permit = self.acquire_unary_permit(operation)?;
let _inflight = InFlightGuard::runtime();
let mapping_span = Stage::RuntimeArgumentsMap.span();
@@ -246,6 +319,7 @@ impl RuntimeExecutor {
prepared_request,
Some(request_context),
Arc::clone(&dispatch_started),
durable_attempt_id.as_deref(),
)
.await;
if let Some(lease) = lease.as_mut() {
@@ -348,6 +422,7 @@ impl RuntimeExecutor {
prepared_request: PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
durable_attempt_id: Option<&str>,
) -> Result<Value, RuntimeError> {
let mut prepared_request = prepared_request;
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
@@ -464,6 +539,7 @@ impl RuntimeExecutor {
operation,
idempotency_key.as_deref(),
),
durable_attempt_id,
)
.await;
if let Ok(response) = &adapter_response {
@@ -604,10 +680,23 @@ impl RuntimeExecutor {
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
verified_idempotency: bool,
durable_attempt_id: Option<&str>,
) -> Result<AdapterResponse, RuntimeError> {
let max_attempts = retry_attempts(operation);
let mut attempt = 1;
loop {
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint; the
// retry index discriminates the attempts inside it. Only paths without a durable attempt
// (synchronous tools/call) mint a synthetic per-attempt id.
let attempt_id = durable_attempt_id
.map(str::to_owned)
.unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7()));
let attempt_span = tracing::info_span!(
"adapter_attempt",
attempt,
max_attempts,
attempt_id = %attempt_id
);
let result = self
.execute_adapter(
operation,
@@ -615,6 +704,7 @@ impl RuntimeExecutor {
request_context,
Arc::clone(&dispatch_started),
)
.instrument(attempt_span)
.await;
match result {
Ok(response) => return Ok(response),
+5
View File
@@ -15,6 +15,7 @@ mod rate_limit;
mod request_context;
mod request_preparation;
mod response_cache;
mod task_worker;
mod tasks;
pub fn validate_resource_template_parameter(
schema: &crank_schema::Schema,
@@ -115,4 +116,8 @@ pub use rate_limit::{
};
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
pub use secret_crypto::SecretCrypto;
pub use task_worker::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
};
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
+573
View File
@@ -0,0 +1,573 @@
//! Fenced durable Task worker loop for Story 4.3.
//!
//! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded,
//! Valkey may only wake it up (never represented here as correctness), and every lease-protected
//! write goes through CAS using owner + fencing token. The actual executable input and revalidation
//! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a
//! redacted projection and is never dispatched.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use crank_core::{TaskEnvelope, TaskError, TaskFinishRequest, TaskLease, TaskState, TaskStore};
use tokio::sync::watch;
use tokio::task::JoinSet;
use tracing::{Instrument, debug, warn};
use uuid::Uuid;
/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets.
#[derive(Clone, Debug)]
pub struct TaskWorkerConfig {
/// Stable random-ish owner identity for this process instance. Required lease owner.
pub owner: String,
/// Maximum number of Tasks claimed in one bounded PG scan.
pub batch_limit: u32,
/// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used.
pub lease_ms: i64,
/// Authoritative PG polling interval when no wake-up hint is available.
pub poll_interval: Duration,
/// Renewal cadence while processing a lease.
pub renewal_interval: Duration,
/// Concurrent in-flight Tasks for this worker.
pub max_inflight: usize,
/// Maximum time to wait for active work during graceful shutdown.
pub shutdown_timeout: Duration,
}
impl Default for TaskWorkerConfig {
fn default() -> Self {
Self {
owner: format!("task-worker-{}", Uuid::now_v7()),
batch_limit: 8,
lease_ms: 30_000,
poll_interval: Duration::from_millis(250),
renewal_interval: Duration::from_millis(5_000),
max_inflight: 4,
shutdown_timeout: Duration::from_secs(30),
}
}
}
/// Result of one durable Task attempt produced by the application layer.
#[derive(Clone, Debug)]
pub enum TaskAttemptOutcome {
Completed(serde_json::Value),
Failed(serde_json::Value),
OutcomeUnknown,
}
/// Application-owned seam for pre-dispatch revalidation and execution.
///
/// The worker owns lease renewal/CAS/fencing and the dispatch checkpoint. The handler only
/// receives the leased record and attempt identity; it must not trust `TaskRecord.input` as
/// executable payload.
#[async_trait]
pub trait TaskHandler: Send + Sync {
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>;
async fn execute(
&self,
lease: &TaskLease,
attempt_id: &str,
cancelled: Arc<AtomicBool>,
permit: TaskDispatchPermit,
) -> TaskAttemptOutcome;
}
/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and
/// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and
/// it binds one durable Task attempt to its execution request.
pub struct TaskDispatchPermit {
task_id: crank_core::TaskId,
operation_id: String,
operation_version: u32,
workspace_id: String,
agent_id: Option<String>,
owner: String,
fence: u64,
attempt_id: String,
consumed: std::sync::atomic::AtomicBool,
}
impl TaskDispatchPermit {
pub(crate) fn new(
task_id: crank_core::TaskId,
operation_id: String,
operation_version: u32,
workspace_id: String,
agent_id: Option<String>,
owner: String,
fence: u64,
attempt_id: String,
) -> Self {
Self {
task_id,
operation_id,
operation_version,
workspace_id,
agent_id,
owner,
fence,
attempt_id,
consumed: std::sync::atomic::AtomicBool::new(false),
}
}
pub fn task_id(&self) -> crank_core::TaskId {
self.task_id
}
pub fn operation_id(&self) -> &str {
&self.operation_id
}
pub fn operation_version(&self) -> u32 {
self.operation_version
}
pub fn workspace_id(&self) -> &str {
&self.workspace_id
}
pub fn agent_id(&self) -> Option<&str> {
self.agent_id.as_deref()
}
pub fn owner(&self) -> &str {
&self.owner
}
pub fn fence(&self) -> u64 {
self.fence
}
pub fn attempt_id(&self) -> &str {
&self.attempt_id
}
pub fn is_consumed(&self) -> bool {
self.consumed.load(std::sync::atomic::Ordering::Acquire)
}
pub fn consume(&self) -> bool {
!self
.consumed
.swap(true, std::sync::atomic::Ordering::AcqRel)
}
}
#[derive(thiserror::Error, Debug)]
pub enum TaskWorkerError {
#[error("task store error: {0}")]
Store(#[from] TaskError),
#[error("invalid worker configuration: {0}")]
Config(String),
#[error("task terminal transition CAS failed")]
CasRejected,
}
pub struct TaskWorker {
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
config: TaskWorkerConfig,
stopping: Arc<AtomicBool>,
}
impl TaskWorker {
pub fn new(
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
config: TaskWorkerConfig,
) -> Result<Self, TaskWorkerError> {
if config.owner.is_empty() {
return Err(TaskWorkerError::Config(
"owner must not be empty".to_owned(),
));
}
if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 {
return Err(TaskWorkerError::Config(
"batch_limit must be 1..=64 and lease_ms must be positive".to_owned(),
));
}
if config.lease_ms > 30_000
|| config.poll_interval.is_zero()
|| config.poll_interval > Duration::from_secs(30)
|| config.renewal_interval.is_zero()
|| config.renewal_interval >= Duration::from_millis(config.lease_ms as u64)
|| config.max_inflight == 0
|| config.shutdown_timeout.is_zero()
|| config.shutdown_timeout > Duration::from_secs(30)
{
return Err(TaskWorkerError::Config(
"lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds"
.to_owned(),
));
}
Ok(Self {
store,
handler,
config,
stopping: Arc::new(AtomicBool::new(false)),
})
}
/// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled.
pub fn stop(&self) {
self.stopping.store(true, Ordering::Release);
}
/// Returns a handle whose drop does not stop the worker; call `stop()` on the worker.
pub fn stop_flag(&self) -> Arc<AtomicBool> {
self.stopping.clone()
}
/// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan.
pub async fn run(&self) -> Result<(), TaskWorkerError> {
self.run_inner(None).await
}
/// Run until either `stop()` or the provided shutdown receiver becomes true.
pub async fn run_until_shutdown(
&self,
mut shutdown: watch::Receiver<bool>,
) -> Result<(), TaskWorkerError> {
self.run_inner(Some(&mut shutdown)).await
}
async fn run_inner(
&self,
mut shutdown: Option<&mut watch::Receiver<bool>>,
) -> Result<(), TaskWorkerError> {
let mut active: JoinSet<()> = JoinSet::new();
loop {
if self.stopping.load(Ordering::Acquire) {
break;
}
if let Some(rx) = shutdown.as_mut() {
if *rx.borrow() {
break;
}
}
// Reap finished attempts before claiming more.
while active.try_join_next().is_some() {}
let available = self.config.max_inflight.saturating_sub(active.len());
if available > 0 {
let batch_limit = self
.config
.batch_limit
.min(u32::try_from(available).unwrap_or(u32::MAX));
match self
.store
.claim_batch(&self.config.owner, batch_limit, self.config.lease_ms)
.await
{
Ok(leases) => {
for lease in leases {
let store = self.store.clone();
let handler = self.handler.clone();
let owner = self.config.owner.clone();
let lease_ms = self.config.lease_ms;
let renewal_interval = self.config.renewal_interval;
let stopping = self.stopping.clone();
active.spawn(async move {
process_lease(
store,
handler,
owner,
lease,
lease_ms,
renewal_interval,
stopping,
)
.await;
});
}
}
Err(error) => {
warn!(error = %error, "bounded task claim failed; retrying on next scan");
}
}
}
tokio::select! {
_ = tokio::time::sleep(self.config.poll_interval) => {}
_ = async {
if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await }
} => {}
}
}
// Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded.
// The same stop flag reaches cooperative cancellation and lease-renewal loops.
self.stopping.store(true, Ordering::Release);
let deadline = Instant::now() + self.config.shutdown_timeout;
while !active.is_empty() {
if Instant::now() >= deadline {
warn!("graceful shutdown timeout reached; aborting remaining task attempts");
active.abort_all();
break;
}
tokio::select! {
_ = active.join_next() => {}
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
}
Ok(())
}
}
async fn process_lease(
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
owner: String,
lease: TaskLease,
lease_ms: i64,
renewal_interval: Duration,
stopping: Arc<AtomicBool>,
) {
let task_id = lease.record.task_id;
let fence = lease.fence;
let cancelled = Arc::new(AtomicBool::new(false));
// Pre-dispatch revalidation under the fresh lease.
if let Err(error) = handler.revalidate(&lease).await {
// A transient authority-store outage is not evidence that the work is invalid: leave the
// lease to expire so the bounded recovery scan can retry it without dispatching now.
if matches!(
&error,
TaskWorkerError::Store(TaskError::Unavailable { .. })
) {
debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery");
return;
}
debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch");
finish_failed(
&store,
&owner,
task_id,
fence,
&lease,
"revalidation_failed",
)
.await;
return;
}
if stopping.load(Ordering::Acquire) {
// Graceful shutdown before the durable dispatch checkpoint: no outbound call can have
// happened, so the Task is left leased. The bounded recovery scan requeues it after the
// lease expires instead of turning a never-started attempt into a user-visible failure.
return;
}
let attempt_id = format!("attempt-{}", Uuid::now_v7());
// Durable checkpoint before any network use.
if store
.checkpoint_dispatch(&owner, task_id, fence, attempt_id.clone())
.await
.is_err()
{
// Lease lost or checkpoint rejected: do not dispatch.
cancelled.store(true, Ordering::Release);
return;
}
if stopping.load(Ordering::Acquire) {
// The checkpoint is durable, but we know the outbound callback has not started.
finish_failed(
&store,
&owner,
task_id,
fence,
&lease,
"shutdown_before_dispatch",
)
.await;
return;
}
// Sealed single-use permit for the trusted runtime dispatch bridge. It is created only after
// the current lease and durable checkpoint above.
let agent_id = if lease.record.owner_kind == crank_core::TaskOwnerKind::Agent {
Some(lease.record.owner_id.clone())
} else {
None
};
let permit = TaskDispatchPermit::new(
lease.record.task_id,
lease.record.operation_id.clone(),
lease.record.pins.operation_version,
lease.record.workspace_id.clone(),
agent_id,
owner.clone(),
fence,
attempt_id.clone(),
);
// Background lease renewal while the attempt runs.
let renewal_store = store.clone();
let renewal_owner = owner.clone();
let renewal_cancelled = cancelled.clone();
let renewal_stopping = stopping.clone();
let renewal_task_id = task_id;
let renewal_handle = tokio::spawn(async move {
let mut interval = tokio::time::interval(renewal_interval);
interval.tick().await;
loop {
interval.tick().await;
if renewal_cancelled.load(Ordering::Acquire) {
break;
}
if renewal_stopping.load(Ordering::Acquire) {
renewal_cancelled.store(true, Ordering::Release);
break;
}
if renewal_store
.renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms)
.await
.is_err()
{
renewal_cancelled.store(true, Ordering::Release);
break;
}
}
});
// Each dispatch gets a child span carrying the durable attempt identity, rooted in the parent
// trace persisted with the accepted Task (never a mutable queue/cache projection).
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, attempt_id = %attempt_id, fence);
if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) {
crank_trace::set_parent_from_trace_context(&span, &parent);
}
let outcome = handler
.execute(&lease, &attempt_id, cancelled.clone(), permit)
.instrument(span)
.await;
cancelled.store(true, Ordering::Release);
renewal_handle.abort();
// Terminal CAS under valid lease. Only one writer wins; stale token is rejected by store.
let access = crank_core::TaskAccess {
workspace_id: lease.record.workspace_id.clone(),
owner_kind: lease.record.owner_kind,
owner_id: lease.record.owner_id.clone(),
authorization_scope: lease.record.authorization_scope.clone(),
};
match outcome {
TaskAttemptOutcome::Completed(value) => {
let Ok(result) = TaskEnvelope::try_redacted(&value) else {
finish_failed(
&store,
&owner,
task_id,
fence,
&lease,
"result_not_redactable",
)
.await;
return;
};
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(&lease),
expected_version: lease.record.version,
target_state: TaskState::Completed,
result: Some(result),
error: None,
},
&owner,
fence,
)
.await;
}
TaskAttemptOutcome::Failed(value) => {
let error = TaskEnvelope::try_redacted(&value).or_else(|_| {
TaskEnvelope::try_redacted(&serde_json::json!({
"code": "task_error_not_redactable"
}))
});
let Ok(error) = error else {
return;
};
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(&lease),
expected_version: lease.record.version,
target_state: TaskState::Failed,
result: None,
error: Some(error),
},
&owner,
fence,
)
.await;
}
TaskAttemptOutcome::OutcomeUnknown => {
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(&lease),
expected_version: lease.record.version,
target_state: TaskState::OutcomeUnknown,
result: None,
error: None,
},
&owner,
fence,
)
.await;
}
}
}
fn leased_expected_state(lease: &TaskLease) -> TaskState {
// Claimed tasks are expected to be in a non-terminal processing state.
if lease.record.state == TaskState::Queued {
TaskState::Queued
} else {
TaskState::Working
}
}
async fn finish_failed(
store: &Arc<dyn TaskStore>,
owner: &str,
task_id: crank_core::TaskId,
fence: u64,
lease: &TaskLease,
code: &str,
) {
let access = crank_core::TaskAccess {
workspace_id: lease.record.workspace_id.clone(),
owner_kind: lease.record.owner_kind,
owner_id: lease.record.owner_id.clone(),
authorization_scope: lease.record.authorization_scope.clone(),
};
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(lease),
expected_version: lease.record.version,
target_state: TaskState::Failed,
result: None,
error: TaskEnvelope::try_redacted(&serde_json::json!({ "code": code })).ok(),
},
owner,
fence,
)
.await;
}
+160 -6
View File
@@ -13,11 +13,13 @@
use std::sync::Arc;
use std::time::Instant;
use crank_core::PlatformApiKeyId;
use crank_core::{
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds,
ToolExecutionMode, current_unix_ms, validate_task_envelope,
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskLease,
TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds, ToolExecutionMode,
contains_sensitive_task_keys, current_unix_ms, validate_task_envelope,
};
use sha2::{Digest, Sha256};
@@ -29,6 +31,9 @@ pub struct TaskAcceptDraft {
pub owner_id: String,
/// Server-derived authorization scope identity of the verified principal.
pub authorization_scope: String,
/// Typed immutable Platform API key identity of the verified machine credential. It is sealed
/// into the executable-input binding and revalidated authoritatively by the worker.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
pub operation_version: u32,
pub policy_mode: ToolExecutionMode,
@@ -68,11 +73,22 @@ impl TaskOutcome {
#[derive(Clone)]
pub struct TaskService {
store: Arc<dyn TaskStore>,
crypto: Option<Arc<crate::SecretCrypto>>,
}
impl TaskService {
pub fn new(store: Arc<dyn TaskStore>) -> Self {
Self { store }
Self {
store,
crypto: None,
}
}
pub fn with_secret_crypto(store: Arc<dyn TaskStore>, crypto: Arc<crate::SecretCrypto>) -> Self {
Self {
store,
crypto: Some(crypto),
}
}
/// Readiness of the mandatory durable Task state.
@@ -83,18 +99,61 @@ impl TaskService {
/// Durable acceptance: the returned identity exists only after the store committed it.
pub async fn accept(&self, draft: TaskAcceptDraft) -> Result<TaskAcceptance, TaskError> {
validate_task_envelope(&draft.input)?;
if contains_sensitive_task_keys(&draft.input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
let (scoped_key, input_fingerprint) = (
scoped_key_digest(&draft)?,
canonical_input_fingerprint(&draft.input)?,
);
let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline);
let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS);
let task_id = TaskId::generate();
let crypto = self.crypto.as_ref().ok_or_else(|| {
TaskError::invalid("secret crypto is required for durable Task acceptance")
})?;
let binding = serde_json::json!({
"task_id": task_id.to_string(),
"workspace_id": draft.workspace_id,
"owner_kind": draft.owner_kind.as_str(),
"owner_id": draft.owner_id,
"authorization_scope": draft.authorization_scope,
"platform_api_key_id": draft
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str),
"operation_id": draft.operation_id,
"operation_version": draft.operation_version,
"policy_mode": draft.policy_mode,
"input_fingerprint": input_fingerprint,
"input": draft.input,
});
let ciphertext = crypto
.encrypt(&binding)
.map_err(|error| TaskError::invalid(error.to_string()))?;
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
return Err(TaskError::invalid(
"encrypted task input exceeds its byte budget",
));
}
let (
executable_input_ciphertext,
executable_input_key_version,
executable_input_master_key_epoch,
) = (
Some(ciphertext),
Some(crypto.key_version().to_owned()),
Some(crypto.master_key_epoch()),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
task_id,
workspace_id: draft.workspace_id,
owner_kind: draft.owner_kind,
owner_id: draft.owner_id,
authorization_scope: draft.authorization_scope,
platform_api_key_id: draft.platform_api_key_id,
operation_id: draft.operation_id,
pins: TaskPins {
operation_version: draft.operation_version,
@@ -110,6 +169,9 @@ impl TaskService {
draft.correlation.request_id().clone(),
draft.correlation.trace_context(),
),
executable_input_ciphertext,
executable_input_key_version,
executable_input_master_key_epoch,
};
self.store.accept(request).await
}
@@ -134,6 +196,97 @@ impl TaskService {
))
}
/// Decrypts and validates the executable input bound to a valid current lease.
pub async fn executable_input(
&self,
lease: &TaskLease,
) -> Result<serde_json::Value, TaskError> {
let crypto = self.crypto.as_ref().ok_or_else(|| {
TaskError::invalid("secret crypto is required for durable Task execution")
})?;
let (ciphertext, key_version, epoch) = self
.store
.read_executable_input(&lease.owner, lease.record.task_id, lease.fence)
.await?;
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
let binding = crypto
.decrypt_for_epoch(&key_version, epoch, &ciphertext)
.map_err(|error| TaskError::invalid(error.to_string()))?;
let object = binding
.as_object()
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
let check_string = |field: &str, expected: &str| -> Result<(), TaskError> {
match object.get(field).and_then(|value| value.as_str()) {
Some(actual) if actual == expected => Ok(()),
_ => Err(TaskError::invalid("task executable input binding mismatch")),
}
};
check_string("task_id", &lease.record.task_id.to_string())?;
check_string("workspace_id", &lease.record.workspace_id)?;
check_string("owner_id", &lease.record.owner_id)?;
check_string("authorization_scope", &lease.record.authorization_scope)?;
let expected_key = lease
.record
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str);
match (object.get("platform_api_key_id"), expected_key) {
(None | Some(serde_json::Value::Null), None) => {}
(Some(serde_json::Value::String(actual)), Some(expected)) if actual == expected => {}
_ => {
return Err(TaskError::invalid("task executable input binding mismatch"));
}
}
check_string("operation_id", &lease.record.operation_id)?;
match object.get("owner_kind").and_then(|value| value.as_str()) {
Some(actual) if actual == lease.record.owner_kind.as_str() => {}
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
}
match object
.get("operation_version")
.and_then(|value| value.as_u64())
{
Some(actual) if actual == u64::from(lease.record.pins.operation_version) => {}
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
}
let expected_policy = match lease.record.pins.policy_mode {
ToolExecutionMode::Synchronous => "synchronous",
ToolExecutionMode::Background => "background",
ToolExecutionMode::Adaptive => "adaptive",
};
match object.get("policy_mode").and_then(|value| value.as_str()) {
Some(actual) if actual == expected_policy => {}
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
}
check_string("input_fingerprint", &lease.record.input_fingerprint)?;
let input = object
.get("input")
.cloned()
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
let canonical = canonical_input_fingerprint(&input)?;
if canonical != lease.record.input_fingerprint {
return Err(TaskError::invalid("task executable input binding mismatch"));
}
validate_task_envelope(&input)?;
let encoded = serde_json::to_vec(&input)
.map_err(|_| TaskError::invalid("task executable input is not serializable"))?;
if encoded.len() > TASK_EXECUTABLE_INPUT_MAX_BYTES {
return Err(TaskError::invalid(
"task executable input exceeds its byte budget",
));
}
if contains_sensitive_task_keys(&input) {
return Err(TaskError::invalid(
"task executable input contains sensitive fields",
));
}
Ok(input)
}
/// Default retention pinned by the application layer when it does not choose one explicitly.
pub fn default_retention_ms() -> i64 {
TASK_DEFAULT_RETENTION_MS
@@ -220,6 +373,7 @@ mod tests {
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
operation_id: "op_report".to_owned(),
operation_version: 2,
policy_mode: ToolExecutionMode::Background,
+86 -5
View File
@@ -106,6 +106,83 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
async fn adapter_retry_child_spans_are_distinct_without_a_durable_attempt() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let executor = RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(FlakyReadAdapter {
call_count: Arc::clone(&call_count),
}))
.build();
let mut operation: crank_runtime::RuntimeOperation = operation().into();
let Target::Rest(target) = &mut operation.target;
target.method = HttpMethod::Get;
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
// The synchronous tools/call path has no durable Attempt: every adapter attempt gets its own
// synthetic id. A Task dispatch instead reuses the durable id (mcp-server `attempt_trace`).
let result = async {
let request_root = tracing::info_span!("request_root");
executor
.execute(&operation, &json!({"name": "retry-read"}))
.instrument(request_root)
.await
}
.with_subscriber(subscriber)
.await
.expect("read retry should recover");
assert_eq!(result, json!({"accepted": true}));
let spans = capture.snapshot();
let runtime_span = spans
.iter()
.find(|span| span.name == "runtime.execute")
.expect("runtime.execute span");
assert_eq!(
runtime_span.parent_name,
Some("request_root"),
"the execution continues the caller's trace root"
);
let attempt_spans: Vec<_> = spans
.iter()
.filter(|span| span.name == "adapter_attempt")
.collect();
assert_eq!(attempt_spans.len(), 2);
for span in &attempt_spans {
assert_eq!(span.parent_name, Some("runtime.execute"));
assert!(
!span
.fields
.get("attempt_id")
.unwrap_or(&"".to_string())
.is_empty()
);
assert!(
!span
.fields
.get("attempt_id")
.unwrap_or(&"".to_string())
.contains("traceparent")
);
}
let attempts: Vec<String> = attempt_spans
.iter()
.map(|span| span.fields.get("attempt").cloned().unwrap_or_default())
.collect();
assert!(attempts.contains(&"1".to_string()));
assert!(attempts.contains(&"2".to_string()));
let ids: Vec<String> = attempt_spans
.iter()
.map(|span| span.fields.get("attempt_id").cloned().unwrap_or_default())
.collect();
assert_ne!(ids[0], ids[1]);
}
#[tokio::test]
async fn read_retry_revalidates_adapter_attempt_and_keeps_one_success_outcome() {
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
@@ -236,11 +313,7 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
.find(|span| span.name == "runtime.execute")
.expect("runtime span");
assert_eq!(runtime.parent_name, Some("mcp.request"));
for child in [
"runtime.arguments.map",
"upstream.http",
"runtime.response.transform",
] {
for child in ["runtime.arguments.map", "runtime.response.transform"] {
assert_eq!(
spans
.iter()
@@ -250,6 +323,14 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
"{child} must be a runtime child"
);
}
assert_eq!(
spans
.iter()
.find(|span| span.name == "upstream.http")
.and_then(|span| span.parent_name),
Some("adapter_attempt"),
"upstream.http must be an adapter child"
);
}
#[tokio::test]
+40 -3
View File
@@ -48,6 +48,43 @@ input fingerprint — типизированный конфликт; явная
authorization scope) входит в каждый read; чужой Task неотличим от отсутствующего.
Границы этой story: публичный MCP wire Tasks/capabilities/activation остаётся
Story 4.7, worker/leases/recovery — Story 4.3. Существующий `tools/call` сохраняет
synchronous/unsupported поведение; durable acceptance не запускает ephemeral
execution и не рекламирует клиенту background capability.
Story 4.7, worker/leases/recovery реализованы в Story 4.3 (ниже). Существующий
`tools/call` сохраняет synchronous/unsupported поведение; durable acceptance не
запускает ephemeral execution и не рекламирует клиенту background capability.
## Исполнение durable Task (Story 4.3)
`mcp-server` запускает fenced worker в том же процессе: bounded claim через
PostgreSQL `FOR UPDATE SKIP LOCKED` над reserved dispatch intent, lease owner/expiry
и monotonic fence; Valkey не является источником истины и может только ускорять
wake-up. Claim использует уже принятый shared admission из Story 4.1
(`lock_admission_in_tx` по controller-строке `*` + `grant_next_task` с его
scoped-лимитами, bounded queue и weighted fairness cursor) и не добавляет второго
scheduler'а: 4.3-дельта лишь fencing/lease-поля и CAS-предикаты поверх того же
grant, поэтому принятые fairness-инварианты 4.1 сохраняются. Progress, renewal,
dispatch checkpoint и terminal outcome пишутся только через CAS по текущему
owner/fence/expiry. До сетевого вызова worker повторно проверяет публикацию и
точную версию capability, активный Platform API key/scope, approval policy и
deadline; невалидная работа завершается без dispatch. Attempt ID и dispatch
checkpoint коммитятся атомарно до outbound; после checkpoint потеря lease означает
uncertain outcome и Task переходит в `outcome_unknown` без автоматического retry.
Исполняемый input берётся только из зашифрованного Task payload через
`TaskService::executable_input` (redacted projection никогда не исполняется).
Parent TraceContext сохраняется с acceptance; durable attempt span продолжает его,
а каждая adapter-попытка (включая retry существующих GET/verified-idempotent write)
получает собственный child span `adapter_attempt` с тем же durable Attempt ID и
номером попытки, поэтому retry не сливается с первым attempt в трассе, а любой
такой span сопоставим с `tasks.current_attempt_id`; синтетический attempt id
остаётся только у синхронного tools/call, где durable попытки нет. Скан PostgreSQL авторитетен:
crash до checkpoint возвращает Task в queue по истечении lease (recovery ≤60s).
Graceful shutdown является процессным, а не только worker-локальным: сигнал
SIGTERM/ctrl-c одновременно прекращает новые claims и запускает HTTP graceful
drain (worker не ждёт завершения долгого соединения), активные attempts
checkpoint-ятся/дренируются в пределах 30s, а ошибочное завершение HTTP-сервера
также останавливает worker. Уже поставленные в queue Tasks не теряются. Остаётся
узкое окно между durable checkpoint и созданием sealed dispatch permit: если stop
приходит именно туда, Task финализируется как failed(`shutdown_before_dispatch`),
не отправляя запрос; это доказано регрессией, которая детерминированно
задерживает checkpoint в его собственной транзакции, проверяет отсутствие вызова
adapter и сохранность ещё queued работы.
+1 -1
View File
@@ -1 +1 @@
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
+7
View File
@@ -78,3 +78,10 @@ run_atdd "runtime task identity" -p crank-runtime --lib tasks:: -- --test-thread
run_atdd "durable task store" -p crank-registry --test integration tasks:: -- --test-threads=1
run_atdd "durable task application entry" -p mcp-server --test integration durable_tasks:: -- --test-threads=1
run_atdd "durable task migration" -p crank-registry --test integration migrations:: -- --test-threads=1
# Story 4.3 (FR-22, FR-23): fenced durable Task execution. The Story 4.2 selectors above already run
# the lease/fencing/recovery store, the Task application entry and the migration guards; these two
# oracles bound the rest of this story's AC: the process-level shutdown ordering (signal stops new
# claims before a slow HTTP drain, not after it) and one child attempt span per adapter retry.
run_atdd "task worker process shutdown" -p mcp-server --bin mcp-server -- --test-threads=1
run_atdd "durable task retry attempt spans" -p crank-runtime --test stages retry -- --test-threads=1