feat(epic4): исполнять durable Task через PG lease/fence и восстанавливать после сбоев (Story 4.3)
CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с fence-токеном, зашифрованный input и повторная проверка доступа до сетевого вызова, checkpoint с AttemptID до dispatch и восстановление только по PG. Устаревший worker не может зафиксировать результат. Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса: стоп в окне после checkpoint и до permit завершает задачу как failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи остаются доступными. Retry-попытки получают отдельные дочерние spans с durable current_attempt_id. Shared admission/fairness из Story 4.1 сохранены. Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47, bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
This commit is contained in:
1 parent
6df6ffac4e
commit
f9f8c625d1
52 files changed
+5142
-232
No files matched your search
@@ -2,6 +2,10 @@
|
||||
mod admin_auth_command;
|
||||
#[path = "crank_migrate/db_connect.rs"]
|
||||
mod db_connect;
|
||||
#[path = "crank_migrate/reporting.rs"]
|
||||
mod reporting;
|
||||
|
||||
use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json};
|
||||
|
||||
use crank_config::{ConfigSource, parse_migrator};
|
||||
use crank_registry::{
|
||||
@@ -11,7 +15,7 @@ use crank_registry::{
|
||||
};
|
||||
use crank_runtime::SecretCrypto;
|
||||
use serde_json::json;
|
||||
use std::{path::Path, process::ExitCode};
|
||||
use std::process::ExitCode;
|
||||
use time::OffsetDateTime;
|
||||
const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000;
|
||||
#[tokio::main]
|
||||
@@ -765,6 +769,59 @@ async fn process_rotation_batch(
|
||||
processed += 1;
|
||||
}
|
||||
}
|
||||
let mut after_task_id: Option<String> = None;
|
||||
loop {
|
||||
let tasks = registry
|
||||
.list_tasks_for_master_key_epoch_page(
|
||||
rotation.source_epoch,
|
||||
after_task_id.as_deref(),
|
||||
MASTER_KEY_ROTATION_PAGE_SIZE,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
if tasks.is_empty() {
|
||||
break;
|
||||
}
|
||||
for task in tasks {
|
||||
after_task_id = Some(task.task_id.clone());
|
||||
if task.target_epoch == Some(rotation.target_epoch) {
|
||||
continue;
|
||||
}
|
||||
if max_versions.is_some_and(|limit| processed as usize >= limit) {
|
||||
return Ok(processed);
|
||||
}
|
||||
let plaintext = current_crypto
|
||||
.decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext)
|
||||
.map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.stage_master_key_rotation_task_ciphertext(
|
||||
&rotation.id,
|
||||
&task.task_id,
|
||||
rotation.source_epoch,
|
||||
&target_ciphertext,
|
||||
target_crypto.key_version(),
|
||||
rotation.target_epoch,
|
||||
&now,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
processed += 1;
|
||||
}
|
||||
}
|
||||
Ok(processed)
|
||||
}
|
||||
|
||||
@@ -818,6 +875,47 @@ async fn verify_staged_targets(
|
||||
verified += 1;
|
||||
}
|
||||
}
|
||||
let mut after_task_id: Option<String> = None;
|
||||
loop {
|
||||
let tasks = registry
|
||||
.list_target_tasks_for_master_key_rotation_page(
|
||||
rotation.target_epoch,
|
||||
after_task_id.as_deref(),
|
||||
MASTER_KEY_ROTATION_PAGE_SIZE,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
if tasks.is_empty() {
|
||||
break;
|
||||
}
|
||||
for task in tasks {
|
||||
after_task_id = Some(task.task_id.clone());
|
||||
let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let key_version = task.target_key_version.as_deref().ok_or_else(|| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
target_crypto
|
||||
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
|
||||
.map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
verified += 1;
|
||||
}
|
||||
}
|
||||
if verified != rotation.total_secret_versions {
|
||||
return Err(CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
@@ -877,89 +975,3 @@ fn decrypt_current(
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_rotation(
|
||||
registry: &PostgresRegistry,
|
||||
allowed_states: &[&str],
|
||||
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
|
||||
let status = registry
|
||||
.master_key_rotation_status()
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
Ok(status
|
||||
.rotations
|
||||
.into_iter()
|
||||
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
|
||||
}
|
||||
|
||||
fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
|
||||
json!({
|
||||
"id": rotation.id,
|
||||
"source_epoch": rotation.source_epoch,
|
||||
"target_epoch": rotation.target_epoch,
|
||||
"state": rotation.state,
|
||||
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
|
||||
"checkpoint_secret_id": rotation.checkpoint_secret_id,
|
||||
"total_secret_versions": rotation.total_secret_versions,
|
||||
"processed_secret_versions": rotation.processed_secret_versions,
|
||||
"verified_secret_versions": rotation.verified_secret_versions,
|
||||
"failure_code": rotation.failure_code,
|
||||
})
|
||||
}
|
||||
|
||||
fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
|
||||
let path = path.ok_or_else(|| {
|
||||
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
|
||||
})?;
|
||||
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if !metadata.is_file() || metadata.len() > 16_384 {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if value.trim().is_empty() {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
|
||||
if options.current_key_file.is_some()
|
||||
|| options.target_key_file.is_some()
|
||||
|| options.backup_ref.is_some()
|
||||
|| options.rotation_id.is_some()
|
||||
|| options.max_versions.is_some()
|
||||
{
|
||||
return Err(CliError::new(
|
||||
"invalid_command",
|
||||
"cli.arguments",
|
||||
"run_master_key_status",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Leaf/reporting helpers of the migrator CLI.
|
||||
//!
|
||||
//! Extracted from `crank-migrate.rs` without behaviour changes so the binary entry stays within
|
||||
//! the code-health line limit.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crank_registry::{MasterKeyRotationRecord, PostgresRegistry};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{CliError, MasterKeyOptions};
|
||||
|
||||
pub(super) async fn active_rotation(
|
||||
registry: &PostgresRegistry,
|
||||
allowed_states: &[&str],
|
||||
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
|
||||
let status = registry
|
||||
.master_key_rotation_status()
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
Ok(status
|
||||
.rotations
|
||||
.into_iter()
|
||||
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
|
||||
}
|
||||
|
||||
pub(super) fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
|
||||
json!({
|
||||
"id": rotation.id,
|
||||
"source_epoch": rotation.source_epoch,
|
||||
"target_epoch": rotation.target_epoch,
|
||||
"state": rotation.state,
|
||||
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
|
||||
"checkpoint_secret_id": rotation.checkpoint_secret_id,
|
||||
"total_secret_versions": rotation.total_secret_versions,
|
||||
"processed_secret_versions": rotation.processed_secret_versions,
|
||||
"verified_secret_versions": rotation.verified_secret_versions,
|
||||
"failure_code": rotation.failure_code,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
|
||||
let path = path.ok_or_else(|| {
|
||||
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
|
||||
})?;
|
||||
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if !metadata.is_file() || metadata.len() > 16_384 {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if value.trim().is_empty() {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub(super) fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
|
||||
if options.current_key_file.is_some()
|
||||
|| options.target_key_file.is_some()
|
||||
|| options.backup_ref.is_some()
|
||||
|| options.rotation_id.is_some()
|
||||
|| options.max_versions.is_some()
|
||||
{
|
||||
return Err(CliError::new(
|
||||
"invalid_command",
|
||||
"cli.arguments",
|
||||
"run_master_key_status",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -29,7 +29,7 @@ sha2.workspace = true
|
||||
sqlx.workspace = true
|
||||
thiserror.workspace = true
|
||||
time.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio = { workspace = true, features = ["sync", "signal"] }
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
@@ -45,5 +45,6 @@ opentelemetry_sdk.workspace = true
|
||||
prost.workspace = true
|
||||
reqwest.workspace = true
|
||||
tower.workspace = true
|
||||
tokio = { workspace = true, features = ["net", "time"] }
|
||||
tracing-opentelemetry.workspace = true
|
||||
metrics-util = "0.20.4"
|
||||
+149
-13
@@ -201,20 +201,45 @@ async fn run(
|
||||
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
|
||||
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
|
||||
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
|
||||
// Story 4.2: the durable Task application entry shares the registry pool, the published catalog
|
||||
// refresh interval and the exact admission limits of the runtime; the public wire stays
|
||||
// synchronous/unsupported and no worker is started.
|
||||
let task_store = PostgresTaskStore::new(
|
||||
// Story 4.2/4.3: the durable Task application entry shares the registry pool, the published
|
||||
// catalog refresh interval and the exact admission limits of the runtime; the public wire stays
|
||||
// synchronous/unsupported while a fenced in-process worker executes accepted Tasks.
|
||||
let task_store = std::sync::Arc::new(PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
admission_limits(runtime_limits),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?;
|
||||
)?);
|
||||
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
cache_stores.coordination.clone(),
|
||||
crank_runtime::TaskService::new(std::sync::Arc::new(task_store)),
|
||||
crank_runtime::TaskService::with_secret_crypto(
|
||||
task_store.clone(),
|
||||
std::sync::Arc::new(secret_crypto.clone()),
|
||||
),
|
||||
));
|
||||
// Story 4.3: the durable Task worker runs in this process over the same PostgreSQL store,
|
||||
// published catalog, executor and encrypted-input authority as the accepting application.
|
||||
let task_handler =
|
||||
std::sync::Arc::new(crank_community_mcp::TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime.clone(),
|
||||
secret_crypto.clone(),
|
||||
));
|
||||
let task_worker = std::sync::Arc::new(crank_runtime::TaskWorker::new(
|
||||
task_store,
|
||||
task_handler,
|
||||
crank_runtime::TaskWorkerConfig {
|
||||
owner: format!("mcp-server-{}", std::process::id()),
|
||||
..crank_runtime::TaskWorkerConfig::default()
|
||||
},
|
||||
)?);
|
||||
let (task_shutdown_tx, task_shutdown_rx) = tokio::sync::watch::channel(false);
|
||||
let task_worker_handle = tokio::spawn({
|
||||
let worker = task_worker.clone();
|
||||
async move { worker.run_until_shutdown(task_shutdown_rx).await }
|
||||
});
|
||||
let app = build_app_with_artifacts_and_tasks(
|
||||
registry,
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
@@ -252,15 +277,77 @@ async fn run(
|
||||
info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening");
|
||||
*startup_completed = true;
|
||||
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = axum::serve(listener, app) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
let serve_result: Result<(), Box<dyn std::error::Error>> = async {
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
}
|
||||
} else {
|
||||
serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone())
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
// Stop new claims; active attempts checkpoint/drain or are cancelled inside the bounded worker
|
||||
// shutdown timeout (<=30s), and queued PostgreSQL work is left for recovery.
|
||||
task_shutdown_tx.send(true).ok();
|
||||
if tokio::time::timeout(Duration::from_secs(30), task_worker_handle)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
name: "mcp.task_worker.shutdown_timeout",
|
||||
"durable Task worker did not stop within the story shutdown budget"
|
||||
);
|
||||
}
|
||||
serve_result
|
||||
}
|
||||
|
||||
/// Gracefully stop the durable Task worker as soon as shutdown is requested, while still
|
||||
/// draining in-flight HTTP connections before returning.
|
||||
async fn serve_with_worker_shutdown<S>(
|
||||
app: axum::Router,
|
||||
listener: tokio::net::TcpListener,
|
||||
signal: S,
|
||||
worker_stop: tokio::sync::watch::Sender<bool>,
|
||||
) -> std::io::Result<()>
|
||||
where
|
||||
S: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let worker_stop_graceful = worker_stop.clone();
|
||||
let graceful = async move {
|
||||
signal.await;
|
||||
let _ = worker_stop_graceful.send(true);
|
||||
};
|
||||
let result = axum::serve(listener, app)
|
||||
.with_graceful_shutdown(graceful)
|
||||
.await;
|
||||
let _ = worker_stop.send(true);
|
||||
result
|
||||
}
|
||||
|
||||
/// Process shutdown trigger for the HTTP surface and the durable Task worker.
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
if let Ok(mut signal) =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
{
|
||||
signal.recv().await;
|
||||
}
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {}
|
||||
_ = terminate => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_observability(
|
||||
@@ -506,3 +593,52 @@ fn runtime_cache_config(
|
||||
.map(|value| value.expose_secret().to_owned()),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{Router, routing::get};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{oneshot, watch};
|
||||
|
||||
async fn slow_handler() -> &'static str {
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
"ok"
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_stops_before_http_drain_completes() {
|
||||
let app = Router::new().route("/slow", get(slow_handler));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (stop_tx, mut stop_rx) = watch::channel(false);
|
||||
let (signal_tx, signal_rx) = oneshot::channel::<()>();
|
||||
|
||||
let handle = tokio::spawn(serve_with_worker_shutdown(
|
||||
app,
|
||||
listener,
|
||||
async {
|
||||
let _ = signal_rx.await;
|
||||
},
|
||||
stop_tx,
|
||||
));
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let request = client.get(format!("http://{}/slow", addr));
|
||||
let in_flight = tokio::spawn(async move { request.send().await });
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
let _ = signal_tx.send(());
|
||||
|
||||
tokio::time::timeout(Duration::from_millis(250), stop_rx.changed())
|
||||
.await
|
||||
.expect("worker stop signal should arrive before HTTP drain")
|
||||
.expect("watch changed");
|
||||
assert!(*stop_rx.borrow());
|
||||
|
||||
let response = in_flight.await.unwrap().unwrap();
|
||||
assert!(response.status().is_success());
|
||||
let serve_result = handle.await.unwrap();
|
||||
assert!(serve_result.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,413 @@
|
||||
//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1).
|
||||
//!
|
||||
//! End-to-end: a real durable Task worker dispatches a retrying GET through a real local
|
||||
//! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is
|
||||
//! persisted in `tasks.current_attempt_id` on both attempts.
|
||||
|
||||
use super::super::common;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::get;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{
|
||||
AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_runtime::{
|
||||
OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy,
|
||||
};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{Id, Subscriber, field::Visit};
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
const TOOL_NAME: &str = "attempt_trace";
|
||||
const KEY: &str = "attempt-trace-key-1";
|
||||
|
||||
async fn publish(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
operation_id: &crank_core::OperationId,
|
||||
version: u32,
|
||||
) {
|
||||
registry
|
||||
.publish_operation(crank_registry::PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id,
|
||||
version,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn application_with_store(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
) -> (
|
||||
crank_community_mcp::TaskApplication,
|
||||
std::sync::Arc<crank_registry::PostgresTaskStore>,
|
||||
SecretCrypto,
|
||||
) {
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = std::sync::Arc::new(
|
||||
crank_registry::PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = crank_community_mcp::TaskApplication::new(
|
||||
registry.clone(),
|
||||
std::time::Duration::ZERO,
|
||||
std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()),
|
||||
crank_runtime::TaskService::with_secret_crypto(
|
||||
store.clone(),
|
||||
std::sync::Arc::new(crypto.clone()),
|
||||
),
|
||||
);
|
||||
(tasks, store, crypto)
|
||||
}
|
||||
|
||||
async fn principal(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
name: &str,
|
||||
scopes: &[crank_core::PlatformApiKeyScope],
|
||||
) -> crank_community_mcp::TaskPrincipal {
|
||||
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("an active platform key resolves");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crank_community_mcp::TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: crank_core::VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn adapter_attempt_span_carries_durable_attempt_id() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
get({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
let n = calls.fetch_add(1, Ordering::SeqCst);
|
||||
if n == 0 {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(serde_json::json!({"error":"boom"})),
|
||||
);
|
||||
}
|
||||
(
|
||||
StatusCode::OK,
|
||||
axum::Json(serde_json::json!({"id":"lead-1"})),
|
||||
)
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
let Target::Rest(target) = &mut operation.target;
|
||||
target.method = crank_core::HttpMethod::Get;
|
||||
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
});
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"attempt-trace",
|
||||
&[
|
||||
crank_core::PlatformApiKeyScope::Read,
|
||||
crank_core::PlatformApiKeyScope::Write,
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let (tasks, store, crypto) = application_with_store(®istry).await;
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&serde_json::json!({"email": "user@example.com"}),
|
||||
&CorrelationContext::generate(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
let runtime =
|
||||
community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let mut config = TaskWorkerConfig::default();
|
||||
config.owner = "runtime-worker-attempt-trace".to_owned();
|
||||
config.batch_limit = 1;
|
||||
config.lease_ms = 10_000;
|
||||
config.poll_interval = std::time::Duration::from_millis(50);
|
||||
config.max_inflight = 1;
|
||||
config.shutdown_timeout = std::time::Duration::from_secs(5);
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
|
||||
let capture = TraceCapture::default();
|
||||
let subscriber = tracing_subscriber::registry().with(capture.clone());
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
server.abort();
|
||||
|
||||
let durable_attempt_id: Option<String> = sqlx::query_scalar::<_, Option<String>>(
|
||||
"select current_attempt_id from tasks where task_id = $1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
|
||||
let spans = capture.snapshot();
|
||||
let attempt_spans: Vec<_> = spans
|
||||
.iter()
|
||||
.filter(|s| s.name == "adapter_attempt")
|
||||
.collect();
|
||||
assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans");
|
||||
for span in &attempt_spans {
|
||||
let aid = span.fields.get("attempt_id").cloned().unwrap_or_default();
|
||||
assert!(!aid.is_empty(), "attempt_id must not be empty");
|
||||
assert!(
|
||||
!aid.contains("traceparent"),
|
||||
"attempt_id must not contain traceparent"
|
||||
);
|
||||
assert_eq!(
|
||||
Some(aid.as_str()),
|
||||
durable_attempt_id.as_deref(),
|
||||
"span attempt_id must equal durable attempt id"
|
||||
);
|
||||
}
|
||||
let ids: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert_eq!(ids[0], ids[1], "same durable id on both retries");
|
||||
assert_eq!(
|
||||
spans
|
||||
.iter()
|
||||
.filter(|span| span.name == "durable_task_attempt")
|
||||
.count(),
|
||||
1,
|
||||
"one durable attempt span per dispatch"
|
||||
);
|
||||
let runtime_span = spans
|
||||
.iter()
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime.execute span");
|
||||
assert_eq!(
|
||||
runtime_span.parent_name,
|
||||
Some("durable_task_attempt"),
|
||||
"the execution continues the durable attempt trace"
|
||||
);
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(
|
||||
span.parent_name,
|
||||
Some("runtime.execute"),
|
||||
"each retry is a child attempt span under the durable execution"
|
||||
);
|
||||
}
|
||||
|
||||
let attempts: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|s| s.fields.get("attempt").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert!(attempts.contains(&"1".to_string()));
|
||||
assert!(attempts.contains(&"2".to_string()));
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(
|
||||
span.fields.get("max_attempts").cloned().unwrap_or_default(),
|
||||
"2"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// TraceCapture minimal port from crates/crank-runtime/tests/stages.rs
|
||||
#[derive(Clone, Default)]
|
||||
struct TraceCapture {
|
||||
spans: Arc<Mutex<Vec<CapturedSpan>>>,
|
||||
}
|
||||
|
||||
impl TraceCapture {
|
||||
fn snapshot(&self) -> Vec<CapturedSpan> {
|
||||
self.spans.lock().expect("span lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturedSpan {
|
||||
name: &'static str,
|
||||
parent_name: Option<&'static str>,
|
||||
fields: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for TraceCapture
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
{
|
||||
fn on_new_span(
|
||||
&self,
|
||||
attributes: &tracing::span::Attributes<'_>,
|
||||
id: &Id,
|
||||
context: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let parent = attributes
|
||||
.parent()
|
||||
.and_then(|parent| context.span(parent))
|
||||
.or_else(|| {
|
||||
attributes
|
||||
.is_contextual()
|
||||
.then(|| context.lookup_current())
|
||||
.flatten()
|
||||
});
|
||||
let mut visitor = FieldVisitor::default();
|
||||
attributes.record(&mut visitor);
|
||||
let mut spans = self.spans.lock().expect("span lock");
|
||||
let index = spans.len();
|
||||
spans.push(CapturedSpan {
|
||||
name: attributes.metadata().name(),
|
||||
parent_name: parent.map(|span| span.metadata().name()),
|
||||
fields: visitor.fields,
|
||||
});
|
||||
context
|
||||
.span(id)
|
||||
.expect("span exists")
|
||||
.extensions_mut()
|
||||
.insert(index);
|
||||
}
|
||||
|
||||
fn on_record(
|
||||
&self,
|
||||
id: &Id,
|
||||
values: &tracing::span::Record<'_>,
|
||||
context: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let mut visitor = FieldVisitor::default();
|
||||
values.record(&mut visitor);
|
||||
let span = context.span(id).expect("span exists");
|
||||
let index = *span.extensions().get::<usize>().expect("capture index");
|
||||
self.spans.lock().expect("span lock")[index]
|
||||
.fields
|
||||
.extend(visitor.fields);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FieldVisitor {
|
||||
fields: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl Visit for FieldVisitor {
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), value.to_owned());
|
||||
}
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), format!("{value:?}"));
|
||||
}
|
||||
}
|
||||
+33
-14
@@ -14,13 +14,14 @@ use std::time::Duration;
|
||||
|
||||
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
use crank_core::{
|
||||
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState,
|
||||
ToolExecutionMode, ToolExecutionPolicy,
|
||||
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId,
|
||||
TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{
|
||||
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
|
||||
};
|
||||
use crank_runtime::{InMemoryCoordinationStateStore, TaskService};
|
||||
use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService};
|
||||
use serde_json::json;
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
@@ -149,7 +150,9 @@ async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn task_service(pool: PgPool) -> TaskService {
|
||||
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
||||
|
||||
fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService {
|
||||
let store = PostgresTaskStore::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
@@ -161,15 +164,27 @@ fn task_service(pool: PgPool) -> TaskService {
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration");
|
||||
TaskService::new(std::sync::Arc::new(store))
|
||||
TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto))
|
||||
}
|
||||
|
||||
fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
/// The application entry under the same encrypted-input authority the production process uses.
|
||||
async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key");
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(0),
|
||||
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
task_service(registry.pool().clone()),
|
||||
task_service(registry.pool().clone(), crypto),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -243,7 +258,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
@@ -273,7 +288,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// A freshly constructed application + service reads the same durable identity and pins.
|
||||
let reader = application(®istry);
|
||||
let reader = application(®istry).await;
|
||||
let record = reader
|
||||
.status(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
@@ -317,7 +332,7 @@ async fn a_new_publication_never_rebinds_an_accepted_task() {
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
@@ -397,7 +412,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&owner,
|
||||
@@ -433,7 +448,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
));
|
||||
|
||||
// A synchronous publication is not accepted as background work at all.
|
||||
let synchronous_app = application(®istry);
|
||||
let synchronous_app = application(®istry).await;
|
||||
let refused = synchronous_app
|
||||
.accept_background(
|
||||
&owner,
|
||||
@@ -453,7 +468,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
let admin = TaskPrincipal::Admin {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
@@ -549,7 +564,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
assert!(app.health().await.is_ok());
|
||||
|
||||
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
|
||||
@@ -589,3 +604,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
|
||||
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
mod attempt_trace;
|
||||
mod shutdown_window;
|
||||
mod task_worker;
|
||||
@@ -0,0 +1,190 @@
|
||||
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
|
||||
//!
|
||||
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
|
||||
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
|
||||
/// does not consume work that is still queued.
|
||||
///
|
||||
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
|
||||
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
|
||||
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
|
||||
#[tokio::test]
|
||||
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
||||
TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
|
||||
// statement keeps the window deterministic without competing for a second connection.
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
|
||||
begin
|
||||
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
||||
perform pg_sleep(1.5);
|
||||
end if;
|
||||
return new;
|
||||
end $$;
|
||||
create trigger delay_task_dispatch_checkpoint before update on tasks
|
||||
for each row execute function delay_task_dispatch_checkpoint();",
|
||||
))
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.expect("checkpoint delay trigger");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let mut accepted = Vec::new();
|
||||
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_stop_window".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
},
|
||||
scoped_key: scoped_key.repeat(64),
|
||||
input_fingerprint: scoped_key.repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
accepted.push(store.accept(request).await.expect("PG accepts"));
|
||||
}
|
||||
|
||||
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
|
||||
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let executed = Arc::new(AtomicUsize::new(0));
|
||||
struct GatedHandler {
|
||||
entered: Arc<tokio::sync::Semaphore>,
|
||||
release: Arc<tokio::sync::Semaphore>,
|
||||
executed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for GatedHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
self.entered.add_permits(1);
|
||||
let _ = self.release.acquire().await;
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
_lease: &TaskLease,
|
||||
_attempt_id: &str,
|
||||
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
||||
_permit: TaskDispatchPermit,
|
||||
) -> TaskAttemptOutcome {
|
||||
self.executed.fetch_add(1, Ordering::SeqCst);
|
||||
TaskAttemptOutcome::Completed(json!({ "ok": true }))
|
||||
}
|
||||
}
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
|
||||
entered: revalidate_entered.clone(),
|
||||
release: revalidate_release.clone(),
|
||||
executed: executed.clone(),
|
||||
});
|
||||
let mut config = TaskWorkerConfig::default();
|
||||
config.owner = "runtime-worker-stop-window".to_owned();
|
||||
config.batch_limit = 1;
|
||||
config.lease_ms = 30_000;
|
||||
config.poll_interval = Duration::from_millis(50);
|
||||
config.renewal_interval = Duration::from_secs(20);
|
||||
config.max_inflight = 1;
|
||||
config.shutdown_timeout = Duration::from_secs(20);
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
|
||||
.await
|
||||
.expect("worker must claim the first Task and enter revalidation")
|
||||
.expect("revalidation gate is open")
|
||||
.forget();
|
||||
|
||||
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
|
||||
revalidate_release.add_permits(1);
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
|
||||
// finalize the Task, but no permit exists yet, so no outbound call can start.
|
||||
let stop_requested = std::time::Instant::now();
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the 30s budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
assert!(
|
||||
stop_requested.elapsed() >= Duration::from_millis(500),
|
||||
"the checkpoint must have still been in flight when the stop arrived"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
executed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"a stop between checkpoint and permit must never reach the upstream adapter"
|
||||
);
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_stop_window".to_owned(),
|
||||
};
|
||||
let record = store.read(&access, accepted[0].task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Failed);
|
||||
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
|
||||
assert!(record.current_attempt_id.is_some());
|
||||
assert_eq!(
|
||||
record
|
||||
.error
|
||||
.as_ref()
|
||||
.and_then(|error| error.value().get("code"))
|
||||
.and_then(|code| code.as_str()),
|
||||
Some("shutdown_before_dispatch")
|
||||
);
|
||||
|
||||
// Work that was still queued is not consumed by the stopped worker.
|
||||
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
|
||||
assert_eq!(queued.state, TaskState::Queued);
|
||||
let recovered = store
|
||||
.claim_batch("worker_after_stop", 4, 10_000)
|
||||
.await
|
||||
.expect("queued work stays claimable");
|
||||
assert_eq!(recovered.len(), 1);
|
||||
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
|
||||
}
|
||||
@@ -0,0 +1,822 @@
|
||||
//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch.
|
||||
//!
|
||||
//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a
|
||||
/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan,
|
||||
/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented.
|
||||
#[tokio::test]
|
||||
async fn task_worker_store_lease_recovery_contract() {
|
||||
let registry = common::test_registry().await;
|
||||
let store = PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration");
|
||||
|
||||
// Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and
|
||||
// leases durable work, preserving the same admission-backed intent.
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
},
|
||||
scoped_key: "a".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
// This store-level control exercises lease/fence behavior, not the encrypted acceptance path.
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
let leases = store
|
||||
.claim_batch("worker_1", 4, 10_000)
|
||||
.await
|
||||
.expect("claim");
|
||||
assert_eq!(leases.len(), 1);
|
||||
assert_eq!(leases[0].record.task_id, accepted.task_id);
|
||||
let lease = leases[0].clone();
|
||||
|
||||
// A stale or competing owner cannot win CAS with the issued fencing token.
|
||||
assert!(
|
||||
store
|
||||
.renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.write_progress(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence + 1,
|
||||
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
|
||||
assert!(
|
||||
store
|
||||
.checkpoint_dispatch(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence + 1,
|
||||
"attempt-stale".to_owned()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
// The valid owner still commits the dispatch checkpoint before any outbound work.
|
||||
assert!(
|
||||
store
|
||||
.checkpoint_dispatch(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence,
|
||||
"attempt-1".to_owned(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.current_attempt_id
|
||||
.as_deref()
|
||||
== Some("attempt-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_worker_runtime_dispatch_and_recovery_contract() {
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
||||
TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let pool = registry.pool().clone();
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
pool.clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// Positive control: PostgreSQL alone accepts the durable Task.
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_runtime".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
},
|
||||
scoped_key: "r".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
let task_id = accepted.task_id;
|
||||
|
||||
struct RuntimeHandler {
|
||||
observed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl TaskHandler for RuntimeHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
_lease: &TaskLease,
|
||||
_attempt_id: &str,
|
||||
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
||||
_permit: TaskDispatchPermit,
|
||||
) -> TaskAttemptOutcome {
|
||||
self.observed.fetch_add(1, Ordering::SeqCst);
|
||||
TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
|
||||
}
|
||||
}
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
|
||||
observed: observed.clone(),
|
||||
});
|
||||
|
||||
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
|
||||
let mut config = TaskWorkerConfig::default();
|
||||
config.owner = "runtime-worker-1".to_owned();
|
||||
config.batch_limit = 1;
|
||||
config.lease_ms = 10_000;
|
||||
config.poll_interval = Duration::from_millis(100);
|
||||
config.renewal_interval = Duration::from_millis(1_000);
|
||||
config.max_inflight = 1;
|
||||
config.shutdown_timeout = Duration::from_secs(5);
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_runtime".to_owned(),
|
||||
};
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
if let Ok(record) = store.read(&access, task_id).await {
|
||||
state = Some(record.state);
|
||||
if record.state == TaskState::Completed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state, Some(TaskState::Completed));
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
1,
|
||||
"exactly one outbound attempt"
|
||||
);
|
||||
// Terminal state is not duplicated.
|
||||
let record = store.read(&access, task_id).await.expect("terminal read");
|
||||
assert_eq!(record.state, TaskState::Completed);
|
||||
|
||||
// Bounded shutdown with no fresh work returns promptly.
|
||||
let mut idle_config = TaskWorkerConfig::default();
|
||||
idle_config.owner = "runtime-worker-idle".to_owned();
|
||||
idle_config.poll_interval = Duration::from_millis(50);
|
||||
idle_config.shutdown_timeout = Duration::from_secs(2);
|
||||
let idle_worker = TaskWorker::new(
|
||||
store.clone(),
|
||||
handler_for_no_work(store.clone()),
|
||||
idle_config,
|
||||
)
|
||||
.expect("valid idle worker");
|
||||
idle_worker.stop();
|
||||
let started = std::time::Instant::now();
|
||||
idle_worker.run().await.expect("idle worker stops");
|
||||
assert!(started.elapsed() <= Duration::from_secs(30));
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call.
|
||||
#[tokio::test]
|
||||
async fn task_worker_revalidation_denial_never_dispatches() {
|
||||
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
||||
TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_denied".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
},
|
||||
scoped_key: "d".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
|
||||
struct DeniedHandler {
|
||||
observed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for DeniedHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Err(TaskWorkerError::Store(TaskError::invalid(
|
||||
"publication or credential revoked",
|
||||
)))
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
_lease: &TaskLease,
|
||||
_attempt_id: &str,
|
||||
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
||||
_permit: TaskDispatchPermit,
|
||||
) -> TaskAttemptOutcome {
|
||||
self.observed.fetch_add(1, Ordering::SeqCst);
|
||||
TaskAttemptOutcome::Completed(json!({"ok": true}))
|
||||
}
|
||||
}
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
|
||||
observed: observed.clone(),
|
||||
});
|
||||
let mut config = TaskWorkerConfig::default();
|
||||
config.owner = "runtime-worker-denied".to_owned();
|
||||
config.batch_limit = 1;
|
||||
config.lease_ms = 10_000;
|
||||
config.poll_interval = Duration::from_millis(50);
|
||||
config.max_inflight = 1;
|
||||
config.shutdown_timeout = Duration::from_secs(5);
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_denied".to_owned(),
|
||||
};
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
if let Ok(record) = store.read(&access, accepted.task_id).await {
|
||||
state = Some(record.state);
|
||||
if record.state == TaskState::Failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state, Some(TaskState::Failed));
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"invalidated work never reaches the upstream"
|
||||
);
|
||||
}
|
||||
|
||||
fn handler_for_no_work(
|
||||
_store: std::sync::Arc<PostgresTaskStore>,
|
||||
) -> std::sync::Arc<dyn crank_runtime::TaskHandler> {
|
||||
struct NoWorkHandler;
|
||||
#[async_trait::async_trait]
|
||||
impl crank_runtime::TaskHandler for NoWorkHandler {
|
||||
async fn revalidate(
|
||||
&self,
|
||||
_lease: &crank_core::TaskLease,
|
||||
) -> Result<(), crank_runtime::TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
_lease: &crank_core::TaskLease,
|
||||
_attempt_id: &str,
|
||||
_cancelled: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
||||
_permit: crank_runtime::TaskDispatchPermit,
|
||||
) -> crank_runtime::TaskAttemptOutcome {
|
||||
crank_runtime::TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
|
||||
}
|
||||
}
|
||||
std::sync::Arc::new(NoWorkHandler)
|
||||
}
|
||||
|
||||
/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a
|
||||
/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the
|
||||
/// accepted parent trace is continued on the wire.
|
||||
#[tokio::test]
|
||||
async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
|
||||
use axum::Router;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let seen_traceparent = Arc::new(Mutex::new(None::<String>));
|
||||
|
||||
let app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
let seen = Arc::clone(&seen_traceparent);
|
||||
move |headers: HeaderMap| {
|
||||
let calls = Arc::clone(&calls);
|
||||
let seen = Arc::clone(&seen);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(value) = headers
|
||||
.get("traceparent")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
{
|
||||
*seen.lock().unwrap() = Some(value.to_owned());
|
||||
}
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-worker-e2e",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"worker-e2e-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance with executable input");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let mut config = TaskWorkerConfig::default();
|
||||
config.owner = "runtime-worker-e2e".to_owned();
|
||||
config.batch_limit = 1;
|
||||
config.lease_ms = 10_000;
|
||||
config.poll_interval = Duration::from_millis(50);
|
||||
config.max_inflight = 1;
|
||||
config.shutdown_timeout = Duration::from_secs(5);
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call");
|
||||
assert_eq!(
|
||||
seen_traceparent.lock().unwrap().as_deref(),
|
||||
Some(correlation.trace_context().traceparent()),
|
||||
"the durable attempt continues the accepted parent trace"
|
||||
);
|
||||
}
|
||||
|
||||
/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as
|
||||
/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically.
|
||||
#[tokio::test]
|
||||
async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-worker-uncertain",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"worker-uncertain-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance with executable input");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let mut config = TaskWorkerConfig::default();
|
||||
config.owner = "runtime-worker-uncertain".to_owned();
|
||||
config.batch_limit = 1;
|
||||
config.lease_ms = 10_000;
|
||||
config.poll_interval = Duration::from_millis(50);
|
||||
config.max_inflight = 1;
|
||||
config.shutdown_timeout = Duration::from_secs(5);
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("outcome_unknown") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("outcome_unknown"));
|
||||
// No automatic retry: another authoritative scan never redispaches the uncertain attempt.
|
||||
assert!(
|
||||
store
|
||||
.claim_batch("worker_after", 4, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced
|
||||
/// terminal commit within the 30s budget.
|
||||
#[tokio::test]
|
||||
async fn task_worker_graceful_stop_drains_active_attempt() {
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
||||
TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_drain".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
},
|
||||
scoped_key: "s".repeat(64),
|
||||
input_fingerprint: "d".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
|
||||
struct SlowHandler {
|
||||
started: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for SlowHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
_lease: &TaskLease,
|
||||
_attempt_id: &str,
|
||||
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
||||
_permit: TaskDispatchPermit,
|
||||
) -> TaskAttemptOutcome {
|
||||
self.started.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
TaskAttemptOutcome::Completed(json!({"ok": true}))
|
||||
}
|
||||
}
|
||||
let started = Arc::new(AtomicUsize::new(0));
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
|
||||
started: started.clone(),
|
||||
});
|
||||
let mut config = TaskWorkerConfig::default();
|
||||
config.owner = "runtime-worker-drain".to_owned();
|
||||
config.batch_limit = 1;
|
||||
config.lease_ms = 10_000;
|
||||
config.poll_interval = Duration::from_millis(50);
|
||||
config.max_inflight = 1;
|
||||
config.shutdown_timeout = Duration::from_secs(20);
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
for _ in 0..100 {
|
||||
if started.load(Ordering::SeqCst) > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started");
|
||||
let stop_requested = std::time::Instant::now();
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the 30s budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
assert!(stop_requested.elapsed() <= Duration::from_secs(30));
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_drain".to_owned(),
|
||||
};
|
||||
let record = store.read(&access, accepted.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Completed);
|
||||
}
|
||||
@@ -272,7 +272,10 @@ fn record_unknown_approval_access_denial(reason: &'static str) {
|
||||
);
|
||||
}
|
||||
|
||||
fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool {
|
||||
pub(super) fn allows_scope(
|
||||
scopes: &[PlatformApiKeyScope],
|
||||
required_scope: PlatformApiKeyScope,
|
||||
) -> bool {
|
||||
match required_scope {
|
||||
PlatformApiKeyScope::Read => scopes.iter().any(|scope| {
|
||||
matches!(
|
||||
|
||||
@@ -51,15 +51,15 @@ mod auth_resolution;
|
||||
mod dispatch;
|
||||
mod endpoint;
|
||||
mod legacy_dispatch;
|
||||
mod task_worker;
|
||||
mod tasks;
|
||||
|
||||
use self::approvals::{
|
||||
approve_request, deny_request, get_approval_request, list_pending_approvals,
|
||||
};
|
||||
pub(super) use self::auth_resolution::resolve_operation_auth;
|
||||
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
|
||||
pub use self::task_worker::TaskWorkerHandler;
|
||||
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
|
||||
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
|
||||
// through `super`, and the plain lib build does not use them (same pattern as
|
||||
// `#[cfg(test)] use invocation_history::observe_invocation_history_outcome;` above).
|
||||
|
||||
@@ -19,18 +19,31 @@ pub(crate) async fn resolve_operation_auth(
|
||||
workspace_id: &crank_core::WorkspaceId,
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<Option<ResolvedAuth>, RuntimeError> {
|
||||
if execution_config.auth_profile_ref.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let span = Stage::AuthResolve.span();
|
||||
let result = resolve_runtime_auth_for_task(
|
||||
resolve_operation_auth_with(
|
||||
&state.registry,
|
||||
&state.secret_crypto,
|
||||
workspace_id,
|
||||
execution_config,
|
||||
)
|
||||
.instrument(span.clone())
|
||||
.await;
|
||||
.await
|
||||
}
|
||||
|
||||
/// Same resolution for callers that hold the authoritative registry/secret-crypto pair without the
|
||||
/// HTTP application state (the durable Task worker).
|
||||
pub(crate) async fn resolve_operation_auth_with(
|
||||
registry: &PostgresRegistry,
|
||||
secret_crypto: &SecretCrypto,
|
||||
workspace_id: &crank_core::WorkspaceId,
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<Option<ResolvedAuth>, RuntimeError> {
|
||||
if execution_config.auth_profile_ref.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let span = Stage::AuthResolve.span();
|
||||
let result =
|
||||
resolve_runtime_auth_for_task(registry, secret_crypto, workspace_id, execution_config)
|
||||
.instrument(span.clone())
|
||||
.await;
|
||||
match &result {
|
||||
Ok(_) => StageOutcome::Success.record(&span),
|
||||
Err(_) => {
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
//! Application-level durable Task handler for Story 4.3.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
|
||||
TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId,
|
||||
current_unix_ms,
|
||||
};
|
||||
use crank_registry::PostgresRegistry;
|
||||
use crank_registry::PublishedAgentTool;
|
||||
use crank_runtime::{
|
||||
ExecutionAuthorization, ExecutionRequestError, RuntimeExecutionRequest, RuntimeExecutor,
|
||||
SecretCrypto, TaskDispatchPermit, TaskHandler, TaskService, TaskWorkerError,
|
||||
};
|
||||
|
||||
use super::TaskApplication;
|
||||
use super::auth_resolution::resolve_operation_auth_with;
|
||||
use super::tool_resolution::runtime_operation;
|
||||
use crate::access::{allows_scope, credential_allows_security_level};
|
||||
use crate::catalog::PublishedToolCatalog;
|
||||
|
||||
pub struct TaskWorkerHandler {
|
||||
registry: PostgresRegistry,
|
||||
catalog: PublishedToolCatalog,
|
||||
runtime: RuntimeExecutor,
|
||||
secret_crypto: SecretCrypto,
|
||||
service: TaskService,
|
||||
}
|
||||
|
||||
impl TaskWorkerHandler {
|
||||
/// Production construction from the same application entry, registry and runtime the MCP
|
||||
/// process serves requests with. The worker needs no HTTP state, only these authoritative
|
||||
/// dependencies.
|
||||
pub fn from_application(
|
||||
tasks: &TaskApplication,
|
||||
registry: PostgresRegistry,
|
||||
runtime: RuntimeExecutor,
|
||||
secret_crypto: SecretCrypto,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
catalog: tasks.catalog(),
|
||||
runtime,
|
||||
secret_crypto,
|
||||
service: tasks.task_service(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
|
||||
///
|
||||
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
|
||||
/// policy and revalidates the typed persisted Platform API key identity against the registry:
|
||||
/// the key must still be active, unexpired and bound to this workspace/agent, and its exact
|
||||
/// scopes/security level must still authorize the operation. Any denial is a typed error, so the
|
||||
/// worker finishes the attempt without dispatch (fail closed before any network use).
|
||||
///
|
||||
/// An administrator Task has no authoritative Task revalidation API in the registry, so it stays
|
||||
/// fail-closed here instead of inventing authority.
|
||||
async fn authorize(&self, lease: &TaskLease) -> Result<PublishedAgentTool, TaskError> {
|
||||
if lease.record.owner_kind != TaskOwnerKind::Agent {
|
||||
return Err(TaskError::invalid(
|
||||
"admin task authority revalidation is not implemented",
|
||||
));
|
||||
}
|
||||
let Some(key_id) = lease.record.platform_api_key_id.as_ref() else {
|
||||
return Err(TaskError::invalid("task authority reference is missing"));
|
||||
};
|
||||
|
||||
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
||||
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
||||
let workspace = self
|
||||
.registry
|
||||
.get_workspace(&workspace_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?
|
||||
.ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?;
|
||||
let agents = self
|
||||
.registry
|
||||
.list_agents(&workspace_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
||||
let agent = agents
|
||||
.into_iter()
|
||||
.find(|agent| agent.id.as_str() == lease.record.owner_id.as_str())
|
||||
.ok_or_else(|| TaskError::invalid("task agent no longer exists"))?;
|
||||
|
||||
let catalog = self
|
||||
.catalog
|
||||
.get_catalog(&workspace.workspace.slug, &agent.slug)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
||||
let tool = catalog
|
||||
.tools
|
||||
.into_iter()
|
||||
.find(|tool| {
|
||||
tool.operation.id.as_str() == lease.record.operation_id.as_str()
|
||||
&& tool.operation.version == lease.record.pins.operation_version
|
||||
})
|
||||
.ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?;
|
||||
|
||||
if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() {
|
||||
return Err(TaskError::invalid("task workspace binding mismatch"));
|
||||
}
|
||||
if tool.agent_id.as_str() != lease.record.owner_id.as_str() {
|
||||
return Err(TaskError::invalid("task agent binding mismatch"));
|
||||
}
|
||||
if tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()
|
||||
.is_some_and(|policy| policy.required)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"missing approval proof for task operation",
|
||||
));
|
||||
}
|
||||
|
||||
// Authoritative revalidation of the exact immutable key identity. A revoked, expired,
|
||||
// foreign-workspace or foreign-agent key is absent here and never reaches the network.
|
||||
let key = self
|
||||
.registry
|
||||
.revalidate_platform_api_key(&workspace_id, &agent_id, key_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("task authority store unavailable"))?
|
||||
.ok_or_else(|| TaskError::invalid("task authority is no longer active"))?;
|
||||
let credential = VerifiedMachineCredential {
|
||||
machine_access_mode: MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: OperationSecurityLevel::Standard,
|
||||
scopes: key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(key.api_key.id.clone()),
|
||||
};
|
||||
if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) {
|
||||
return Err(TaskError::invalid("task authority scope is insufficient"));
|
||||
}
|
||||
if !credential_allows_security_level(&credential, tool.operation.security_level) {
|
||||
return Err(TaskError::invalid(
|
||||
"task authority security level is insufficient",
|
||||
));
|
||||
}
|
||||
Ok(tool)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskHandler for TaskWorkerHandler {
|
||||
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
self.authorize(lease)
|
||||
.await
|
||||
.map_err(TaskWorkerError::Store)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
lease: &TaskLease,
|
||||
_attempt_id: &str,
|
||||
cancelled: Arc<std::sync::atomic::AtomicBool>,
|
||||
permit: TaskDispatchPermit,
|
||||
) -> crank_runtime::TaskAttemptOutcome {
|
||||
use crank_runtime::TaskAttemptOutcome;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
// The lease may have been lost between the checkpoint and the outbound call; never dispatch
|
||||
// on behalf of a worker that no longer owns the Task.
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return TaskAttemptOutcome::OutcomeUnknown;
|
||||
}
|
||||
|
||||
let tool = match self.authorize(lease).await {
|
||||
Ok(tool) => tool,
|
||||
Err(error) => {
|
||||
return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()}));
|
||||
}
|
||||
};
|
||||
|
||||
let input = match self.service.executable_input(lease).await {
|
||||
Ok(input) => input,
|
||||
Err(error) => {
|
||||
return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()}));
|
||||
}
|
||||
};
|
||||
|
||||
let operation = runtime_operation(&tool);
|
||||
let now_ms = current_unix_ms();
|
||||
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
|
||||
if deadline_ms <= now_ms {
|
||||
return TaskAttemptOutcome::Failed(
|
||||
serde_json::json!({"code": "task deadline elapsed"}),
|
||||
);
|
||||
}
|
||||
let deadline = Instant::now()
|
||||
+ Duration::from_millis((deadline_ms - now_ms).min(i64::MAX as i64) as u64);
|
||||
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
||||
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
||||
let resolved_auth = match resolve_operation_auth_with(
|
||||
&self.registry,
|
||||
&self.secret_crypto,
|
||||
&workspace_id,
|
||||
&operation.execution_config,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(auth) => auth,
|
||||
Err(error) => {
|
||||
return TaskAttemptOutcome::Failed(
|
||||
serde_json::json!({ "code": error.to_string() }),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Continue the parent trace persisted with the accepted Task; the worker's attempt span is a
|
||||
// child of it, and the executor records the attempt identity. Trace data stays out of labels.
|
||||
let trace_context = match TraceContext::parse(&lease.record.support.traceparent) {
|
||||
Ok(context) => context,
|
||||
Err(_) => {
|
||||
return TaskAttemptOutcome::Failed(
|
||||
serde_json::json!({ "code": "task trace parent is invalid" }),
|
||||
);
|
||||
}
|
||||
};
|
||||
let request_context = crank_runtime::RuntimeRequestContext::new(
|
||||
lease.record.support.request_id.clone(),
|
||||
trace_context,
|
||||
)
|
||||
.with_metering_context(
|
||||
workspace_id.clone(),
|
||||
Some(agent_id.clone()),
|
||||
crank_core::InvocationSource::AgentToolCall,
|
||||
);
|
||||
|
||||
let request = match RuntimeExecutionRequest::try_new(
|
||||
&workspace_id,
|
||||
ExecutionOrigin::AgentSnapshot,
|
||||
Some(&agent_id),
|
||||
&operation,
|
||||
&input,
|
||||
ExecutionAuthorization::Authorized,
|
||||
resolved_auth.as_ref(),
|
||||
&request_context,
|
||||
deadline,
|
||||
) {
|
||||
Ok(request) => request,
|
||||
Err(error) => {
|
||||
return TaskAttemptOutcome::Failed(match error {
|
||||
ExecutionRequestError::InvalidOrigin => {
|
||||
serde_json::json!({"code": "invalid execution origin"})
|
||||
}
|
||||
ExecutionRequestError::InvalidOperationVersion => {
|
||||
serde_json::json!({"code": "invalid operation version"})
|
||||
}
|
||||
ExecutionRequestError::DeadlineElapsed => {
|
||||
serde_json::json!({"code": "task deadline elapsed"})
|
||||
}
|
||||
ExecutionRequestError::InvalidMeteringScope => {
|
||||
serde_json::json!({"code": "invalid metering scope"})
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return TaskAttemptOutcome::OutcomeUnknown;
|
||||
}
|
||||
match self.runtime.execute_task_outcome(request, permit).await {
|
||||
Ok(success) => TaskAttemptOutcome::Completed(success.output),
|
||||
Err(failure) => {
|
||||
// An ambiguous outbound failure is recorded as uncertainty, never as a retryable
|
||||
// failure: the durable checkpoint already exists, so Story 4.4 may reconcile it.
|
||||
if failure.outcome_certainty() == crank_core::OutcomeCertainty::OutcomeUnknown {
|
||||
return TaskAttemptOutcome::OutcomeUnknown;
|
||||
}
|
||||
TaskAttemptOutcome::Failed(
|
||||
serde_json::json!({ "code": format!("{:?}", failure.error_code()) }),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,8 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord,
|
||||
ToolExecutionMode,
|
||||
CorrelationContext, PlatformApiKeyId, TaskAcceptance, TaskAccess, TaskError, TaskId,
|
||||
TaskOwnerKind, TaskRecord, ToolExecutionMode,
|
||||
};
|
||||
use crank_registry::{PostgresRegistry, PublishedAgentTool};
|
||||
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
|
||||
@@ -64,6 +64,20 @@ impl TaskPrincipal {
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed immutable Platform API key identity of this principal. A verified agent credential
|
||||
/// without a key ID cannot be revalidated by a worker, so it is refused instead of accepted;
|
||||
/// an administrator has no machine key and is stored as `None` (the worker fails closed).
|
||||
fn platform_api_key_id(&self) -> Result<Option<PlatformApiKeyId>, TaskApplicationError> {
|
||||
match self {
|
||||
Self::Agent { credential, .. } => credential
|
||||
.platform_api_key_id
|
||||
.clone()
|
||||
.map(Some)
|
||||
.ok_or(TaskApplicationError::Denied),
|
||||
Self::Admin { .. } => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded server-derived authorization scope identity of this principal.
|
||||
fn authorization_scope(&self) -> String {
|
||||
let identity = match self {
|
||||
@@ -131,6 +145,17 @@ impl TaskApplication {
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime task service carrying encrypted executable input.
|
||||
pub fn task_service(&self) -> TaskService {
|
||||
self.service.clone()
|
||||
}
|
||||
|
||||
/// Published catalog of the same application entry, for the durable Task worker's pre-dispatch
|
||||
/// revalidation.
|
||||
pub(super) fn catalog(&self) -> PublishedToolCatalog {
|
||||
self.catalog.clone()
|
||||
}
|
||||
|
||||
/// Readiness of the durable Task store this entry depends on.
|
||||
pub async fn health(&self) -> Result<(), TaskError> {
|
||||
self.service.health().await
|
||||
@@ -170,6 +195,7 @@ impl TaskApplication {
|
||||
// of accepting work a worker could never safely execute.
|
||||
return Err(TaskApplicationError::ApprovalRequired);
|
||||
}
|
||||
let platform_api_key_id = principal.platform_api_key_id()?;
|
||||
let draft = |operation_version: u32| {
|
||||
let (owner_kind, owner_id) = principal.owner(&tool);
|
||||
TaskAcceptDraft {
|
||||
@@ -177,6 +203,7 @@ impl TaskApplication {
|
||||
owner_kind,
|
||||
owner_id,
|
||||
authorization_scope: principal.authorization_scope(),
|
||||
platform_api_key_id: platform_api_key_id.clone(),
|
||||
operation_id: tool.operation.id.as_str().to_owned(),
|
||||
operation_version,
|
||||
policy_mode: policy.mode,
|
||||
|
||||
@@ -16,7 +16,7 @@ mod tool_search;
|
||||
mod transport;
|
||||
|
||||
pub use app::{
|
||||
TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts,
|
||||
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
|
||||
build_app_with_background_workers_and_limits,
|
||||
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler, build_app,
|
||||
build_app_with_artifacts, build_app_with_artifacts_and_tasks,
|
||||
build_app_with_background_workers, build_app_with_background_workers_and_limits,
|
||||
};
|
||||
@@ -76,10 +76,12 @@ pub mod domain {
|
||||
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use crate::task::{
|
||||
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
|
||||
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES,
|
||||
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
|
||||
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds,
|
||||
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES,
|
||||
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
|
||||
TASK_IDENTITY_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
|
||||
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
|
||||
TaskState, TaskSupportIds,
|
||||
};
|
||||
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
|
||||
pub use crate::tool_quality::{
|
||||
@@ -202,11 +204,12 @@ pub use product_event::{
|
||||
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
|
||||
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use task::{
|
||||
TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
|
||||
TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
|
||||
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
|
||||
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
|
||||
TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key,
|
||||
validate_task_envelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport,
|
||||
TaskRecord, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
|
||||
redact_task_envelope, sensitive_task_key, validate_task_envelope,
|
||||
};
|
||||
pub use tool_catalog::{
|
||||
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
|
||||
|
||||
@@ -15,10 +15,15 @@ use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::correlation::{RequestId, TraceContext, TraceId};
|
||||
use crate::ids::PlatformApiKeyId;
|
||||
use crate::operation::ToolExecutionMode;
|
||||
|
||||
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
|
||||
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
|
||||
/// Maximum byte budget of the decrypted executable Task input.
|
||||
pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES;
|
||||
/// Maximum byte budget of the stored encrypted executable Task input ciphertext.
|
||||
pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304;
|
||||
/// Maximum nesting depth of a persisted Task envelope.
|
||||
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
|
||||
/// Maximum number of JSON members/nodes of a persisted Task envelope.
|
||||
@@ -351,6 +356,16 @@ pub fn redact_task_envelope(value: &Value) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
/// Recursive check used to fail closed on sensitive fields before encryption.
|
||||
pub fn contains_sensitive_task_keys(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Object(object) => object
|
||||
.iter()
|
||||
.any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)),
|
||||
Value::Array(values) => values.iter().any(contains_sensitive_task_keys),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
|
||||
/// queue capacity commit together, and no acknowledgement exists before that commit.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@@ -360,6 +375,10 @@ pub struct TaskAcceptRequest {
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
/// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the
|
||||
/// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively.
|
||||
/// `None` means the accepting principal has no machine key (administrator) and fail-closes.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
pub pins: TaskPins,
|
||||
/// Digest of the caller key over the full owner/authorization/Tool scope.
|
||||
@@ -368,6 +387,10 @@ pub struct TaskAcceptRequest {
|
||||
pub input_fingerprint: String,
|
||||
pub input: TaskEnvelope,
|
||||
pub support: TaskSupportIds,
|
||||
/// Bounded encrypted executable input, persisted alongside the redacted input projection.
|
||||
pub executable_input_ciphertext: Option<String>,
|
||||
pub executable_input_key_version: Option<String>,
|
||||
pub executable_input_master_key_epoch: Option<i64>,
|
||||
}
|
||||
|
||||
impl TaskAcceptRequest {
|
||||
@@ -382,6 +405,13 @@ impl TaskAcceptRequest {
|
||||
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
||||
}
|
||||
}
|
||||
if let Some(key_id) = self.platform_api_key_id.as_ref()
|
||||
&& (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task platform api key reference is invalid",
|
||||
));
|
||||
}
|
||||
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task scoped key digest is invalid"));
|
||||
}
|
||||
@@ -394,7 +424,40 @@ impl TaskAcceptRequest {
|
||||
return Err(TaskError::invalid("task input fingerprint is invalid"));
|
||||
}
|
||||
self.pins.validate()?;
|
||||
self.support.validate()
|
||||
self.support.validate()?;
|
||||
let has_ciphertext = self.executable_input_ciphertext.is_some();
|
||||
let has_key = self.executable_input_key_version.is_some();
|
||||
let has_epoch = self.executable_input_master_key_epoch.is_some();
|
||||
match (has_ciphertext, has_key, has_epoch) {
|
||||
(false, false, false) => {}
|
||||
(true, true, true) => {
|
||||
let ciphertext = self.executable_input_ciphertext.as_deref().unwrap();
|
||||
if ciphertext.is_empty()
|
||||
|| ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
let key_version = self.executable_input_key_version.as_deref().unwrap();
|
||||
if key_version.is_empty() || key_version.len() > 64 {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
if self.executable_input_master_key_epoch.unwrap_or_default() < 1 {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -448,6 +511,9 @@ pub struct TaskRecord {
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
/// Typed immutable identity of the Platform API key that authorized the acceptance; the worker
|
||||
/// revalidates this exact key against the registry before any outbound dispatch.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
|
||||
pub input_fingerprint: String,
|
||||
@@ -458,6 +524,16 @@ pub struct TaskRecord {
|
||||
pub result: Option<TaskEnvelope>,
|
||||
pub error: Option<TaskEnvelope>,
|
||||
pub attempts: u32,
|
||||
/// Monotonic lease/fencing token. It never resets and protects Task state mutations.
|
||||
pub fence: u64,
|
||||
/// Random process-instance lease owner, if a worker currently owns the Task.
|
||||
pub lease_owner: Option<String>,
|
||||
/// Lease expiry epoch milliseconds, if the Task is claimed.
|
||||
pub lease_until_unix_ms: Option<i64>,
|
||||
/// Current dispatch attempt identity, committed before any outbound adapter call.
|
||||
pub current_attempt_id: Option<String>,
|
||||
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
|
||||
pub dispatch_started_at_unix_ms: Option<i64>,
|
||||
/// Monotonic lifecycle version used by the terminal CAS.
|
||||
pub version: u32,
|
||||
pub created_at_unix_ms: i64,
|
||||
@@ -542,6 +618,15 @@ impl TaskError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Leased view of a claimed Task: its durable record plus lease fencing identity.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TaskLease {
|
||||
pub record: TaskRecord,
|
||||
pub owner: String,
|
||||
pub fence: u64,
|
||||
pub lease_until: i64,
|
||||
}
|
||||
|
||||
/// Authoritative durable Task port.
|
||||
#[async_trait]
|
||||
pub trait TaskStore: Send + Sync {
|
||||
@@ -566,9 +651,59 @@ pub trait TaskStore: Send + Sync {
|
||||
/// Applies exactly one conditional terminal transition and releases its intent atomically.
|
||||
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes
|
||||
/// expired terminal records. Live and non-terminal work is never deleted.
|
||||
/// Reads encrypted executable input only under a valid, unexpired current lease.
|
||||
async fn read_executable_input(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
) -> Result<(String, String, i64), TaskError>;
|
||||
|
||||
/// Bounded retention...
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
|
||||
|
||||
/// Claims a bounded batch of queued or safely recoverable Tasks under leases.
|
||||
async fn claim_batch(
|
||||
&self,
|
||||
owner: &str,
|
||||
batch_limit: u32,
|
||||
lease_ms: i64,
|
||||
) -> Result<Vec<TaskLease>, TaskError>;
|
||||
|
||||
/// Renews a lease owned by `owner` when fence and expiry still match.
|
||||
async fn renew_lease(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
lease_ms: i64,
|
||||
) -> Result<TaskLease, TaskError>;
|
||||
|
||||
/// Persists redacted progress under a valid lease.
|
||||
async fn write_progress(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
progress: TaskEnvelope,
|
||||
) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Commits an outbound attempt identity and dispatch timestamp before network use.
|
||||
async fn checkpoint_dispatch(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
attempt_id: String,
|
||||
) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Applies a terminal transition only under a valid lease and releases intent atomically.
|
||||
async fn finish_leased(
|
||||
&self,
|
||||
request: TaskFinishRequest,
|
||||
owner: &str,
|
||||
fence: u64,
|
||||
) -> Result<TaskRecord, TaskError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -583,6 +718,7 @@ mod tests {
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 3,
|
||||
@@ -599,6 +735,9 @@ mod tests {
|
||||
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -694,6 +833,11 @@ mod tests {
|
||||
let mut bad_retention = request();
|
||||
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
|
||||
assert!(bad_retention.validate().is_err());
|
||||
|
||||
// The persisted authority reference is bounded and never empty.
|
||||
let mut empty_key_reference = request();
|
||||
empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new(""));
|
||||
assert!(empty_key_reference.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,6 +7,9 @@ mod artifact_metadata_v12;
|
||||
mod authority;
|
||||
mod baseline_v1;
|
||||
mod durable_tasks_v25;
|
||||
mod durable_tasks_v26;
|
||||
mod durable_tasks_v27;
|
||||
mod durable_tasks_v28;
|
||||
mod execution_admission_v24;
|
||||
mod execution_outcome_v5;
|
||||
mod import_plan_v14;
|
||||
@@ -39,6 +42,9 @@ mod schema_guard_v22;
|
||||
mod schema_guard_v23;
|
||||
mod schema_guard_v24;
|
||||
mod schema_guard_v25;
|
||||
mod schema_guard_v26;
|
||||
mod schema_guard_v27;
|
||||
mod schema_guard_v28;
|
||||
mod schema_guard_v7;
|
||||
mod schema_guard_v8;
|
||||
mod schema_guard_v9;
|
||||
|
||||
@@ -5,6 +5,9 @@ use super::approval_side_effects_v10;
|
||||
use super::artifact_cleanup_indexes_v13;
|
||||
use super::artifact_metadata_v12;
|
||||
use super::durable_tasks_v25;
|
||||
use super::durable_tasks_v26;
|
||||
use super::durable_tasks_v27;
|
||||
use super::durable_tasks_v28;
|
||||
use super::execution_admission_v24;
|
||||
use super::execution_outcome_v5;
|
||||
use super::import_plan_v14;
|
||||
@@ -41,9 +44,10 @@ use preflight::*;
|
||||
use contract::baseline_source_digest;
|
||||
use contract::validate_descriptors;
|
||||
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
|
||||
const CURRENT_VERSION: i64 = 25;
|
||||
const CURRENT_VERSION: i64 = 28;
|
||||
const IMPLEMENTED_VERSIONS: &[i64] = &[
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
|
||||
27, 28,
|
||||
];
|
||||
const BASELINE_SOURCE_SHA256: &str =
|
||||
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
|
||||
@@ -344,6 +348,24 @@ impl MigrationAuthority {
|
||||
durable_tasks_v25::SOURCE_SHA256,
|
||||
24,
|
||||
),
|
||||
expand_descriptor(
|
||||
26,
|
||||
durable_tasks_v26::NAME,
|
||||
durable_tasks_v26::SOURCE_SHA256,
|
||||
25,
|
||||
),
|
||||
expand_descriptor(
|
||||
27,
|
||||
durable_tasks_v27::NAME,
|
||||
durable_tasks_v27::SOURCE_SHA256,
|
||||
26,
|
||||
),
|
||||
expand_descriptor(
|
||||
28,
|
||||
durable_tasks_v28::NAME,
|
||||
durable_tasks_v28::SOURCE_SHA256,
|
||||
27,
|
||||
),
|
||||
]
|
||||
}
|
||||
pub fn validate_sequence() -> Result<(), MigrationError> {
|
||||
@@ -509,6 +531,15 @@ impl MigrationAuthority {
|
||||
if from < 25 {
|
||||
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
|
||||
}
|
||||
if from < 26 {
|
||||
durable_tasks_v26::apply(&mut transaction, &Self::sequence()[25]).await?;
|
||||
}
|
||||
if from < 27 {
|
||||
durable_tasks_v27::apply(&mut transaction, &Self::sequence()[26]).await?;
|
||||
}
|
||||
if from < 28 {
|
||||
durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?;
|
||||
}
|
||||
match inspect(&mut transaction).await? {
|
||||
MigrationPreflight::Current {
|
||||
version: CURRENT_VERSION,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v26";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v26.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v26",
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v26",
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
-- V26: PostgreSQL lease/fencing/CAS additions for durable background Tasks (Story 4.3).
|
||||
alter table tasks add column if not exists fence bigint not null default 0;
|
||||
alter table tasks add column if not exists lease_owner text;
|
||||
alter table tasks add column if not exists lease_until timestamptz;
|
||||
alter table tasks add column if not exists current_attempt_id text;
|
||||
alter table tasks add column if not exists dispatch_started_at timestamptz;
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v27";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v27.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v27",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v27",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
-- V27: encrypted executable Task input (Story 4.3).
|
||||
alter table tasks add column if not exists executable_input_ciphertext text;
|
||||
alter table tasks add column if not exists executable_input_key_version text;
|
||||
alter table tasks add column if not exists executable_input_master_key_epoch bigint;
|
||||
|
||||
alter table tasks add constraint tasks_executable_input_key_version_check check (
|
||||
executable_input_ciphertext is null
|
||||
or (executable_input_key_version is not null and octet_length(executable_input_key_version) between 1 and 64)
|
||||
);
|
||||
|
||||
alter table tasks add column if not exists target_executable_input_ciphertext text;
|
||||
alter table tasks add column if not exists target_executable_input_key_version text;
|
||||
alter table tasks add column if not exists target_executable_input_master_key_epoch bigint;
|
||||
|
||||
alter table tasks add constraint tasks_executable_input_epoch_check check (
|
||||
executable_input_ciphertext is null
|
||||
or (executable_input_master_key_epoch is not null
|
||||
and executable_input_master_key_epoch >= 1)
|
||||
);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v28";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v28.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v28",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v28",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
-- V28: typed Platform API key authority reference of a durable Task (Story 4.3).
|
||||
-- Only the immutable key identifier is stored, never the bearer secret.
|
||||
alter table tasks add column if not exists platform_api_key_id text;
|
||||
|
||||
alter table tasks add constraint tasks_platform_api_key_id_check check (
|
||||
platform_api_key_id is null
|
||||
or octet_length(platform_api_key_id) between 1 and 160
|
||||
);
|
||||
@@ -480,8 +480,40 @@ pub(super) async fn validate_schema_fingerprint(
|
||||
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 26 {
|
||||
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 27 {
|
||||
super::schema_guard_v26::validate_v26_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 28 {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v27::validate_v27_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v28::validate_v28_durable_tasks(connection).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//! V26 schema guard: lease/fencing columns for durable background Tasks (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("task_id", "uuid", false),
|
||||
("workspace_id", "text", false),
|
||||
("owner_kind", "text", false),
|
||||
("owner_id", "text", false),
|
||||
("authorization_scope", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("scoped_key", "text", false),
|
||||
("input_fingerprint", "text", false),
|
||||
("request_id", "text", false),
|
||||
("trace_id", "text", false),
|
||||
("traceparent", "text", false),
|
||||
("trace_sampled", "boolean", false),
|
||||
("policy_mode", "text", false),
|
||||
("approval_reference", "text", true),
|
||||
("state", "text", false),
|
||||
("version", "integer", false),
|
||||
("attempts", "integer", false),
|
||||
("input_envelope", "jsonb", false),
|
||||
("progress_envelope", "jsonb", true),
|
||||
("result_envelope", "jsonb", true),
|
||||
("error_envelope", "jsonb", true),
|
||||
("execution_deadline", "timestamp with time zone", false),
|
||||
("retain_until", "timestamp with time zone", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
("fence", "bigint", false),
|
||||
("lease_owner", "text", true),
|
||||
("lease_until", "timestamp with time zone", true),
|
||||
("current_attempt_id", "text", true),
|
||||
("dispatch_started_at", "timestamp with time zone", true),
|
||||
];
|
||||
|
||||
fn v26_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
// delegate to v25 absent for relation mostly; V26 columns must not exist.
|
||||
for column in [
|
||||
"fence",
|
||||
"lease_owner",
|
||||
"lease_until",
|
||||
"current_attempt_id",
|
||||
"dispatch_started_at",
|
||||
] {
|
||||
if super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_base_columns(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
if !relation_exists(connection, TASKS).await? {
|
||||
return Err(v26_error("v26.relations"));
|
||||
}
|
||||
for (column, data_type, nullable) in TASK_COLUMNS {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v26_error("v26.columns"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
validate_v26_base_columns(connection).await?;
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
|
||||
return Err(v26_error("v26.columns"));
|
||||
}
|
||||
// Constraints/index checks inherited conceptually from V25; V26 adds columns only.
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
//! V27 schema guard: encrypted executable Task input columns (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
use super::schema_guard_v26::validate_v26_base_columns;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
pub(super) async fn validate_v27_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
for column in [
|
||||
"executable_input_ciphertext",
|
||||
"executable_input_key_version",
|
||||
"executable_input_master_key_epoch",
|
||||
] {
|
||||
if super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v27_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
validate_v26_base_columns(connection).await?;
|
||||
// Additional V27 columns.
|
||||
let extra = [
|
||||
("executable_input_ciphertext", "text", true),
|
||||
("executable_input_key_version", "text", true),
|
||||
("executable_input_master_key_epoch", "bigint", true),
|
||||
("target_executable_input_ciphertext", "text", true),
|
||||
("target_executable_input_key_version", "text", true),
|
||||
("target_executable_input_master_key_epoch", "bigint", true),
|
||||
];
|
||||
for (column, data_type, nullable) in extra {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v27.columns",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
}
|
||||
// Exact total column count for version 27: 43 V26 columns + 3 V27 columns.
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(37).unwrap_or_default() {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v27.columns",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
//! V28 schema guard: typed Platform API key authority reference of a durable Task (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
pub(super) async fn validate_v28_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
if super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v28_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
// The V27 column set is still required and unchanged; V28 only appends one column.
|
||||
validate_v27_base_columns(connection).await?;
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind("platform_api_key_id")
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some("text")
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref() == Some("YES")
|
||||
});
|
||||
if !valid {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v28.columns",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
// Exact total column count for version 28: the 37 V27 columns + 1 V28 column.
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(38).unwrap_or_default() {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v28.columns",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// V27 columns must still be exactly present before V28 is considered applied.
|
||||
async fn validate_v27_base_columns(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
||||
for column in [
|
||||
"executable_input_ciphertext",
|
||||
"executable_input_key_version",
|
||||
"executable_input_master_key_epoch",
|
||||
"target_executable_input_ciphertext",
|
||||
"target_executable_input_key_version",
|
||||
"target_executable_input_master_key_epoch",
|
||||
] {
|
||||
if !super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v28.columns",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -577,6 +577,145 @@ pub(super) async fn insert_admission_entry(
|
||||
Ok(u64::try_from(ticket_id).unwrap_or_default())
|
||||
}
|
||||
|
||||
/// Shared grant step used by the durable Task store: grants one reserved Task intent if capacity allows,
|
||||
/// marks it `executing`, persists the weighted round-robin cursor, and returns its Task id. It reuses
|
||||
/// the same selector/capacity logic as synchronous admission so no second policy exists.
|
||||
pub(super) async fn grant_next_task(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
cursor: &AdmissionCursor,
|
||||
limits: &AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
) -> Result<(Option<Uuid>, AdmissionCursor), AttemptError> {
|
||||
let rows = sqlx::query(
|
||||
"select ticket_id, agent_key, operation_id, task_id
|
||||
from execution_admission_entries
|
||||
where state = 'reserved' and task_id is not null
|
||||
order by ticket_id
|
||||
limit $1
|
||||
for update skip locked",
|
||||
)
|
||||
.bind(QUEUE_READ_LIMIT)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
if rows.is_empty() {
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
struct Candidate {
|
||||
entry: AdmissionQueueEntry,
|
||||
task_id: Uuid,
|
||||
}
|
||||
let mut candidates = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
|
||||
let task_id: Uuid = row.try_get("task_id").map_err(AttemptError::from_sql)?;
|
||||
candidates.push(Candidate {
|
||||
entry: AdmissionQueueEntry {
|
||||
ticket_id: u64::try_from(ticket_id).unwrap_or_default(),
|
||||
sequence: 0,
|
||||
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
|
||||
operation_id: row
|
||||
.try_get("operation_id")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
},
|
||||
task_id,
|
||||
});
|
||||
}
|
||||
let queue: Vec<AdmissionQueueEntry> = candidates.iter().map(|c| c.entry.clone()).collect();
|
||||
let Some(selection) = select_next_grant(&queue, cursor, scheduler) else {
|
||||
return Ok((None, cursor.clone()));
|
||||
};
|
||||
let head_index = selection.index;
|
||||
let head = &candidates[head_index];
|
||||
|
||||
let global_executing: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries where state = 'executing'",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let agent_counts = sqlx::query(
|
||||
"select agent_key as key, count(*)::bigint as count
|
||||
from execution_admission_entries where state = 'executing' group by agent_key",
|
||||
)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let mut agent_map = std::collections::BTreeMap::new();
|
||||
for row in agent_counts {
|
||||
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
|
||||
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
|
||||
agent_map.insert(key, u64::try_from(count).unwrap_or_default());
|
||||
}
|
||||
let tool_counts = sqlx::query(
|
||||
"select operation_id as key, count(*)::bigint as count
|
||||
from execution_admission_entries where state = 'executing' group by operation_id",
|
||||
)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let mut tool_map = std::collections::BTreeMap::new();
|
||||
for row in tool_counts {
|
||||
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
|
||||
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
|
||||
tool_map.insert(key, u64::try_from(count).unwrap_or_default());
|
||||
}
|
||||
|
||||
if u64::try_from(global_executing).unwrap_or(u64::MAX)
|
||||
>= u64::try_from(limits.global_limit).unwrap_or(u64::MAX)
|
||||
{
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
let agent_executing = agent_map
|
||||
.get(head.entry.agent_key.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if agent_executing >= u64::try_from(limits.agent_limit).unwrap_or(u64::MAX) {
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
let tool_executing = tool_map
|
||||
.get(head.entry.operation_id.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if tool_executing >= u64::try_from(limits.tool_limit).unwrap_or(u64::MAX) {
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"update execution_admission_entries
|
||||
set state = 'executing', granted_at = clock_timestamp()
|
||||
where ticket_id = $1 and state = 'reserved'",
|
||||
)
|
||||
.bind(i64::try_from(head.entry.ticket_id).unwrap_or(i64::MAX))
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
sqlx::query(
|
||||
"update execution_admission_controllers
|
||||
set cursor_agent = $2,
|
||||
cursor_agent_credit = $3,
|
||||
cursor_tool = $4,
|
||||
cursor_tool_credit = $5,
|
||||
updated_at = now()
|
||||
where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(selection.cursor.agent.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
|
||||
.bind(selection.cursor.tool.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
Ok((Some(head.task_id), selection.cursor))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum AttemptError {
|
||||
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
|
||||
Contended,
|
||||
|
||||
@@ -170,6 +170,69 @@ impl PostgresRegistry {
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Authoritative revalidation of the Platform API key that authorized a durable Task (Story 4.3).
|
||||
///
|
||||
/// The key is returned only when the exact immutable ID is still active, unexpired and bound to
|
||||
/// the same workspace and agent; otherwise `None`, so a worker fails closed. Only the immutable
|
||||
/// identity is supplied here — the bearer secret is never stored or re-read.
|
||||
pub async fn revalidate_platform_api_key(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
agent_id: &AgentId,
|
||||
key_id: &PlatformApiKeyId,
|
||||
) -> Result<Option<PlatformApiKeyRecord>, RegistryError> {
|
||||
let row = sqlx::query(
|
||||
"select
|
||||
k.id,
|
||||
k.workspace_id,
|
||||
k.agent_id,
|
||||
k.key_kind,
|
||||
k.name,
|
||||
k.prefix,
|
||||
k.scopes_json,
|
||||
k.status,
|
||||
k.created_at,
|
||||
k.last_used_at,
|
||||
k.expires_at,
|
||||
k.allowed_origins_json
|
||||
from platform_api_keys k
|
||||
where k.id = $1
|
||||
and k.workspace_id = $2
|
||||
and k.agent_id = $3
|
||||
and k.key_kind = 'mcp_client'
|
||||
and k.status = 'active'
|
||||
and (k.expires_at is null or k.expires_at > now())
|
||||
limit 1",
|
||||
)
|
||||
.bind(key_id.as_str())
|
||||
.bind(workspace_id.as_str())
|
||||
.bind(agent_id.as_str())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
row.map(|row| {
|
||||
Ok(PlatformApiKeyRecord {
|
||||
api_key: PlatformApiKey {
|
||||
id: PlatformApiKeyId::new(row.get::<String, _>("id")),
|
||||
workspace_id: WorkspaceId::new(row.get::<String, _>("workspace_id")),
|
||||
agent_id: row.get::<Option<String>, _>("agent_id").map(AgentId::new),
|
||||
key_kind: deserialize_enum_text(&row.get::<String, _>("key_kind"), "key_kind")?,
|
||||
name: row.get("name"),
|
||||
prefix: row.get("prefix"),
|
||||
scopes: deserialize_json_value(row.get::<Value, _>("scopes_json"))?,
|
||||
status: deserialize_enum_text(&row.get::<String, _>("status"), "status")?,
|
||||
created_at: row.get("created_at"),
|
||||
last_used_at: row.get("last_used_at"),
|
||||
expires_at: row.get("expires_at"),
|
||||
allowed_origins: deserialize_json_value(
|
||||
row.get::<Value, _>("allowed_origins_json"),
|
||||
)?,
|
||||
},
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub async fn get_approval_api_key_by_secret_for_agent_slug(
|
||||
&self,
|
||||
workspace_slug: &str,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::*;
|
||||
|
||||
const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
|
||||
pub(super) const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
|
||||
|
||||
impl PostgresRegistry {
|
||||
pub async fn verify_or_register_master_key_identity(
|
||||
@@ -356,12 +356,19 @@ impl PostgresRegistry {
|
||||
return Err(RegistryError::MasterKeyRotationConflict);
|
||||
}
|
||||
|
||||
let total = sqlx::query_scalar::<_, i64>(
|
||||
let secret_total = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from secret_versions where master_key_epoch = $1",
|
||||
)
|
||||
.bind(source_epoch)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
let task_total = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from tasks where executable_input_master_key_epoch = $1",
|
||||
)
|
||||
.bind(source_epoch)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
let total = secret_total + task_total;
|
||||
let rotation_id = format!("master-key-e{source_epoch}-to-e{target_epoch}");
|
||||
let aborted = sqlx::query(
|
||||
"select
|
||||
@@ -403,6 +410,16 @@ impl PostgresRegistry {
|
||||
.bind(target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set target_executable_input_ciphertext = null,
|
||||
target_executable_input_key_version = null,
|
||||
target_executable_input_master_key_epoch = null
|
||||
where target_executable_input_master_key_epoch = $1",
|
||||
)
|
||||
.bind(target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set state = 'running',
|
||||
@@ -662,9 +679,24 @@ impl PostgresRegistry {
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set executable_input_ciphertext = target_executable_input_ciphertext,
|
||||
executable_input_key_version = target_executable_input_key_version,
|
||||
executable_input_master_key_epoch = target_executable_input_master_key_epoch,
|
||||
target_executable_input_ciphertext = null,
|
||||
target_executable_input_key_version = null,
|
||||
target_executable_input_master_key_epoch = null
|
||||
where executable_input_master_key_epoch = $1
|
||||
and target_executable_input_master_key_epoch = $2",
|
||||
)
|
||||
.bind(rotation.source_epoch)
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set state = 'promoted',
|
||||
set state = 'promoted',
|
||||
updated_at = $2::timestamptz
|
||||
where id = $1",
|
||||
)
|
||||
@@ -697,6 +729,16 @@ impl PostgresRegistry {
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set target_executable_input_ciphertext = null,
|
||||
target_executable_input_key_version = null,
|
||||
target_executable_input_master_key_epoch = null
|
||||
where target_executable_input_master_key_epoch = $1",
|
||||
)
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set state = 'aborted',
|
||||
@@ -883,7 +925,7 @@ async fn select_active_master_key_identity_for_update(
|
||||
Ok(row.map(map_master_key_identity))
|
||||
}
|
||||
|
||||
async fn select_master_key_rotation_for_update(
|
||||
pub(super) async fn select_master_key_rotation_for_update(
|
||||
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
rotation_id: &str,
|
||||
) -> Result<MasterKeyRotationRecord, RegistryError> {
|
||||
@@ -915,16 +957,19 @@ async fn select_master_key_rotation_for_update(
|
||||
})
|
||||
}
|
||||
|
||||
async fn count_staged_rotation_targets(
|
||||
pub(super) async fn count_staged_rotation_targets(
|
||||
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
source_epoch: i64,
|
||||
target_epoch: i64,
|
||||
) -> Result<i64, RegistryError> {
|
||||
let count = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*)
|
||||
from secret_versions
|
||||
where master_key_epoch = $1
|
||||
and target_master_key_epoch = $2",
|
||||
"select
|
||||
(select count(*) from secret_versions
|
||||
where master_key_epoch = $1
|
||||
and target_master_key_epoch = $2) +
|
||||
(select count(*) from tasks
|
||||
where executable_input_master_key_epoch = $1
|
||||
and target_executable_input_master_key_epoch = $2)",
|
||||
)
|
||||
.bind(source_epoch)
|
||||
.bind(target_epoch)
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
//! Task ciphertext paging/staging of the master-key rotation (Story 4.3).
|
||||
//!
|
||||
//! Extracted from `master_key.rs` without logic changes so that module stays within the
|
||||
//! code-health line limit; the operations and their SQL are unchanged.
|
||||
|
||||
use super::master_key::{
|
||||
MASTER_KEY_PAGE_LIMIT_MAX, count_staged_rotation_targets, select_master_key_rotation_for_update,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
fn map_master_key_task_version(
|
||||
row: sqlx::postgres::PgRow,
|
||||
) -> Result<MasterKeyTaskVersion, RegistryError> {
|
||||
Ok(MasterKeyTaskVersion {
|
||||
task_id: row.get::<String, _>("task_id"),
|
||||
ciphertext: row
|
||||
.get::<Option<String>, _>("executable_input_ciphertext")
|
||||
.unwrap_or_default(),
|
||||
key_version: row
|
||||
.get::<Option<String>, _>("executable_input_key_version")
|
||||
.unwrap_or_default(),
|
||||
epoch: row
|
||||
.get::<Option<i64>, _>("executable_input_master_key_epoch")
|
||||
.unwrap_or_default(),
|
||||
target_ciphertext: row.get::<Option<String>, _>("target_executable_input_ciphertext"),
|
||||
target_key_version: row.get::<Option<String>, _>("target_executable_input_key_version"),
|
||||
target_epoch: row.get::<Option<i64>, _>("target_executable_input_master_key_epoch"),
|
||||
})
|
||||
}
|
||||
|
||||
impl PostgresRegistry {
|
||||
pub async fn list_tasks_for_master_key_epoch_page(
|
||||
&self,
|
||||
epoch: i64,
|
||||
after_task_id: Option<&str>,
|
||||
limit: i64,
|
||||
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
|
||||
if epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
|
||||
return Err(RegistryError::InvalidMasterKeyIdentity);
|
||||
}
|
||||
let rows = sqlx::query(
|
||||
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
|
||||
)
|
||||
.bind(epoch)
|
||||
.bind(after_task_id)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
rows.into_iter().map(map_master_key_task_version).collect()
|
||||
}
|
||||
|
||||
pub async fn list_target_tasks_for_master_key_rotation_page(
|
||||
&self,
|
||||
target_epoch: i64,
|
||||
after_task_id: Option<&str>,
|
||||
limit: i64,
|
||||
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
|
||||
if target_epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
|
||||
return Err(RegistryError::InvalidMasterKeyIdentity);
|
||||
}
|
||||
let rows = sqlx::query(
|
||||
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where target_executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
|
||||
)
|
||||
.bind(target_epoch)
|
||||
.bind(after_task_id)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
rows.into_iter().map(map_master_key_task_version).collect()
|
||||
}
|
||||
|
||||
pub async fn stage_master_key_rotation_task_ciphertext(
|
||||
&self,
|
||||
rotation_id: &str,
|
||||
task_id: &str,
|
||||
source_epoch: i64,
|
||||
target_ciphertext: &str,
|
||||
target_key_version: &str,
|
||||
target_epoch: i64,
|
||||
now: &OffsetDateTime,
|
||||
) -> Result<MasterKeyRotationRecord, RegistryError> {
|
||||
let mut tx = self.pool.begin().await?;
|
||||
let rotation = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
|
||||
if rotation.state != "running"
|
||||
|| rotation.source_epoch != source_epoch
|
||||
|| rotation.target_epoch != target_epoch
|
||||
{
|
||||
return Err(RegistryError::MasterKeyRotationConflict);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"update tasks
|
||||
set target_executable_input_ciphertext = $3,
|
||||
target_executable_input_key_version = $4,
|
||||
target_executable_input_master_key_epoch = $5
|
||||
where task_id::text = $1
|
||||
and executable_input_master_key_epoch = $2
|
||||
and (
|
||||
target_executable_input_master_key_epoch is null
|
||||
or target_executable_input_master_key_epoch = $5
|
||||
)
|
||||
and (
|
||||
target_executable_input_ciphertext is null
|
||||
or target_executable_input_ciphertext = $3
|
||||
)",
|
||||
)
|
||||
.bind(task_id)
|
||||
.bind(source_epoch)
|
||||
.bind(target_ciphertext)
|
||||
.bind(target_key_version)
|
||||
.bind(target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
if result.rows_affected() != 1 {
|
||||
return Err(RegistryError::MasterKeyRotationConflict);
|
||||
}
|
||||
let processed = count_staged_rotation_targets(&mut tx, source_epoch, target_epoch).await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set checkpoint_secret_id = checkpoint_secret_id,
|
||||
processed_secret_versions = $2,
|
||||
updated_at = $3::timestamptz
|
||||
where id = $1",
|
||||
)
|
||||
.bind(rotation_id)
|
||||
.bind(processed)
|
||||
.bind(now)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let updated = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
|
||||
tx.commit().await?;
|
||||
Ok(updated)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MasterKeyTaskVersion {
|
||||
pub task_id: String,
|
||||
pub ciphertext: String,
|
||||
pub key_version: String,
|
||||
pub epoch: i64,
|
||||
pub target_ciphertext: Option<String>,
|
||||
pub target_key_version: Option<String>,
|
||||
pub target_epoch: Option<i64>,
|
||||
}
|
||||
@@ -10,6 +10,7 @@ mod auth;
|
||||
mod connection;
|
||||
mod import_job;
|
||||
mod master_key;
|
||||
mod master_key_tasks;
|
||||
mod observability;
|
||||
mod onboarding;
|
||||
mod onboarding_import;
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
|
||||
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode,
|
||||
PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError,
|
||||
TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
|
||||
TaskState, TaskStore, TaskSupportIds, ToolExecutionMode,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
@@ -27,10 +27,14 @@ use super::admission::{
|
||||
};
|
||||
|
||||
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
|
||||
authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode,
|
||||
authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version,
|
||||
policy_mode,
|
||||
approval_reference,
|
||||
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
|
||||
result_envelope, error_envelope, attempts, version,
|
||||
result_envelope, error_envelope, attempts, fence, lease_owner, lease_until,
|
||||
(extract(epoch from lease_until) * 1000)::bigint as lease_until_ms,
|
||||
current_attempt_id, dispatch_started_at,
|
||||
(extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version,
|
||||
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
|
||||
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
|
||||
(extract(epoch from created_at) * 1000)::bigint as created_ms,
|
||||
@@ -40,6 +44,7 @@ const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_i
|
||||
pub struct PostgresTaskStore {
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: crank_core::AdmissionSchedulerConfig,
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
@@ -54,7 +59,11 @@ impl PostgresTaskStore {
|
||||
scheduler
|
||||
.validate()
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
Ok(Self { pool, limits })
|
||||
Ok(Self {
|
||||
pool,
|
||||
limits,
|
||||
scheduler,
|
||||
})
|
||||
}
|
||||
|
||||
fn unavailable(error: impl std::fmt::Display) -> TaskError {
|
||||
@@ -151,6 +160,20 @@ impl TaskStore for PostgresTaskStore {
|
||||
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
if let Some(epoch) = request.executable_input_master_key_epoch {
|
||||
super::master_key::ensure_active_master_key_epoch(&mut transaction, epoch)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
}
|
||||
let outcome = self.accept_locked(&mut transaction, &request).await;
|
||||
match outcome {
|
||||
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
|
||||
@@ -252,6 +275,7 @@ impl TaskStore for PostgresTaskStore {
|
||||
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
||||
and owner_id = $7 and authorization_scope = $8
|
||||
and state = $9 and version = $10
|
||||
and (lease_owner is null or fence = 0)
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(request.target_state.as_str())
|
||||
@@ -285,6 +309,12 @@ impl TaskStore for PostgresTaskStore {
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
|
||||
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
|
||||
let mut transaction = self.start().await?;
|
||||
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
let expired = sqlx::query(
|
||||
"with expired as (
|
||||
update tasks
|
||||
@@ -295,6 +325,7 @@ impl TaskStore for PostgresTaskStore {
|
||||
where task_id in (
|
||||
select task_id from tasks
|
||||
where state = 'queued' and execution_deadline <= now()
|
||||
and lease_owner is null
|
||||
order by task_id
|
||||
limit $1
|
||||
)
|
||||
@@ -332,6 +363,351 @@ impl TaskStore for PostgresTaskStore {
|
||||
expired_terminal: purged.rows_affected(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn claim_batch(
|
||||
&self,
|
||||
owner: &str,
|
||||
batch_limit: u32,
|
||||
lease_ms: i64,
|
||||
) -> Result<Vec<TaskLease>, TaskError> {
|
||||
let batch = (batch_limit.clamp(1, 64)) as usize;
|
||||
// A checkpointed attempt is never redispatched: once its lease expires the outbound outcome
|
||||
// is uncertain, so it resolves to outcome_unknown and releases its admission slot. Story 4.4
|
||||
// owns any later retry/cancellation of that uncertain outcome.
|
||||
let mut transaction = self.start().await?;
|
||||
sqlx::query(
|
||||
"with candidates as (
|
||||
select task_id from tasks
|
||||
where state = 'working' and lease_owner is not null
|
||||
and current_attempt_id is not null
|
||||
and lease_until < clock_timestamp()
|
||||
order by task_id
|
||||
limit 64
|
||||
), resolved as (
|
||||
update tasks
|
||||
set state = 'outcome_unknown',
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id in (select task_id from candidates)
|
||||
and state = 'working'
|
||||
returning task_id
|
||||
)
|
||||
delete from execution_admission_entries
|
||||
where task_id in (select task_id from resolved)",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
|
||||
// A crash before the durable dispatch checkpoint left the Task queued for execution: its
|
||||
// admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is
|
||||
// bounded by the <=30s lease expiry, inside the 60s story budget.
|
||||
sqlx::query(
|
||||
"with candidates as (
|
||||
select task_id from tasks
|
||||
where state = 'working' and lease_owner is not null
|
||||
and current_attempt_id is null
|
||||
and lease_until < clock_timestamp()
|
||||
order by task_id
|
||||
limit 64
|
||||
), requeued as (
|
||||
update tasks
|
||||
set state = 'queued',
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id in (select task_id from candidates)
|
||||
and state = 'working'
|
||||
returning task_id
|
||||
)
|
||||
update execution_admission_entries
|
||||
set state = 'reserved', granted_at = null
|
||||
where task_id in (select task_id from requeued) and state = 'executing'",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
|
||||
let mut leases = Vec::new();
|
||||
for _ in 0..batch {
|
||||
let mut transaction = self.start().await?;
|
||||
let workspace_cursor =
|
||||
match super::admission::lock_admission_in_tx(&mut transaction, "*").await {
|
||||
Ok(cursor) => cursor,
|
||||
Err(error) => {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::unavailable(format!("{error:?}")));
|
||||
}
|
||||
};
|
||||
let (task_id, _updated_cursor) = match super::admission::grant_next_task(
|
||||
&mut transaction,
|
||||
"*",
|
||||
&workspace_cursor,
|
||||
&self.limits,
|
||||
self.scheduler,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::unavailable(format!("{error:?}")));
|
||||
}
|
||||
};
|
||||
// Persist the updated controller cursor even when no task is granted.
|
||||
let Some(task_id) = task_id else {
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
break;
|
||||
};
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = 'working',
|
||||
lease_owner = $1,
|
||||
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
|
||||
fence = fence + 1,
|
||||
updated_at = now()
|
||||
where task_id = $3
|
||||
and (
|
||||
(state = 'queued' and lease_owner is null)
|
||||
or (state = 'working' and lease_owner is not null
|
||||
and lease_until < clock_timestamp() and current_attempt_id is null)
|
||||
)
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(owner)
|
||||
.bind(lease_ms)
|
||||
.bind(task_id)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
// Persist cursor update (grant_next_task already did it) regardless.
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
break;
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
leases.push(TaskLease {
|
||||
owner: owner.to_owned(),
|
||||
fence: record.fence,
|
||||
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
|
||||
record,
|
||||
});
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
}
|
||||
Ok(leases)
|
||||
}
|
||||
|
||||
async fn renew_lease(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
lease_ms: i64,
|
||||
) -> Result<TaskLease, TaskError> {
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set lease_until = clock_timestamp() + ($1::bigint || ' milliseconds')::interval,
|
||||
updated_at = now()
|
||||
where task_id = $2 and lease_owner = $3 and fence = $4
|
||||
and lease_until > clock_timestamp()
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(lease_ms)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
let lease = TaskLease {
|
||||
owner: owner.to_owned(),
|
||||
fence: record.fence,
|
||||
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
|
||||
record,
|
||||
};
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(lease)
|
||||
}
|
||||
|
||||
async fn write_progress(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
progress: TaskEnvelope,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set progress_envelope = $1, updated_at = now()
|
||||
where task_id = $2 and lease_owner = $3 and fence = $4
|
||||
and lease_until > clock_timestamp()
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(progress.value().clone())
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn read_executable_input(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
) -> Result<(String, String, i64), TaskError> {
|
||||
let row = sqlx::query(
|
||||
"select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch
|
||||
from tasks
|
||||
where task_id = $1 and state = 'working' and lease_owner = $2 and fence = $3
|
||||
and lease_until > clock_timestamp()",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let ciphertext: Option<String> = row
|
||||
.try_get("executable_input_ciphertext")
|
||||
.map_err(Self::unavailable)?;
|
||||
let key_version: Option<String> = row
|
||||
.try_get("executable_input_key_version")
|
||||
.map_err(Self::unavailable)?;
|
||||
let epoch: Option<i64> = row
|
||||
.try_get("executable_input_master_key_epoch")
|
||||
.map_err(Self::unavailable)?;
|
||||
if let Some(epoch) = epoch {
|
||||
let active = sqlx::query_scalar::<_, i64>(
|
||||
"select epoch from master_key_identities where status = 'active' order by epoch desc limit 1",
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
if let Some(active) = active {
|
||||
if active != epoch {
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
}
|
||||
}
|
||||
match (ciphertext, key_version, epoch) {
|
||||
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
|
||||
Ok((ciphertext, key_version, epoch))
|
||||
}
|
||||
_ => Err(TaskError::NotFound),
|
||||
}
|
||||
}
|
||||
|
||||
async fn checkpoint_dispatch(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
attempt_id: String,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set current_attempt_id = $1,
|
||||
dispatch_started_at = now(),
|
||||
attempts = attempts + 1,
|
||||
updated_at = now()
|
||||
where task_id = $2 and lease_owner = $3 and fence = $4
|
||||
and lease_until > clock_timestamp() and current_attempt_id is null
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(&attempt_id)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn finish_leased(
|
||||
&self,
|
||||
request: TaskFinishRequest,
|
||||
owner: &str,
|
||||
fence: u64,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
let access = &request.access;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = $1,
|
||||
result_envelope = $2,
|
||||
error_envelope = $3,
|
||||
version = version + 1,
|
||||
updated_at = now(),
|
||||
lease_owner = null,
|
||||
lease_until = null
|
||||
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
||||
and owner_id = $7 and authorization_scope = $8
|
||||
and state = $9 and version = $10
|
||||
and lease_owner = $11 and fence = $12
|
||||
and lease_until > clock_timestamp()
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(request.target_state.as_str())
|
||||
.bind(request.result.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.error.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(request.expected_state.as_str())
|
||||
.bind(i64::from(request.expected_version))
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(request.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
@@ -377,11 +753,12 @@ impl PostgresTaskStore {
|
||||
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
|
||||
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
|
||||
trace_sampled, policy_mode, approval_reference, state, input_envelope,
|
||||
execution_deadline, retain_until
|
||||
execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version,
|
||||
executable_input_master_key_epoch, platform_api_key_id
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued',
|
||||
$16, to_timestamp($17::double precision / 1000.0),
|
||||
to_timestamp($18::double precision / 1000.0)
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
|
||||
to_timestamp($17::double precision / 1000.0),
|
||||
to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22
|
||||
)
|
||||
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
|
||||
)
|
||||
@@ -403,6 +780,15 @@ impl PostgresTaskStore {
|
||||
.bind(request.input.value().clone())
|
||||
.bind(request.pins.execution_deadline_unix_ms)
|
||||
.bind(request.pins.retain_until_unix_ms)
|
||||
.bind(request.executable_input_ciphertext.as_deref())
|
||||
.bind(request.executable_input_key_version.as_deref())
|
||||
.bind(request.executable_input_master_key_epoch)
|
||||
.bind(
|
||||
request
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(PlatformApiKeyId::as_str),
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(task_sql_error)?
|
||||
@@ -540,6 +926,10 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError>
|
||||
authorization_scope: row
|
||||
.try_get("authorization_scope")
|
||||
.map_err(unavailable_row)?,
|
||||
platform_api_key_id: row
|
||||
.try_get::<Option<String>, _>("platform_api_key_id")
|
||||
.map_err(unavailable_row)?
|
||||
.map(PlatformApiKeyId::new),
|
||||
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
|
||||
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
|
||||
pins: TaskPins {
|
||||
@@ -574,6 +964,14 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError>
|
||||
error: envelope("error_envelope")?,
|
||||
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
lease_owner: row.try_get("lease_owner").map_err(unavailable_row)?,
|
||||
lease_until_unix_ms: row.try_get("lease_until_ms").map_err(unavailable_row)?,
|
||||
current_attempt_id: row.try_get("current_attempt_id").map_err(unavailable_row)?,
|
||||
dispatch_started_at_unix_ms: row
|
||||
.try_get("dispatch_started_at_ms")
|
||||
.map_err(unavailable_row)?,
|
||||
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
|
||||
|
||||
@@ -2,6 +2,7 @@ use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry};
|
||||
use sqlx::Row;
|
||||
|
||||
mod artifact_metadata;
|
||||
mod durable_tasks_guard;
|
||||
mod import_plan;
|
||||
mod legacy_adoption;
|
||||
mod preflight;
|
||||
@@ -72,6 +73,9 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
|
||||
(23, "prompt-dependencies-v23"),
|
||||
(24, "execution-admission-v24"),
|
||||
(25, "durable-tasks-v25"),
|
||||
(26, "durable-tasks-v26"),
|
||||
(27, "durable-tasks-v27"),
|
||||
(28, "durable-tasks-v28"),
|
||||
];
|
||||
assert_eq!(rows.len(), expected.len());
|
||||
for (row, (version, name)) in rows.iter().zip(expected) {
|
||||
@@ -182,7 +186,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(first.pool()).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
@@ -230,7 +234,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
|
||||
.expect("the V18-local guard must be restorable");
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"restoring the exact V18 contract must return the current verdict"
|
||||
);
|
||||
}
|
||||
@@ -356,7 +360,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -469,13 +473,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 2,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
let trace_column: bool = sqlx::query_scalar(
|
||||
"select exists (
|
||||
@@ -518,7 +522,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 3,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -578,7 +582,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 4,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -832,6 +836,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 11,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
|
||||
for (drift, restore) in [
|
||||
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
|
||||
+8
-8
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"a rejected primary-key nullability change must leave the schema current"
|
||||
);
|
||||
|
||||
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
|
||||
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"the baseline before the drift matrix must be the current V16 schema"
|
||||
);
|
||||
let mut observations: Vec<String> = Vec::new();
|
||||
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
|
||||
.await
|
||||
.expect("restoring the exact definition must apply");
|
||||
let restored = MigrationAuthority::preflight(&pool).await;
|
||||
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 }));
|
||||
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 28 }));
|
||||
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
|
||||
// мнением того же guard-а, который принимает неправильные определения (F4).
|
||||
let restored_exact = catalog_definition_snapshot(&pool).await == before;
|
||||
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"baseline before inserting the shadow schema"
|
||||
);
|
||||
|
||||
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
// Тень не меняет вердикт guard-а по текущей схеме.
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"a same-named object in another schema must not change the verdict for the current schema"
|
||||
);
|
||||
|
||||
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 28 },
|
||||
"the exact restoration must return the current V16 schema"
|
||||
);
|
||||
|
||||
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
use super::*;
|
||||
use crank_registry::MigrationApplyResult;
|
||||
|
||||
#[tokio::test]
|
||||
async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_v43_clean_legacy").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
super::remove_v25_schema(&pool).await;
|
||||
// remove_v25_schema drops `tasks` and ledger >= 25, but V25 also added the
|
||||
// nullable dispatch-intent column/index/FK to execution_admission_entries;
|
||||
// restore those to the V24 shape so the legacy ledger is self-consistent.
|
||||
sqlx::raw_sql(
|
||||
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
|
||||
drop index if exists execution_admission_entries_task_uidx;
|
||||
alter table execution_admission_entries drop column if exists task_id;",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 24,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::apply(&pool).await.unwrap(),
|
||||
MigrationApplyResult::Applied { from: 24, to: 28 }
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_v27_on_legacy_ledger_is_rejected_by_preflight() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_v43_partial_v27").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
super::remove_v25_schema(&pool).await;
|
||||
sqlx::raw_sql(
|
||||
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
|
||||
drop index if exists execution_admission_entries_task_uidx;
|
||||
alter table execution_admission_entries drop column if exists task_id;",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// Reintroduce a partial V27-shaped `tasks` table on top of the legacy (v24) ledger.
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"create table tasks (
|
||||
task_id uuid primary key,
|
||||
executable_input_ciphertext text,
|
||||
executable_input_key_version text,
|
||||
executable_input_master_key_epoch bigint
|
||||
);",
|
||||
))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||
assert_eq!(error.code(), "partial_sequence");
|
||||
// The rejection today is attributed to V25's absent-schema guard because the
|
||||
// `tasks` relation already exists, so a behavioral RED pin for the missing
|
||||
// V27 guard line cannot be reached; this test fixes the rejection contract
|
||||
// instead of claiming new behavior.
|
||||
assert_eq!(error.version(), Some(24));
|
||||
assert_eq!(error.stage(), "preflight.schema");
|
||||
|
||||
let count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from __crank_migrations where version >= 25")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
count, 0,
|
||||
"nothing at or above V25 should have been applied over the partial fixture"
|
||||
);
|
||||
}
|
||||
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
let tables = sqlx::query(
|
||||
"select table_name
|
||||
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 13,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
|
||||
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||
assert_eq!(operation_count, 1);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
|
||||
assert!(supports_elicitation);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+13
-7
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
|
||||
.fetch_one(&pool)
|
||||
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 18,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::apply(&pool).await.unwrap(),
|
||||
MigrationApplyResult::Applied { from: 18, to: 25 }
|
||||
MigrationApplyResult::Applied { from: 18, to: 28 }
|
||||
);
|
||||
let after: Vec<(i64, String, String)> =
|
||||
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
|
||||
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
|
||||
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 28 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -86,16 +86,22 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
// The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer
|
||||
// database; a synthetic version above the current one exercises it without a second binary.
|
||||
let current: i64 = sqlx::query_scalar("select max(version) from __crank_migrations")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let future = current + 1;
|
||||
sqlx::query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')",
|
||||
values ($1, 'future-synthetic', repeat('a', 64), 'expand', 'n-minus-one-readable')",
|
||||
)
|
||||
.bind(future)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||
assert_eq!(error.code(), "future_version");
|
||||
assert_eq!(error.version(), Some(26));
|
||||
assert_eq!(error.version(), Some(future));
|
||||
assert_eq!(error.recovery(), "install_matching_application");
|
||||
assert!(
|
||||
MigrationAuthority::require_current(&pool).await.is_err(),
|
||||
@@ -155,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 18,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 11,
|
||||
target: 25,
|
||||
target: 28,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -330,9 +330,10 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
|
||||
|
||||
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
|
||||
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
|
||||
// V25 created `tasks`; V26–V28 only altered it, so dropping the table removes their schema too.
|
||||
sqlx::raw_sql(
|
||||
"drop table if exists tasks cascade;
|
||||
delete from __crank_migrations where version = 25;",
|
||||
delete from __crank_migrations where version >= 25;",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
|
||||
+228
-5
@@ -8,14 +8,18 @@
|
||||
use super::common;
|
||||
|
||||
use crank_core::{
|
||||
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence,
|
||||
ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms,
|
||||
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, AgentId, ApprovalEvidence,
|
||||
ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind,
|
||||
PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest, TaskAccess,
|
||||
TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState,
|
||||
TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId, current_unix_ms,
|
||||
};
|
||||
use crank_registry::{
|
||||
AdmissionLimits, CreatePlatformApiKeyRequest, PostgresExecutionAdmission, PostgresTaskStore,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore};
|
||||
use serde_json::json;
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::{Duration as TimeDuration, OffsetDateTime};
|
||||
|
||||
const WORKSPACE: &str = "ws_default";
|
||||
const AGENT: &str = "agent_a";
|
||||
@@ -48,6 +52,7 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: agent.to_owned(),
|
||||
authorization_scope: format!("credential:pk_{agent}"),
|
||||
platform_api_key_id: Some(PlatformApiKeyId::new(format!("pk_{agent}"))),
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 3,
|
||||
@@ -60,6 +65,9 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
|
||||
input_fingerprint: fingerprint.to_owned(),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(),
|
||||
support: unsampled_support(),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -664,6 +672,70 @@ async fn invalid_bounded_envelope_is_rejected_before_persistence() {
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_worker_lease_and_fencing_contract() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("lease1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let leases = store.claim_batch("worker_1", 4, 10_000).await.unwrap();
|
||||
assert_eq!(leases.len(), 1);
|
||||
assert_eq!(leases[0].record.task_id, created.task_id);
|
||||
assert_eq!(leases[0].record.state, TaskState::Working);
|
||||
assert_eq!(leases[0].owner, "worker_1");
|
||||
|
||||
let lease = &leases[0];
|
||||
let fenced_progress = store
|
||||
.write_progress(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence,
|
||||
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(fenced_progress.progress.is_some());
|
||||
|
||||
// stale fence rejected
|
||||
let bad = store
|
||||
.renew_lease("worker_1", lease.record.task_id, lease.fence + 1, 10_000)
|
||||
.await;
|
||||
assert!(bad.is_err());
|
||||
|
||||
// checkpoint dispatch once
|
||||
let checkpointed = store
|
||||
.checkpoint_dispatch(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence,
|
||||
"attempt-1".to_owned(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
checkpointed.current_attempt_id.as_deref(),
|
||||
Some("attempt-1")
|
||||
);
|
||||
assert!(checkpointed.dispatch_started_at_unix_ms.is_some());
|
||||
|
||||
// second checkpoint rejected
|
||||
let second = store
|
||||
.checkpoint_dispatch(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence,
|
||||
"attempt-2".to_owned(),
|
||||
)
|
||||
.await;
|
||||
assert!(second.is_err());
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_failure_fails_closed_without_records() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
@@ -680,3 +752,154 @@ async fn store_failure_fails_closed_without_records() {
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: the typed immutable Platform API key reference of an accepted Task is persisted,
|
||||
/// read back through the lease, and only an active, unexpired, exactly bound key is authoritatively
|
||||
/// revalidated. Revocation, expiry and a foreign workspace/agent binding all close; an administrator
|
||||
/// Task stores no machine key identity at all, so a worker fails closed on it.
|
||||
#[tokio::test]
|
||||
async fn accepted_task_authority_reference_is_persisted_and_revalidated() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let workspace = WorkspaceId::new(WORKSPACE);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
// `ws_default` is seeded by the baseline migration; the keys only need that workspace row.
|
||||
let agent = AgentId::new(AGENT);
|
||||
|
||||
let active_id = PlatformApiKeyId::new("pk_task_active");
|
||||
for key in [
|
||||
authority_key(
|
||||
"pk_task_active",
|
||||
&workspace,
|
||||
AGENT,
|
||||
PlatformApiKeyStatus::Active,
|
||||
None,
|
||||
),
|
||||
authority_key(
|
||||
"pk_task_revoked",
|
||||
&workspace,
|
||||
AGENT,
|
||||
PlatformApiKeyStatus::Revoked,
|
||||
None,
|
||||
),
|
||||
authority_key(
|
||||
"pk_task_expired",
|
||||
&workspace,
|
||||
AGENT,
|
||||
PlatformApiKeyStatus::Active,
|
||||
Some(now - TimeDuration::hours(1)),
|
||||
),
|
||||
] {
|
||||
let secret_hash = format!("secret-hash-{}", key.id.as_str());
|
||||
registry
|
||||
.create_platform_api_key(CreatePlatformApiKeyRequest {
|
||||
secret_hash: &secret_hash,
|
||||
api_key: &key,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Acceptance persists only the immutable typed reference, never the bearer secret.
|
||||
let mut accepted = request(&key("k_authority"), &digest("fa"), AGENT, 300_000);
|
||||
accepted.platform_api_key_id = Some(active_id.clone());
|
||||
let task_id = accepted.task_id;
|
||||
store.accept(accepted).await.expect("acceptance commits");
|
||||
|
||||
let leases = store
|
||||
.claim_batch("worker_authority", 8, 30_000)
|
||||
.await
|
||||
.expect("claim");
|
||||
let lease = leases
|
||||
.into_iter()
|
||||
.find(|lease| lease.record.task_id == task_id)
|
||||
.expect("accepted task is leased");
|
||||
assert_eq!(lease.record.platform_api_key_id.as_ref(), Some(&active_id));
|
||||
|
||||
// Positive: the exact active key of this workspace/agent is revalidated authoritatively.
|
||||
let revalidated = registry
|
||||
.revalidate_platform_api_key(&workspace, &agent, &active_id)
|
||||
.await
|
||||
.expect("authoritative read")
|
||||
.expect("active key is authoritative");
|
||||
assert!(
|
||||
revalidated
|
||||
.api_key
|
||||
.scopes
|
||||
.contains(&PlatformApiKeyScope::Write)
|
||||
);
|
||||
|
||||
// Negatives: revocation, expiry and a foreign workspace/agent binding all fail closed.
|
||||
let foreign_agent = AgentId::new("agent_b");
|
||||
let foreign_workspace = WorkspaceId::new("ws_elsewhere");
|
||||
for (other_workspace, other_agent, other_key) in [
|
||||
(&workspace, &agent, PlatformApiKeyId::new("pk_task_revoked")),
|
||||
(&workspace, &agent, PlatformApiKeyId::new("pk_task_expired")),
|
||||
(&workspace, &foreign_agent, active_id.clone()),
|
||||
(&foreign_workspace, &agent, active_id.clone()),
|
||||
] {
|
||||
let found = registry
|
||||
.revalidate_platform_api_key(other_workspace, other_agent, &other_key)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
found.is_none(),
|
||||
"authority must fail closed for {}",
|
||||
other_key.as_str()
|
||||
);
|
||||
}
|
||||
|
||||
// An administrator Task carries no machine key identity: the worker has nothing to revalidate.
|
||||
let mut admin_request = request(&key("k_admin"), &digest("fb"), AGENT, 300_000);
|
||||
admin_request.owner_kind = TaskOwnerKind::Admin;
|
||||
admin_request.owner_id = "user_admin".to_owned();
|
||||
admin_request.authorization_scope = "admin:user_admin".to_owned();
|
||||
admin_request.platform_api_key_id = None;
|
||||
let admin_task = admin_request.task_id;
|
||||
store
|
||||
.accept(admin_request)
|
||||
.await
|
||||
.expect("administrator acceptance commits");
|
||||
let admin_record = store
|
||||
.read(
|
||||
&TaskAccess {
|
||||
workspace_id: WORKSPACE.to_owned(),
|
||||
owner_kind: TaskOwnerKind::Admin,
|
||||
owner_id: "user_admin".to_owned(),
|
||||
authorization_scope: "admin:user_admin".to_owned(),
|
||||
},
|
||||
admin_task,
|
||||
)
|
||||
.await
|
||||
.expect("owner read");
|
||||
assert_eq!(admin_record.platform_api_key_id, None);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
fn authority_key(
|
||||
id: &str,
|
||||
workspace_id: &WorkspaceId,
|
||||
agent: &str,
|
||||
status: PlatformApiKeyStatus,
|
||||
expires_at: Option<OffsetDateTime>,
|
||||
) -> PlatformApiKey {
|
||||
PlatformApiKey {
|
||||
id: PlatformApiKeyId::new(id),
|
||||
workspace_id: workspace_id.clone(),
|
||||
agent_id: Some(AgentId::new(agent)),
|
||||
key_kind: PlatformApiKeyKind::McpClient,
|
||||
name: id.to_owned(),
|
||||
prefix: "crk_live".to_owned(),
|
||||
scopes: vec![PlatformApiKeyScope::Write],
|
||||
status,
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
last_used_at: None,
|
||||
expires_at,
|
||||
allowed_origins: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
mod task_worker;
|
||||
@@ -0,0 +1,163 @@
|
||||
//! Story 4.3 ATDD: durable Task leases, fencing, crash recovery and bounded claim behavior.
|
||||
//!
|
||||
//! Split from `tasks.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
/// Story 4.3 ATDD: a worker crash before the durable dispatch checkpoint leaves only an expired
|
||||
/// lease. PostgreSQL is authoritative, so another worker recovers the Task and the dead owner can
|
||||
/// never commit a terminal outcome with its old fence.
|
||||
#[tokio::test]
|
||||
async fn task_worker_expired_pre_checkpoint_lease_is_recovered_and_stale_owner_cannot_commit() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("recover1"), &digest("f2"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let first = store.claim_batch("worker_dead", 1, 150).await.unwrap();
|
||||
assert_eq!(first.len(), 1);
|
||||
assert_eq!(first[0].record.task_id, created.task_id);
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
|
||||
// Recovery on the next bounded authoritative scan, with no Valkey wake-up.
|
||||
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
|
||||
let lease = recovered
|
||||
.into_iter()
|
||||
.find(|lease| lease.record.task_id == created.task_id)
|
||||
.expect("expired pre-checkpoint Task is recovered");
|
||||
assert!(lease.fence > first[0].fence, "recovery bumps the fence");
|
||||
|
||||
// The previous owner's terminal CAS is rejected; the live owner wins exactly once.
|
||||
let stale = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_state: first[0].record.state,
|
||||
expected_version: first[0].record.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"stale": true})).unwrap()),
|
||||
error: None,
|
||||
},
|
||||
"worker_dead",
|
||||
first[0].fence,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(stale, Err(TaskError::StateConflict)));
|
||||
|
||||
let finished = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_state: lease.record.state,
|
||||
expected_version: lease.record.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
error: None,
|
||||
},
|
||||
"worker_live",
|
||||
lease.fence,
|
||||
)
|
||||
.await
|
||||
.expect("current lease wins the terminal CAS");
|
||||
assert_eq!(finished.state, TaskState::Completed);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: after the dispatch checkpoint the outbound outcome is uncertain. An expired
|
||||
/// attempt resolves to `outcome_unknown`, is never redispatched automatically and releases the
|
||||
/// admission slot it held.
|
||||
#[tokio::test]
|
||||
async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_without_redispatch() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("uncertain1"), &digest("f3"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap();
|
||||
let lease = leases.into_iter().next().expect("claim");
|
||||
store
|
||||
.checkpoint_dispatch(
|
||||
"worker_dead",
|
||||
lease.record.task_id,
|
||||
lease.fence,
|
||||
"attempt-uncertain".to_owned(),
|
||||
)
|
||||
.await
|
||||
.expect("durable dispatch checkpoint");
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
|
||||
// The authority scan resolves uncertainty instead of handing the attempt to another worker.
|
||||
let after = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
|
||||
assert!(
|
||||
after.is_empty(),
|
||||
"a checkpointed attempt must never be redispatched"
|
||||
);
|
||||
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
||||
assert_eq!(
|
||||
record.current_attempt_id.as_deref(),
|
||||
Some("attempt-uncertain")
|
||||
);
|
||||
let intents: i64 = sqlx::query_scalar(
|
||||
"select count(*)::bigint from execution_admission_entries where task_id = $1",
|
||||
)
|
||||
.bind(created.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(intents, 0, "uncertain terminal outcome releases capacity");
|
||||
|
||||
// Still no automatic retry on the following scans.
|
||||
assert!(
|
||||
store
|
||||
.claim_batch("worker_live", 4, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: concurrent claimers never hold the same Task at once.
|
||||
#[tokio::test]
|
||||
async fn task_worker_concurrent_claimers_never_double_lease_one_task() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = std::sync::Arc::new(store_with(registry.pool().clone(), 8));
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("concurrent1"), &digest("f4"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut handles = Vec::new();
|
||||
for index in 0..4 {
|
||||
let store = store.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
store
|
||||
.claim_batch(&format!("worker_{index}"), 1, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
}));
|
||||
}
|
||||
let mut leased = Vec::new();
|
||||
for handle in handles {
|
||||
leased.extend(handle.await.unwrap());
|
||||
}
|
||||
assert_eq!(leased.len(), 1, "exactly one owner wins the claim");
|
||||
assert_eq!(leased[0].record.task_id, created.task_id);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
@@ -22,7 +22,9 @@ use serde_json::{Map, Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
use tracing::{Instrument, Span, debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::task_worker::TaskDispatchPermit;
|
||||
use crate::{
|
||||
AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError,
|
||||
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
|
||||
@@ -178,8 +180,72 @@ impl RuntimeExecutor {
|
||||
pub(crate) async fn execute_request(
|
||||
&self,
|
||||
request: RuntimeExecutionRequest<'_>,
|
||||
) -> Result<(Value, Value), RuntimeError> {
|
||||
self.execute_request_impl(request, None).await
|
||||
}
|
||||
|
||||
pub async fn execute_task_outcome(
|
||||
&self,
|
||||
request: RuntimeExecutionRequest<'_>,
|
||||
permit: TaskDispatchPermit,
|
||||
) -> Result<crank_core::ExecutionSuccess, crank_core::ExecutionFailure> {
|
||||
let operation = request.operation().clone();
|
||||
let operation_id = operation.operation_id.clone();
|
||||
let operation_version = operation.operation_version;
|
||||
let origin = request.origin();
|
||||
let correlation = crank_core::CorrelationContext::new(
|
||||
request.request_context().request_id.clone(),
|
||||
request.request_context().trace_context.clone(),
|
||||
);
|
||||
|
||||
// Validate permit binding before any adapter dispatch.
|
||||
if permit.operation_id() != operation_id.as_str()
|
||||
|| permit.operation_version() != operation_version
|
||||
|| permit.workspace_id() != request.workspace_id().as_str()
|
||||
|| permit.agent_id() != request.agent_id().map(|value| value.as_str())
|
||||
{
|
||||
return Err(crate::normalize_runtime_error_for_operation(
|
||||
&RuntimeError::InvalidExecutionRequest,
|
||||
&correlation,
|
||||
&operation,
|
||||
));
|
||||
}
|
||||
if !permit.consume() {
|
||||
return Err(crate::normalize_runtime_error_for_operation(
|
||||
&RuntimeError::InvalidExecutionRequest,
|
||||
&correlation,
|
||||
&operation,
|
||||
));
|
||||
}
|
||||
|
||||
match self.execute_request_impl(request, Some(permit)).await {
|
||||
Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess {
|
||||
operation_id: operation_id.clone(),
|
||||
operation_version,
|
||||
origin,
|
||||
correlation,
|
||||
request_preview,
|
||||
output,
|
||||
}),
|
||||
Err(error) => Err(crate::normalize_runtime_error_for_operation(
|
||||
&error,
|
||||
&correlation,
|
||||
&operation,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_request_impl(
|
||||
&self,
|
||||
request: RuntimeExecutionRequest<'_>,
|
||||
permit: Option<TaskDispatchPermit>,
|
||||
) -> Result<(Value, Value), RuntimeError> {
|
||||
let operation = request.operation();
|
||||
// Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission.
|
||||
let trusted_task_dispatch = permit.is_some();
|
||||
// The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the
|
||||
// trace names the real persisted attempt instead of inventing one (Story 4.3 AC).
|
||||
let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned());
|
||||
let input = request.input();
|
||||
let request_context = request.request_context();
|
||||
debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized);
|
||||
@@ -205,7 +271,9 @@ impl RuntimeExecutor {
|
||||
let dispatch_started = Arc::new(AtomicBool::new(false));
|
||||
// True until the admission gate answered. A deadline that fires while the request is still
|
||||
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
|
||||
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
|
||||
let admission_pending = Arc::new(AtomicBool::new(
|
||||
!trusted_task_dispatch && self.admission.is_some(),
|
||||
));
|
||||
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
|
||||
metric_invocation_source(Some(request_context)),
|
||||
request_context
|
||||
@@ -222,14 +290,19 @@ impl RuntimeExecutor {
|
||||
// One admission gate for every executable path (Story 4.1): the published policy decides
|
||||
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
|
||||
// is held for the whole local execution future, including retries.
|
||||
let mut lease = crate::admission::acquire(
|
||||
self.admission.as_ref(),
|
||||
self.coordination_store.as_deref(),
|
||||
&request,
|
||||
execution_deadline,
|
||||
)
|
||||
.await?;
|
||||
admission_pending.store(false, Ordering::Release);
|
||||
let mut lease = if trusted_task_dispatch {
|
||||
None
|
||||
} else {
|
||||
let result = crate::admission::acquire(
|
||||
self.admission.as_ref(),
|
||||
self.coordination_store.as_deref(),
|
||||
&request,
|
||||
execution_deadline,
|
||||
)
|
||||
.await?;
|
||||
admission_pending.store(false, Ordering::Release);
|
||||
result
|
||||
};
|
||||
let _permit = self.acquire_unary_permit(operation)?;
|
||||
let _inflight = InFlightGuard::runtime();
|
||||
let mapping_span = Stage::RuntimeArgumentsMap.span();
|
||||
@@ -246,6 +319,7 @@ impl RuntimeExecutor {
|
||||
prepared_request,
|
||||
Some(request_context),
|
||||
Arc::clone(&dispatch_started),
|
||||
durable_attempt_id.as_deref(),
|
||||
)
|
||||
.await;
|
||||
if let Some(lease) = lease.as_mut() {
|
||||
@@ -348,6 +422,7 @@ impl RuntimeExecutor {
|
||||
prepared_request: PreparedRequest,
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
dispatch_started: Arc<AtomicBool>,
|
||||
durable_attempt_id: Option<&str>,
|
||||
) -> Result<Value, RuntimeError> {
|
||||
let mut prepared_request = prepared_request;
|
||||
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
|
||||
@@ -464,6 +539,7 @@ impl RuntimeExecutor {
|
||||
operation,
|
||||
idempotency_key.as_deref(),
|
||||
),
|
||||
durable_attempt_id,
|
||||
)
|
||||
.await;
|
||||
if let Ok(response) = &adapter_response {
|
||||
@@ -604,10 +680,23 @@ impl RuntimeExecutor {
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
dispatch_started: Arc<AtomicBool>,
|
||||
verified_idempotency: bool,
|
||||
durable_attempt_id: Option<&str>,
|
||||
) -> Result<AdapterResponse, RuntimeError> {
|
||||
let max_attempts = retry_attempts(operation);
|
||||
let mut attempt = 1;
|
||||
loop {
|
||||
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint; the
|
||||
// retry index discriminates the attempts inside it. Only paths without a durable attempt
|
||||
// (synchronous tools/call) mint a synthetic per-attempt id.
|
||||
let attempt_id = durable_attempt_id
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7()));
|
||||
let attempt_span = tracing::info_span!(
|
||||
"adapter_attempt",
|
||||
attempt,
|
||||
max_attempts,
|
||||
attempt_id = %attempt_id
|
||||
);
|
||||
let result = self
|
||||
.execute_adapter(
|
||||
operation,
|
||||
@@ -615,6 +704,7 @@ impl RuntimeExecutor {
|
||||
request_context,
|
||||
Arc::clone(&dispatch_started),
|
||||
)
|
||||
.instrument(attempt_span)
|
||||
.await;
|
||||
match result {
|
||||
Ok(response) => return Ok(response),
|
||||
|
||||
@@ -15,6 +15,7 @@ mod rate_limit;
|
||||
mod request_context;
|
||||
mod request_preparation;
|
||||
mod response_cache;
|
||||
mod task_worker;
|
||||
mod tasks;
|
||||
pub fn validate_resource_template_parameter(
|
||||
schema: &crank_schema::Schema,
|
||||
@@ -115,4 +116,8 @@ pub use rate_limit::{
|
||||
};
|
||||
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
|
||||
pub use secret_crypto::SecretCrypto;
|
||||
pub use task_worker::{
|
||||
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
||||
TaskWorkerError,
|
||||
};
|
||||
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
|
||||
@@ -0,0 +1,573 @@
|
||||
//! Fenced durable Task worker loop for Story 4.3.
|
||||
//!
|
||||
//! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded,
|
||||
//! Valkey may only wake it up (never represented here as correctness), and every lease-protected
|
||||
//! write goes through CAS using owner + fencing token. The actual executable input and revalidation
|
||||
//! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a
|
||||
//! redacted projection and is never dispatched.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{TaskEnvelope, TaskError, TaskFinishRequest, TaskLease, TaskState, TaskStore};
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::JoinSet;
|
||||
use tracing::{Instrument, debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct TaskWorkerConfig {
|
||||
/// Stable random-ish owner identity for this process instance. Required lease owner.
|
||||
pub owner: String,
|
||||
/// Maximum number of Tasks claimed in one bounded PG scan.
|
||||
pub batch_limit: u32,
|
||||
/// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used.
|
||||
pub lease_ms: i64,
|
||||
/// Authoritative PG polling interval when no wake-up hint is available.
|
||||
pub poll_interval: Duration,
|
||||
/// Renewal cadence while processing a lease.
|
||||
pub renewal_interval: Duration,
|
||||
/// Concurrent in-flight Tasks for this worker.
|
||||
pub max_inflight: usize,
|
||||
/// Maximum time to wait for active work during graceful shutdown.
|
||||
pub shutdown_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for TaskWorkerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
owner: format!("task-worker-{}", Uuid::now_v7()),
|
||||
batch_limit: 8,
|
||||
lease_ms: 30_000,
|
||||
poll_interval: Duration::from_millis(250),
|
||||
renewal_interval: Duration::from_millis(5_000),
|
||||
max_inflight: 4,
|
||||
shutdown_timeout: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of one durable Task attempt produced by the application layer.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum TaskAttemptOutcome {
|
||||
Completed(serde_json::Value),
|
||||
Failed(serde_json::Value),
|
||||
OutcomeUnknown,
|
||||
}
|
||||
|
||||
/// Application-owned seam for pre-dispatch revalidation and execution.
|
||||
///
|
||||
/// The worker owns lease renewal/CAS/fencing and the dispatch checkpoint. The handler only
|
||||
/// receives the leased record and attempt identity; it must not trust `TaskRecord.input` as
|
||||
/// executable payload.
|
||||
#[async_trait]
|
||||
pub trait TaskHandler: Send + Sync {
|
||||
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>;
|
||||
async fn execute(
|
||||
&self,
|
||||
lease: &TaskLease,
|
||||
attempt_id: &str,
|
||||
cancelled: Arc<AtomicBool>,
|
||||
permit: TaskDispatchPermit,
|
||||
) -> TaskAttemptOutcome;
|
||||
}
|
||||
|
||||
/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and
|
||||
/// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and
|
||||
/// it binds one durable Task attempt to its execution request.
|
||||
pub struct TaskDispatchPermit {
|
||||
task_id: crank_core::TaskId,
|
||||
operation_id: String,
|
||||
operation_version: u32,
|
||||
workspace_id: String,
|
||||
agent_id: Option<String>,
|
||||
owner: String,
|
||||
fence: u64,
|
||||
attempt_id: String,
|
||||
consumed: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl TaskDispatchPermit {
|
||||
pub(crate) fn new(
|
||||
task_id: crank_core::TaskId,
|
||||
operation_id: String,
|
||||
operation_version: u32,
|
||||
workspace_id: String,
|
||||
agent_id: Option<String>,
|
||||
owner: String,
|
||||
fence: u64,
|
||||
attempt_id: String,
|
||||
) -> Self {
|
||||
Self {
|
||||
task_id,
|
||||
operation_id,
|
||||
operation_version,
|
||||
workspace_id,
|
||||
agent_id,
|
||||
owner,
|
||||
fence,
|
||||
attempt_id,
|
||||
consumed: std::sync::atomic::AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn task_id(&self) -> crank_core::TaskId {
|
||||
self.task_id
|
||||
}
|
||||
|
||||
pub fn operation_id(&self) -> &str {
|
||||
&self.operation_id
|
||||
}
|
||||
|
||||
pub fn operation_version(&self) -> u32 {
|
||||
self.operation_version
|
||||
}
|
||||
|
||||
pub fn workspace_id(&self) -> &str {
|
||||
&self.workspace_id
|
||||
}
|
||||
|
||||
pub fn agent_id(&self) -> Option<&str> {
|
||||
self.agent_id.as_deref()
|
||||
}
|
||||
|
||||
pub fn owner(&self) -> &str {
|
||||
&self.owner
|
||||
}
|
||||
|
||||
pub fn fence(&self) -> u64 {
|
||||
self.fence
|
||||
}
|
||||
|
||||
pub fn attempt_id(&self) -> &str {
|
||||
&self.attempt_id
|
||||
}
|
||||
|
||||
pub fn is_consumed(&self) -> bool {
|
||||
self.consumed.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn consume(&self) -> bool {
|
||||
!self
|
||||
.consumed
|
||||
.swap(true, std::sync::atomic::Ordering::AcqRel)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum TaskWorkerError {
|
||||
#[error("task store error: {0}")]
|
||||
Store(#[from] TaskError),
|
||||
#[error("invalid worker configuration: {0}")]
|
||||
Config(String),
|
||||
#[error("task terminal transition CAS failed")]
|
||||
CasRejected,
|
||||
}
|
||||
|
||||
pub struct TaskWorker {
|
||||
store: Arc<dyn TaskStore>,
|
||||
handler: Arc<dyn TaskHandler>,
|
||||
config: TaskWorkerConfig,
|
||||
stopping: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl TaskWorker {
|
||||
pub fn new(
|
||||
store: Arc<dyn TaskStore>,
|
||||
handler: Arc<dyn TaskHandler>,
|
||||
config: TaskWorkerConfig,
|
||||
) -> Result<Self, TaskWorkerError> {
|
||||
if config.owner.is_empty() {
|
||||
return Err(TaskWorkerError::Config(
|
||||
"owner must not be empty".to_owned(),
|
||||
));
|
||||
}
|
||||
if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 {
|
||||
return Err(TaskWorkerError::Config(
|
||||
"batch_limit must be 1..=64 and lease_ms must be positive".to_owned(),
|
||||
));
|
||||
}
|
||||
if config.lease_ms > 30_000
|
||||
|| config.poll_interval.is_zero()
|
||||
|| config.poll_interval > Duration::from_secs(30)
|
||||
|| config.renewal_interval.is_zero()
|
||||
|| config.renewal_interval >= Duration::from_millis(config.lease_ms as u64)
|
||||
|| config.max_inflight == 0
|
||||
|| config.shutdown_timeout.is_zero()
|
||||
|| config.shutdown_timeout > Duration::from_secs(30)
|
||||
{
|
||||
return Err(TaskWorkerError::Config(
|
||||
"lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds"
|
||||
.to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
store,
|
||||
handler,
|
||||
config,
|
||||
stopping: Arc::new(AtomicBool::new(false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled.
|
||||
pub fn stop(&self) {
|
||||
self.stopping.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Returns a handle whose drop does not stop the worker; call `stop()` on the worker.
|
||||
pub fn stop_flag(&self) -> Arc<AtomicBool> {
|
||||
self.stopping.clone()
|
||||
}
|
||||
|
||||
/// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan.
|
||||
pub async fn run(&self) -> Result<(), TaskWorkerError> {
|
||||
self.run_inner(None).await
|
||||
}
|
||||
|
||||
/// Run until either `stop()` or the provided shutdown receiver becomes true.
|
||||
pub async fn run_until_shutdown(
|
||||
&self,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) -> Result<(), TaskWorkerError> {
|
||||
self.run_inner(Some(&mut shutdown)).await
|
||||
}
|
||||
|
||||
async fn run_inner(
|
||||
&self,
|
||||
mut shutdown: Option<&mut watch::Receiver<bool>>,
|
||||
) -> Result<(), TaskWorkerError> {
|
||||
let mut active: JoinSet<()> = JoinSet::new();
|
||||
loop {
|
||||
if self.stopping.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
if let Some(rx) = shutdown.as_mut() {
|
||||
if *rx.borrow() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Reap finished attempts before claiming more.
|
||||
while active.try_join_next().is_some() {}
|
||||
|
||||
let available = self.config.max_inflight.saturating_sub(active.len());
|
||||
if available > 0 {
|
||||
let batch_limit = self
|
||||
.config
|
||||
.batch_limit
|
||||
.min(u32::try_from(available).unwrap_or(u32::MAX));
|
||||
match self
|
||||
.store
|
||||
.claim_batch(&self.config.owner, batch_limit, self.config.lease_ms)
|
||||
.await
|
||||
{
|
||||
Ok(leases) => {
|
||||
for lease in leases {
|
||||
let store = self.store.clone();
|
||||
let handler = self.handler.clone();
|
||||
let owner = self.config.owner.clone();
|
||||
let lease_ms = self.config.lease_ms;
|
||||
let renewal_interval = self.config.renewal_interval;
|
||||
let stopping = self.stopping.clone();
|
||||
active.spawn(async move {
|
||||
process_lease(
|
||||
store,
|
||||
handler,
|
||||
owner,
|
||||
lease,
|
||||
lease_ms,
|
||||
renewal_interval,
|
||||
stopping,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "bounded task claim failed; retrying on next scan");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(self.config.poll_interval) => {}
|
||||
_ = async {
|
||||
if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await }
|
||||
} => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded.
|
||||
// The same stop flag reaches cooperative cancellation and lease-renewal loops.
|
||||
self.stopping.store(true, Ordering::Release);
|
||||
let deadline = Instant::now() + self.config.shutdown_timeout;
|
||||
while !active.is_empty() {
|
||||
if Instant::now() >= deadline {
|
||||
warn!("graceful shutdown timeout reached; aborting remaining task attempts");
|
||||
active.abort_all();
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
_ = active.join_next() => {}
|
||||
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_lease(
|
||||
store: Arc<dyn TaskStore>,
|
||||
handler: Arc<dyn TaskHandler>,
|
||||
owner: String,
|
||||
lease: TaskLease,
|
||||
lease_ms: i64,
|
||||
renewal_interval: Duration,
|
||||
stopping: Arc<AtomicBool>,
|
||||
) {
|
||||
let task_id = lease.record.task_id;
|
||||
let fence = lease.fence;
|
||||
let cancelled = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Pre-dispatch revalidation under the fresh lease.
|
||||
if let Err(error) = handler.revalidate(&lease).await {
|
||||
// A transient authority-store outage is not evidence that the work is invalid: leave the
|
||||
// lease to expire so the bounded recovery scan can retry it without dispatching now.
|
||||
if matches!(
|
||||
&error,
|
||||
TaskWorkerError::Store(TaskError::Unavailable { .. })
|
||||
) {
|
||||
debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery");
|
||||
return;
|
||||
}
|
||||
debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch");
|
||||
finish_failed(
|
||||
&store,
|
||||
&owner,
|
||||
task_id,
|
||||
fence,
|
||||
&lease,
|
||||
"revalidation_failed",
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
if stopping.load(Ordering::Acquire) {
|
||||
// Graceful shutdown before the durable dispatch checkpoint: no outbound call can have
|
||||
// happened, so the Task is left leased. The bounded recovery scan requeues it after the
|
||||
// lease expires instead of turning a never-started attempt into a user-visible failure.
|
||||
return;
|
||||
}
|
||||
|
||||
let attempt_id = format!("attempt-{}", Uuid::now_v7());
|
||||
|
||||
// Durable checkpoint before any network use.
|
||||
if store
|
||||
.checkpoint_dispatch(&owner, task_id, fence, attempt_id.clone())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
// Lease lost or checkpoint rejected: do not dispatch.
|
||||
cancelled.store(true, Ordering::Release);
|
||||
return;
|
||||
}
|
||||
if stopping.load(Ordering::Acquire) {
|
||||
// The checkpoint is durable, but we know the outbound callback has not started.
|
||||
finish_failed(
|
||||
&store,
|
||||
&owner,
|
||||
task_id,
|
||||
fence,
|
||||
&lease,
|
||||
"shutdown_before_dispatch",
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
// Sealed single-use permit for the trusted runtime dispatch bridge. It is created only after
|
||||
// the current lease and durable checkpoint above.
|
||||
let agent_id = if lease.record.owner_kind == crank_core::TaskOwnerKind::Agent {
|
||||
Some(lease.record.owner_id.clone())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let permit = TaskDispatchPermit::new(
|
||||
lease.record.task_id,
|
||||
lease.record.operation_id.clone(),
|
||||
lease.record.pins.operation_version,
|
||||
lease.record.workspace_id.clone(),
|
||||
agent_id,
|
||||
owner.clone(),
|
||||
fence,
|
||||
attempt_id.clone(),
|
||||
);
|
||||
|
||||
// Background lease renewal while the attempt runs.
|
||||
let renewal_store = store.clone();
|
||||
let renewal_owner = owner.clone();
|
||||
let renewal_cancelled = cancelled.clone();
|
||||
let renewal_stopping = stopping.clone();
|
||||
let renewal_task_id = task_id;
|
||||
let renewal_handle = tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(renewal_interval);
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if renewal_cancelled.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
if renewal_stopping.load(Ordering::Acquire) {
|
||||
renewal_cancelled.store(true, Ordering::Release);
|
||||
break;
|
||||
}
|
||||
if renewal_store
|
||||
.renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
renewal_cancelled.store(true, Ordering::Release);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Each dispatch gets a child span carrying the durable attempt identity, rooted in the parent
|
||||
// trace persisted with the accepted Task (never a mutable queue/cache projection).
|
||||
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, attempt_id = %attempt_id, fence);
|
||||
if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) {
|
||||
crank_trace::set_parent_from_trace_context(&span, &parent);
|
||||
}
|
||||
let outcome = handler
|
||||
.execute(&lease, &attempt_id, cancelled.clone(), permit)
|
||||
.instrument(span)
|
||||
.await;
|
||||
|
||||
cancelled.store(true, Ordering::Release);
|
||||
renewal_handle.abort();
|
||||
|
||||
// Terminal CAS under valid lease. Only one writer wins; stale token is rejected by store.
|
||||
let access = crank_core::TaskAccess {
|
||||
workspace_id: lease.record.workspace_id.clone(),
|
||||
owner_kind: lease.record.owner_kind,
|
||||
owner_id: lease.record.owner_id.clone(),
|
||||
authorization_scope: lease.record.authorization_scope.clone(),
|
||||
};
|
||||
|
||||
match outcome {
|
||||
TaskAttemptOutcome::Completed(value) => {
|
||||
let Ok(result) = TaskEnvelope::try_redacted(&value) else {
|
||||
finish_failed(
|
||||
&store,
|
||||
&owner,
|
||||
task_id,
|
||||
fence,
|
||||
&lease,
|
||||
"result_not_redactable",
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
};
|
||||
let _ = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access,
|
||||
task_id,
|
||||
expected_state: leased_expected_state(&lease),
|
||||
expected_version: lease.record.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(result),
|
||||
error: None,
|
||||
},
|
||||
&owner,
|
||||
fence,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
TaskAttemptOutcome::Failed(value) => {
|
||||
let error = TaskEnvelope::try_redacted(&value).or_else(|_| {
|
||||
TaskEnvelope::try_redacted(&serde_json::json!({
|
||||
"code": "task_error_not_redactable"
|
||||
}))
|
||||
});
|
||||
let Ok(error) = error else {
|
||||
return;
|
||||
};
|
||||
let _ = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access,
|
||||
task_id,
|
||||
expected_state: leased_expected_state(&lease),
|
||||
expected_version: lease.record.version,
|
||||
target_state: TaskState::Failed,
|
||||
result: None,
|
||||
error: Some(error),
|
||||
},
|
||||
&owner,
|
||||
fence,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
TaskAttemptOutcome::OutcomeUnknown => {
|
||||
let _ = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access,
|
||||
task_id,
|
||||
expected_state: leased_expected_state(&lease),
|
||||
expected_version: lease.record.version,
|
||||
target_state: TaskState::OutcomeUnknown,
|
||||
result: None,
|
||||
error: None,
|
||||
},
|
||||
&owner,
|
||||
fence,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn leased_expected_state(lease: &TaskLease) -> TaskState {
|
||||
// Claimed tasks are expected to be in a non-terminal processing state.
|
||||
if lease.record.state == TaskState::Queued {
|
||||
TaskState::Queued
|
||||
} else {
|
||||
TaskState::Working
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_failed(
|
||||
store: &Arc<dyn TaskStore>,
|
||||
owner: &str,
|
||||
task_id: crank_core::TaskId,
|
||||
fence: u64,
|
||||
lease: &TaskLease,
|
||||
code: &str,
|
||||
) {
|
||||
let access = crank_core::TaskAccess {
|
||||
workspace_id: lease.record.workspace_id.clone(),
|
||||
owner_kind: lease.record.owner_kind,
|
||||
owner_id: lease.record.owner_id.clone(),
|
||||
authorization_scope: lease.record.authorization_scope.clone(),
|
||||
};
|
||||
let _ = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access,
|
||||
task_id,
|
||||
expected_state: leased_expected_state(lease),
|
||||
expected_version: lease.record.version,
|
||||
target_state: TaskState::Failed,
|
||||
result: None,
|
||||
error: TaskEnvelope::try_redacted(&serde_json::json!({ "code": code })).ok(),
|
||||
},
|
||||
owner,
|
||||
fence,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -13,11 +13,13 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use crank_core::PlatformApiKeyId;
|
||||
use crank_core::{
|
||||
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS,
|
||||
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds,
|
||||
ToolExecutionMode, current_unix_ms, validate_task_envelope,
|
||||
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
|
||||
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_SCOPED_KEY_MAX_BYTES,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskLease,
|
||||
TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds, ToolExecutionMode,
|
||||
contains_sensitive_task_keys, current_unix_ms, validate_task_envelope,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -29,6 +31,9 @@ pub struct TaskAcceptDraft {
|
||||
pub owner_id: String,
|
||||
/// Server-derived authorization scope identity of the verified principal.
|
||||
pub authorization_scope: String,
|
||||
/// Typed immutable Platform API key identity of the verified machine credential. It is sealed
|
||||
/// into the executable-input binding and revalidated authoritatively by the worker.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
pub operation_version: u32,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
@@ -68,11 +73,22 @@ impl TaskOutcome {
|
||||
#[derive(Clone)]
|
||||
pub struct TaskService {
|
||||
store: Arc<dyn TaskStore>,
|
||||
crypto: Option<Arc<crate::SecretCrypto>>,
|
||||
}
|
||||
|
||||
impl TaskService {
|
||||
pub fn new(store: Arc<dyn TaskStore>) -> Self {
|
||||
Self { store }
|
||||
Self {
|
||||
store,
|
||||
crypto: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_secret_crypto(store: Arc<dyn TaskStore>, crypto: Arc<crate::SecretCrypto>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
crypto: Some(crypto),
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness of the mandatory durable Task state.
|
||||
@@ -83,18 +99,61 @@ impl TaskService {
|
||||
/// Durable acceptance: the returned identity exists only after the store committed it.
|
||||
pub async fn accept(&self, draft: TaskAcceptDraft) -> Result<TaskAcceptance, TaskError> {
|
||||
validate_task_envelope(&draft.input)?;
|
||||
if contains_sensitive_task_keys(&draft.input) {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input contains sensitive fields",
|
||||
));
|
||||
}
|
||||
let (scoped_key, input_fingerprint) = (
|
||||
scoped_key_digest(&draft)?,
|
||||
canonical_input_fingerprint(&draft.input)?,
|
||||
);
|
||||
let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline);
|
||||
let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS);
|
||||
let task_id = TaskId::generate();
|
||||
let crypto = self.crypto.as_ref().ok_or_else(|| {
|
||||
TaskError::invalid("secret crypto is required for durable Task acceptance")
|
||||
})?;
|
||||
let binding = serde_json::json!({
|
||||
"task_id": task_id.to_string(),
|
||||
"workspace_id": draft.workspace_id,
|
||||
"owner_kind": draft.owner_kind.as_str(),
|
||||
"owner_id": draft.owner_id,
|
||||
"authorization_scope": draft.authorization_scope,
|
||||
"platform_api_key_id": draft
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(PlatformApiKeyId::as_str),
|
||||
"operation_id": draft.operation_id,
|
||||
"operation_version": draft.operation_version,
|
||||
"policy_mode": draft.policy_mode,
|
||||
"input_fingerprint": input_fingerprint,
|
||||
"input": draft.input,
|
||||
});
|
||||
let ciphertext = crypto
|
||||
.encrypt(&binding)
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
|
||||
return Err(TaskError::invalid(
|
||||
"encrypted task input exceeds its byte budget",
|
||||
));
|
||||
}
|
||||
let (
|
||||
executable_input_ciphertext,
|
||||
executable_input_key_version,
|
||||
executable_input_master_key_epoch,
|
||||
) = (
|
||||
Some(ciphertext),
|
||||
Some(crypto.key_version().to_owned()),
|
||||
Some(crypto.master_key_epoch()),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
task_id,
|
||||
workspace_id: draft.workspace_id,
|
||||
owner_kind: draft.owner_kind,
|
||||
owner_id: draft.owner_id,
|
||||
authorization_scope: draft.authorization_scope,
|
||||
platform_api_key_id: draft.platform_api_key_id,
|
||||
operation_id: draft.operation_id,
|
||||
pins: TaskPins {
|
||||
operation_version: draft.operation_version,
|
||||
@@ -110,6 +169,9 @@ impl TaskService {
|
||||
draft.correlation.request_id().clone(),
|
||||
draft.correlation.trace_context(),
|
||||
),
|
||||
executable_input_ciphertext,
|
||||
executable_input_key_version,
|
||||
executable_input_master_key_epoch,
|
||||
};
|
||||
self.store.accept(request).await
|
||||
}
|
||||
@@ -134,6 +196,97 @@ impl TaskService {
|
||||
))
|
||||
}
|
||||
|
||||
/// Decrypts and validates the executable input bound to a valid current lease.
|
||||
pub async fn executable_input(
|
||||
&self,
|
||||
lease: &TaskLease,
|
||||
) -> Result<serde_json::Value, TaskError> {
|
||||
let crypto = self.crypto.as_ref().ok_or_else(|| {
|
||||
TaskError::invalid("secret crypto is required for durable Task execution")
|
||||
})?;
|
||||
let (ciphertext, key_version, epoch) = self
|
||||
.store
|
||||
.read_executable_input(&lease.owner, lease.record.task_id, lease.fence)
|
||||
.await?;
|
||||
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
let binding = crypto
|
||||
.decrypt_for_epoch(&key_version, epoch, &ciphertext)
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
let object = binding
|
||||
.as_object()
|
||||
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
|
||||
let check_string = |field: &str, expected: &str| -> Result<(), TaskError> {
|
||||
match object.get(field).and_then(|value| value.as_str()) {
|
||||
Some(actual) if actual == expected => Ok(()),
|
||||
_ => Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
};
|
||||
check_string("task_id", &lease.record.task_id.to_string())?;
|
||||
check_string("workspace_id", &lease.record.workspace_id)?;
|
||||
check_string("owner_id", &lease.record.owner_id)?;
|
||||
check_string("authorization_scope", &lease.record.authorization_scope)?;
|
||||
let expected_key = lease
|
||||
.record
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(PlatformApiKeyId::as_str);
|
||||
match (object.get("platform_api_key_id"), expected_key) {
|
||||
(None | Some(serde_json::Value::Null), None) => {}
|
||||
(Some(serde_json::Value::String(actual)), Some(expected)) if actual == expected => {}
|
||||
_ => {
|
||||
return Err(TaskError::invalid("task executable input binding mismatch"));
|
||||
}
|
||||
}
|
||||
check_string("operation_id", &lease.record.operation_id)?;
|
||||
match object.get("owner_kind").and_then(|value| value.as_str()) {
|
||||
Some(actual) if actual == lease.record.owner_kind.as_str() => {}
|
||||
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
match object
|
||||
.get("operation_version")
|
||||
.and_then(|value| value.as_u64())
|
||||
{
|
||||
Some(actual) if actual == u64::from(lease.record.pins.operation_version) => {}
|
||||
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
let expected_policy = match lease.record.pins.policy_mode {
|
||||
ToolExecutionMode::Synchronous => "synchronous",
|
||||
ToolExecutionMode::Background => "background",
|
||||
ToolExecutionMode::Adaptive => "adaptive",
|
||||
};
|
||||
match object.get("policy_mode").and_then(|value| value.as_str()) {
|
||||
Some(actual) if actual == expected_policy => {}
|
||||
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
check_string("input_fingerprint", &lease.record.input_fingerprint)?;
|
||||
let input = object
|
||||
.get("input")
|
||||
.cloned()
|
||||
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
|
||||
let canonical = canonical_input_fingerprint(&input)?;
|
||||
if canonical != lease.record.input_fingerprint {
|
||||
return Err(TaskError::invalid("task executable input binding mismatch"));
|
||||
}
|
||||
validate_task_envelope(&input)?;
|
||||
let encoded = serde_json::to_vec(&input)
|
||||
.map_err(|_| TaskError::invalid("task executable input is not serializable"))?;
|
||||
if encoded.len() > TASK_EXECUTABLE_INPUT_MAX_BYTES {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input exceeds its byte budget",
|
||||
));
|
||||
}
|
||||
if contains_sensitive_task_keys(&input) {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input contains sensitive fields",
|
||||
));
|
||||
}
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
/// Default retention pinned by the application layer when it does not choose one explicitly.
|
||||
pub fn default_retention_ms() -> i64 {
|
||||
TASK_DEFAULT_RETENTION_MS
|
||||
@@ -220,6 +373,7 @@ mod tests {
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
|
||||
operation_id: "op_report".to_owned(),
|
||||
operation_version: 2,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
|
||||
@@ -106,6 +106,83 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
|
||||
assert_eq!(failure.stage(), ExecutionStage::Upstream);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adapter_retry_child_spans_are_distinct_without_a_durable_attempt() {
|
||||
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
|
||||
let capture = TraceCapture::default();
|
||||
let subscriber = tracing_subscriber::registry().with(capture.clone());
|
||||
|
||||
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let executor = RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(FlakyReadAdapter {
|
||||
call_count: Arc::clone(&call_count),
|
||||
}))
|
||||
.build();
|
||||
let mut operation: crank_runtime::RuntimeOperation = operation().into();
|
||||
let Target::Rest(target) = &mut operation.target;
|
||||
target.method = HttpMethod::Get;
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
|
||||
// The synchronous tools/call path has no durable Attempt: every adapter attempt gets its own
|
||||
// synthetic id. A Task dispatch instead reuses the durable id (mcp-server `attempt_trace`).
|
||||
let result = async {
|
||||
let request_root = tracing::info_span!("request_root");
|
||||
executor
|
||||
.execute(&operation, &json!({"name": "retry-read"}))
|
||||
.instrument(request_root)
|
||||
.await
|
||||
}
|
||||
.with_subscriber(subscriber)
|
||||
.await
|
||||
.expect("read retry should recover");
|
||||
|
||||
assert_eq!(result, json!({"accepted": true}));
|
||||
|
||||
let spans = capture.snapshot();
|
||||
let runtime_span = spans
|
||||
.iter()
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime.execute span");
|
||||
assert_eq!(
|
||||
runtime_span.parent_name,
|
||||
Some("request_root"),
|
||||
"the execution continues the caller's trace root"
|
||||
);
|
||||
let attempt_spans: Vec<_> = spans
|
||||
.iter()
|
||||
.filter(|span| span.name == "adapter_attempt")
|
||||
.collect();
|
||||
assert_eq!(attempt_spans.len(), 2);
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(span.parent_name, Some("runtime.execute"));
|
||||
assert!(
|
||||
!span
|
||||
.fields
|
||||
.get("attempt_id")
|
||||
.unwrap_or(&"".to_string())
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!span
|
||||
.fields
|
||||
.get("attempt_id")
|
||||
.unwrap_or(&"".to_string())
|
||||
.contains("traceparent")
|
||||
);
|
||||
}
|
||||
let attempts: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|span| span.fields.get("attempt").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert!(attempts.contains(&"1".to_string()));
|
||||
assert!(attempts.contains(&"2".to_string()));
|
||||
let ids: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|span| span.fields.get("attempt_id").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert_ne!(ids[0], ids[1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_retry_revalidates_adapter_attempt_and_keeps_one_success_outcome() {
|
||||
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
@@ -236,11 +313,7 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime span");
|
||||
assert_eq!(runtime.parent_name, Some("mcp.request"));
|
||||
for child in [
|
||||
"runtime.arguments.map",
|
||||
"upstream.http",
|
||||
"runtime.response.transform",
|
||||
] {
|
||||
for child in ["runtime.arguments.map", "runtime.response.transform"] {
|
||||
assert_eq!(
|
||||
spans
|
||||
.iter()
|
||||
@@ -250,6 +323,14 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
|
||||
"{child} must be a runtime child"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
spans
|
||||
.iter()
|
||||
.find(|span| span.name == "upstream.http")
|
||||
.and_then(|span| span.parent_name),
|
||||
Some("adapter_attempt"),
|
||||
"upstream.http must be an adapter child"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -48,6 +48,43 @@ input fingerprint — типизированный конфликт; явная
|
||||
authorization scope) входит в каждый read; чужой Task неотличим от отсутствующего.
|
||||
|
||||
Границы этой story: публичный MCP wire Tasks/capabilities/activation остаётся
|
||||
Story 4.7, worker/leases/recovery — Story 4.3. Существующий `tools/call` сохраняет
|
||||
synchronous/unsupported поведение; durable acceptance не запускает ephemeral
|
||||
execution и не рекламирует клиенту background capability.
|
||||
Story 4.7, worker/leases/recovery реализованы в Story 4.3 (ниже). Существующий
|
||||
`tools/call` сохраняет synchronous/unsupported поведение; durable acceptance не
|
||||
запускает ephemeral execution и не рекламирует клиенту background capability.
|
||||
|
||||
## Исполнение durable Task (Story 4.3)
|
||||
|
||||
`mcp-server` запускает fenced worker в том же процессе: bounded claim через
|
||||
PostgreSQL `FOR UPDATE SKIP LOCKED` над reserved dispatch intent, lease owner/expiry
|
||||
и monotonic fence; Valkey не является источником истины и может только ускорять
|
||||
wake-up. Claim использует уже принятый shared admission из Story 4.1
|
||||
(`lock_admission_in_tx` по controller-строке `*` + `grant_next_task` с его
|
||||
scoped-лимитами, bounded queue и weighted fairness cursor) и не добавляет второго
|
||||
scheduler'а: 4.3-дельта лишь fencing/lease-поля и CAS-предикаты поверх того же
|
||||
grant, поэтому принятые fairness-инварианты 4.1 сохраняются. Progress, renewal,
|
||||
dispatch checkpoint и terminal outcome пишутся только через CAS по текущему
|
||||
owner/fence/expiry. До сетевого вызова worker повторно проверяет публикацию и
|
||||
точную версию capability, активный Platform API key/scope, approval policy и
|
||||
deadline; невалидная работа завершается без dispatch. Attempt ID и dispatch
|
||||
checkpoint коммитятся атомарно до outbound; после checkpoint потеря lease означает
|
||||
uncertain outcome и Task переходит в `outcome_unknown` без автоматического retry.
|
||||
Исполняемый input берётся только из зашифрованного Task payload через
|
||||
`TaskService::executable_input` (redacted projection никогда не исполняется).
|
||||
Parent TraceContext сохраняется с acceptance; durable attempt span продолжает его,
|
||||
а каждая adapter-попытка (включая retry существующих GET/verified-idempotent write)
|
||||
получает собственный child span `adapter_attempt` с тем же durable Attempt ID и
|
||||
номером попытки, поэтому retry не сливается с первым attempt в трассе, а любой
|
||||
такой span сопоставим с `tasks.current_attempt_id`; синтетический attempt id
|
||||
остаётся только у синхронного tools/call, где durable попытки нет. Скан PostgreSQL авторитетен:
|
||||
crash до checkpoint возвращает Task в queue по истечении lease (recovery ≤60s).
|
||||
|
||||
Graceful shutdown является процессным, а не только worker-локальным: сигнал
|
||||
SIGTERM/ctrl-c одновременно прекращает новые claims и запускает HTTP graceful
|
||||
drain (worker не ждёт завершения долгого соединения), активные attempts
|
||||
checkpoint-ятся/дренируются в пределах 30s, а ошибочное завершение HTTP-сервера
|
||||
также останавливает worker. Уже поставленные в queue Tasks не теряются. Остаётся
|
||||
узкое окно между durable checkpoint и созданием sealed dispatch permit: если stop
|
||||
приходит именно туда, Task финализируется как failed(`shutdown_before_dispatch`),
|
||||
не отправляя запрос; это доказано регрессией, которая детерминированно
|
||||
задерживает checkpoint в его собственной транзакции, проверяет отсутствие вызова
|
||||
adapter и сохранность ещё queued работы.
|
||||
@@ -78,3 +78,10 @@ run_atdd "runtime task identity" -p crank-runtime --lib tasks:: -- --test-thread
|
||||
run_atdd "durable task store" -p crank-registry --test integration tasks:: -- --test-threads=1
|
||||
run_atdd "durable task application entry" -p mcp-server --test integration durable_tasks:: -- --test-threads=1
|
||||
run_atdd "durable task migration" -p crank-registry --test integration migrations:: -- --test-threads=1
|
||||
|
||||
# Story 4.3 (FR-22, FR-23): fenced durable Task execution. The Story 4.2 selectors above already run
|
||||
# the lease/fencing/recovery store, the Task application entry and the migration guards; these two
|
||||
# oracles bound the rest of this story's AC: the process-level shutdown ordering (signal stops new
|
||||
# claims before a slow HTTP drain, not after it) and one child attempt span per adapter retry.
|
||||
run_atdd "task worker process shutdown" -p mcp-server --bin mcp-server -- --test-threads=1
|
||||
run_atdd "durable task retry attempt spans" -p crank-runtime --test stages retry -- --test-threads=1
|
||||
Reference in new issue
Block a user