CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с fence-токеном, зашифрованный input и повторная проверка доступа до сетевого вызова, checkpoint с AttemptID до dispatch и восстановление только по PG. Устаревший worker не может зафиксировать результат. Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса: стоп в окне после checkpoint и до permit завершает задачу как failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи остаются доступными. Retry-попытки получают отдельные дочерние spans с durable current_attempt_id. Shared admission/fairness из Story 4.1 сохранены. Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47, bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
823 lines
30 KiB
Rust
823 lines
30 KiB
Rust
//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch.
|
|
//!
|
|
//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit.
|
|
|
|
use super::*;
|
|
|
|
/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a
|
|
/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan,
|
|
/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented.
|
|
#[tokio::test]
|
|
async fn task_worker_store_lease_recovery_contract() {
|
|
let registry = common::test_registry().await;
|
|
let store = PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration");
|
|
|
|
// Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and
|
|
// leases durable work, preserving the same admission-backed intent.
|
|
let request = TaskAcceptRequest {
|
|
task_id: TaskId::generate(),
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_1".to_owned(),
|
|
platform_api_key_id: None,
|
|
operation_id: "op_report".to_owned(),
|
|
pins: TaskPins {
|
|
operation_version: 1,
|
|
policy_mode: ToolExecutionMode::Background,
|
|
execution_deadline_unix_ms: 1_900_000_000_000,
|
|
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
|
approval_reference: None,
|
|
},
|
|
scoped_key: "a".repeat(64),
|
|
input_fingerprint: "b".repeat(64),
|
|
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
|
support: TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&crank_core::TraceContext::parse(
|
|
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
|
)
|
|
.unwrap(),
|
|
),
|
|
// This store-level control exercises lease/fence behavior, not the encrypted acceptance path.
|
|
executable_input_ciphertext: None,
|
|
executable_input_key_version: None,
|
|
executable_input_master_key_epoch: None,
|
|
};
|
|
let accepted = store.accept(request).await.expect("PG accepts");
|
|
let leases = store
|
|
.claim_batch("worker_1", 4, 10_000)
|
|
.await
|
|
.expect("claim");
|
|
assert_eq!(leases.len(), 1);
|
|
assert_eq!(leases[0].record.task_id, accepted.task_id);
|
|
let lease = leases[0].clone();
|
|
|
|
// A stale or competing owner cannot win CAS with the issued fencing token.
|
|
assert!(
|
|
store
|
|
.renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000)
|
|
.await
|
|
.is_err()
|
|
);
|
|
assert!(
|
|
store
|
|
.write_progress(
|
|
"worker_1",
|
|
lease.record.task_id,
|
|
lease.fence + 1,
|
|
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
|
|
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
|
|
assert!(
|
|
store
|
|
.checkpoint_dispatch(
|
|
"worker_1",
|
|
lease.record.task_id,
|
|
lease.fence + 1,
|
|
"attempt-stale".to_owned()
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
// The valid owner still commits the dispatch checkpoint before any outbound work.
|
|
assert!(
|
|
store
|
|
.checkpoint_dispatch(
|
|
"worker_1",
|
|
lease.record.task_id,
|
|
lease.fence,
|
|
"attempt-1".to_owned(),
|
|
)
|
|
.await
|
|
.unwrap()
|
|
.current_attempt_id
|
|
.as_deref()
|
|
== Some("attempt-1")
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn task_worker_runtime_dispatch_and_recovery_contract() {
|
|
use async_trait::async_trait;
|
|
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
|
use crank_runtime::{
|
|
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
|
TaskWorkerError,
|
|
};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
|
|
let registry = common::test_registry().await;
|
|
let pool = registry.pool().clone();
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
pool.clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let observed = Arc::new(AtomicUsize::new(0));
|
|
|
|
// Positive control: PostgreSQL alone accepts the durable Task.
|
|
let request = TaskAcceptRequest {
|
|
task_id: TaskId::generate(),
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_runtime".to_owned(),
|
|
platform_api_key_id: None,
|
|
operation_id: "op_report".to_owned(),
|
|
pins: TaskPins {
|
|
operation_version: 1,
|
|
policy_mode: ToolExecutionMode::Background,
|
|
execution_deadline_unix_ms: 1_900_000_000_000,
|
|
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
|
approval_reference: None,
|
|
},
|
|
scoped_key: "r".repeat(64),
|
|
input_fingerprint: "b".repeat(64),
|
|
input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(),
|
|
support: TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&crank_core::TraceContext::parse(
|
|
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
|
)
|
|
.unwrap(),
|
|
),
|
|
executable_input_ciphertext: None,
|
|
executable_input_key_version: None,
|
|
executable_input_master_key_epoch: None,
|
|
};
|
|
let accepted = store.accept(request).await.expect("PG accepts");
|
|
let task_id = accepted.task_id;
|
|
|
|
struct RuntimeHandler {
|
|
observed: Arc<AtomicUsize>,
|
|
}
|
|
#[async_trait]
|
|
impl TaskHandler for RuntimeHandler {
|
|
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
|
Ok(())
|
|
}
|
|
async fn execute(
|
|
&self,
|
|
_lease: &TaskLease,
|
|
_attempt_id: &str,
|
|
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
|
_permit: TaskDispatchPermit,
|
|
) -> TaskAttemptOutcome {
|
|
self.observed.fetch_add(1, Ordering::SeqCst);
|
|
TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
|
|
}
|
|
}
|
|
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
|
|
observed: observed.clone(),
|
|
});
|
|
|
|
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
|
|
let mut config = TaskWorkerConfig::default();
|
|
config.owner = "runtime-worker-1".to_owned();
|
|
config.batch_limit = 1;
|
|
config.lease_ms = 10_000;
|
|
config.poll_interval = Duration::from_millis(100);
|
|
config.renewal_interval = Duration::from_millis(1_000);
|
|
config.max_inflight = 1;
|
|
config.shutdown_timeout = Duration::from_secs(5);
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
let access = TaskAccess {
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_runtime".to_owned(),
|
|
};
|
|
let mut state = None;
|
|
for _ in 0..100 {
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
if let Ok(record) = store.read(&access, task_id).await {
|
|
state = Some(record.state);
|
|
if record.state == TaskState::Completed {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
|
|
assert_eq!(state, Some(TaskState::Completed));
|
|
assert_eq!(
|
|
observed.load(Ordering::SeqCst),
|
|
1,
|
|
"exactly one outbound attempt"
|
|
);
|
|
// Terminal state is not duplicated.
|
|
let record = store.read(&access, task_id).await.expect("terminal read");
|
|
assert_eq!(record.state, TaskState::Completed);
|
|
|
|
// Bounded shutdown with no fresh work returns promptly.
|
|
let mut idle_config = TaskWorkerConfig::default();
|
|
idle_config.owner = "runtime-worker-idle".to_owned();
|
|
idle_config.poll_interval = Duration::from_millis(50);
|
|
idle_config.shutdown_timeout = Duration::from_secs(2);
|
|
let idle_worker = TaskWorker::new(
|
|
store.clone(),
|
|
handler_for_no_work(store.clone()),
|
|
idle_config,
|
|
)
|
|
.expect("valid idle worker");
|
|
idle_worker.stop();
|
|
let started = std::time::Instant::now();
|
|
idle_worker.run().await.expect("idle worker stops");
|
|
assert!(started.elapsed() <= Duration::from_secs(30));
|
|
}
|
|
|
|
/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call.
|
|
#[tokio::test]
|
|
async fn task_worker_revalidation_denial_never_dispatches() {
|
|
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
|
|
use crank_runtime::{
|
|
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
|
TaskWorkerError,
|
|
};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
|
|
let registry = common::test_registry().await;
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let request = TaskAcceptRequest {
|
|
task_id: TaskId::generate(),
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_denied".to_owned(),
|
|
platform_api_key_id: None,
|
|
operation_id: "op_report".to_owned(),
|
|
pins: TaskPins {
|
|
operation_version: 1,
|
|
policy_mode: ToolExecutionMode::Background,
|
|
execution_deadline_unix_ms: 1_900_000_000_000,
|
|
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
|
approval_reference: None,
|
|
},
|
|
scoped_key: "d".repeat(64),
|
|
input_fingerprint: "b".repeat(64),
|
|
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
|
support: TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&crank_core::TraceContext::parse(
|
|
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
|
|
)
|
|
.unwrap(),
|
|
),
|
|
executable_input_ciphertext: None,
|
|
executable_input_key_version: None,
|
|
executable_input_master_key_epoch: None,
|
|
};
|
|
let accepted = store.accept(request).await.expect("PG accepts");
|
|
|
|
struct DeniedHandler {
|
|
observed: Arc<AtomicUsize>,
|
|
}
|
|
#[async_trait::async_trait]
|
|
impl TaskHandler for DeniedHandler {
|
|
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
|
Err(TaskWorkerError::Store(TaskError::invalid(
|
|
"publication or credential revoked",
|
|
)))
|
|
}
|
|
async fn execute(
|
|
&self,
|
|
_lease: &TaskLease,
|
|
_attempt_id: &str,
|
|
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
|
_permit: TaskDispatchPermit,
|
|
) -> TaskAttemptOutcome {
|
|
self.observed.fetch_add(1, Ordering::SeqCst);
|
|
TaskAttemptOutcome::Completed(json!({"ok": true}))
|
|
}
|
|
}
|
|
let observed = Arc::new(AtomicUsize::new(0));
|
|
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
|
|
observed: observed.clone(),
|
|
});
|
|
let mut config = TaskWorkerConfig::default();
|
|
config.owner = "runtime-worker-denied".to_owned();
|
|
config.batch_limit = 1;
|
|
config.lease_ms = 10_000;
|
|
config.poll_interval = Duration::from_millis(50);
|
|
config.max_inflight = 1;
|
|
config.shutdown_timeout = Duration::from_secs(5);
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
let access = TaskAccess {
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_denied".to_owned(),
|
|
};
|
|
let mut state = None;
|
|
for _ in 0..100 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
if let Ok(record) = store.read(&access, accepted.task_id).await {
|
|
state = Some(record.state);
|
|
if record.state == TaskState::Failed {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
|
|
assert_eq!(state, Some(TaskState::Failed));
|
|
assert_eq!(
|
|
observed.load(Ordering::SeqCst),
|
|
0,
|
|
"invalidated work never reaches the upstream"
|
|
);
|
|
}
|
|
|
|
fn handler_for_no_work(
|
|
_store: std::sync::Arc<PostgresTaskStore>,
|
|
) -> std::sync::Arc<dyn crank_runtime::TaskHandler> {
|
|
struct NoWorkHandler;
|
|
#[async_trait::async_trait]
|
|
impl crank_runtime::TaskHandler for NoWorkHandler {
|
|
async fn revalidate(
|
|
&self,
|
|
_lease: &crank_core::TaskLease,
|
|
) -> Result<(), crank_runtime::TaskWorkerError> {
|
|
Ok(())
|
|
}
|
|
async fn execute(
|
|
&self,
|
|
_lease: &crank_core::TaskLease,
|
|
_attempt_id: &str,
|
|
_cancelled: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
|
_permit: crank_runtime::TaskDispatchPermit,
|
|
) -> crank_runtime::TaskAttemptOutcome {
|
|
crank_runtime::TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
|
|
}
|
|
}
|
|
std::sync::Arc::new(NoWorkHandler)
|
|
}
|
|
|
|
/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a
|
|
/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the
|
|
/// accepted parent trace is continued on the wire.
|
|
#[tokio::test]
|
|
async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
|
|
use axum::Router;
|
|
use axum::http::{HeaderMap, StatusCode};
|
|
use axum::routing::post;
|
|
use crank_community_mcp::TaskWorkerHandler;
|
|
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
|
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let seen_traceparent = Arc::new(Mutex::new(None::<String>));
|
|
|
|
let app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
let seen = Arc::clone(&seen_traceparent);
|
|
move |headers: HeaderMap| {
|
|
let calls = Arc::clone(&calls);
|
|
let seen = Arc::clone(&seen);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
if let Some(value) = headers
|
|
.get("traceparent")
|
|
.and_then(|value| value.to_str().ok())
|
|
{
|
|
*seen.lock().unwrap() = Some(value.to_owned());
|
|
}
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(background_policy());
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
publish(®istry, &operation.id, 1).await;
|
|
common::publish_agent_with_bindings(
|
|
®istry,
|
|
AGENT_SLUG,
|
|
vec![AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
|
|
let caller = principal(
|
|
®istry,
|
|
"task-worker-e2e",
|
|
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
|
)
|
|
.await;
|
|
let correlation = unsampled_correlation();
|
|
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
|
let observed_at = OffsetDateTime::now_utc();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &observed_at,
|
|
})
|
|
.await
|
|
.expect("test master-key identity registered");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let tasks = TaskApplication::new(
|
|
registry.clone(),
|
|
Duration::ZERO,
|
|
Arc::new(InMemoryCoordinationStateStore::default()),
|
|
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
|
);
|
|
let acceptance = tasks
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"worker-e2e-key",
|
|
&json!({"email": "user@example.com"}),
|
|
&correlation,
|
|
)
|
|
.await
|
|
.expect("durable acceptance with executable input");
|
|
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.build();
|
|
let handler = Arc::new(TaskWorkerHandler::from_application(
|
|
&tasks,
|
|
registry.clone(),
|
|
runtime,
|
|
crypto,
|
|
));
|
|
let mut config = TaskWorkerConfig::default();
|
|
config.owner = "runtime-worker-e2e".to_owned();
|
|
config.batch_limit = 1;
|
|
config.lease_ms = 10_000;
|
|
config.poll_interval = Duration::from_millis(50);
|
|
config.max_inflight = 1;
|
|
config.shutdown_timeout = Duration::from_secs(5);
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
let mut state = None;
|
|
for _ in 0..200 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
let current: Option<String> =
|
|
sqlx::query_scalar("select state from tasks where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
state = current;
|
|
if state.as_deref() == Some("completed") {
|
|
break;
|
|
}
|
|
}
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
server.abort();
|
|
|
|
assert_eq!(state.as_deref(), Some("completed"));
|
|
assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call");
|
|
assert_eq!(
|
|
seen_traceparent.lock().unwrap().as_deref(),
|
|
Some(correlation.trace_context().traceparent()),
|
|
"the durable attempt continues the accepted parent trace"
|
|
);
|
|
}
|
|
|
|
/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as
|
|
/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically.
|
|
#[tokio::test]
|
|
async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
|
|
use axum::Router;
|
|
use axum::http::StatusCode;
|
|
use axum::routing::post;
|
|
use crank_community_mcp::TaskWorkerHandler;
|
|
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
|
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::Ordering;
|
|
|
|
let registry = common::test_registry().await;
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(background_policy());
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
publish(®istry, &operation.id, 1).await;
|
|
common::publish_agent_with_bindings(
|
|
®istry,
|
|
AGENT_SLUG,
|
|
vec![AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
|
|
let caller = principal(
|
|
®istry,
|
|
"task-worker-uncertain",
|
|
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
|
)
|
|
.await;
|
|
let correlation = unsampled_correlation();
|
|
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
|
let observed_at = OffsetDateTime::now_utc();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &observed_at,
|
|
})
|
|
.await
|
|
.expect("test master-key identity registered");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let tasks = TaskApplication::new(
|
|
registry.clone(),
|
|
Duration::ZERO,
|
|
Arc::new(InMemoryCoordinationStateStore::default()),
|
|
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
|
);
|
|
let acceptance = tasks
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"worker-uncertain-key",
|
|
&json!({"email": "user@example.com"}),
|
|
&correlation,
|
|
)
|
|
.await
|
|
.expect("durable acceptance with executable input");
|
|
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.build();
|
|
let handler = Arc::new(TaskWorkerHandler::from_application(
|
|
&tasks,
|
|
registry.clone(),
|
|
runtime,
|
|
crypto,
|
|
));
|
|
let mut config = TaskWorkerConfig::default();
|
|
config.owner = "runtime-worker-uncertain".to_owned();
|
|
config.batch_limit = 1;
|
|
config.lease_ms = 10_000;
|
|
config.poll_interval = Duration::from_millis(50);
|
|
config.max_inflight = 1;
|
|
config.shutdown_timeout = Duration::from_secs(5);
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
let mut state = None;
|
|
for _ in 0..200 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
let current: Option<String> =
|
|
sqlx::query_scalar("select state from tasks where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
state = current;
|
|
if state.as_deref() == Some("outcome_unknown") {
|
|
break;
|
|
}
|
|
}
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
server.abort();
|
|
|
|
assert_eq!(state.as_deref(), Some("outcome_unknown"));
|
|
// No automatic retry: another authoritative scan never redispaches the uncertain attempt.
|
|
assert!(
|
|
store
|
|
.claim_batch("worker_after", 4, 10_000)
|
|
.await
|
|
.unwrap()
|
|
.is_empty()
|
|
);
|
|
}
|
|
|
|
/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced
|
|
/// terminal commit within the 30s budget.
|
|
#[tokio::test]
|
|
async fn task_worker_graceful_stop_drains_active_attempt() {
|
|
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
|
use crank_runtime::{
|
|
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
|
TaskWorkerError,
|
|
};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
|
|
let registry = common::test_registry().await;
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let request = TaskAcceptRequest {
|
|
task_id: TaskId::generate(),
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_drain".to_owned(),
|
|
platform_api_key_id: None,
|
|
operation_id: "op_report".to_owned(),
|
|
pins: TaskPins {
|
|
operation_version: 1,
|
|
policy_mode: ToolExecutionMode::Background,
|
|
execution_deadline_unix_ms: 1_900_000_000_000,
|
|
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
|
approval_reference: None,
|
|
},
|
|
scoped_key: "s".repeat(64),
|
|
input_fingerprint: "d".repeat(64),
|
|
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
|
support: TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&crank_core::TraceContext::parse(
|
|
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
|
)
|
|
.unwrap(),
|
|
),
|
|
executable_input_ciphertext: None,
|
|
executable_input_key_version: None,
|
|
executable_input_master_key_epoch: None,
|
|
};
|
|
let accepted = store.accept(request).await.expect("PG accepts");
|
|
|
|
struct SlowHandler {
|
|
started: Arc<AtomicUsize>,
|
|
}
|
|
#[async_trait::async_trait]
|
|
impl TaskHandler for SlowHandler {
|
|
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
|
Ok(())
|
|
}
|
|
async fn execute(
|
|
&self,
|
|
_lease: &TaskLease,
|
|
_attempt_id: &str,
|
|
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
|
_permit: TaskDispatchPermit,
|
|
) -> TaskAttemptOutcome {
|
|
self.started.fetch_add(1, Ordering::SeqCst);
|
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
TaskAttemptOutcome::Completed(json!({"ok": true}))
|
|
}
|
|
}
|
|
let started = Arc::new(AtomicUsize::new(0));
|
|
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
|
|
started: started.clone(),
|
|
});
|
|
let mut config = TaskWorkerConfig::default();
|
|
config.owner = "runtime-worker-drain".to_owned();
|
|
config.batch_limit = 1;
|
|
config.lease_ms = 10_000;
|
|
config.poll_interval = Duration::from_millis(50);
|
|
config.max_inflight = 1;
|
|
config.shutdown_timeout = Duration::from_secs(20);
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
for _ in 0..100 {
|
|
if started.load(Ordering::SeqCst) > 0 {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started");
|
|
let stop_requested = std::time::Instant::now();
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
|
.await
|
|
.expect("worker stops inside the 30s budget")
|
|
.expect("worker exits cleanly")
|
|
.expect("worker reports no error");
|
|
assert!(stop_requested.elapsed() <= Duration::from_secs(30));
|
|
|
|
let access = TaskAccess {
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_drain".to_owned(),
|
|
};
|
|
let record = store.read(&access, accepted.task_id).await.unwrap();
|
|
assert_eq!(record.state, TaskState::Completed);
|
|
}
|