Files
crank/apps/mcp-server/tests/integration/durable_tasks/cancel_chain.rs
T
bsodfather a3057de51e
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): безопасные повторы и отмена durable Task (Story 4.4)
Durable per-outbound attempt до отправки запроса, bounded retry/read,
server-owned verified writes, честный cancel/unknown, deadline/fencing,
additive schema v29. Staged renames task.rs/tasks.rs разложены на модули.

Includes disclosed cosmetic parity exception: единственная правка вне
принятой дельты — удалена лишняя пустая строка в конце
scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
2026-10-30 12:21:57 +00:00

847 lines
30 KiB
Rust

//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream.
use super::common;
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
use crank_core::{
AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState,
TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential,
};
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
};
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "report_gen";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
}
async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(
registry,
AGENT_SLUG,
name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("active key");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store, Arc::new(crypto)),
)
}
#[tokio::test]
async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
let signal = Arc::new(std::sync::Mutex::new(Some(tx)));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
move || {
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
async move {
calls.fetch_add(1, Ordering::SeqCst);
if let Some(tx) = signal.lock().unwrap().take() {
let _ = tx.send(());
}
tokio::time::sleep(Duration::from_millis(1500)).await;
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s1").await;
let foreign = principal(&registry, "cancel-s1-foreign").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s1-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&app,
registry.clone(),
runtime,
crypto.clone(),
));
let worker = TaskWorker::new(
store,
handler,
TaskWorkerConfig {
owner: "worker-s1".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
renewal_interval: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(5),
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let _ = tokio::time::timeout(Duration::from_secs(5), rx).await;
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, TaskState::OutcomeUnknown);
}
other => panic!("expected OutcomeUnknown got {other:?}"),
}
let foreign_cancel = app
.cancel(&foreign, TOOL_NAME, acceptance.task_id, None)
.await;
assert!(matches!(
foreign_cancel,
Err(crank_community_mcp::TaskApplicationError::NotFound)
));
tokio::time::sleep(Duration::from_millis(1600)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1);
let cancel2 = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel2");
assert!(matches!(
cancel2,
crank_core::TaskCancelOutcome::OutcomeUnknown(_)
));
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
}
#[tokio::test]
async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s3").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s3-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::Cancelled(record) => {
assert_eq!(record.state, TaskState::Cancelled)
}
other => panic!("expected Cancelled got {other:?}"),
}
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("cancelled")
);
let attempts = app
.attempts(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("attempts");
assert!(attempts.is_empty());
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s2").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s2-key",
&json!({"email":"u","idempotency_key":"k-2"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
// Reproduce deterministically the durable state the executor commits after an ambiguous attempt
// whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and
// `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a
// running executor against the cancel would not be a repeatable oracle.
let lease = store
.claim_batch("worker-s2-prep", 1, 10_000)
.await
.expect("claim")
.pop()
.expect("one claimed lease");
let attempt = store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: acceptance.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("durable checkpoint");
let resolution = store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Uncertain {
error: crank_core::TaskEnvelope::try_redacted(
&json!({"code": "upstream_ack_lost", "phase": "dispatched"}),
)
.expect("bounded attempt error"),
repeatable: true,
phase: crank_core::AttemptPhase::Dispatched,
},
)
.await
.expect("attempt resolution");
assert!(
matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }),
"an ambiguous but safely repeatable attempt arms the pinned retry"
);
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown)
}
other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"),
}
// A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not
// claimable and no second terminal write is possible.
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s2".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::sleep(Duration::from_secs(1)).await;
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
requests.load(Ordering::SeqCst),
0,
"no outbound send after the cancel"
);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
}
#[tokio::test]
async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s4").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s4-key",
&json!({"email":"u","idempotency_key":"k-4"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s4".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
// Revoke the key the Task was accepted with while the first outbound write is provably in flight:
// that attempt still resolves as an uncertain write, and the next attempt is refused by the
// authority before its checkpoint.
for _ in 0..200 {
if requests.load(Ordering::SeqCst) >= 1 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the first attempt is dispatched before the refusal"
);
registry
.revoke_platform_api_key_for_agent(
&common::test_workspace_id(),
&common::test_agent_id(AGENT_SLUG),
&PlatformApiKeyId::new("pk_cancel-s4".to_owned()),
&OffsetDateTime::now_utc(),
)
.await
.expect("revoke");
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("outcome_unknown") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
state.as_deref(),
Some("outcome_unknown"),
"a refused attempt after a lost acknowledgement must stay uncertain"
);
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the refused attempt is never dispatched"
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let phase: Option<String> = sqlx::query_scalar(
"select phase from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(phase.as_deref(), Some("dispatched"));
let error: Option<(Option<String>, Option<String>)> = sqlx::query_as(
"select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
let (code, error_phase) = error.expect("the refused Task carries its bounded error");
assert_eq!(code.as_deref(), Some("task_authority_rejected"));
assert_eq!(
error_phase.as_deref(),
Some("dispatched"),
"the error keeps the phase of the attempt that may have reached the upstream"
);
}