CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable per-outbound attempt до отправки запроса, bounded retry/read, server-owned verified writes, честный cancel/unknown, deadline/fencing, additive schema v29. Staged renames task.rs/tasks.rs разложены на модули. Includes disclosed cosmetic parity exception: единственная правка вне принятой дельты — удалена лишняя пустая строка в конце scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
847 lines
30 KiB
Rust
847 lines
30 KiB
Rust
//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream.
|
|
|
|
use super::common;
|
|
use super::unsampled_correlation;
|
|
|
|
use axum::Router;
|
|
use axum::http::StatusCode;
|
|
use axum::routing::post;
|
|
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
|
|
use crank_core::{
|
|
AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState,
|
|
TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential,
|
|
};
|
|
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
|
|
use crank_runtime::{
|
|
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
|
|
};
|
|
use serde_json::json;
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::time::Duration;
|
|
use time::OffsetDateTime;
|
|
|
|
const AGENT_SLUG: &str = "reports";
|
|
const TOOL_NAME: &str = "report_gen";
|
|
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
|
|
|
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
|
|
sqlx::query_scalar("select state from tasks where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap()
|
|
}
|
|
|
|
async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) {
|
|
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(policy);
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
registry
|
|
.publish_operation(PublishRequest {
|
|
workspace_id: &common::test_workspace_id(),
|
|
operation_id: &operation.id,
|
|
version: 1,
|
|
published_at: &OffsetDateTime::parse(
|
|
"2026-03-26T10:00:00Z",
|
|
&time::format_description::well_known::Rfc3339,
|
|
)
|
|
.unwrap(),
|
|
published_by: Some("alice"),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
common::publish_agent_with_bindings(
|
|
registry,
|
|
AGENT_SLUG,
|
|
vec![AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
}
|
|
|
|
async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal {
|
|
let token = common::create_platform_api_key(
|
|
registry,
|
|
AGENT_SLUG,
|
|
name,
|
|
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
|
)
|
|
.await;
|
|
let secret_hash = common::hash_access_secret(&token);
|
|
let api_key = registry
|
|
.get_platform_api_key_by_secret_for_agent_slug(
|
|
common::test_workspace_slug(),
|
|
AGENT_SLUG,
|
|
&secret_hash,
|
|
)
|
|
.await
|
|
.unwrap()
|
|
.expect("active key");
|
|
registry
|
|
.touch_platform_api_key(
|
|
&api_key.api_key.workspace_id,
|
|
&api_key.api_key.id,
|
|
&OffsetDateTime::now_utc(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
crank_community_mcp::TaskPrincipal::Agent {
|
|
workspace_slug: common::test_workspace_slug().to_owned(),
|
|
agent_slug: AGENT_SLUG.to_owned(),
|
|
credential: VerifiedMachineCredential {
|
|
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
|
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
|
scopes: api_key.api_key.scopes.clone(),
|
|
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
|
},
|
|
}
|
|
}
|
|
|
|
async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
let observed_at = OffsetDateTime::now_utc();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &observed_at,
|
|
})
|
|
.await
|
|
.expect("master key");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("store"),
|
|
);
|
|
TaskApplication::new(
|
|
registry.clone(),
|
|
Duration::ZERO,
|
|
Arc::new(InMemoryCoordinationStateStore::default()),
|
|
TaskService::with_secret_crypto(store, Arc::new(crypto)),
|
|
)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
|
|
let signal = Arc::new(std::sync::Mutex::new(Some(tx)));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
let signal = Arc::clone(&signal);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
let signal = Arc::clone(&signal);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
if let Some(tx) = signal.lock().unwrap().take() {
|
|
let _ = tx.send(());
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(1500)).await;
|
|
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 30_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &upstream, policy).await;
|
|
let caller = principal(®istry, "cancel-s1").await;
|
|
let foreign = principal(®istry, "cancel-s1-foreign").await;
|
|
let app = application(®istry).await;
|
|
|
|
let acceptance = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"s1-key",
|
|
&json!({"email":"u"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept");
|
|
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &OffsetDateTime::now_utc(),
|
|
})
|
|
.await
|
|
.expect("master key");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("store"),
|
|
);
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.build();
|
|
let handler = Arc::new(TaskWorkerHandler::from_application(
|
|
&app,
|
|
registry.clone(),
|
|
runtime,
|
|
crypto.clone(),
|
|
));
|
|
let worker = TaskWorker::new(
|
|
store,
|
|
handler,
|
|
TaskWorkerConfig {
|
|
owner: "worker-s1".to_owned(),
|
|
batch_limit: 1,
|
|
lease_ms: 10_000,
|
|
poll_interval: Duration::from_millis(50),
|
|
max_inflight: 1,
|
|
renewal_interval: Duration::from_millis(100),
|
|
shutdown_timeout: Duration::from_secs(5),
|
|
},
|
|
)
|
|
.expect("worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
let _ = tokio::time::timeout(Duration::from_secs(5), rx).await;
|
|
|
|
let cancel = app
|
|
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
|
.await
|
|
.expect("cancel");
|
|
match cancel {
|
|
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
|
|
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
|
}
|
|
other => panic!("expected OutcomeUnknown got {other:?}"),
|
|
}
|
|
|
|
let foreign_cancel = app
|
|
.cancel(&foreign, TOOL_NAME, acceptance.task_id, None)
|
|
.await;
|
|
assert!(matches!(
|
|
foreign_cancel,
|
|
Err(crank_community_mcp::TaskApplicationError::NotFound)
|
|
));
|
|
|
|
tokio::time::sleep(Duration::from_millis(1600)).await;
|
|
|
|
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
|
assert_eq!(
|
|
state_of(®istry, acceptance.task_id).await.as_deref(),
|
|
Some("outcome_unknown")
|
|
);
|
|
let certainty: Option<String> = sqlx::query_scalar(
|
|
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
|
)
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(certainty.as_deref(), Some("unknown"));
|
|
let code: Option<String> =
|
|
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
|
|
let attempts_count: i64 =
|
|
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempts_count, 1);
|
|
|
|
let cancel2 = app
|
|
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
|
.await
|
|
.expect("cancel2");
|
|
assert!(matches!(
|
|
cancel2,
|
|
crank_core::TaskCancelOutcome::OutcomeUnknown(_)
|
|
));
|
|
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 30_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &upstream, policy).await;
|
|
let caller = principal(®istry, "cancel-s3").await;
|
|
let app = application(®istry).await;
|
|
|
|
let acceptance = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"s3-key",
|
|
&json!({"email":"u"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept");
|
|
|
|
let cancel = app
|
|
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
|
.await
|
|
.expect("cancel");
|
|
match cancel {
|
|
crank_core::TaskCancelOutcome::Cancelled(record) => {
|
|
assert_eq!(record.state, TaskState::Cancelled)
|
|
}
|
|
other => panic!("expected Cancelled got {other:?}"),
|
|
}
|
|
assert_eq!(
|
|
state_of(®istry, acceptance.task_id).await.as_deref(),
|
|
Some("cancelled")
|
|
);
|
|
let attempts = app
|
|
.attempts(&caller, TOOL_NAME, acceptance.task_id)
|
|
.await
|
|
.expect("attempts");
|
|
assert!(attempts.is_empty());
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() {
|
|
use crank_community_mcp::VerifiedIdempotentWrite;
|
|
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
|
|
|
let registry = common::test_registry().await;
|
|
let requests = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let requests = Arc::clone(&requests);
|
|
move |headers: axum::http::HeaderMap| {
|
|
let requests = Arc::clone(&requests);
|
|
async move {
|
|
let _ = headers.get("idempotency-key");
|
|
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
|
if attempt == 0 {
|
|
// Applied upstream, acknowledgement lost after the caller had time to observe that
|
|
// the request really left the worker.
|
|
tokio::time::sleep(Duration::from_millis(600)).await;
|
|
return (
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
axum::Json(json!({"error":"acknowledgement lost"})),
|
|
);
|
|
}
|
|
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 30_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
});
|
|
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
|
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
|
mode: IdempotencyMode::Required,
|
|
ttl_ms: 60_000,
|
|
input_field: Some("idempotency_key".to_owned()),
|
|
header_name: Some("Idempotency-Key".to_owned()),
|
|
});
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
registry
|
|
.publish_operation(PublishRequest {
|
|
workspace_id: &common::test_workspace_id(),
|
|
operation_id: &operation.id,
|
|
version: 1,
|
|
published_at: &OffsetDateTime::parse(
|
|
"2026-03-26T10:00:00Z",
|
|
&time::format_description::well_known::Rfc3339,
|
|
)
|
|
.unwrap(),
|
|
published_by: Some("alice"),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
common::publish_agent_with_bindings(
|
|
®istry,
|
|
AGENT_SLUG,
|
|
vec![AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
let caller = principal(®istry, "cancel-s2").await;
|
|
let app = application(®istry).await;
|
|
let acceptance = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"s2-key",
|
|
&json!({"email":"u","idempotency_key":"k-2"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept");
|
|
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &OffsetDateTime::now_utc(),
|
|
})
|
|
.await
|
|
.expect("master key");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("store"),
|
|
);
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
|
.build();
|
|
let handler = Arc::new(
|
|
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
|
|
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
|
operation_id: operation.id.as_str().to_owned(),
|
|
operation_version: 1,
|
|
key_header: "Idempotency-Key".to_owned(),
|
|
validity_ms: 60_000,
|
|
}]),
|
|
);
|
|
// Reproduce deterministically the durable state the executor commits after an ambiguous attempt
|
|
// whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and
|
|
// `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a
|
|
// running executor against the cancel would not be a repeatable oracle.
|
|
let lease = store
|
|
.claim_batch("worker-s2-prep", 1, 10_000)
|
|
.await
|
|
.expect("claim")
|
|
.pop()
|
|
.expect("one claimed lease");
|
|
let attempt = store
|
|
.checkpoint_dispatch(crank_core::BeginAttempt {
|
|
task_id: acceptance.task_id,
|
|
owner: lease.owner.clone(),
|
|
fence: lease.fence,
|
|
expected_version: lease.record.version,
|
|
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
|
|
input_fingerprint: lease.record.input_fingerprint.clone(),
|
|
upstream_key_digest: None,
|
|
retry_allowed: true,
|
|
})
|
|
.await
|
|
.expect("durable checkpoint");
|
|
let resolution = store
|
|
.resolve_attempt(
|
|
&attempt,
|
|
crank_core::AttemptOutcome::Uncertain {
|
|
error: crank_core::TaskEnvelope::try_redacted(
|
|
&json!({"code": "upstream_ack_lost", "phase": "dispatched"}),
|
|
)
|
|
.expect("bounded attempt error"),
|
|
repeatable: true,
|
|
phase: crank_core::AttemptPhase::Dispatched,
|
|
},
|
|
)
|
|
.await
|
|
.expect("attempt resolution");
|
|
assert!(
|
|
matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }),
|
|
"an ambiguous but safely repeatable attempt arms the pinned retry"
|
|
);
|
|
|
|
let cancel = app
|
|
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
|
.await
|
|
.expect("cancel");
|
|
match cancel {
|
|
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
|
|
assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown)
|
|
}
|
|
other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"),
|
|
}
|
|
|
|
// A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not
|
|
// claimable and no second terminal write is possible.
|
|
let worker = TaskWorker::new(
|
|
store.clone(),
|
|
handler,
|
|
TaskWorkerConfig {
|
|
owner: "worker-s2".to_owned(),
|
|
batch_limit: 1,
|
|
lease_ms: 10_000,
|
|
poll_interval: Duration::from_millis(50),
|
|
max_inflight: 1,
|
|
shutdown_timeout: Duration::from_secs(5),
|
|
..TaskWorkerConfig::default()
|
|
},
|
|
)
|
|
.expect("worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
server.abort();
|
|
|
|
assert_eq!(
|
|
requests.load(Ordering::SeqCst),
|
|
0,
|
|
"no outbound send after the cancel"
|
|
);
|
|
assert_eq!(
|
|
state_of(®istry, acceptance.task_id).await.as_deref(),
|
|
Some("outcome_unknown")
|
|
);
|
|
let attempts_count: i64 =
|
|
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempts_count, 1, "no second durable attempt");
|
|
let certainty: Option<String> = sqlx::query_scalar(
|
|
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
|
)
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(certainty.as_deref(), Some("unknown"));
|
|
let code: Option<String> =
|
|
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() {
|
|
use crank_community_mcp::VerifiedIdempotentWrite;
|
|
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
|
|
|
let registry = common::test_registry().await;
|
|
let requests = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let requests = Arc::clone(&requests);
|
|
move |headers: axum::http::HeaderMap| {
|
|
let requests = Arc::clone(&requests);
|
|
async move {
|
|
let _ = headers.get("idempotency-key");
|
|
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
|
if attempt == 0 {
|
|
// Applied upstream, acknowledgement lost after the caller had time to observe that
|
|
// the request really left the worker.
|
|
tokio::time::sleep(Duration::from_millis(600)).await;
|
|
return (
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
axum::Json(json!({"error":"acknowledgement lost"})),
|
|
);
|
|
}
|
|
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 30_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
});
|
|
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
|
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
|
mode: IdempotencyMode::Required,
|
|
ttl_ms: 60_000,
|
|
input_field: Some("idempotency_key".to_owned()),
|
|
header_name: Some("Idempotency-Key".to_owned()),
|
|
});
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
registry
|
|
.publish_operation(PublishRequest {
|
|
workspace_id: &common::test_workspace_id(),
|
|
operation_id: &operation.id,
|
|
version: 1,
|
|
published_at: &OffsetDateTime::parse(
|
|
"2026-03-26T10:00:00Z",
|
|
&time::format_description::well_known::Rfc3339,
|
|
)
|
|
.unwrap(),
|
|
published_by: Some("alice"),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
common::publish_agent_with_bindings(
|
|
®istry,
|
|
AGENT_SLUG,
|
|
vec![AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
let caller = principal(®istry, "cancel-s4").await;
|
|
let app = application(®istry).await;
|
|
let acceptance = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"s4-key",
|
|
&json!({"email":"u","idempotency_key":"k-4"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept");
|
|
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &OffsetDateTime::now_utc(),
|
|
})
|
|
.await
|
|
.expect("master key");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("store"),
|
|
);
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
|
.build();
|
|
let handler = Arc::new(
|
|
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
|
|
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
|
operation_id: operation.id.as_str().to_owned(),
|
|
operation_version: 1,
|
|
key_header: "Idempotency-Key".to_owned(),
|
|
validity_ms: 60_000,
|
|
}]),
|
|
);
|
|
let worker = TaskWorker::new(
|
|
store.clone(),
|
|
handler,
|
|
TaskWorkerConfig {
|
|
owner: "worker-s4".to_owned(),
|
|
batch_limit: 1,
|
|
lease_ms: 10_000,
|
|
poll_interval: Duration::from_millis(50),
|
|
max_inflight: 1,
|
|
shutdown_timeout: Duration::from_secs(5),
|
|
..TaskWorkerConfig::default()
|
|
},
|
|
)
|
|
.expect("worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
// Revoke the key the Task was accepted with while the first outbound write is provably in flight:
|
|
// that attempt still resolves as an uncertain write, and the next attempt is refused by the
|
|
// authority before its checkpoint.
|
|
for _ in 0..200 {
|
|
if requests.load(Ordering::SeqCst) >= 1 {
|
|
break;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
assert_eq!(
|
|
requests.load(Ordering::SeqCst),
|
|
1,
|
|
"the first attempt is dispatched before the refusal"
|
|
);
|
|
registry
|
|
.revoke_platform_api_key_for_agent(
|
|
&common::test_workspace_id(),
|
|
&common::test_agent_id(AGENT_SLUG),
|
|
&PlatformApiKeyId::new("pk_cancel-s4".to_owned()),
|
|
&OffsetDateTime::now_utc(),
|
|
)
|
|
.await
|
|
.expect("revoke");
|
|
|
|
let mut state = None;
|
|
for _ in 0..200 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
state = state_of(®istry, acceptance.task_id).await;
|
|
if state.as_deref() == Some("outcome_unknown") {
|
|
break;
|
|
}
|
|
}
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
server.abort();
|
|
|
|
assert_eq!(
|
|
state.as_deref(),
|
|
Some("outcome_unknown"),
|
|
"a refused attempt after a lost acknowledgement must stay uncertain"
|
|
);
|
|
assert_eq!(
|
|
requests.load(Ordering::SeqCst),
|
|
1,
|
|
"the refused attempt is never dispatched"
|
|
);
|
|
let attempts_count: i64 =
|
|
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempts_count, 1, "no second durable attempt");
|
|
let certainty: Option<String> = sqlx::query_scalar(
|
|
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
|
)
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(certainty.as_deref(), Some("unknown"));
|
|
let phase: Option<String> = sqlx::query_scalar(
|
|
"select phase from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
|
)
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(phase.as_deref(), Some("dispatched"));
|
|
let error: Option<(Option<String>, Option<String>)> = sqlx::query_as(
|
|
"select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1",
|
|
)
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
let (code, error_phase) = error.expect("the refused Task carries its bounded error");
|
|
assert_eq!(code.as_deref(), Some("task_authority_rejected"));
|
|
assert_eq!(
|
|
error_phase.as_deref(),
|
|
Some("dispatched"),
|
|
"the error keeps the phase of the attempt that may have reached the upstream"
|
|
);
|
|
}
|