CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable per-outbound attempt до отправки запроса, bounded retry/read, server-owned verified writes, честный cancel/unknown, deadline/fencing, additive schema v29. Staged renames task.rs/tasks.rs разложены на модули. Includes disclosed cosmetic parity exception: единственная правка вне принятой дельты — удалена лишняя пустая строка в конце scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
556 lines
20 KiB
Rust
556 lines
20 KiB
Rust
//! Story 4.4 ATDD: no outbound send exists without a committed durable attempt, and an elapsed
|
|
//! accepted deadline fails before dispatch.
|
|
//!
|
|
//! Both oracles drive the production composition (application entry → PG store → runtime executor →
|
|
//! real local HTTP upstream); only the upstream and the checkpoint failure are test-controlled.
|
|
|
|
use super::*;
|
|
|
|
use axum::Router;
|
|
use axum::http::StatusCode;
|
|
use axum::routing::post;
|
|
use crank_community_mcp::TaskWorkerHandler;
|
|
use crank_core::{
|
|
AgentOperationBinding, PlatformApiKeyScope, TaskState, ToolExecutionMode, ToolExecutionPolicy,
|
|
};
|
|
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use time::OffsetDateTime;
|
|
|
|
async fn composition(
|
|
registry: &PostgresRegistry,
|
|
upstream: &str,
|
|
policy: ToolExecutionPolicy,
|
|
caller_name: &str,
|
|
scoped_key: &str,
|
|
) -> (
|
|
Arc<PostgresTaskStore>,
|
|
crank_community_mcp::TaskApplication,
|
|
SecretCrypto,
|
|
crank_community_mcp::TaskPrincipal,
|
|
) {
|
|
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(policy);
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
publish(registry, &operation.id, 1).await;
|
|
common::publish_agent_with_bindings(
|
|
registry,
|
|
AGENT_SLUG,
|
|
vec![AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
let caller = principal(
|
|
registry,
|
|
caller_name,
|
|
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
|
)
|
|
.await;
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &OffsetDateTime::now_utc(),
|
|
})
|
|
.await
|
|
.expect("test master-key identity registered");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let tasks = TaskApplication::new(
|
|
registry.clone(),
|
|
Duration::ZERO,
|
|
Arc::new(InMemoryCoordinationStateStore::default()),
|
|
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
|
);
|
|
let _ = scoped_key;
|
|
(store, tasks, crypto, caller)
|
|
}
|
|
|
|
fn worker_for(
|
|
registry: &PostgresRegistry,
|
|
store: &Arc<PostgresTaskStore>,
|
|
tasks: &TaskApplication,
|
|
crypto: &SecretCrypto,
|
|
owner: &str,
|
|
) -> TaskWorker {
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.build();
|
|
let handler = Arc::new(TaskWorkerHandler::from_application(
|
|
tasks,
|
|
registry.clone(),
|
|
runtime,
|
|
crypto.clone(),
|
|
));
|
|
let config = TaskWorkerConfig {
|
|
owner: owner.to_owned(),
|
|
batch_limit: 1,
|
|
lease_ms: 10_000,
|
|
poll_interval: Duration::from_millis(50),
|
|
max_inflight: 1,
|
|
shutdown_timeout: Duration::from_secs(5),
|
|
..TaskWorkerConfig::default()
|
|
};
|
|
TaskWorker::new(store.clone(), handler, config).expect("valid worker")
|
|
}
|
|
|
|
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
|
|
sqlx::query_scalar("select state from tasks where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap()
|
|
}
|
|
|
|
/// AC1: a dispatch checkpoint that cannot commit produces zero outbound calls and no attempt row.
|
|
#[tokio::test]
|
|
async fn checkpoint_commit_failure_never_reaches_upstream() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let (store, tasks, crypto, caller) = composition(
|
|
®istry,
|
|
&upstream,
|
|
background_policy(),
|
|
"attempt-guards-checkpoint",
|
|
"guards-checkpoint-key",
|
|
)
|
|
.await;
|
|
let acceptance = tasks
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"guards-checkpoint-key",
|
|
&json!({"email": "user@example.com"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("durable acceptance");
|
|
|
|
// The checkpoint is the only statement that commits an attempt; it is refused here.
|
|
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
|
"create function refuse_task_checkpoint() returns trigger language plpgsql as $$
|
|
begin
|
|
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
|
raise exception 'durable checkpoint refused for test';
|
|
end if;
|
|
return new;
|
|
end $$;
|
|
create trigger refuse_task_checkpoint before update on tasks
|
|
for each row execute function refuse_task_checkpoint();",
|
|
))
|
|
.execute(registry.pool())
|
|
.await
|
|
.expect("checkpoint refusal trigger");
|
|
|
|
let worker = worker_for(
|
|
®istry,
|
|
&store,
|
|
&tasks,
|
|
&crypto,
|
|
"worker-guards-checkpoint",
|
|
);
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
|
|
assert_eq!(
|
|
calls.load(Ordering::SeqCst),
|
|
0,
|
|
"an uncommitted durable checkpoint must never produce an outbound call"
|
|
);
|
|
let attempts: i64 =
|
|
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
attempts, 0,
|
|
"no attempt history without a committed checkpoint"
|
|
);
|
|
assert_ne!(
|
|
state_of(®istry, acceptance.task_id).await.as_deref(),
|
|
Some("completed"),
|
|
"the refused attempt cannot complete the Task"
|
|
);
|
|
|
|
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
|
"drop trigger refuse_task_checkpoint on tasks; drop function refuse_task_checkpoint();",
|
|
))
|
|
.execute(registry.pool())
|
|
.await
|
|
.expect("checkpoint refusal trigger removed");
|
|
server.abort();
|
|
}
|
|
|
|
/// AC6: an accepted absolute deadline that already elapsed fails the Task before any dispatch.
|
|
#[tokio::test]
|
|
async fn elapsed_deadline_fails_before_dispatch_without_upstream_call() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let short_deadline = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 300,
|
|
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
|
};
|
|
let (store, tasks, crypto, caller) = composition(
|
|
®istry,
|
|
&upstream,
|
|
short_deadline,
|
|
"attempt-guards-deadline",
|
|
"guards-deadline-key",
|
|
)
|
|
.await;
|
|
let acceptance = tasks
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"guards-deadline-key",
|
|
&json!({"email": "user@example.com"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("durable acceptance");
|
|
|
|
// The accepted deadline elapses before any worker claims the Task.
|
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
|
|
|
let worker = worker_for(®istry, &store, &tasks, &crypto, "worker-guards-deadline");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
let mut state = None;
|
|
for _ in 0..100 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
state = state_of(®istry, acceptance.task_id).await;
|
|
if state.as_deref() == Some("failed") {
|
|
break;
|
|
}
|
|
}
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
|
|
assert_eq!(state.as_deref(), Some("failed"));
|
|
assert_eq!(
|
|
calls.load(Ordering::SeqCst),
|
|
0,
|
|
"an elapsed deadline must fail before dispatch"
|
|
);
|
|
let attempts: i64 =
|
|
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempts, 0, "no attempt is checkpointed after the deadline");
|
|
let code: Option<String> =
|
|
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(code.as_deref(), Some("task_deadline_elapsed"));
|
|
assert_eq!(
|
|
sqlx::query_scalar::<_, String>("select state from tasks where task_id = $1")
|
|
.bind(acceptance.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap(),
|
|
TaskState::Failed.as_str()
|
|
);
|
|
|
|
server.abort();
|
|
}
|
|
|
|
/// AC2: a write whose upstream idempotency is backed by server-owned evidence is retried with the
|
|
/// same stable key, in a bounded number of distinct durable attempts, and applies exactly one effect.
|
|
#[tokio::test]
|
|
async fn verified_idempotent_write_retries_with_stable_key_and_one_effect() {
|
|
use crank_community_mcp::VerifiedIdempotentWrite;
|
|
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
|
use std::collections::HashSet;
|
|
use std::sync::Mutex;
|
|
|
|
let registry = common::test_registry().await;
|
|
let requests = Arc::new(AtomicUsize::new(0));
|
|
let effects = Arc::new(AtomicUsize::new(0));
|
|
let keys = Arc::new(Mutex::new(Vec::<String>::new()));
|
|
let seen = Arc::new(Mutex::new(HashSet::<String>::new()));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let requests = Arc::clone(&requests);
|
|
let effects = Arc::clone(&effects);
|
|
let keys = Arc::clone(&keys);
|
|
let seen = Arc::clone(&seen);
|
|
move |headers: axum::http::HeaderMap| {
|
|
let requests = Arc::clone(&requests);
|
|
let effects = Arc::clone(&effects);
|
|
let keys = Arc::clone(&keys);
|
|
let seen = Arc::clone(&seen);
|
|
async move {
|
|
let key = headers
|
|
.get("idempotency-key")
|
|
.and_then(|value| value.to_str().ok())
|
|
.unwrap_or_default()
|
|
.to_owned();
|
|
keys.lock().expect("keys").push(key.clone());
|
|
let first_application = seen.lock().expect("seen").insert(key);
|
|
if first_application {
|
|
effects.fetch_add(1, Ordering::SeqCst);
|
|
}
|
|
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
|
if attempt == 0 {
|
|
// The first application is stored but its acknowledgement is lost.
|
|
return (
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
axum::Json(json!({"error": "acknowledgement lost"})),
|
|
);
|
|
}
|
|
(
|
|
StatusCode::OK,
|
|
axum::Json(json!({"id": "lead-1", "deduped": !first_application})),
|
|
)
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(background_policy());
|
|
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
|
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
|
mode: IdempotencyMode::Required,
|
|
ttl_ms: 60_000,
|
|
input_field: Some("idempotency_key".to_owned()),
|
|
header_name: Some("Idempotency-Key".to_owned()),
|
|
});
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
publish(®istry, &operation.id, 1).await;
|
|
common::publish_agent_with_bindings(
|
|
®istry,
|
|
AGENT_SLUG,
|
|
vec![AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
let caller = principal(
|
|
®istry,
|
|
"attempt-guards-verified",
|
|
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
|
)
|
|
.await;
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &OffsetDateTime::now_utc(),
|
|
})
|
|
.await
|
|
.expect("test master-key identity registered");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let tasks = TaskApplication::new(
|
|
registry.clone(),
|
|
Duration::ZERO,
|
|
Arc::new(InMemoryCoordinationStateStore::default()),
|
|
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
|
);
|
|
let acceptance = tasks
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"guards-verified-key",
|
|
&json!({"email": "user@example.com", "idempotency_key": "k-1"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("durable acceptance");
|
|
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
|
.build();
|
|
let handler = Arc::new(
|
|
TaskWorkerHandler::from_application(&tasks, registry.clone(), runtime, crypto.clone())
|
|
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
|
operation_id: operation.id.as_str().to_owned(),
|
|
operation_version: 1,
|
|
key_header: "Idempotency-Key".to_owned(),
|
|
validity_ms: 60_000,
|
|
}]),
|
|
);
|
|
let config = TaskWorkerConfig {
|
|
owner: "worker-guards-verified".to_owned(),
|
|
batch_limit: 1,
|
|
lease_ms: 10_000,
|
|
poll_interval: Duration::from_millis(50),
|
|
max_inflight: 1,
|
|
shutdown_timeout: Duration::from_secs(5),
|
|
..TaskWorkerConfig::default()
|
|
};
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
let mut state = None;
|
|
for _ in 0..200 {
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
state = state_of(®istry, acceptance.task_id).await;
|
|
if state.as_deref() == Some("completed") {
|
|
break;
|
|
}
|
|
}
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
server.abort();
|
|
|
|
assert_eq!(state.as_deref(), Some("completed"));
|
|
assert_eq!(
|
|
requests.load(Ordering::SeqCst),
|
|
2,
|
|
"exactly one retry of the write"
|
|
);
|
|
assert_eq!(
|
|
effects.load(Ordering::SeqCst),
|
|
1,
|
|
"the upstream applied the write exactly once"
|
|
);
|
|
assert_eq!(
|
|
keys.lock().expect("keys").as_slice(),
|
|
["k-1".to_owned(), "k-1".to_owned()],
|
|
"the same stable key is delivered on both attempts"
|
|
);
|
|
|
|
let attempts = store
|
|
.attempts(
|
|
&crank_core::TaskAccess {
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: crank_core::TaskOwnerKind::Agent,
|
|
owner_id: "agent_attempt_guards".to_owned(),
|
|
authorization_scope: String::new(),
|
|
},
|
|
acceptance.task_id,
|
|
)
|
|
.await;
|
|
// Owner scope of this principal is the agent of the published binding; read the history through
|
|
// the application entry instead of guessing the authorization scope string.
|
|
assert!(attempts.is_err(), "a foreign scope never reads the history");
|
|
let attempts = tasks
|
|
.attempts(&caller, TOOL_NAME, acceptance.task_id)
|
|
.await
|
|
.expect("owner-scoped attempt history");
|
|
assert_eq!(attempts.len(), 2, "two distinct durable attempts");
|
|
assert_eq!(attempts[0].ordinal, 1);
|
|
assert_eq!(attempts[1].ordinal, 2);
|
|
assert!(attempts.iter().all(|attempt| attempt.retry_allowed));
|
|
assert_eq!(
|
|
attempts[0].outcome,
|
|
Some(crank_core::AttemptOutcomeClass::RetryScheduled)
|
|
);
|
|
assert_eq!(
|
|
attempts[1].outcome,
|
|
Some(crank_core::AttemptOutcomeClass::Completed)
|
|
);
|
|
assert_eq!(
|
|
attempts[0].upstream_key_digest, attempts[1].upstream_key_digest,
|
|
"the same stable key digest is recorded on both attempts"
|
|
);
|
|
assert!(attempts[0].upstream_key_digest.is_some());
|
|
let record = tasks
|
|
.status(&caller, TOOL_NAME, acceptance.task_id)
|
|
.await
|
|
.expect("status");
|
|
assert_eq!(record.pins.retry.max_attempts, 2);
|
|
}
|