CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если закреплённая схема требует отсутствующее поле; authorized schema-validated update атомарно и с аудитом возобновляет тот же Task ровно один раз. Replay/conflict/TTL безопасны, ожидание удерживает admission permit. Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
966 lines
31 KiB
Rust
966 lines
31 KiB
Rust
//! ATDD Story 4.5 (Beam): durable input continuation through TaskApplication -> PG -> worker -> upstream.
|
|
|
|
use super::common;
|
|
use super::unsampled_correlation;
|
|
|
|
use axum::Router;
|
|
use axum::http::StatusCode;
|
|
use axum::routing::post;
|
|
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
|
|
use crank_core::{
|
|
NoTasksFallback, PlatformApiKeyScope, TaskId, TaskState, ToolExecutionMode, ToolExecutionPolicy,
|
|
};
|
|
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
|
|
use crank_runtime::{
|
|
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
|
|
};
|
|
use serde_json::json;
|
|
use sqlx::Row;
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::time::Duration;
|
|
use time::OffsetDateTime;
|
|
|
|
const AGENT_SLUG: &str = "reports";
|
|
pub(crate) const TOOL_NAME: &str = "report_gen";
|
|
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
|
|
|
pub(crate) async fn state_of(registry: &PostgresRegistry, task_id: TaskId) -> Option<String> {
|
|
sqlx::query_scalar("select state from tasks where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_optional(registry.pool())
|
|
.await
|
|
.unwrap()
|
|
}
|
|
|
|
pub(crate) async fn wait_for_state(
|
|
registry: &PostgresRegistry,
|
|
task_id: TaskId,
|
|
desired: &str,
|
|
limit: Duration,
|
|
) -> bool {
|
|
let deadline = tokio::time::Instant::now() + limit;
|
|
while tokio::time::Instant::now() < deadline {
|
|
if state_of(registry, task_id).await.as_deref() == Some(desired) {
|
|
return true;
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
}
|
|
state_of(registry, task_id).await.as_deref() == Some(desired)
|
|
}
|
|
|
|
pub(crate) async fn seed_tool(
|
|
registry: &PostgresRegistry,
|
|
upstream: &str,
|
|
policy: ToolExecutionPolicy,
|
|
) {
|
|
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
|
operation.execution_config.execution_policy = Some(policy);
|
|
registry
|
|
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
|
.await
|
|
.unwrap();
|
|
registry
|
|
.publish_operation(PublishRequest {
|
|
workspace_id: &common::test_workspace_id(),
|
|
operation_id: &operation.id,
|
|
version: 1,
|
|
published_at: &OffsetDateTime::parse(
|
|
"2026-03-26T10:00:00Z",
|
|
&time::format_description::well_known::Rfc3339,
|
|
)
|
|
.unwrap(),
|
|
published_by: Some("alice"),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
common::publish_agent_with_bindings(
|
|
registry,
|
|
AGENT_SLUG,
|
|
vec![crank_core::AgentOperationBinding {
|
|
operation_id: operation.id.clone(),
|
|
operation_version: 1,
|
|
tool_name: operation.name.clone(),
|
|
tool_title: operation.tool_description.title.clone(),
|
|
tool_description_override: None,
|
|
enabled: true,
|
|
}],
|
|
)
|
|
.await;
|
|
}
|
|
|
|
pub(crate) async fn principal(
|
|
registry: &PostgresRegistry,
|
|
name: &str,
|
|
) -> crank_community_mcp::TaskPrincipal {
|
|
let token = common::create_platform_api_key(
|
|
registry,
|
|
AGENT_SLUG,
|
|
name,
|
|
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
|
)
|
|
.await;
|
|
let secret_hash = common::hash_access_secret(&token);
|
|
let api_key = registry
|
|
.get_platform_api_key_by_secret_for_agent_slug(
|
|
common::test_workspace_slug(),
|
|
AGENT_SLUG,
|
|
&secret_hash,
|
|
)
|
|
.await
|
|
.unwrap()
|
|
.expect("active key");
|
|
registry
|
|
.touch_platform_api_key(
|
|
&api_key.api_key.workspace_id,
|
|
&api_key.api_key.id,
|
|
&OffsetDateTime::now_utc(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
crank_community_mcp::TaskPrincipal::Agent {
|
|
workspace_slug: common::test_workspace_slug().to_owned(),
|
|
agent_slug: AGENT_SLUG.to_owned(),
|
|
credential: crank_core::VerifiedMachineCredential {
|
|
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
|
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
|
scopes: api_key.api_key.scopes.clone(),
|
|
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
|
},
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
let observed_at = OffsetDateTime::now_utc();
|
|
registry
|
|
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
|
epoch: crypto.master_key_epoch(),
|
|
fingerprint: crypto.master_key_fingerprint(),
|
|
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &observed_at,
|
|
})
|
|
.await
|
|
.expect("master key");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("store"),
|
|
);
|
|
TaskApplication::new(
|
|
registry.clone(),
|
|
Duration::ZERO,
|
|
Arc::new(InMemoryCoordinationStateStore::default()),
|
|
TaskService::with_secret_crypto(store, Arc::new(crypto)),
|
|
)
|
|
}
|
|
|
|
pub(crate) fn spawn_worker(
|
|
registry: &PostgresRegistry,
|
|
app: &TaskApplication,
|
|
) -> (
|
|
Arc<std::sync::atomic::AtomicBool>,
|
|
tokio::task::JoinHandle<()>,
|
|
) {
|
|
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
|
let pool = registry.pool().clone();
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
pool,
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("store"),
|
|
);
|
|
let runtime = crank_runtime::community_with_outbound_policy(
|
|
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
|
)
|
|
.build();
|
|
let handler = Arc::new(TaskWorkerHandler::from_application(
|
|
app,
|
|
registry.clone(),
|
|
runtime,
|
|
crypto,
|
|
));
|
|
let worker = TaskWorker::new(
|
|
store,
|
|
handler,
|
|
TaskWorkerConfig {
|
|
owner: "worker-input".to_owned(),
|
|
batch_limit: 1,
|
|
lease_ms: 10_000,
|
|
poll_interval: Duration::from_millis(50),
|
|
max_inflight: 1,
|
|
renewal_interval: Duration::from_millis(100),
|
|
shutdown_timeout: Duration::from_secs(5),
|
|
},
|
|
)
|
|
.expect("worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let jh = tokio::spawn(async move {
|
|
let _ = worker.run().await;
|
|
});
|
|
(stop_flag, jh)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn input_continuation_happy_path_accept_update_resume_replay() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let upstream = format!("http://{address}");
|
|
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 60_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &upstream, policy).await;
|
|
let caller = principal(®istry, "input-happy").await;
|
|
let app = application(®istry).await;
|
|
|
|
let acceptance = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"input-key-1",
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("acceptance");
|
|
assert_eq!(acceptance.state, TaskState::Queued);
|
|
let task_id = acceptance.task_id;
|
|
|
|
let (stop, worker_task) = spawn_worker(®istry, &app);
|
|
assert!(
|
|
wait_for_state(
|
|
®istry,
|
|
task_id,
|
|
"input_required",
|
|
Duration::from_secs(10)
|
|
)
|
|
.await,
|
|
"worker must suspend for missing email"
|
|
);
|
|
// positive control: before update no outbound request
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
// persisted request fields
|
|
let row = sqlx::query(
|
|
"select wait_id, expected_version, schema_snapshot, projected_schema, prompt_code, expires_at from task_input_requests where task_id = $1",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
let wait_id: String = row.get("wait_id");
|
|
let expected_version: i32 = row.get("expected_version");
|
|
let schema_snapshot: serde_json::Value = row.get("schema_snapshot");
|
|
let projected_schema: serde_json::Value = row.get("projected_schema");
|
|
let prompt_code: String = row.get("prompt_code");
|
|
let _expires_at: OffsetDateTime = row.get("expires_at");
|
|
assert!(!wait_id.is_empty());
|
|
assert!(expected_version > 0);
|
|
assert!(schema_snapshot.is_object());
|
|
assert!(projected_schema.is_object());
|
|
assert_eq!(prompt_code, "required_input_missing");
|
|
let audit_count_before: i64 = sqlx::query_scalar(
|
|
"select count(*)::bigint from task_input_audit where task_id = $1 and event = 'input_required'",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert!(audit_count_before >= 1);
|
|
|
|
// stop worker before update to avoid races
|
|
stop.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
|
|
let outcome = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version as u32,
|
|
"update-key-1",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect("valid update");
|
|
assert_eq!(outcome.state, TaskState::Working);
|
|
assert!(!outcome.replayed);
|
|
assert_eq!(outcome.task_id, task_id);
|
|
assert_eq!(outcome.wait_id, wait_id);
|
|
|
|
let (stop2, worker_task2) = spawn_worker(®istry, &app);
|
|
assert!(
|
|
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
|
"worker must complete after update"
|
|
);
|
|
stop2.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
|
|
|
|
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
|
|
|
let attempts = app
|
|
.attempts(&caller, TOOL_NAME, task_id)
|
|
.await
|
|
.expect("attempts");
|
|
assert_eq!(attempts.len(), 1);
|
|
let effective_fp: String =
|
|
sqlx::query_scalar("select effective_input_fingerprint from tasks where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempts[0].input_fingerprint, effective_fp);
|
|
let input_fp: String =
|
|
sqlx::query_scalar("select input_fingerprint from tasks where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_ne!(effective_fp, input_fp);
|
|
|
|
// An identical answer replay still returns the committed outcome after the Task completed:
|
|
// no second transition, audit row or outbound call.
|
|
let completed_replay = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version as u32,
|
|
"update-key-1",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect("identical replay after completion");
|
|
assert!(completed_replay.replayed);
|
|
assert_eq!(completed_replay.version, outcome.version);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
|
|
|
// replay with original scoped key/partial input: same Task, no fingerprint conflict
|
|
let replay = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"input-key-1",
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("replay accepted");
|
|
assert_eq!(replay.task_id, task_id);
|
|
|
|
server.abort();
|
|
}
|
|
|
|
pub(crate) async fn start_and_suspend(
|
|
registry: &PostgresRegistry,
|
|
app: &TaskApplication,
|
|
caller: &crank_community_mcp::TaskPrincipal,
|
|
scoped_key: &str,
|
|
) -> (TaskId, String, u32) {
|
|
let acceptance = app
|
|
.accept_background(
|
|
caller,
|
|
TOOL_NAME,
|
|
None,
|
|
scoped_key,
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("acceptance");
|
|
let (stop, worker_task) = spawn_worker(registry, app);
|
|
assert!(
|
|
wait_for_state(
|
|
registry,
|
|
acceptance.task_id,
|
|
"input_required",
|
|
Duration::from_secs(10)
|
|
)
|
|
.await,
|
|
"worker must suspend for missing email"
|
|
);
|
|
stop.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
let (wait_id, expected_version) = wait_row(registry, acceptance.task_id).await;
|
|
(acceptance.task_id, wait_id, expected_version)
|
|
}
|
|
|
|
async fn wait_row(registry: &PostgresRegistry, task_id: TaskId) -> (String, u32) {
|
|
let row =
|
|
sqlx::query("select wait_id, expected_version from task_input_requests where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
(
|
|
row.get::<String, _>("wait_id"),
|
|
u32::try_from(row.get::<i32, _>("expected_version")).unwrap_or_default(),
|
|
)
|
|
}
|
|
|
|
pub(crate) async fn audit_events(
|
|
registry: &PostgresRegistry,
|
|
task_id: TaskId,
|
|
) -> Vec<(String, Option<String>)> {
|
|
sqlx::query("select event, reason_code from task_input_audit where task_id = $1 order by id")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_all(registry.pool())
|
|
.await
|
|
.unwrap()
|
|
.into_iter()
|
|
.map(|row| (row.get("event"), row.get("reason_code")))
|
|
.collect()
|
|
}
|
|
|
|
pub(crate) async fn admission_entries(
|
|
registry: &PostgresRegistry,
|
|
task_id: TaskId,
|
|
) -> Vec<(String, i64)> {
|
|
sqlx::query(
|
|
"select state, count(*)::bigint as count from execution_admission_entries
|
|
where task_id = $1 group by state order by state",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.fetch_all(registry.pool())
|
|
.await
|
|
.unwrap()
|
|
.into_iter()
|
|
.map(|row| (row.get("state"), row.get("count")))
|
|
.collect()
|
|
}
|
|
|
|
fn key_id(principal: &crank_community_mcp::TaskPrincipal) -> crank_core::PlatformApiKeyId {
|
|
match principal {
|
|
crank_community_mcp::TaskPrincipal::Agent { credential, .. } => credential
|
|
.platform_api_key_id
|
|
.clone()
|
|
.expect("agent principal carries a key id"),
|
|
crank_community_mcp::TaskPrincipal::Admin { .. } => panic!("agent principal expected"),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn invalid_schema_foreign_scope_and_revoked_key_never_transition_or_dispatch() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 60_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
|
let caller = principal(®istry, "input-scope").await;
|
|
let app = application(®istry).await;
|
|
let (task_id, wait_id, expected_version) =
|
|
start_and_suspend(®istry, &app, &caller, "input-key-scope").await;
|
|
|
|
// Invalid answer: server-side schema validation refuses it, nothing transitions and nothing leaks.
|
|
let invalid = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-invalid",
|
|
&json!({}),
|
|
)
|
|
.await
|
|
.expect_err("the pinned schema requires email");
|
|
assert!(
|
|
matches!(
|
|
invalid,
|
|
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::Invalid { .. })
|
|
),
|
|
"unexpected error: {invalid:?}"
|
|
);
|
|
assert_eq!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("input_required")
|
|
);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
assert!(
|
|
audit_events(®istry, task_id)
|
|
.await
|
|
.iter()
|
|
.all(|(event, _)| event == "input_required"),
|
|
"a rejected answer must not be audited as accepted"
|
|
);
|
|
let stored_snapshot: serde_json::Value =
|
|
sqlx::query_scalar("select schema_snapshot from task_input_requests where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert!(!stored_snapshot.to_string().contains("user@example.com"));
|
|
|
|
// A different key of the same agent carries a different authorization scope: opaque not-found.
|
|
let foreign = principal(®istry, "input-foreign").await;
|
|
let foreign_result = app
|
|
.update_input(
|
|
&foreign,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-foreign",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("a foreign scope cannot continue the Task");
|
|
assert_eq!(
|
|
foreign_result,
|
|
crank_community_mcp::TaskApplicationError::NotFound
|
|
);
|
|
assert_eq!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("input_required")
|
|
);
|
|
|
|
// Positive control: the accepting principal (same key/scope) completes the transition.
|
|
let accepted = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-valid",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect("valid answer for the accepting scope");
|
|
assert_eq!(accepted.state, TaskState::Working);
|
|
assert!(!accepted.replayed);
|
|
// Finish the positive-control Task through a real worker so the revocation case starts from a
|
|
// known dispatch count instead of leaving resumable work behind.
|
|
let (finish_stop, finish_worker) = spawn_worker(®istry, &app);
|
|
assert!(
|
|
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
|
"the positive-control Task completes"
|
|
);
|
|
finish_stop.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), finish_worker).await;
|
|
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
|
|
|
// Revocation race: the accepting key is revoked after suspension. The same waiting scope must
|
|
// fail closed inside the update transaction and never revive or dispatch the Task.
|
|
let revocable = principal(®istry, "input-revoked").await;
|
|
let (revoked_task, revoked_wait, revoked_version) =
|
|
start_and_suspend(®istry, &app, &revocable, "input-key-revoked").await;
|
|
registry
|
|
.revoke_platform_api_key(
|
|
&common::test_workspace_id(),
|
|
&key_id(&revocable),
|
|
&OffsetDateTime::now_utc(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let revoked_result = app
|
|
.update_input(
|
|
&revocable,
|
|
TOOL_NAME,
|
|
revoked_task,
|
|
&revoked_wait,
|
|
revoked_version,
|
|
"update-revoked",
|
|
&json!({"email": "revoked@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("a revoked key cannot continue the Task");
|
|
assert_eq!(
|
|
revoked_result,
|
|
crank_community_mcp::TaskApplicationError::NotFound
|
|
);
|
|
assert_eq!(
|
|
state_of(®istry, revoked_task).await.as_deref(),
|
|
Some("input_required")
|
|
);
|
|
assert_eq!(
|
|
calls.load(Ordering::SeqCst),
|
|
1,
|
|
"the revoked answer must not add an outbound call"
|
|
);
|
|
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn replay_conflict_and_stale_answers_have_one_typed_outcome() {
|
|
let registry = common::test_registry().await;
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 60_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, "http://127.0.0.1:9", policy).await;
|
|
let caller = principal(®istry, "input-replay").await;
|
|
let app = application(®istry).await;
|
|
let (task_id, wait_id, expected_version) =
|
|
start_and_suspend(®istry, &app, &caller, "input-key-replay").await;
|
|
|
|
let first = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-key",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect("first valid answer");
|
|
assert!(!first.replayed);
|
|
assert_eq!(first.state, TaskState::Working);
|
|
|
|
// Identical replay returns the stored outcome without a second CAS/audit.
|
|
let replay = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-key",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect("identical replay");
|
|
assert!(replay.replayed);
|
|
assert_eq!(replay.version, first.version);
|
|
assert_eq!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("working")
|
|
);
|
|
let accepted_audit = audit_events(®istry, task_id)
|
|
.await
|
|
.into_iter()
|
|
.filter(|(event, _)| event == "input_accepted")
|
|
.count();
|
|
assert_eq!(accepted_audit, 1);
|
|
|
|
// Same key with another payload is a fingerprint conflict.
|
|
let conflict = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-key",
|
|
&json!({"email": "other@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("same key with another payload conflicts");
|
|
assert_eq!(
|
|
conflict,
|
|
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::FingerprintConflict)
|
|
);
|
|
|
|
// Another key after a committed answer, and a foreign wait id, are state conflicts.
|
|
let other_key = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
first.version,
|
|
"update-key-2",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("another key after commit conflicts");
|
|
assert_eq!(
|
|
other_key,
|
|
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
|
);
|
|
let foreign_wait = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
"wait-does-not-exist",
|
|
first.version,
|
|
"update-key-3",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("a foreign wait id conflicts");
|
|
assert_eq!(
|
|
foreign_wait,
|
|
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
|
);
|
|
|
|
// A stale generation on a fresh wait is refused; the valid generation still works.
|
|
let (stale_task, stale_wait, stale_version) =
|
|
start_and_suspend(®istry, &app, &caller, "input-key-stale").await;
|
|
let stale = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
stale_task,
|
|
&stale_wait,
|
|
stale_version.saturating_sub(1).max(1),
|
|
"update-stale",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("a stale generation cannot be continued");
|
|
assert_eq!(
|
|
stale,
|
|
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
|
);
|
|
let repaired = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
stale_task,
|
|
&stale_wait,
|
|
stale_version,
|
|
"update-stale",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect("the current generation still accepts the answer");
|
|
assert!(!repaired.replayed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn ttl_expiry_fails_with_audit_and_releases_capacity_without_dispatch() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
// The wait TTL is the remaining pinned absolute execution deadline.
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 1_200,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
|
let caller = principal(®istry, "input-ttl").await;
|
|
let app = application(®istry).await;
|
|
let acceptance = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"input-key-ttl",
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("acceptance");
|
|
let task_id = acceptance.task_id;
|
|
let (stop, worker_task) = spawn_worker(®istry, &app);
|
|
assert!(
|
|
wait_for_state(®istry, task_id, "input_required", Duration::from_secs(5)).await,
|
|
"worker must suspend"
|
|
);
|
|
assert!(
|
|
wait_for_state(®istry, task_id, "failed", Duration::from_secs(6)).await,
|
|
"the per-poll sweeper must fail the expired wait"
|
|
);
|
|
stop.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
|
|
let (wait_id, expected_version) = wait_row(®istry, task_id).await;
|
|
let events = audit_events(®istry, task_id).await;
|
|
assert!(
|
|
events.iter().any(|(event, reason)| event == "input_expired"
|
|
&& reason.as_deref() == Some("input_ttl_expired")),
|
|
"expiry reason must be audited: {events:?}"
|
|
);
|
|
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
// A late answer cannot revive the failed Task.
|
|
let late = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-late",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("an expired wait never revives");
|
|
assert!(
|
|
matches!(
|
|
late,
|
|
crank_community_mcp::TaskApplicationError::Task(
|
|
crank_core::TaskError::InputExpired | crank_core::TaskError::StateConflict
|
|
)
|
|
),
|
|
"unexpected late answer error: {late:?}"
|
|
);
|
|
assert_eq!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("failed")
|
|
);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn competing_identical_updates_commit_once_and_keep_one_permit() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 60_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
|
let caller = principal(®istry, "input-race").await;
|
|
let app = application(®istry).await;
|
|
let (task_id, wait_id, expected_version) =
|
|
start_and_suspend(®istry, &app, &caller, "input-key-race").await;
|
|
assert_eq!(
|
|
admission_entries(®istry, task_id).await,
|
|
vec![("executing".to_owned(), 1)],
|
|
"the waiting Task keeps exactly one executing permit"
|
|
);
|
|
|
|
let payload = json!({"email": "user@example.com"});
|
|
let first = app.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-race",
|
|
&payload,
|
|
);
|
|
let second = app.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-race",
|
|
&payload,
|
|
);
|
|
let (first, second) = tokio::join!(first, second);
|
|
let mut outcomes = [
|
|
first.expect("competing answer"),
|
|
second.expect("competing answer"),
|
|
];
|
|
outcomes.sort_by_key(|outcome| outcome.replayed);
|
|
assert!(!outcomes[0].replayed);
|
|
assert!(outcomes[1].replayed);
|
|
assert_eq!(outcomes[0].version, outcomes[1].version);
|
|
assert_eq!(
|
|
audit_events(®istry, task_id)
|
|
.await
|
|
.into_iter()
|
|
.filter(|(event, _)| event == "input_accepted")
|
|
.count(),
|
|
1
|
|
);
|
|
assert_eq!(
|
|
admission_entries(®istry, task_id).await,
|
|
vec![("executing".to_owned(), 1)],
|
|
"the update must not grant a second permit"
|
|
);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
let (stop, worker_task) = spawn_worker(®istry, &app);
|
|
assert!(
|
|
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
|
"the resumed Task completes"
|
|
);
|
|
stop.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
|
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
|
server.abort();
|
|
}
|