CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если закреплённая схема требует отсутствующее поле; authorized schema-validated update атомарно и с аудитом возобновляет тот же Task ровно один раз. Replay/conflict/TTL безопасны, ожидание удерживает admission permit. Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
458 lines
15 KiB
Rust
458 lines
15 KiB
Rust
//! ATDD Story 4.5 QA1 guards (F1/F3): one pinned deadline for several waiting Tasks, cancellation
|
|
//! while the Task waits for input and refusal of a stale lease/fence around the continuation seam.
|
|
|
|
use super::common;
|
|
use super::input_continuation::{
|
|
TOOL_NAME, admission_entries, application, audit_events, principal, seed_tool, spawn_worker,
|
|
start_and_suspend, state_of, wait_for_state,
|
|
};
|
|
use super::unsampled_correlation;
|
|
|
|
use axum::Router;
|
|
use axum::http::StatusCode;
|
|
use axum::routing::post;
|
|
use crank_core::{
|
|
BeginAttempt, NoTasksFallback, TASK_INPUT_SCHEMA_FORMAT_VERSION, TaskInputRequest,
|
|
TaskInputStore, TaskState, TaskStore, ToolExecutionMode, ToolExecutionPolicy,
|
|
};
|
|
use crank_registry::{AdmissionLimits, PostgresTaskStore};
|
|
use serde_json::json;
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::time::Duration;
|
|
use time::OffsetDateTime;
|
|
|
|
#[tokio::test]
|
|
async fn two_waits_with_one_pinned_deadline_and_cancelled_leftover_do_not_block_suspension() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 60_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
|
let caller = principal(®istry, "deadline-cancel").await;
|
|
let app = application(®istry).await;
|
|
|
|
let acc_a = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"deadline-key-a",
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept a");
|
|
let acc_b = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"deadline-key-b",
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept b");
|
|
let task_a = acc_a.task_id;
|
|
let task_b = acc_b.task_id;
|
|
|
|
let pinned_ms = (OffsetDateTime::now_utc().unix_timestamp() as f64 * 1000.0) + 60_000.0;
|
|
for tid in [task_a, task_b] {
|
|
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
|
|
.bind(pinned_ms)
|
|
.bind(tid.as_uuid())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
let (stop, worker_task) = spawn_worker(®istry, &app);
|
|
assert!(
|
|
wait_for_state(®istry, task_a, "input_required", Duration::from_secs(10)).await,
|
|
"a must suspend"
|
|
);
|
|
assert!(
|
|
wait_for_state(®istry, task_b, "input_required", Duration::from_secs(10)).await,
|
|
"b must suspend"
|
|
);
|
|
stop.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
// Both active waits carry the same absolute TTL value: the per-Task uniqueness above is the only
|
|
// uniqueness, so this must not be a UNIQUE index on expires_at.
|
|
let same_deadline_wait: i64 = sqlx::query_scalar(
|
|
"select count(*)::bigint from task_input_requests a
|
|
join task_input_requests b on b.task_id = $2
|
|
where a.task_id = $1 and a.accepted_at is null and b.accepted_at is null
|
|
and a.expires_at = b.expires_at",
|
|
)
|
|
.bind(task_a.as_uuid())
|
|
.bind(task_b.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
same_deadline_wait, 1,
|
|
"both waits share the pinned deadline"
|
|
);
|
|
|
|
let outcome = app
|
|
.cancel(&caller, TOOL_NAME, task_a, None)
|
|
.await
|
|
.expect("cancel a");
|
|
assert!(matches!(
|
|
outcome,
|
|
crank_core::TaskCancelOutcome::Cancelled(_)
|
|
));
|
|
assert_eq!(
|
|
state_of(®istry, task_a).await.as_deref(),
|
|
Some("cancelled")
|
|
);
|
|
assert_eq!(admission_entries(®istry, task_a).await, Vec::new());
|
|
|
|
// accept C with same pinned deadline
|
|
let acc_c = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"deadline-key-c",
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept c");
|
|
let task_c = acc_c.task_id;
|
|
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
|
|
.bind(pinned_ms)
|
|
.bind(task_c.as_uuid())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
|
|
let (stop2, worker_task2) = spawn_worker(®istry, &app);
|
|
assert!(
|
|
wait_for_state(®istry, task_c, "input_required", Duration::from_secs(10)).await,
|
|
"c must suspend despite leftover a"
|
|
);
|
|
stop2.store(true, Ordering::SeqCst);
|
|
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
|
|
|
|
assert_eq!(
|
|
state_of(®istry, task_b).await.as_deref(),
|
|
Some("input_required")
|
|
);
|
|
let c_deadline_ms: i64 = sqlx::query_scalar(
|
|
"select (extract(epoch from expires_at) * 1000)::bigint
|
|
from task_input_requests where task_id = $1",
|
|
)
|
|
.bind(task_c.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
c_deadline_ms, pinned_ms as i64,
|
|
"the third wait reuses the same pinned deadline value"
|
|
);
|
|
let a_accepted: Option<OffsetDateTime> =
|
|
sqlx::query_scalar("select accepted_at from task_input_requests where task_id = $1")
|
|
.bind(task_a.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
a_accepted.is_none(),
|
|
"cancellation leaves the wait row without an accepted receipt"
|
|
);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn cancel_while_input_required_never_revives_or_dispatches() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 60_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
|
let caller = principal(®istry, "cancel-wait").await;
|
|
let app = application(®istry).await;
|
|
|
|
let (task_id, wait_id, expected_version) =
|
|
start_and_suspend(®istry, &app, &caller, "input-key-cancel").await;
|
|
assert_eq!(
|
|
admission_entries(®istry, task_id).await,
|
|
vec![("executing".to_owned(), 1)]
|
|
);
|
|
|
|
let outcome = app
|
|
.cancel(&caller, TOOL_NAME, task_id, None)
|
|
.await
|
|
.expect("cancel");
|
|
assert!(matches!(
|
|
outcome,
|
|
crank_core::TaskCancelOutcome::Cancelled(_)
|
|
));
|
|
assert_eq!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("cancelled")
|
|
);
|
|
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
|
|
|
let late = app
|
|
.update_input(
|
|
&caller,
|
|
TOOL_NAME,
|
|
task_id,
|
|
&wait_id,
|
|
expected_version,
|
|
"update-late",
|
|
&json!({"email": "user@example.com"}),
|
|
)
|
|
.await
|
|
.expect_err("late answer must not revive");
|
|
assert!(
|
|
matches!(
|
|
late,
|
|
crank_community_mcp::TaskApplicationError::Task(
|
|
crank_core::TaskError::StateConflict | crank_core::TaskError::InputExpired
|
|
)
|
|
),
|
|
"unexpected: {late:?}"
|
|
);
|
|
assert_eq!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("cancelled")
|
|
);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
let events = audit_events(®istry, task_id).await;
|
|
assert!(events.iter().any(|(e, _)| e == "input_required"));
|
|
assert!(!events.iter().any(|(e, _)| e == "input_accepted"));
|
|
let attempts = app
|
|
.attempts(&caller, TOOL_NAME, task_id)
|
|
.await
|
|
.expect("attempts");
|
|
assert!(attempts.is_empty());
|
|
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn stale_lease_fence_cannot_suspend_or_dispatch() {
|
|
let registry = common::test_registry().await;
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let upstream_app = Router::new().route(
|
|
"/crm/leads",
|
|
post({
|
|
let calls = Arc::clone(&calls);
|
|
move || {
|
|
let calls = Arc::clone(&calls);
|
|
async move {
|
|
calls.fetch_add(1, Ordering::SeqCst);
|
|
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
|
let policy = ToolExecutionPolicy {
|
|
mode: ToolExecutionMode::Background,
|
|
deadline_ms: 60_000,
|
|
no_tasks_fallback: NoTasksFallback::Unsupported,
|
|
};
|
|
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
|
let caller = principal(®istry, "stale-fence").await;
|
|
let app = application(®istry).await;
|
|
let acceptance = app
|
|
.accept_background(
|
|
&caller,
|
|
TOOL_NAME,
|
|
None,
|
|
"stale-key",
|
|
&json!({"report": "q3"}),
|
|
&unsampled_correlation(),
|
|
)
|
|
.await
|
|
.expect("accept");
|
|
let task_id = acceptance.task_id;
|
|
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("store"),
|
|
);
|
|
let mut old_claims = store
|
|
.claim_batch("worker-old", 1, 50)
|
|
.await
|
|
.expect("old claim");
|
|
assert!(!old_claims.is_empty());
|
|
let old = old_claims.remove(0);
|
|
|
|
tokio::time::sleep(Duration::from_millis(120)).await;
|
|
let mut new_claims = store
|
|
.claim_batch("worker-new", 1, 10_000)
|
|
.await
|
|
.expect("new claim");
|
|
assert!(!new_claims.is_empty());
|
|
let new = new_claims.remove(0);
|
|
assert!(new.fence > old.fence);
|
|
assert_eq!(new.record.task_id, task_id);
|
|
|
|
let request = TaskInputRequest {
|
|
wait_id: "stale-wait".to_string(),
|
|
expected_version: old.record.version + 1,
|
|
schema_revision: 1,
|
|
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
|
schema_snapshot: json!({}),
|
|
projected_schema: json!({}),
|
|
prompt_code: "required_input_missing".to_string(),
|
|
missing_fields: vec!["$.email".to_string()],
|
|
expires_at_unix_ms: OffsetDateTime::now_utc().unix_timestamp() * 1000 + 30_000,
|
|
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
|
};
|
|
|
|
let suspended =
|
|
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &old, request).await;
|
|
assert!(
|
|
suspended.is_err(),
|
|
"stale fence must not suspend: {suspended:?}"
|
|
);
|
|
|
|
let attempt_id = "attempt-stale".to_string();
|
|
let checkpoint = store
|
|
.checkpoint_dispatch(BeginAttempt {
|
|
task_id,
|
|
owner: old.owner.clone(),
|
|
fence: old.fence,
|
|
expected_version: old.record.version,
|
|
attempt_id,
|
|
input_fingerprint: old.record.input_fingerprint.clone(),
|
|
upstream_key_digest: None,
|
|
retry_allowed: false,
|
|
})
|
|
.await;
|
|
assert!(
|
|
checkpoint.is_err(),
|
|
"stale fence must not checkpoint: {checkpoint:?}"
|
|
);
|
|
|
|
assert_ne!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("input_required")
|
|
);
|
|
let attempt_count: i64 =
|
|
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempt_count, 0);
|
|
let req_count: i64 =
|
|
sqlx::query_scalar("select count(*)::bigint from task_input_requests where task_id = $1")
|
|
.bind(task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(req_count, 0);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
// Positive control: the same request under the current lease suspends the Task exactly once, so
|
|
// the refusal above is the stale fence and not an invalid request shape.
|
|
let request = TaskInputRequest {
|
|
wait_id: "current-wait".to_string(),
|
|
expected_version: new.record.version + 1,
|
|
schema_revision: 1,
|
|
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
|
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
|
schema_snapshot: json!({}),
|
|
projected_schema: json!({}),
|
|
prompt_code: "required_input_missing".to_string(),
|
|
missing_fields: vec!["$.email".to_string()],
|
|
expires_at_unix_ms: new.record.pins.execution_deadline_unix_ms,
|
|
};
|
|
let suspended =
|
|
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &new, request)
|
|
.await
|
|
.expect("the current lease suspends");
|
|
assert_eq!(suspended.state, TaskState::InputRequired);
|
|
assert_eq!(
|
|
state_of(®istry, task_id).await.as_deref(),
|
|
Some("input_required")
|
|
);
|
|
assert_eq!(
|
|
admission_entries(®istry, task_id).await,
|
|
vec![("executing".to_owned(), 1)]
|
|
);
|
|
// The stale fence still cannot checkpoint a dispatch of this generation.
|
|
let checkpoint = store
|
|
.checkpoint_dispatch(BeginAttempt {
|
|
task_id,
|
|
owner: old.owner.clone(),
|
|
fence: old.fence,
|
|
expected_version: old.record.version,
|
|
attempt_id: "attempt-stale-after-suspend".to_string(),
|
|
input_fingerprint: old.record.input_fingerprint.clone(),
|
|
upstream_key_digest: None,
|
|
retry_allowed: false,
|
|
})
|
|
.await;
|
|
assert!(
|
|
checkpoint.is_err(),
|
|
"stale fence must not checkpoint: {checkpoint:?}"
|
|
);
|
|
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
|
|
|
server.abort();
|
|
}
|