Files
crank/crates/crank-registry/tests/integration/tasks/task_worker.rs
T
bsodfather a3057de51e
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): безопасные повторы и отмена durable Task (Story 4.4)
Durable per-outbound attempt до отправки запроса, bounded retry/read,
server-owned verified writes, честный cancel/unknown, deadline/fencing,
additive schema v29. Staged renames task.rs/tasks.rs разложены на модули.

Includes disclosed cosmetic parity exception: единственная правка вне
принятой дельты — удалена лишняя пустая строка в конце
scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
2026-10-30 12:21:57 +00:00

168 lines
6.2 KiB
Rust

//! Story 4.3 ATDD: durable Task leases, fencing, crash recovery and bounded claim behavior.
//!
//! Split from `tasks.rs` to keep both integration files inside the code-health limit.
use super::*;
use serde_json::json;
/// Story 4.3 ATDD: a worker crash before the durable dispatch checkpoint leaves only an expired
/// lease. PostgreSQL is authoritative, so another worker recovers the Task and the dead owner can
/// never commit a terminal outcome with its old fence.
#[tokio::test]
async fn task_worker_expired_pre_checkpoint_lease_is_recovered_and_stale_owner_cannot_commit() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("recover1"), &digest("f2"), AGENT, 30_000))
.await
.unwrap();
let first = store.claim_batch("worker_dead", 1, 150).await.unwrap();
assert_eq!(first.len(), 1);
assert_eq!(first[0].record.task_id, created.task_id);
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
// Recovery on the next bounded authoritative scan, with no Valkey wake-up.
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
let lease = recovered
.into_iter()
.find(|lease| lease.record.task_id == created.task_id)
.expect("expired pre-checkpoint Task is recovered");
assert!(lease.fence > first[0].fence, "recovery bumps the fence");
// The previous owner's terminal CAS is rejected; the live owner wins exactly once.
let stale = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: first[0].record.state,
expected_version: first[0].record.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"stale": true})).unwrap()),
error: None,
},
"worker_dead",
first[0].fence,
)
.await;
assert!(matches!(stale, Err(TaskError::StateConflict)));
let finished = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: lease.record.state,
expected_version: lease.record.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
},
"worker_live",
lease.fence,
)
.await
.expect("current lease wins the terminal CAS");
assert_eq!(finished.state, TaskState::Completed);
database.cleanup().await;
}
/// Story 4.3 ATDD: after the dispatch checkpoint the outbound outcome is uncertain. An expired
/// attempt resolves to `outcome_unknown`, is never redispatched automatically and releases the
/// admission slot it held.
#[tokio::test]
async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_without_redispatch() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("uncertain1"), &digest("f3"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_dead".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-uncertain".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("durable dispatch checkpoint");
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
// The authority scan resolves uncertainty instead of handing the attempt to another worker.
let after = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
assert!(
after.is_empty(),
"a checkpointed attempt must never be redispatched"
);
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::OutcomeUnknown);
assert_eq!(
record.current_attempt_id.as_deref(),
Some("attempt-uncertain")
);
let intents: i64 = sqlx::query_scalar(
"select count(*)::bigint from execution_admission_entries where task_id = $1",
)
.bind(created.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(intents, 0, "uncertain terminal outcome releases capacity");
// Still no automatic retry on the following scans.
assert!(
store
.claim_batch("worker_live", 4, 10_000)
.await
.unwrap()
.is_empty()
);
database.cleanup().await;
}
/// Story 4.3 ATDD: concurrent claimers never hold the same Task at once.
#[tokio::test]
async fn task_worker_concurrent_claimers_never_double_lease_one_task() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = std::sync::Arc::new(store_with(registry.pool().clone(), 8));
let created = store
.accept(request(&key("concurrent1"), &digest("f4"), AGENT, 30_000))
.await
.unwrap();
let mut handles = Vec::new();
for index in 0..4 {
let store = store.clone();
handles.push(tokio::spawn(async move {
store
.claim_batch(&format!("worker_{index}"), 1, 10_000)
.await
.unwrap()
}));
}
let mut leased = Vec::new();
for handle in handles {
leased.extend(handle.await.unwrap());
}
assert_eq!(leased.len(), 1, "exactly one owner wins the claim");
assert_eq!(leased[0].record.task_id, created.task_id);
database.cleanup().await;
}