CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable per-outbound attempt до отправки запроса, bounded retry/read, server-owned verified writes, честный cancel/unknown, deadline/fencing, additive schema v29. Staged renames task.rs/tasks.rs разложены на модули. Includes disclosed cosmetic parity exception: единственная правка вне принятой дельты — удалена лишняя пустая строка в конце scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
168 lines
6.2 KiB
Rust
168 lines
6.2 KiB
Rust
//! Story 4.3 ATDD: durable Task leases, fencing, crash recovery and bounded claim behavior.
|
|
//!
|
|
//! Split from `tasks.rs` to keep both integration files inside the code-health limit.
|
|
|
|
use super::*;
|
|
use serde_json::json;
|
|
|
|
/// Story 4.3 ATDD: a worker crash before the durable dispatch checkpoint leaves only an expired
|
|
/// lease. PostgreSQL is authoritative, so another worker recovers the Task and the dead owner can
|
|
/// never commit a terminal outcome with its old fence.
|
|
#[tokio::test]
|
|
async fn task_worker_expired_pre_checkpoint_lease_is_recovered_and_stale_owner_cannot_commit() {
|
|
let database = common::TestDatabase::new().await;
|
|
let registry = database.registry().await;
|
|
let store = store_with(registry.pool().clone(), 8);
|
|
|
|
let created = store
|
|
.accept(request(&key("recover1"), &digest("f2"), AGENT, 30_000))
|
|
.await
|
|
.unwrap();
|
|
let first = store.claim_batch("worker_dead", 1, 150).await.unwrap();
|
|
assert_eq!(first.len(), 1);
|
|
assert_eq!(first[0].record.task_id, created.task_id);
|
|
|
|
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
|
|
|
// Recovery on the next bounded authoritative scan, with no Valkey wake-up.
|
|
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
|
|
let lease = recovered
|
|
.into_iter()
|
|
.find(|lease| lease.record.task_id == created.task_id)
|
|
.expect("expired pre-checkpoint Task is recovered");
|
|
assert!(lease.fence > first[0].fence, "recovery bumps the fence");
|
|
|
|
// The previous owner's terminal CAS is rejected; the live owner wins exactly once.
|
|
let stale = store
|
|
.finish_leased(
|
|
TaskFinishRequest {
|
|
access: access(AGENT),
|
|
task_id: created.task_id,
|
|
expected_state: first[0].record.state,
|
|
expected_version: first[0].record.version,
|
|
target_state: TaskState::Completed,
|
|
result: Some(TaskEnvelope::try_redacted(&json!({"stale": true})).unwrap()),
|
|
error: None,
|
|
},
|
|
"worker_dead",
|
|
first[0].fence,
|
|
)
|
|
.await;
|
|
assert!(matches!(stale, Err(TaskError::StateConflict)));
|
|
|
|
let finished = store
|
|
.finish_leased(
|
|
TaskFinishRequest {
|
|
access: access(AGENT),
|
|
task_id: created.task_id,
|
|
expected_state: lease.record.state,
|
|
expected_version: lease.record.version,
|
|
target_state: TaskState::Completed,
|
|
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
|
error: None,
|
|
},
|
|
"worker_live",
|
|
lease.fence,
|
|
)
|
|
.await
|
|
.expect("current lease wins the terminal CAS");
|
|
assert_eq!(finished.state, TaskState::Completed);
|
|
|
|
database.cleanup().await;
|
|
}
|
|
|
|
/// Story 4.3 ATDD: after the dispatch checkpoint the outbound outcome is uncertain. An expired
|
|
/// attempt resolves to `outcome_unknown`, is never redispatched automatically and releases the
|
|
/// admission slot it held.
|
|
#[tokio::test]
|
|
async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_without_redispatch() {
|
|
let database = common::TestDatabase::new().await;
|
|
let registry = database.registry().await;
|
|
let store = store_with(registry.pool().clone(), 8);
|
|
|
|
let created = store
|
|
.accept(request(&key("uncertain1"), &digest("f3"), AGENT, 30_000))
|
|
.await
|
|
.unwrap();
|
|
let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap();
|
|
let lease = leases.into_iter().next().expect("claim");
|
|
store
|
|
.checkpoint_dispatch(BeginAttempt {
|
|
task_id: lease.record.task_id,
|
|
owner: "worker_dead".to_owned(),
|
|
fence: lease.fence,
|
|
expected_version: lease.record.version,
|
|
attempt_id: "attempt-uncertain".to_owned(),
|
|
input_fingerprint: lease.record.input_fingerprint.clone(),
|
|
upstream_key_digest: None,
|
|
retry_allowed: false,
|
|
})
|
|
.await
|
|
.expect("durable dispatch checkpoint");
|
|
|
|
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
|
|
|
// The authority scan resolves uncertainty instead of handing the attempt to another worker.
|
|
let after = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
|
|
assert!(
|
|
after.is_empty(),
|
|
"a checkpointed attempt must never be redispatched"
|
|
);
|
|
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
|
|
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
|
assert_eq!(
|
|
record.current_attempt_id.as_deref(),
|
|
Some("attempt-uncertain")
|
|
);
|
|
let intents: i64 = sqlx::query_scalar(
|
|
"select count(*)::bigint from execution_admission_entries where task_id = $1",
|
|
)
|
|
.bind(created.task_id.as_uuid())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(intents, 0, "uncertain terminal outcome releases capacity");
|
|
|
|
// Still no automatic retry on the following scans.
|
|
assert!(
|
|
store
|
|
.claim_batch("worker_live", 4, 10_000)
|
|
.await
|
|
.unwrap()
|
|
.is_empty()
|
|
);
|
|
|
|
database.cleanup().await;
|
|
}
|
|
|
|
/// Story 4.3 ATDD: concurrent claimers never hold the same Task at once.
|
|
#[tokio::test]
|
|
async fn task_worker_concurrent_claimers_never_double_lease_one_task() {
|
|
let database = common::TestDatabase::new().await;
|
|
let registry = database.registry().await;
|
|
let store = std::sync::Arc::new(store_with(registry.pool().clone(), 8));
|
|
|
|
let created = store
|
|
.accept(request(&key("concurrent1"), &digest("f4"), AGENT, 30_000))
|
|
.await
|
|
.unwrap();
|
|
let mut handles = Vec::new();
|
|
for index in 0..4 {
|
|
let store = store.clone();
|
|
handles.push(tokio::spawn(async move {
|
|
store
|
|
.claim_batch(&format!("worker_{index}"), 1, 10_000)
|
|
.await
|
|
.unwrap()
|
|
}));
|
|
}
|
|
let mut leased = Vec::new();
|
|
for handle in handles {
|
|
leased.extend(handle.await.unwrap());
|
|
}
|
|
assert_eq!(leased.len(), 1, "exactly one owner wins the claim");
|
|
assert_eq!(leased[0].record.task_id, created.task_id);
|
|
|
|
database.cleanup().await;
|
|
}
|