Files
crank/apps/mcp-server/tests/integration/durable_tasks/shutdown_window.rs
T
bsodfather f9f8c625d1
CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
feat(epic4): исполнять durable Task через PG lease/fence и восстанавливать после сбоев (Story 4.3)
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с
fence-токеном, зашифрованный input и повторная проверка доступа до сетевого
вызова, checkpoint с AttemptID до dispatch и восстановление только по PG.
Устаревший worker не может зафиксировать результат.

Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой
схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса:
стоп в окне после checkpoint и до permit завершает задачу как
failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи
остаются доступными. Retry-попытки получают отдельные дочерние spans с durable
current_attempt_id. Shared admission/fairness из Story 4.1 сохранены.

Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47,
bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA
и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
2026-10-29 22:26:52 +00:00

191 lines
7.9 KiB
Rust

//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
//!
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
use super::*;
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
/// does not consume work that is still queued.
///
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
#[tokio::test]
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
// statement keeps the window deterministic without competing for a second connection.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
begin
if new.current_attempt_id is not null and old.current_attempt_id is null then
perform pg_sleep(1.5);
end if;
return new;
end $$;
create trigger delay_task_dispatch_checkpoint before update on tasks
for each row execute function delay_task_dispatch_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint delay trigger");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let mut accepted = Vec::new();
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
},
scoped_key: scoped_key.repeat(64),
input_fingerprint: scoped_key.repeat(64),
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
accepted.push(store.accept(request).await.expect("PG accepts"));
}
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
let executed = Arc::new(AtomicUsize::new(0));
struct GatedHandler {
entered: Arc<tokio::sync::Semaphore>,
release: Arc<tokio::sync::Semaphore>,
executed: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for GatedHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
self.entered.add_permits(1);
let _ = self.release.acquire().await;
Ok(())
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
self.executed.fetch_add(1, Ordering::SeqCst);
TaskAttemptOutcome::Completed(json!({ "ok": true }))
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
entered: revalidate_entered.clone(),
release: revalidate_release.clone(),
executed: executed.clone(),
});
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-stop-window".to_owned();
config.batch_limit = 1;
config.lease_ms = 30_000;
config.poll_interval = Duration::from_millis(50);
config.renewal_interval = Duration::from_secs(20);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(20);
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
.await
.expect("worker must claim the first Task and enter revalidation")
.expect("revalidation gate is open")
.forget();
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
revalidate_release.add_permits(1);
tokio::time::sleep(Duration::from_millis(300)).await;
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
// finalize the Task, but no permit exists yet, so no outbound call can start.
let stop_requested = std::time::Instant::now();
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(Duration::from_secs(30), worker_task)
.await
.expect("worker stops inside the 30s budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
assert!(
stop_requested.elapsed() >= Duration::from_millis(500),
"the checkpoint must have still been in flight when the stop arrived"
);
assert_eq!(
executed.load(Ordering::SeqCst),
0,
"a stop between checkpoint and permit must never reach the upstream adapter"
);
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
};
let record = store.read(&access, accepted[0].task_id).await.unwrap();
assert_eq!(record.state, TaskState::Failed);
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
assert!(record.current_attempt_id.is_some());
assert_eq!(
record
.error
.as_ref()
.and_then(|error| error.value().get("code"))
.and_then(|code| code.as_str()),
Some("shutdown_before_dispatch")
);
// Work that was still queued is not consumed by the stopped worker.
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
assert_eq!(queued.state, TaskState::Queued);
let recovered = store
.claim_batch("worker_after_stop", 4, 10_000)
.await
.expect("queued work stays claimable");
assert_eq!(recovered.len(), 1);
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
}