CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с fence-токеном, зашифрованный input и повторная проверка доступа до сетевого вызова, checkpoint с AttemptID до dispatch и восстановление только по PG. Устаревший worker не может зафиксировать результат. Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса: стоп в окне после checkpoint и до permit завершает задачу как failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи остаются доступными. Retry-попытки получают отдельные дочерние spans с durable current_attempt_id. Shared admission/fairness из Story 4.1 сохранены. Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47, bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
191 lines
7.9 KiB
Rust
191 lines
7.9 KiB
Rust
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
|
|
//!
|
|
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
|
|
|
|
use super::*;
|
|
|
|
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
|
|
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
|
|
/// does not consume work that is still queued.
|
|
///
|
|
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
|
|
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
|
|
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
|
|
#[tokio::test]
|
|
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
|
|
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
|
use crank_runtime::{
|
|
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
|
|
TaskWorkerError,
|
|
};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
|
|
let registry = common::test_registry().await;
|
|
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
|
|
// statement keeps the window deterministic without competing for a second connection.
|
|
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
|
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
|
|
begin
|
|
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
|
perform pg_sleep(1.5);
|
|
end if;
|
|
return new;
|
|
end $$;
|
|
create trigger delay_task_dispatch_checkpoint before update on tasks
|
|
for each row execute function delay_task_dispatch_checkpoint();",
|
|
))
|
|
.execute(registry.pool())
|
|
.await
|
|
.expect("checkpoint delay trigger");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let mut accepted = Vec::new();
|
|
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
|
|
let request = TaskAcceptRequest {
|
|
task_id: TaskId::generate(),
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_stop_window".to_owned(),
|
|
platform_api_key_id: None,
|
|
operation_id: "op_report".to_owned(),
|
|
pins: TaskPins {
|
|
operation_version: 1,
|
|
policy_mode: ToolExecutionMode::Background,
|
|
execution_deadline_unix_ms: 1_900_000_000_000,
|
|
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
|
approval_reference: None,
|
|
},
|
|
scoped_key: scoped_key.repeat(64),
|
|
input_fingerprint: scoped_key.repeat(64),
|
|
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
|
|
support: TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&crank_core::TraceContext::parse(
|
|
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
|
)
|
|
.unwrap(),
|
|
),
|
|
executable_input_ciphertext: None,
|
|
executable_input_key_version: None,
|
|
executable_input_master_key_epoch: None,
|
|
};
|
|
accepted.push(store.accept(request).await.expect("PG accepts"));
|
|
}
|
|
|
|
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
|
|
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
|
|
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
|
|
let executed = Arc::new(AtomicUsize::new(0));
|
|
struct GatedHandler {
|
|
entered: Arc<tokio::sync::Semaphore>,
|
|
release: Arc<tokio::sync::Semaphore>,
|
|
executed: Arc<AtomicUsize>,
|
|
}
|
|
#[async_trait::async_trait]
|
|
impl TaskHandler for GatedHandler {
|
|
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
|
self.entered.add_permits(1);
|
|
let _ = self.release.acquire().await;
|
|
Ok(())
|
|
}
|
|
async fn execute(
|
|
&self,
|
|
_lease: &TaskLease,
|
|
_attempt_id: &str,
|
|
_cancelled: Arc<std::sync::atomic::AtomicBool>,
|
|
_permit: TaskDispatchPermit,
|
|
) -> TaskAttemptOutcome {
|
|
self.executed.fetch_add(1, Ordering::SeqCst);
|
|
TaskAttemptOutcome::Completed(json!({ "ok": true }))
|
|
}
|
|
}
|
|
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
|
|
entered: revalidate_entered.clone(),
|
|
release: revalidate_release.clone(),
|
|
executed: executed.clone(),
|
|
});
|
|
let mut config = TaskWorkerConfig::default();
|
|
config.owner = "runtime-worker-stop-window".to_owned();
|
|
config.batch_limit = 1;
|
|
config.lease_ms = 30_000;
|
|
config.poll_interval = Duration::from_millis(50);
|
|
config.renewal_interval = Duration::from_secs(20);
|
|
config.max_inflight = 1;
|
|
config.shutdown_timeout = Duration::from_secs(20);
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
|
|
.await
|
|
.expect("worker must claim the first Task and enter revalidation")
|
|
.expect("revalidation gate is open")
|
|
.forget();
|
|
|
|
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
|
|
revalidate_release.add_permits(1);
|
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
|
|
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
|
|
// finalize the Task, but no permit exists yet, so no outbound call can start.
|
|
let stop_requested = std::time::Instant::now();
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
|
|
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
|
.await
|
|
.expect("worker stops inside the 30s budget")
|
|
.expect("worker exits cleanly")
|
|
.expect("worker reports no error");
|
|
assert!(
|
|
stop_requested.elapsed() >= Duration::from_millis(500),
|
|
"the checkpoint must have still been in flight when the stop arrived"
|
|
);
|
|
|
|
assert_eq!(
|
|
executed.load(Ordering::SeqCst),
|
|
0,
|
|
"a stop between checkpoint and permit must never reach the upstream adapter"
|
|
);
|
|
let access = TaskAccess {
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_stop_window".to_owned(),
|
|
};
|
|
let record = store.read(&access, accepted[0].task_id).await.unwrap();
|
|
assert_eq!(record.state, TaskState::Failed);
|
|
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
|
|
assert!(record.current_attempt_id.is_some());
|
|
assert_eq!(
|
|
record
|
|
.error
|
|
.as_ref()
|
|
.and_then(|error| error.value().get("code"))
|
|
.and_then(|code| code.as_str()),
|
|
Some("shutdown_before_dispatch")
|
|
);
|
|
|
|
// Work that was still queued is not consumed by the stopped worker.
|
|
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
|
|
assert_eq!(queued.state, TaskState::Queued);
|
|
let recovered = store
|
|
.claim_batch("worker_after_stop", 4, 10_000)
|
|
.await
|
|
.expect("queued work stays claimable");
|
|
assert_eq!(recovered.len(), 1);
|
|
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
|
|
}
|