CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable per-outbound attempt до отправки запроса, bounded retry/read, server-owned verified writes, честный cancel/unknown, deadline/fencing, additive schema v29. Staged renames task.rs/tasks.rs разложены на модули. Includes disclosed cosmetic parity exception: единственная правка вне принятой дельты — удалена лишняя пустая строка в конце scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
221 lines
9.5 KiB
Rust
221 lines
9.5 KiB
Rust
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
|
|
//!
|
|
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
|
|
|
|
use super::*;
|
|
|
|
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
|
|
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
|
|
/// does not consume work that is still queued.
|
|
///
|
|
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
|
|
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
|
|
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
|
|
#[tokio::test]
|
|
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
|
|
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
|
use crank_runtime::{
|
|
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
|
};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
|
|
let registry = common::test_registry().await;
|
|
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
|
|
// statement keeps the window deterministic without competing for a second connection.
|
|
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
|
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
|
|
begin
|
|
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
|
perform pg_sleep(1.5);
|
|
end if;
|
|
return new;
|
|
end $$;
|
|
create trigger delay_task_dispatch_checkpoint before update on tasks
|
|
for each row execute function delay_task_dispatch_checkpoint();",
|
|
))
|
|
.execute(registry.pool())
|
|
.await
|
|
.expect("checkpoint delay trigger");
|
|
let store = Arc::new(
|
|
PostgresTaskStore::new(
|
|
registry.pool().clone(),
|
|
AdmissionLimits {
|
|
global_limit: 4,
|
|
agent_limit: 4,
|
|
tool_limit: 4,
|
|
queue_limit: 8,
|
|
},
|
|
Default::default(),
|
|
)
|
|
.expect("valid task store configuration"),
|
|
);
|
|
let mut accepted = Vec::new();
|
|
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
|
|
let request = TaskAcceptRequest {
|
|
task_id: TaskId::generate(),
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_stop_window".to_owned(),
|
|
platform_api_key_id: None,
|
|
operation_id: "op_report".to_owned(),
|
|
pins: TaskPins {
|
|
operation_version: 1,
|
|
policy_mode: ToolExecutionMode::Background,
|
|
execution_deadline_unix_ms: 1_900_000_000_000,
|
|
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
|
approval_reference: None,
|
|
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
|
},
|
|
scoped_key: scoped_key.repeat(64),
|
|
input_fingerprint: scoped_key.repeat(64),
|
|
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
|
|
support: TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&crank_core::TraceContext::parse(
|
|
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
|
)
|
|
.unwrap(),
|
|
),
|
|
executable_input_ciphertext: None,
|
|
executable_input_key_version: None,
|
|
executable_input_master_key_epoch: None,
|
|
};
|
|
accepted.push(store.accept(request).await.expect("PG accepts"));
|
|
}
|
|
|
|
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
|
|
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
|
|
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
|
|
let executed = Arc::new(AtomicUsize::new(0));
|
|
struct GatedHandler {
|
|
entered: Arc<tokio::sync::Semaphore>,
|
|
release: Arc<tokio::sync::Semaphore>,
|
|
executed: Arc<AtomicUsize>,
|
|
}
|
|
#[async_trait::async_trait]
|
|
impl TaskHandler for GatedHandler {
|
|
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
|
self.entered.add_permits(1);
|
|
let _ = self.release.acquire().await;
|
|
Ok(())
|
|
}
|
|
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
|
// The trigger delays the durable checkpoint; a stop that lands while it is in flight must
|
|
// finalize failed(shutdown_before_dispatch) without an outbound send.
|
|
let begin = crank_core::BeginAttempt {
|
|
task_id: context.lease().record.task_id,
|
|
owner: context.lease().owner.clone(),
|
|
fence: context.lease().fence,
|
|
expected_version: context.lease().record.version,
|
|
attempt_id: "attempt-stop-window".to_owned(),
|
|
input_fingerprint: context.lease().record.input_fingerprint.clone(),
|
|
upstream_key_digest: None,
|
|
retry_allowed: false,
|
|
};
|
|
let Ok(attempt) = context.store().checkpoint_dispatch(begin).await else {
|
|
return TaskAttemptOutcome::Stopped;
|
|
};
|
|
if context.is_stopping() {
|
|
let _ = context
|
|
.store()
|
|
.resolve_attempt(
|
|
&attempt,
|
|
crank_core::AttemptOutcome::Failed {
|
|
error: crank_core::TaskEnvelope::try_redacted(&json!({
|
|
"code": "shutdown_before_dispatch",
|
|
"phase": "before_dispatch",
|
|
}))
|
|
.expect("bounded error envelope"),
|
|
retryable: false,
|
|
phase: crank_core::AttemptPhase::BeforeDispatch,
|
|
},
|
|
)
|
|
.await;
|
|
return TaskAttemptOutcome::Failed {
|
|
code: "shutdown_before_dispatch".to_owned(),
|
|
};
|
|
}
|
|
self.executed.fetch_add(1, Ordering::SeqCst);
|
|
commit_attempt(&context, completed_outcome(json!({ "ok": true }))).await
|
|
}
|
|
}
|
|
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
|
|
entered: revalidate_entered.clone(),
|
|
release: revalidate_release.clone(),
|
|
executed: executed.clone(),
|
|
});
|
|
let config = TaskWorkerConfig {
|
|
owner: "runtime-worker-stop-window".to_owned(),
|
|
batch_limit: 1,
|
|
lease_ms: 30_000,
|
|
poll_interval: Duration::from_millis(50),
|
|
renewal_interval: Duration::from_secs(20),
|
|
max_inflight: 1,
|
|
shutdown_timeout: Duration::from_secs(20),
|
|
};
|
|
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
|
let stop_flag = worker.stop_flag();
|
|
let worker_task = tokio::spawn(async move { worker.run().await });
|
|
|
|
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
|
|
.await
|
|
.expect("worker must claim the first Task and enter revalidation")
|
|
.expect("revalidation gate is open")
|
|
.forget();
|
|
|
|
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
|
|
revalidate_release.add_permits(1);
|
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
|
|
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
|
|
// finalize the Task, but no permit exists yet, so no outbound call can start.
|
|
let stop_requested = std::time::Instant::now();
|
|
stop_flag.store(true, Ordering::SeqCst);
|
|
|
|
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
|
.await
|
|
.expect("worker stops inside the 30s budget")
|
|
.expect("worker exits cleanly")
|
|
.expect("worker reports no error");
|
|
assert!(
|
|
stop_requested.elapsed() >= Duration::from_millis(500),
|
|
"the checkpoint must have still been in flight when the stop arrived"
|
|
);
|
|
|
|
assert_eq!(
|
|
executed.load(Ordering::SeqCst),
|
|
0,
|
|
"a stop between checkpoint and permit must never reach the upstream adapter"
|
|
);
|
|
let access = TaskAccess {
|
|
workspace_id: common::test_workspace_id().to_string(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_stop_window".to_owned(),
|
|
};
|
|
let record = store.read(&access, accepted[0].task_id).await.unwrap();
|
|
assert_eq!(record.state, TaskState::Failed);
|
|
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
|
|
assert!(record.current_attempt_id.is_some());
|
|
assert_eq!(
|
|
record
|
|
.error
|
|
.as_ref()
|
|
.and_then(|error| error.value().get("code"))
|
|
.and_then(|code| code.as_str()),
|
|
Some("shutdown_before_dispatch")
|
|
);
|
|
|
|
// Work that was still queued is not consumed by the stopped worker.
|
|
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
|
|
assert_eq!(queued.state, TaskState::Queued);
|
|
let recovered = store
|
|
.claim_batch("worker_after_stop", 4, 10_000)
|
|
.await
|
|
.expect("queued work stays claimable");
|
|
assert_eq!(recovered.len(), 1);
|
|
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
|
|
}
|