CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 12m45s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Community Release / release (push) Successful in 40m35s
Tool Draft фиксирует synchronous/background/adaptive, deadline и fallback; runtime применяет единый снимок политики (concurrency, очередь, approval, deadline) до внешнего действия. Насыщение отклоняется typed overload с retry guidance, mandatory safety store fail closed, background/adaptive используют опубликованный synchronous/unsupported fallback. Refs: #29
256 lines
10 KiB
Rust
256 lines
10 KiB
Rust
//! Common execution admission seam of the runtime (Story 4.1, FR-21/FR-24).
|
|
//!
|
|
//! Every real execution — synchronous, or a published `background`/`adaptive` policy that falls back
|
|
//! to synchronous because no durable Task transport exists yet — passes this one gate before any
|
|
//! adapter work. The gate asks the authoritative store for a scoped permit bounded by the published
|
|
//! Tool policy, the verified approval state of the exact Published Version/scope/input and the caller
|
|
//! deadline, and it releases the permit explicitly when the local execution future ends.
|
|
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use crank_core::{
|
|
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
|
|
AdmissionScope, CoordinationStateStore, ExecutionAdmission, NoTasksFallback, ToolExecutionMode,
|
|
current_unix_ms,
|
|
};
|
|
use crank_metrics::{
|
|
AdmissionModeLabel, RejectionReason, ScopeKind, record_admission_queue_wait,
|
|
record_admission_rejection,
|
|
};
|
|
use tracing::warn;
|
|
|
|
use crate::{RuntimeError, RuntimeOperation, RuntimeRequestContext};
|
|
|
|
/// Effective absolute deadline of one execution: the published policy deadline never extends the
|
|
/// caller deadline and always includes queue waiting time.
|
|
pub(crate) fn effective_deadline(
|
|
operation: &RuntimeOperation,
|
|
caller_deadline: Instant,
|
|
) -> Instant {
|
|
match operation.execution_config.execution_policy {
|
|
Some(policy) => {
|
|
caller_deadline.min(Instant::now() + Duration::from_millis(policy.deadline_ms))
|
|
}
|
|
None => caller_deadline,
|
|
}
|
|
}
|
|
|
|
/// Effective execution path for this deployment.
|
|
///
|
|
/// Story 4.1 has no Task transport, so a `background` or `adaptive` policy resolves through its
|
|
/// published `no_tasks_fallback`: either the same synchronous admission path, or a typed
|
|
/// unsupported refusal with no admission entry, no Task id and no outbound call.
|
|
pub(crate) fn effective_mode(
|
|
operation: &RuntimeOperation,
|
|
) -> Result<ToolExecutionMode, RuntimeError> {
|
|
let Some(policy) = operation.execution_config.execution_policy else {
|
|
return Ok(ToolExecutionMode::Synchronous);
|
|
};
|
|
match policy.mode {
|
|
ToolExecutionMode::Synchronous => Ok(ToolExecutionMode::Synchronous),
|
|
mode => match policy.no_tasks_fallback {
|
|
NoTasksFallback::Synchronous => Ok(ToolExecutionMode::Synchronous),
|
|
NoTasksFallback::Unsupported => Err(RuntimeError::ExecutionPolicyUnsupported {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
mode,
|
|
}),
|
|
},
|
|
}
|
|
}
|
|
|
|
pub(crate) struct AdmissionLease {
|
|
store: Arc<dyn ExecutionAdmission>,
|
|
permit: Option<AdmissionPermit>,
|
|
runtime: tokio::runtime::Handle,
|
|
}
|
|
|
|
impl AdmissionLease {
|
|
fn new(store: Arc<dyn ExecutionAdmission>, permit: AdmissionPermit) -> Self {
|
|
Self {
|
|
store,
|
|
permit: Some(permit),
|
|
runtime: tokio::runtime::Handle::current(),
|
|
}
|
|
}
|
|
|
|
/// Explicit release on the normal/error path of the local execution. The `Drop` cleanup stays as
|
|
/// the cancellation path; both CAS on the same owner token, so only one of them can free it.
|
|
pub(crate) async fn release(&mut self) {
|
|
let Some(permit) = self.permit.take() else {
|
|
return;
|
|
};
|
|
if let Err(error) = self.store.release(&permit).await {
|
|
warn!(
|
|
name: "runtime.admission.release_failed",
|
|
error = %error,
|
|
"failed to release an admission permit; the reservation stays fail-closed"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Drop for AdmissionLease {
|
|
fn drop(&mut self) {
|
|
let Some(permit) = self.permit.take() else {
|
|
return;
|
|
};
|
|
let store = Arc::clone(&self.store);
|
|
// Bounded cleanup: the explicit release path already ran on success/error; cancellation
|
|
// reaches here and releases by owner token without ever freeing a foreign permit. A failed
|
|
// release keeps the reservation fail-closed (no TTL reuse of an active permit).
|
|
self.runtime.spawn(async move {
|
|
match store.release(&permit).await {
|
|
Ok(true) => {}
|
|
Ok(false) => warn!(
|
|
name: "runtime.admission.release_skipped",
|
|
"admission permit was already released"
|
|
),
|
|
Err(error) => warn!(
|
|
name: "runtime.admission.release_failed",
|
|
error = %error,
|
|
"failed to release an admission permit; the reservation stays fail-closed"
|
|
),
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
/// Acquires a scoped permit from the authoritative store. Returns `None` when this deployment has no
|
|
/// admission store wired (embedded/unit-test executors keep the pre-4.1 local limiter).
|
|
pub(crate) async fn acquire(
|
|
store: Option<&Arc<dyn ExecutionAdmission>>,
|
|
coordination_store: Option<&dyn CoordinationStateStore>,
|
|
request: &crate::RuntimeExecutionRequest<'_>,
|
|
deadline: Instant,
|
|
) -> Result<Option<AdmissionLease>, RuntimeError> {
|
|
let Some(store) = store else {
|
|
return Ok(None);
|
|
};
|
|
let operation = request.operation();
|
|
let input = request.input();
|
|
let request_context = request.request_context();
|
|
let operation_id = operation.operation_id.as_str().to_owned();
|
|
let policy_mode = effective_mode(operation)?;
|
|
// The one admission snapshot carries the authoritative approval state of this exact Published
|
|
// Version, scope, input and deadline. A confirmation-requiring request without a verified
|
|
// approval never takes capacity and never dispatches: the pending human decision is answered
|
|
// here with the existing challenge, and the single-use token is consumed only at dispatch.
|
|
let approval = crate::confirmation::approval_evidence(
|
|
coordination_store,
|
|
operation,
|
|
input,
|
|
Some(request_context),
|
|
)
|
|
.await?;
|
|
let agent_key = request.agent_id().map(|agent| agent.as_str());
|
|
let workspace_id = request.workspace_id().as_str().to_owned();
|
|
let request = AdmissionRequest {
|
|
scope: AdmissionScope {
|
|
workspace_id,
|
|
agent_key: agent_key.unwrap_or(ADMIN_ORIGIN_KEY).to_owned(),
|
|
operation_id: operation_id.clone(),
|
|
operation_version: operation.operation_version,
|
|
},
|
|
mode: AdmissionMode::Synchronous,
|
|
policy_mode,
|
|
no_tasks_fallback: operation
|
|
.execution_config
|
|
.execution_policy
|
|
.map(|policy| policy.no_tasks_fallback)
|
|
.unwrap_or(NoTasksFallback::Synchronous),
|
|
approval,
|
|
fingerprint: admission_fingerprint(request_context, operation),
|
|
expires_at_unix_ms: current_unix_ms().saturating_add(
|
|
deadline_remaining_ms(deadline)
|
|
.try_into()
|
|
.unwrap_or(i64::MAX),
|
|
),
|
|
};
|
|
let started = Instant::now();
|
|
let outcome = store.admit(request).await.map_err(|error| {
|
|
record_admission_rejection(RejectionReason::DependencyUnavailable, ScopeKind::Global);
|
|
RuntimeError::AdmissionStoreUnavailable {
|
|
operation_id: operation_id.clone(),
|
|
details: error.to_string(),
|
|
}
|
|
})?;
|
|
let waited_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or_default();
|
|
match outcome {
|
|
AdmissionOutcome::Permitted(permit) => {
|
|
record_admission_queue_wait(AdmissionModeLabel::Synchronous, waited_ms);
|
|
Ok(Some(AdmissionLease::new(Arc::clone(store), permit)))
|
|
}
|
|
AdmissionOutcome::Overloaded { retry_after_ms } => {
|
|
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
|
|
Err(RuntimeError::AdmissionOverloaded {
|
|
operation_id,
|
|
retry_after_ms,
|
|
})
|
|
}
|
|
AdmissionOutcome::DeadlineElapsed => {
|
|
// Queue wait exhausted the absolute deadline before a permit was granted: nothing was
|
|
// dispatched, so this is an admission refusal, not an upstream timeout.
|
|
Err(deadline_refusal(operation))
|
|
}
|
|
AdmissionOutcome::ApprovalRequired => {
|
|
// The store refused before reserving any capacity or queue entry. Answer with the
|
|
// existing single-use challenge so the caller can confirm out of band.
|
|
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
|
|
Err(crate::confirmation::challenge(
|
|
coordination_store,
|
|
operation,
|
|
input,
|
|
Some(request_context),
|
|
)
|
|
.await)
|
|
}
|
|
AdmissionOutcome::Unsupported | AdmissionOutcome::Enqueued(_) => {
|
|
// No Task transport exists in this deployment: an enqueue decision can never become a
|
|
// runnable execution here, so it must not be reported as accepted work.
|
|
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
|
|
Err(RuntimeError::ExecutionPolicyUnsupported {
|
|
operation_id,
|
|
mode: policy_mode,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Typed refusal for an admission that never granted a permit within the absolute deadline. Reuses the
|
|
/// existing `execution_overloaded` vocabulary (stage `admission`, retry guidance, certain outcome) so
|
|
/// no new client error code or API shape appears, while reporting zero dispatch truthfully.
|
|
pub(crate) fn deadline_refusal(operation: &RuntimeOperation) -> RuntimeError {
|
|
record_admission_rejection(RejectionReason::Deadline, ScopeKind::Global);
|
|
RuntimeError::AdmissionOverloaded {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
|
}
|
|
}
|
|
|
|
fn deadline_remaining_ms(deadline: Instant) -> u64 {
|
|
u64::try_from(
|
|
deadline
|
|
.saturating_duration_since(Instant::now())
|
|
.as_millis(),
|
|
)
|
|
.unwrap_or_default()
|
|
.max(1)
|
|
}
|
|
|
|
/// Bounded request identity for correlating one admission entry. Never contains payload, headers or
|
|
/// credentials.
|
|
fn admission_fingerprint(
|
|
request_context: &RuntimeRequestContext,
|
|
operation: &RuntimeOperation,
|
|
) -> String {
|
|
let mut fingerprint = format!(
|
|
"{}:{}:{}",
|
|
operation.operation_id.as_str(),
|
|
operation.operation_version,
|
|
request_context.request_id
|
|
);
|
|
fingerprint.truncate(crank_core::ADMISSION_FINGERPRINT_MAX_BYTES);
|
|
fingerprint
|
|
}
|