Story 4.1 — публиковать execution mode и применять admission limits (FR-21, FR-24) #104
No files matched your search
Generated
+1
@@ -826,6 +826,7 @@ dependencies = [
|
||||
name = "crank-registry"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"crank-artifacts",
|
||||
"crank-core",
|
||||
"crank-mapping",
|
||||
|
||||
@@ -181,6 +181,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
|
||||
use admin_api::{
|
||||
app::build_app,
|
||||
auth::{AuthSettings, BootstrapAdminConfig},
|
||||
pool_metrics::spawn_postgres_pool_metrics,
|
||||
pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics},
|
||||
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
|
||||
service::AdminServiceBuilder,
|
||||
state::AppState,
|
||||
@@ -169,6 +169,7 @@ async fn run(
|
||||
};
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
spawn_admission_metrics(registry.pool().clone());
|
||||
}
|
||||
let base_url = config
|
||||
.runtime
|
||||
@@ -214,6 +215,16 @@ async fn run(
|
||||
.with_limits(runtime_limits)
|
||||
.with_response_cache(cache_stores.response.clone())
|
||||
.with_coordination_store(cache_stores.coordination.clone())
|
||||
.with_admission(std::sync::Arc::new(
|
||||
crank_registry::PostgresExecutionAdmission::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?,
|
||||
))
|
||||
.build();
|
||||
let identity_provider =
|
||||
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
|
||||
@@ -321,6 +332,12 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con
|
||||
config.runtime.max_concurrent_sessions,
|
||||
)
|
||||
.map_err(|_| invalid("runtime.limits"))?;
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: config.runtime.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
}
|
||||
.validate()
|
||||
.map_err(|_| invalid("runtime.admission"))?;
|
||||
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
|
||||
RequestRateLimitConfig::new(
|
||||
config.rate_limit.requests_per_second,
|
||||
|
||||
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
|
||||
/// to zero) without waiting for a new request.
|
||||
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
|
||||
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -37,13 +37,22 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
let execution_admission = if checks.execution_admission {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
if checks.is_ready() {
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
"checks": {
|
||||
"postgres": postgres,
|
||||
"artifact_storage": artifact_storage,
|
||||
"execution_admission": execution_admission
|
||||
}
|
||||
})),
|
||||
)
|
||||
} else {
|
||||
@@ -52,7 +61,11 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
"checks": {
|
||||
"postgres": postgres,
|
||||
"artifact_storage": artifact_storage,
|
||||
"execution_admission": execution_admission
|
||||
}
|
||||
})),
|
||||
)
|
||||
}
|
||||
@@ -67,6 +80,7 @@ mod tests {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: false,
|
||||
artifact_storage: true,
|
||||
execution_admission: true,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
@@ -92,6 +106,7 @@ mod tests {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: true,
|
||||
artifact_storage: false,
|
||||
execution_admission: true,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
@@ -111,4 +126,14 @@ mod tests {
|
||||
"not_ready"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readiness_checks_identify_an_admission_state_failure() {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: true,
|
||||
artifact_storage: true,
|
||||
execution_admission: false,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
}
|
||||
}
|
||||
@@ -55,8 +55,8 @@ use crate::{
|
||||
storage::LocalArtifactStorage,
|
||||
};
|
||||
use operation_validation::{
|
||||
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
|
||||
validate_protocol_target, validate_response_cache_policy,
|
||||
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
|
||||
validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -104,11 +104,12 @@ pub use crate::dto::*;
|
||||
pub struct ReadinessChecks {
|
||||
pub postgres: bool,
|
||||
pub artifact_storage: bool,
|
||||
pub execution_admission: bool,
|
||||
}
|
||||
|
||||
impl ReadinessChecks {
|
||||
pub const fn is_ready(self) -> bool {
|
||||
self.postgres && self.artifact_storage
|
||||
self.postgres && self.artifact_storage && self.execution_admission
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +151,8 @@ pub(super) struct CredentialAuditRecord<'a> {
|
||||
impl AdminService {
|
||||
pub async fn readiness(&self) -> ReadinessChecks {
|
||||
let postgres = self.registry.ping().await.is_ok();
|
||||
// Story 4.1: the mandatory admission state is a readiness dependency of every execution.
|
||||
let execution_admission = self.registry.execution_admission_health().await.is_ok();
|
||||
let store = Arc::clone(&self.artifact_store);
|
||||
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
|
||||
.await
|
||||
@@ -157,6 +160,7 @@ impl AdminService {
|
||||
ReadinessChecks {
|
||||
postgres,
|
||||
artifact_storage,
|
||||
execution_admission,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -725,6 +729,7 @@ impl AdminService {
|
||||
validate_response_cache_policy(&payload.target, &payload.execution_config)?;
|
||||
validate_idempotency_policy(&payload.target, &payload.execution_config)?;
|
||||
validate_approval_policy(&payload.execution_config)?;
|
||||
validate_execution_policy(&payload.execution_config)?;
|
||||
payload.input_mapping.validate_paths()?;
|
||||
payload.output_mapping.validate_paths()?;
|
||||
Ok(())
|
||||
@@ -738,6 +743,7 @@ impl AdminService {
|
||||
validate_response_cache_policy(&operation.target, &operation.execution_config)?;
|
||||
validate_idempotency_policy(&operation.target, &operation.execution_config)?;
|
||||
validate_approval_policy(&operation.execution_config)?;
|
||||
validate_execution_policy(&operation.execution_config)?;
|
||||
operation.input_mapping.validate_paths()?;
|
||||
operation.output_mapping.validate_paths()?;
|
||||
Ok(())
|
||||
|
||||
@@ -441,6 +441,7 @@ fn demo_rest_operation_payload() -> OperationPayload {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -635,6 +635,7 @@ impl AdminService {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -610,6 +610,7 @@ impl AdminService {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -160,6 +160,23 @@ pub(super) fn validate_approval_policy(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_execution_policy(
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<(), ApiError> {
|
||||
let Some(policy) = execution_config.execution_policy.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Err(error) = policy.validate() {
|
||||
return Err(ApiError::validation_with_context(
|
||||
error.to_string(),
|
||||
json!({
|
||||
"field": "execution_config.execution_policy.deadline_ms",
|
||||
}),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
@@ -171,8 +188,8 @@ mod tests {
|
||||
};
|
||||
|
||||
use super::{
|
||||
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
|
||||
validate_response_cache_policy,
|
||||
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
|
||||
validate_idempotency_policy, validate_response_cache_policy,
|
||||
};
|
||||
|
||||
fn cacheable_execution_config() -> ExecutionConfig {
|
||||
@@ -183,6 +200,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
@@ -201,6 +219,7 @@ mod tests {
|
||||
}),
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
@@ -365,4 +384,36 @@ mod tests {
|
||||
"approval ttl must be between 1 and 300 seconds"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_published_execution_policy_and_absent_legacy_policy() {
|
||||
let mut config = cacheable_execution_config();
|
||||
validate_execution_policy(&config).unwrap();
|
||||
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Adaptive,
|
||||
deadline_ms: 120_000,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
});
|
||||
validate_execution_policy(&config).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_and_overflowing_execution_policy_deadline() {
|
||||
let mut config = cacheable_execution_config();
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 0,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
|
||||
});
|
||||
let error = validate_execution_policy(&config).unwrap_err();
|
||||
assert!(matches!(error, crate::error::ApiError::Validation { .. }));
|
||||
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Background,
|
||||
deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
|
||||
});
|
||||
assert!(validate_execution_policy(&config).is_err());
|
||||
}
|
||||
}
|
||||
@@ -495,6 +495,7 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -14,13 +14,14 @@ use crank_observability::{
|
||||
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
|
||||
};
|
||||
use crank_registry::{
|
||||
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
|
||||
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
|
||||
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry,
|
||||
};
|
||||
use crank_runtime::{
|
||||
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
|
||||
RuntimeLimits, SecretCrypto,
|
||||
};
|
||||
use mcp_server::pool_metrics::spawn_postgres_pool_metrics;
|
||||
use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics};
|
||||
use sqlx::postgres::PgConnectOptions;
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::info;
|
||||
@@ -175,6 +176,7 @@ async fn run(
|
||||
};
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
spawn_admission_metrics(registry.pool().clone());
|
||||
}
|
||||
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
|
||||
let secret_crypto =
|
||||
@@ -190,6 +192,10 @@ async fn run(
|
||||
.with_limits(runtime_limits)
|
||||
.with_response_cache(cache_stores.response.clone())
|
||||
.with_coordination_store(cache_stores.coordination.clone())
|
||||
.with_admission(execution_admission(
|
||||
registry.pool().clone(),
|
||||
runtime_limits,
|
||||
)?)
|
||||
.build();
|
||||
// Published static Resources are read from the same immutable content-addressed root the
|
||||
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
|
||||
@@ -287,6 +293,21 @@ fn init_observability(
|
||||
)?)
|
||||
}
|
||||
|
||||
fn execution_admission(
|
||||
pool: sqlx::PgPool,
|
||||
runtime_limits: RuntimeLimits,
|
||||
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
|
||||
let store = PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..AdmissionLimits::default()
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?;
|
||||
Ok(std::sync::Arc::new(store))
|
||||
}
|
||||
|
||||
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
|
||||
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
|
||||
database_options(&config.database).map_err(|_| invalid("database.source"))?;
|
||||
@@ -307,6 +328,12 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi
|
||||
config.runtime.max_concurrent_sessions,
|
||||
)
|
||||
.map_err(|_| invalid("runtime.limits"))?;
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: config.runtime.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
}
|
||||
.validate()
|
||||
.map_err(|_| invalid("runtime.admission"))?;
|
||||
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
|
||||
RequestRateLimitConfig::new(
|
||||
config.rate_limit.requests_per_second,
|
||||
|
||||
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
|
||||
/// to zero) without waiting for a new request.
|
||||
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
|
||||
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,874 @@
|
||||
//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission.
|
||||
//!
|
||||
//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a
|
||||
//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported
|
||||
//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal.
|
||||
//!
|
||||
//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this
|
||||
//! app already wires both and this is the real deployment shape.
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence,
|
||||
ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
|
||||
ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation,
|
||||
OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
|
||||
OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget,
|
||||
RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy,
|
||||
WorkspaceId,
|
||||
};
|
||||
use crank_mapping::MappingSet;
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
|
||||
use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
struct CountingAdapter {
|
||||
calls: Arc<AtomicUsize>,
|
||||
current: Arc<AtomicUsize>,
|
||||
peak: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProtocolAdapter for CountingAdapter {
|
||||
fn protocol(&self) -> Protocol {
|
||||
Protocol::Rest
|
||||
}
|
||||
|
||||
fn supports_mode(&self, mode: ExecutionMode) -> bool {
|
||||
mode == ExecutionMode::Unary
|
||||
}
|
||||
|
||||
async fn invoke_unary(
|
||||
&self,
|
||||
_target: &Target,
|
||||
_prepared: &crank_core::PreparedRequest,
|
||||
context: &RuntimeRequestContext,
|
||||
) -> Result<AdapterResponse, crank_core::ProtocolAdapterError> {
|
||||
context.mark_dispatch_started();
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let current = self.current.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
self.peak.fetch_max(current, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
self.current.fetch_sub(1, Ordering::SeqCst);
|
||||
Ok(AdapterResponse {
|
||||
status_code: 200,
|
||||
headers: BTreeMap::new(),
|
||||
body: json!({"ok": true}),
|
||||
data: json!({"ok": true}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn schema(kind: SchemaKind) -> Schema {
|
||||
Schema {
|
||||
kind,
|
||||
format: None,
|
||||
description: None,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: None,
|
||||
fields: BTreeMap::new(),
|
||||
items: None,
|
||||
enum_values: Vec::new(),
|
||||
variants: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn operation(policy: Option<ToolExecutionPolicy>) -> Operation<Schema, MappingSet> {
|
||||
Operation {
|
||||
id: OperationId::new("op_admission_budget"),
|
||||
name: "admission_budget".to_owned(),
|
||||
display_name: "Admission budget".to_owned(),
|
||||
category: "general".to_owned(),
|
||||
protocol: Protocol::Rest,
|
||||
security_level: OperationSecurityLevel::Standard,
|
||||
status: OperationStatus::Published,
|
||||
version: 7,
|
||||
target: Target::Rest(RestTarget {
|
||||
base_url: "https://api.example.com".to_owned(),
|
||||
method: HttpMethod::Get,
|
||||
path_template: "/v1/items".to_owned(),
|
||||
static_headers: BTreeMap::new(),
|
||||
request_contract: None,
|
||||
response_contract: Vec::new(),
|
||||
multipart_field_mime_types: BTreeMap::new(),
|
||||
}),
|
||||
input_schema: schema(SchemaKind::Object),
|
||||
output_schema: schema(SchemaKind::Object),
|
||||
input_mapping: MappingSet::default(),
|
||||
output_mapping: MappingSet::default(),
|
||||
execution_config: ExecutionConfig {
|
||||
timeout_ms: 5_000,
|
||||
execution_policy: policy,
|
||||
..ExecutionConfig::default()
|
||||
},
|
||||
tool_description: ToolDescription {
|
||||
title: "Admission".to_owned(),
|
||||
description: "Admission budget probe".to_owned(),
|
||||
tags: Vec::new(),
|
||||
examples: Vec::new(),
|
||||
},
|
||||
samples: None,
|
||||
generated_draft: None,
|
||||
config_export: None,
|
||||
wizard_state: None,
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
updated_at: OffsetDateTime::now_utc(),
|
||||
published_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
executor: &RuntimeExecutor,
|
||||
operation: &crank_runtime::RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
|
||||
let workspace = WorkspaceId::new("ws_test");
|
||||
let context = crank_runtime::RuntimeRequestContext::from_correlation(
|
||||
&crank_core::CorrelationContext::generate(),
|
||||
)
|
||||
.with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun);
|
||||
execute_in(executor, operation, input, &context).await
|
||||
}
|
||||
|
||||
async fn execute_in(
|
||||
executor: &RuntimeExecutor,
|
||||
operation: &crank_runtime::RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
context: &crank_runtime::RuntimeRequestContext,
|
||||
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
|
||||
let workspace = context
|
||||
.metering_context()
|
||||
.map(|metering| metering.workspace_id.clone())
|
||||
.unwrap_or_else(|| WorkspaceId::new("ws_test"));
|
||||
let request = crank_runtime::RuntimeExecutionRequest::try_new(
|
||||
&workspace,
|
||||
ExecutionOrigin::AdminDraft,
|
||||
None,
|
||||
operation,
|
||||
input,
|
||||
crank_runtime::ExecutionAuthorization::Authorized,
|
||||
None,
|
||||
context,
|
||||
Instant::now() + Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| {
|
||||
crank_core::ExecutionFailure::new(
|
||||
ExecutionErrorCode::RuntimeInternal,
|
||||
crank_core::CorrelationContext::new(
|
||||
context.request_id.clone(),
|
||||
context.trace_context.clone(),
|
||||
),
|
||||
)
|
||||
})?;
|
||||
executor
|
||||
.execute_outcome(request)
|
||||
.await
|
||||
.map(|success| success.output)
|
||||
}
|
||||
|
||||
async fn schema_pool(schema_url: &str) -> PgPool {
|
||||
let pool = PgPool::connect(schema_url)
|
||||
.await
|
||||
.expect("admission schema pool");
|
||||
crank_registry::MigrationAuthority::apply(&pool)
|
||||
.await
|
||||
.expect("migration authority applies the admission schema");
|
||||
pool
|
||||
}
|
||||
|
||||
fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc<PostgresExecutionAdmission> {
|
||||
Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: global,
|
||||
agent_limit: 8,
|
||||
tool_limit: 8,
|
||||
queue_limit: queue,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)
|
||||
}
|
||||
|
||||
fn executor(
|
||||
store: Arc<PostgresExecutionAdmission>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
current: Arc<AtomicUsize>,
|
||||
peak: Arc<AtomicUsize>,
|
||||
) -> RuntimeExecutor {
|
||||
RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(CountingAdapter {
|
||||
calls,
|
||||
current,
|
||||
peak,
|
||||
}))
|
||||
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
|
||||
.with_admission(store)
|
||||
.build()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_executors_share_one_global_budget() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_budget").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool.clone(), 2, 8);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let first = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
let second = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(None).into();
|
||||
let mut tasks = Vec::new();
|
||||
for index in 0..4 {
|
||||
let executor = if index % 2 == 0 {
|
||||
first.clone()
|
||||
} else {
|
||||
second.clone()
|
||||
};
|
||||
let operation = operation.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
execute(&executor, &operation, &json!({})).await
|
||||
}));
|
||||
}
|
||||
let mut completed = 0;
|
||||
for task in tasks {
|
||||
match task.await.expect("execution task") {
|
||||
Ok(_) => completed += 1,
|
||||
Err(failure) => panic!("admission must grant every request: {failure:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(completed, 4);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 4);
|
||||
assert!(
|
||||
peak.load(Ordering::SeqCst) <= 2,
|
||||
"peak concurrency {} exceeded the shared global limit",
|
||||
peak.load(Ordering::SeqCst)
|
||||
);
|
||||
assert_eq!(store.snapshot().await.unwrap().executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturation_returns_typed_overload_without_an_outbound_call() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_overload").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 1, 1);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
// Another process already holds the only global permit.
|
||||
let held = match store
|
||||
.admit(AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: "ws_test".to_owned(),
|
||||
agent_key: "admin".to_owned(),
|
||||
operation_id: "op_admission_budget".to_owned(),
|
||||
operation_version: 7,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_holder".to_owned(),
|
||||
expires_at_unix_ms: crank_core::current_unix_ms() + 10_000,
|
||||
})
|
||||
.await
|
||||
.expect("admission store is reachable")
|
||||
{
|
||||
crank_core::AdmissionOutcome::Permitted(permit) => permit,
|
||||
other => panic!("expected a permit, got {other:?}"),
|
||||
};
|
||||
|
||||
let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 250,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}))
|
||||
.into();
|
||||
let waiting_runtime = runtime.clone();
|
||||
let first_waiting_operation = waiting_operation.clone();
|
||||
let waiting = tokio::spawn(async move {
|
||||
execute(&waiting_runtime, &first_waiting_operation, &json!({})).await
|
||||
});
|
||||
// Let the first request occupy the bounded queue before the next one arrives.
|
||||
while store.snapshot().await.unwrap().waiting == 0 {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
let overloaded = execute(&runtime, &waiting_operation, &json!({})).await;
|
||||
let failure = overloaded.expect_err("a full queue must refuse the request");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::ExecutionOverloaded
|
||||
);
|
||||
assert_eq!(failure.stage(), ExecutionStage::Admission);
|
||||
assert_eq!(failure.retry_after_ms(), Some(250));
|
||||
|
||||
// The queued request hits its absolute deadline instead of dispatching. That refusal happened
|
||||
// inside admission (nothing was ever dispatched), so the client must see an admission-stage
|
||||
// overload with retry guidance — not an `upstream_timeout`.
|
||||
let expired = waiting
|
||||
.await
|
||||
.expect("waiting task")
|
||||
.expect_err("deadline must refuse");
|
||||
assert_eq!(
|
||||
expired.error_code(),
|
||||
ExecutionErrorCode::ExecutionOverloaded,
|
||||
"queue waiting counts against the published deadline"
|
||||
);
|
||||
assert_eq!(expired.stage(), ExecutionStage::Admission);
|
||||
assert_eq!(expired.retry_after_ms(), Some(250));
|
||||
assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain);
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"no outbound call on refusal"
|
||||
);
|
||||
assert!(store.release(&held).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_policy_without_a_task_transport_is_typed_unsupported() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 2_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}))
|
||||
.into();
|
||||
let failure = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect_err("no Task transport exists in Story 4.1");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::ExecutionModeUnsupported
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
let snapshot = store.snapshot().await.unwrap();
|
||||
assert_eq!(snapshot.waiting, 0);
|
||||
assert_eq!(snapshot.executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Adaptive,
|
||||
deadline_ms: 2_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}))
|
||||
.into();
|
||||
let output = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect("adaptive resolves through the published synchronous fallback");
|
||||
// The response shape is unchanged by admission: an operation without output mapping still
|
||||
// returns the same mapped output object it returned before Story 4.1.
|
||||
assert_eq!(output, json!({}));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(store.snapshot().await.unwrap().executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admission_store_outage_fails_closed_before_dispatch() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_outage").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool.clone(), 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
// The mandatory state disappears after the store was built: every execution must fail closed.
|
||||
sqlx::query("drop table execution_admission_entries")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("drop the mandatory admission state");
|
||||
let operation: crank_runtime::RuntimeOperation = operation(None).into();
|
||||
let failure = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect_err("a missing mandatory store must fail closed");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert!(store.health().await.is_err());
|
||||
}
|
||||
|
||||
/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2).
|
||||
struct RecordingAdmission {
|
||||
inner: Arc<PostgresExecutionAdmission>,
|
||||
requests: Mutex<Vec<AdmissionRequest>>,
|
||||
permits: AtomicUsize,
|
||||
releases: AtomicUsize,
|
||||
}
|
||||
|
||||
impl RecordingAdmission {
|
||||
fn new(inner: Arc<PostgresExecutionAdmission>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
permits: AtomicUsize::new(0),
|
||||
releases: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn requests(&self) -> Vec<AdmissionRequest> {
|
||||
self.requests.lock().expect("admission log").clone()
|
||||
}
|
||||
|
||||
fn permits(&self) -> usize {
|
||||
self.permits.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn releases(&self) -> usize {
|
||||
self.releases.load(Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionAdmission for RecordingAdmission {
|
||||
async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> {
|
||||
self.inner.health().await
|
||||
}
|
||||
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<crank_core::AdmissionOutcome, crank_core::AdmissionStoreError> {
|
||||
self.requests
|
||||
.lock()
|
||||
.expect("admission log")
|
||||
.push(request.clone());
|
||||
let outcome = self.inner.admit(request).await?;
|
||||
if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) {
|
||||
self.permits.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn release(
|
||||
&self,
|
||||
permit: &crank_core::AdmissionPermit,
|
||||
) -> Result<bool, crank_core::AdmissionStoreError> {
|
||||
let released = self.inner.release(permit).await?;
|
||||
if released {
|
||||
self.releases.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(released)
|
||||
}
|
||||
|
||||
async fn cancel(
|
||||
&self,
|
||||
ticket_id: u64,
|
||||
owner_token: &str,
|
||||
) -> Result<bool, crank_core::AdmissionStoreError> {
|
||||
self.inner.cancel(ticket_id, owner_token).await
|
||||
}
|
||||
|
||||
async fn snapshot(
|
||||
&self,
|
||||
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
|
||||
self.inner.snapshot().await
|
||||
}
|
||||
}
|
||||
|
||||
fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation {
|
||||
let mut source = operation(None);
|
||||
source.id = OperationId::new("op_admission_confirmation");
|
||||
source.name = "admission_confirmation".to_owned();
|
||||
source.version = version;
|
||||
let Target::Rest(target) = &mut source.target;
|
||||
target.method = HttpMethod::Delete;
|
||||
source.execution_config.safety = Some(OperationSafetyPolicy {
|
||||
class: OperationSafetyClass::Destructive,
|
||||
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
|
||||
});
|
||||
source.into()
|
||||
}
|
||||
|
||||
fn confirmation_runtime(
|
||||
admission: Arc<RecordingAdmission>,
|
||||
coordination: Option<Arc<dyn crank_core::CoordinationStateStore>>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
) -> RuntimeExecutor {
|
||||
let store: Arc<dyn ExecutionAdmission> = admission;
|
||||
let builder = RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(CountingAdapter {
|
||||
calls,
|
||||
current: Arc::new(AtomicUsize::new(0)),
|
||||
peak: Arc::new(AtomicUsize::new(0)),
|
||||
}))
|
||||
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
|
||||
.with_admission(store);
|
||||
match coordination {
|
||||
Some(coordination) => builder.with_coordination_store(coordination).build(),
|
||||
None => builder.build(),
|
||||
}
|
||||
}
|
||||
|
||||
fn confirmation_context() -> crank_runtime::RuntimeRequestContext {
|
||||
crank_runtime::RuntimeRequestContext::from_correlation(
|
||||
&crank_core::CorrelationContext::generate(),
|
||||
)
|
||||
.with_response_cache_scope("ws_test", "agent_a")
|
||||
.with_metering_context(
|
||||
WorkspaceId::new("ws_test"),
|
||||
None,
|
||||
InvocationSource::AdminTestRun,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let recording = Arc::new(RecordingAdmission::new(Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)));
|
||||
let coordination: Arc<dyn crank_core::CoordinationStateStore> =
|
||||
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = confirmation_runtime(
|
||||
Arc::clone(&recording),
|
||||
Some(Arc::clone(&coordination)),
|
||||
Arc::clone(&calls),
|
||||
);
|
||||
let operation = confirmation_operation(3);
|
||||
let context = confirmation_context();
|
||||
let input = json!({ "reason": "remove the stale item" });
|
||||
|
||||
// 1. No approval yet: the existing challenge is returned and no execution capacity is taken.
|
||||
let challenge = execute_in(&runtime, &operation, &input, &context)
|
||||
.await
|
||||
.expect_err("an unapproved destructive call must only stage confirmation");
|
||||
assert_eq!(
|
||||
challenge.error_code(),
|
||||
ExecutionErrorCode::ConfirmationRequired
|
||||
);
|
||||
let token = challenge
|
||||
.confirmation()
|
||||
.expect("expected the confirmation challenge")
|
||||
.token()
|
||||
.to_owned();
|
||||
assert_eq!(
|
||||
recording.permits(),
|
||||
0,
|
||||
"an unapproved request must not take an execution permit"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(
|
||||
requests.len(),
|
||||
1,
|
||||
"the one admission snapshot is consulted once"
|
||||
);
|
||||
assert_eq!(
|
||||
requests[0].approval,
|
||||
ApprovalEvidence::NotGranted,
|
||||
"admission must see the real approval state, not a fixed NotRequired"
|
||||
);
|
||||
assert_eq!(requests[0].scope.operation_version, 3);
|
||||
let snapshot = recording.snapshot().await.expect("admission snapshot");
|
||||
assert_eq!((snapshot.executing, snapshot.waiting), (0, 0));
|
||||
|
||||
// 2. An approval issued for another published version never opens this execution.
|
||||
let other_version = confirmation_operation(4);
|
||||
let foreign = execute_in(
|
||||
&runtime,
|
||||
&other_version,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect_err("a foreign approval must not authorise this version");
|
||||
assert_eq!(
|
||||
foreign.error_code(),
|
||||
ExecutionErrorCode::ConfirmationInvalid
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
1,
|
||||
"an unusable approval is refused before any capacity request"
|
||||
);
|
||||
|
||||
// 3. An approval is bound to its exact input as well.
|
||||
let mismatched_input = json!({ "reason": "remove another item" });
|
||||
let mismatched = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&mismatched_input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect_err("a confirmation token must not authorise different input");
|
||||
assert_eq!(
|
||||
mismatched.error_code(),
|
||||
ExecutionErrorCode::ConfirmationInvalid
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(recording.requests().len(), 1);
|
||||
|
||||
// 4. The matching approval passes admission with truthful evidence and executes exactly once.
|
||||
let executed = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("a verified approval executes");
|
||||
assert!(executed.is_object());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(requests[1].approval, ApprovalEvidence::Granted);
|
||||
assert_eq!(requests[1].scope.operation_version, 3);
|
||||
assert_eq!(recording.permits(), 1);
|
||||
assert_eq!(
|
||||
recording.releases(),
|
||||
1,
|
||||
"the permit is released after execution"
|
||||
);
|
||||
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
|
||||
|
||||
// 5. Replay of the consumed confirmation is refused before capacity and without a side effect.
|
||||
let replay = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token),
|
||||
)
|
||||
.await
|
||||
.expect_err("a confirmation token is single-use");
|
||||
assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(recording.permits(), 1);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
2,
|
||||
"a replayed approval never reaches admission"
|
||||
);
|
||||
|
||||
// 6. The trusted approved-recovery path still passes the same gate with real evidence.
|
||||
let approved = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&json!({ "reason": "approved by an operator" }),
|
||||
&context.with_approval_granted(),
|
||||
)
|
||||
.await;
|
||||
assert!(approved.is_ok());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert_eq!(requests[2].approval, ApprovalEvidence::Granted);
|
||||
assert_eq!(recording.permits(), 2);
|
||||
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mandatory_safety_store_outage_fails_closed_before_admission() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let recording = Arc::new(RecordingAdmission::new(Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)));
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
// Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail
|
||||
// closed instead of reserving capacity and hoping for an in-memory fallback.
|
||||
let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls));
|
||||
let operation = confirmation_operation(3);
|
||||
|
||||
let failure = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&json!({ "reason": "no store" }),
|
||||
&confirmation_context(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a mandatory safety store outage must fail closed");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
let after_missing_store = recording.requests().len();
|
||||
|
||||
// The same Tool with a claimed approval and a *failing* safety store never reaches the capacity
|
||||
// store at all: the approval claim stays unverified, so no permit and no outbound call exist.
|
||||
let failing = confirmation_runtime(
|
||||
Arc::clone(&recording),
|
||||
Some(Arc::new(UnavailableSafetyStore)),
|
||||
Arc::clone(&calls),
|
||||
);
|
||||
let outage = execute_in(
|
||||
&failing,
|
||||
&operation,
|
||||
&json!({ "reason": "failing store" }),
|
||||
&confirmation_context().with_confirmation_token("ct_unavailable"),
|
||||
)
|
||||
.await
|
||||
.expect_err("an unreachable safety store must refuse the claimed approval");
|
||||
assert_eq!(
|
||||
outage.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
after_missing_store,
|
||||
"an unverified approval never becomes a capacity request"
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Coordination store whose mandatory backend is down: every operation fails closed.
|
||||
struct UnavailableSafetyStore;
|
||||
|
||||
impl UnavailableSafetyStore {
|
||||
fn error() -> crank_core::CacheStoreError {
|
||||
crank_core::CacheStoreError::Unavailable {
|
||||
message: "test backend unavailable".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crank_core::CoordinationStateStore for UnavailableSafetyStore {
|
||||
async fn get_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn put_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<(), crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn delete_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<(), crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn take_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn reserve_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<crank_core::CoordinationStateReservation, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn compare_and_set_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_expected: &crank_core::CoordinationStateValue,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<bool, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
}
|
||||
@@ -621,6 +621,7 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation<Schema, Ma
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -1036,6 +1036,7 @@ var TRANSLATIONS = {
|
||||
'wizard.error.secret_name_required': 'Secret name is required.',
|
||||
'wizard.error.secret_value_required': 'Secret value is required.',
|
||||
'wizard.error.save_upstream_first': 'Save the API host after creating a new authorization profile before continuing.',
|
||||
'wizard.error.execution_policy_invalid': 'Execution policy is invalid: mode, deadline from 1 to 3600000 ms and no-Tasks fallback are required.',
|
||||
'wizard.toast.auth_profile_created_title': 'Authorization profile created',
|
||||
'wizard.toast.auth_profile_created_body': 'New authorization profile is ready and selected.',
|
||||
'wizard.toast.auth_profile_error_title': 'Authorization profile creation failed',
|
||||
@@ -2353,6 +2354,7 @@ var TRANSLATIONS = {
|
||||
'wizard.error.secret_name_required': 'Нужно указать имя секрета.',
|
||||
'wizard.error.secret_value_required': 'Нужно указать значение секрета.',
|
||||
'wizard.error.save_upstream_first': 'Сначала сохраните API-хост после создания нового профиля авторизации.',
|
||||
'wizard.error.execution_policy_invalid': 'Политика выполнения некорректна: нужны режим, deadline от 1 до 3600000 мс и поведение без Tasks.',
|
||||
'wizard.toast.auth_profile_created_title': 'Профиль авторизации создан',
|
||||
'wizard.toast.auth_profile_created_body': 'Новый профиль авторизации готов и уже выбран.',
|
||||
'wizard.toast.auth_profile_error_title': 'Не удалось создать профиль авторизации',
|
||||
|
||||
@@ -214,10 +214,33 @@ function parseExecutionConfig(text) {
|
||||
auth_profile_ref: authProfileRef,
|
||||
headers: headers,
|
||||
};
|
||||
var executionPolicy = parseExecutionPolicy(value.execution_policy);
|
||||
if (executionPolicy) {
|
||||
config.execution_policy = executionPolicy;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
// Story 4.1: the published execution policy must survive a wizard save. The editor is a free-form
|
||||
// document, so unknown/legacy shapes stay untouched: only a fully valid policy is projected, and an
|
||||
// explicit invalid value is reported instead of being silently replaced by defaults.
|
||||
function parseExecutionPolicy(raw) {
|
||||
if (raw === undefined || raw === null) return null;
|
||||
if (typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
throw new Error(tKey('wizard.error.execution_policy_invalid'));
|
||||
}
|
||||
var mode = raw.mode === undefined ? 'synchronous' : raw.mode;
|
||||
var fallback = raw.no_tasks_fallback === undefined ? 'synchronous' : raw.no_tasks_fallback;
|
||||
var deadline = Number(raw.deadline_ms);
|
||||
if (['synchronous', 'background', 'adaptive'].indexOf(mode) < 0
|
||||
|| ['synchronous', 'unsupported'].indexOf(fallback) < 0
|
||||
|| !Number.isInteger(deadline) || deadline < 1 || deadline > 3600000) {
|
||||
throw new Error(tKey('wizard.error.execution_policy_invalid'));
|
||||
}
|
||||
return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback };
|
||||
}
|
||||
|
||||
function currentUpstream() {
|
||||
var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; });
|
||||
if (selected) return selected;
|
||||
@@ -415,6 +438,13 @@ function executionConfigToEditorValue(config) {
|
||||
var value = {
|
||||
timeout_ms: config.timeout_ms,
|
||||
};
|
||||
if (config.execution_policy) {
|
||||
value.execution_policy = {
|
||||
mode: config.execution_policy.mode,
|
||||
deadline_ms: config.execution_policy.deadline_ms,
|
||||
no_tasks_fallback: config.execution_policy.no_tasks_fallback,
|
||||
};
|
||||
}
|
||||
if (config.retry_policy) {
|
||||
value.retry = { max_attempts: config.retry_policy.max_attempts };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
const { test, expect } = require('@playwright/test');
|
||||
const { getCurrentWorkspace, login } = require('./helpers');
|
||||
|
||||
// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a
|
||||
// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit
|
||||
// invalid value must be reported instead of being silently replaced by defaults.
|
||||
function draftSnapshot(operationId) {
|
||||
return {
|
||||
id: operationId,
|
||||
name: operationId,
|
||||
display_name: 'Policy rates',
|
||||
category: 'general',
|
||||
protocol: 'rest',
|
||||
target: {
|
||||
kind: 'rest',
|
||||
base_url: 'https://api.example.test',
|
||||
method: 'GET',
|
||||
path_template: '/rates',
|
||||
static_headers: {},
|
||||
},
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
description: null,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: null,
|
||||
fields: {},
|
||||
items: null,
|
||||
enum_values: [],
|
||||
variants: [],
|
||||
},
|
||||
output_schema: {
|
||||
type: 'object',
|
||||
description: null,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: null,
|
||||
fields: {},
|
||||
items: null,
|
||||
enum_values: [],
|
||||
variants: [],
|
||||
},
|
||||
input_mapping: { rules: [] },
|
||||
output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] },
|
||||
execution_config: {
|
||||
timeout_ms: 10000,
|
||||
retry_policy: null,
|
||||
auth_profile_ref: null,
|
||||
headers: {},
|
||||
approval_policy: null,
|
||||
execution_policy: {
|
||||
mode: 'adaptive',
|
||||
deadline_ms: 45000,
|
||||
no_tasks_fallback: 'unsupported',
|
||||
},
|
||||
},
|
||||
tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] },
|
||||
wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] },
|
||||
};
|
||||
}
|
||||
|
||||
test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => {
|
||||
await login(page);
|
||||
const workspace = await getCurrentWorkspace(page);
|
||||
const operationId = 'policy_rates_41';
|
||||
const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`;
|
||||
const savedPayloads = [];
|
||||
|
||||
await page.route(operationPath, async (route) => {
|
||||
if (route.request().method() === 'PATCH') {
|
||||
savedPayloads.push(route.request().postDataJSON());
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
id: operationId,
|
||||
workspace_id: workspace.id,
|
||||
name: operationId,
|
||||
display_name: 'Policy rates',
|
||||
category: 'general',
|
||||
protocol: 'rest',
|
||||
status: 'draft',
|
||||
current_draft_version: 1,
|
||||
draft_version_ref: { version: 1, status: 'draft' },
|
||||
published_version_ref: null,
|
||||
agent_refs: [],
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route(`${operationPath}/versions/*`, async (route) => {
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
operation_id: operationId,
|
||||
workspace_id: workspace.id,
|
||||
version: 1,
|
||||
status: 'draft',
|
||||
snapshot: draftSnapshot(operationId),
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route('**/operations/analyze-quality', async (route) => {
|
||||
await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) });
|
||||
});
|
||||
page.on('dialog', (dialog) => dialog.accept());
|
||||
|
||||
await page.goto(`/wizard/?mode=edit&operationId=${operationId}`);
|
||||
await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates');
|
||||
|
||||
const editorValue = await page.locator('#tool-exec-config').inputValue();
|
||||
expect(editorValue).toContain('execution_policy');
|
||||
expect(editorValue).toContain('adaptive');
|
||||
expect(editorValue).toContain('45000');
|
||||
expect(editorValue).toContain('unsupported');
|
||||
|
||||
// A real edit still saves the pinned policy unchanged.
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(4));
|
||||
await page.locator('#tool-display-name').fill('Policy rates edited');
|
||||
await page.locator('.btn-save-draft').click();
|
||||
await expect.poll(() => savedPayloads.length).toBe(1);
|
||||
expect(savedPayloads[0].execution_config.execution_policy).toEqual({
|
||||
mode: 'adaptive',
|
||||
deadline_ms: 45000,
|
||||
no_tasks_fallback: 'unsupported',
|
||||
});
|
||||
|
||||
// An explicit invalid policy is refused instead of being silently rewritten to defaults.
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
|
||||
await expect(page.locator('#tool-exec-config')).toBeVisible();
|
||||
await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n');
|
||||
await page.locator('.btn-save-draft').click();
|
||||
await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/);
|
||||
expect(savedPayloads.length).toBe(1);
|
||||
});
|
||||
@@ -311,24 +311,41 @@ async fn health() -> Json<Value> {
|
||||
}
|
||||
|
||||
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
|
||||
match state.registry.ping().await {
|
||||
Ok(()) => Json(json!({
|
||||
let postgres_ready = state.registry.ping().await.is_ok();
|
||||
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
|
||||
// reflect it rather than reporting ready on a bare `select 1`.
|
||||
let admission_ready = state.registry.execution_admission_health().await.is_ok();
|
||||
if postgres_ready && admission_ready {
|
||||
return Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": "ready" }
|
||||
"checks": { "postgres": "ready", "execution_admission": "ready" }
|
||||
}))
|
||||
.into_response(),
|
||||
Err(_) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": "not_ready" },
|
||||
"error": "database is unavailable"
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
.into_response();
|
||||
}
|
||||
let mut checks = serde_json::Map::new();
|
||||
checks.insert(
|
||||
"postgres".to_owned(),
|
||||
json!(if postgres_ready { "ready" } else { "not_ready" }),
|
||||
);
|
||||
checks.insert(
|
||||
"execution_admission".to_owned(),
|
||||
json!(if admission_ready {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
}),
|
||||
);
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "not_ready",
|
||||
"checks": checks,
|
||||
"error": "a mandatory dependency is unavailable"
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -550,6 +550,7 @@ pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -136,6 +136,7 @@ fn published_tool(name: &str, title: &str, description: &str, version: u32) -> P
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -284,6 +284,7 @@ fn operation() -> RegistryOperation {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
//! Typed execution-admission seam (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Admission is the single gate in front of every real Tool execution: it decides whether the
|
||||
//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide
|
||||
//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives
|
||||
//! in PostgreSQL (`crank-registry`), so several processes share one budget.
|
||||
//!
|
||||
//! Deliberately absent from this contract: credentials, connection or protocol revision (they must
|
||||
//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2).
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{NoTasksFallback, ToolExecutionMode};
|
||||
|
||||
pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128;
|
||||
pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64;
|
||||
pub const MAX_RETRY_AFTER_MS: u64 = 60_000;
|
||||
/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for
|
||||
/// the store and every runtime so the client contract stays stable.
|
||||
pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250;
|
||||
|
||||
/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms).
|
||||
pub fn current_unix_ms() -> i64 {
|
||||
i64::try_from(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|elapsed| elapsed.as_millis())
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
.unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin
|
||||
/// bucket that is still bounded by the global limit.
|
||||
pub const ADMIN_ORIGIN_KEY: &str = "admin";
|
||||
|
||||
/// Exact execution identity pinned at admission time.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AdmissionScope {
|
||||
pub workspace_id: String,
|
||||
/// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution.
|
||||
pub agent_key: String,
|
||||
pub operation_id: String,
|
||||
/// Quota keys exclude the version; the version is pinned for execution semantics only.
|
||||
pub operation_version: u32,
|
||||
}
|
||||
|
||||
impl AdmissionScope {
|
||||
/// Quota key of the Agent bucket: `(workspace, agent)`.
|
||||
pub fn agent_bucket(&self) -> String {
|
||||
format!("{}:{}", self.workspace_id, self.agent_key)
|
||||
}
|
||||
|
||||
/// Quota key of the Tool bucket: `(workspace, operation)` without version.
|
||||
pub fn tool_bucket(&self) -> String {
|
||||
format!("{}:{}", self.workspace_id, self.operation_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AdmissionMode {
|
||||
/// The caller waits for a scoped permit and dispatches inside it.
|
||||
Synchronous,
|
||||
/// The store commits a durable enqueue decision and reserves queue capacity.
|
||||
Background,
|
||||
}
|
||||
|
||||
impl AdmissionMode {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Synchronous => "synchronous",
|
||||
Self::Background => "background",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-consuming approval evidence gathered by the runtime before admission.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ApprovalEvidence {
|
||||
/// The operation does not require confirmation.
|
||||
NotRequired,
|
||||
/// A verified approval exists for this exact scope and input.
|
||||
Granted,
|
||||
/// Confirmation is required and no verified approval exists yet.
|
||||
NotGranted,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionRequest {
|
||||
pub scope: AdmissionScope,
|
||||
pub mode: AdmissionMode,
|
||||
/// Published policy mode that produced this request.
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
pub no_tasks_fallback: NoTasksFallback,
|
||||
pub approval: ApprovalEvidence,
|
||||
/// Bounded request identity used only to correlate one admission entry.
|
||||
pub fingerprint: String,
|
||||
/// Absolute deadline (unix milliseconds), already including queue waiting time.
|
||||
pub expires_at_unix_ms: i64,
|
||||
}
|
||||
|
||||
impl AdmissionRequest {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
if self.fingerprint.is_empty()
|
||||
|| self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES
|
||||
|| self
|
||||
.fingerprint
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_control() || byte == b' ')
|
||||
{
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission fingerprint is outside its allowed bounds",
|
||||
));
|
||||
}
|
||||
if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission workspace is invalid",
|
||||
));
|
||||
}
|
||||
if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid("admission agent is invalid"));
|
||||
}
|
||||
if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission operation is invalid",
|
||||
));
|
||||
}
|
||||
if self.scope.operation_version == 0 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission operation version must be positive",
|
||||
));
|
||||
}
|
||||
if self.expires_at_unix_ms <= 0 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission deadline is invalid",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Scoped permit. Held for the whole local execution future (including retries) and released
|
||||
/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionPermit {
|
||||
pub ticket_id: u64,
|
||||
pub owner_token: String,
|
||||
pub scope: AdmissionScope,
|
||||
pub mode: AdmissionMode,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
}
|
||||
|
||||
/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics.
|
||||
/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionDecision {
|
||||
pub ticket_id: u64,
|
||||
pub scope: AdmissionScope,
|
||||
pub expires_at_unix_ms: i64,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AdmissionOutcome {
|
||||
Permitted(AdmissionPermit),
|
||||
Enqueued(AdmissionDecision),
|
||||
/// Capacity or queue bounds were exhausted before any unbounded allocation.
|
||||
Overloaded {
|
||||
retry_after_ms: u64,
|
||||
},
|
||||
/// The queue wait reached the absolute deadline; nothing was dispatched.
|
||||
DeadlineElapsed,
|
||||
/// Confirmation is mandatory for this operation and no verified approval exists.
|
||||
ApprovalRequired,
|
||||
/// The published policy does not advertise durable Tasks and refuses the fallback.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct AdmissionQueueSnapshot {
|
||||
pub waiting: u64,
|
||||
pub executing: u64,
|
||||
/// Age of the oldest waiting request in the authoritative queue, in milliseconds.
|
||||
pub oldest_waiting_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum AdmissionStoreError {
|
||||
/// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed.
|
||||
#[error("execution admission store is unavailable: {message}")]
|
||||
Unavailable { message: String },
|
||||
/// The store rejected a malformed admission request.
|
||||
#[error("execution admission request is invalid: {message}")]
|
||||
Invalid { message: String },
|
||||
}
|
||||
|
||||
impl AdmissionStoreError {
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::Unavailable {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn invalid(message: impl Into<String>) -> Self {
|
||||
Self::Invalid {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ExecutionAdmission: Send + Sync {
|
||||
/// Proves the mandatory admission state is usable (schema and operations reachable).
|
||||
async fn health(&self) -> Result<(), AdmissionStoreError>;
|
||||
|
||||
/// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue
|
||||
/// decision for a background request, or returns a typed refusal.
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<AdmissionOutcome, AdmissionStoreError>;
|
||||
|
||||
/// Releases the caller's own permit. Idempotent: a repeated release returns `false`.
|
||||
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError>;
|
||||
|
||||
/// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket.
|
||||
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError>;
|
||||
|
||||
/// Bounded authoritative queue readings for low-cardinality metrics.
|
||||
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError>;
|
||||
}
|
||||
|
||||
/// One waiting entry as seen by the fairness scheduler.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionQueueEntry {
|
||||
pub ticket_id: u64,
|
||||
/// Authoritative insertion order inside `(Agent, Tool)`; never rewritten.
|
||||
pub sequence: u64,
|
||||
pub agent_key: String,
|
||||
pub operation_id: String,
|
||||
}
|
||||
|
||||
/// Weighted round-robin state persisted with the authoritative queue.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct AdmissionCursor {
|
||||
pub agent: Option<String>,
|
||||
pub agent_credit: u32,
|
||||
pub tool: Option<String>,
|
||||
pub tool_credit: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionSchedulerConfig {
|
||||
pub agent_weight: u32,
|
||||
pub tool_weight: u32,
|
||||
}
|
||||
|
||||
impl Default for AdmissionSchedulerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
agent_weight: 1,
|
||||
tool_weight: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdmissionSchedulerConfig {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission weights must be between 1 and 64",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the
|
||||
/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its
|
||||
/// turn instead of being skipped by newer arrivals.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionSelection {
|
||||
pub index: usize,
|
||||
pub cursor: AdmissionCursor,
|
||||
}
|
||||
|
||||
/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside
|
||||
/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit.
|
||||
pub fn select_next_grant(
|
||||
queue: &[AdmissionQueueEntry],
|
||||
cursor: &AdmissionCursor,
|
||||
config: AdmissionSchedulerConfig,
|
||||
) -> Option<AdmissionSelection> {
|
||||
if queue.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let agent_weight = config.agent_weight.max(1);
|
||||
let tool_weight = config.tool_weight.max(1);
|
||||
let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect();
|
||||
agents.sort_unstable();
|
||||
agents.dedup();
|
||||
|
||||
let mut next = cursor.clone();
|
||||
let agent = match cursor.agent.as_deref() {
|
||||
Some(current) if cursor.agent_credit > 0 && agents.contains(¤t) => current.to_owned(),
|
||||
_ => {
|
||||
let chosen = match cursor.agent.as_deref() {
|
||||
None => agents[0].to_owned(),
|
||||
Some(current) => agents
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| *candidate > current)
|
||||
.unwrap_or(agents[0])
|
||||
.to_owned(),
|
||||
};
|
||||
if cursor.agent.as_deref() != Some(chosen.as_str()) {
|
||||
next.tool = None;
|
||||
next.tool_credit = 0;
|
||||
}
|
||||
next.agent = Some(chosen.clone());
|
||||
next.agent_credit = agent_weight;
|
||||
chosen
|
||||
}
|
||||
};
|
||||
|
||||
let mut tools: Vec<&str> = queue
|
||||
.iter()
|
||||
.filter(|entry| entry.agent_key == agent)
|
||||
.map(|entry| entry.operation_id.as_str())
|
||||
.collect();
|
||||
tools.sort_unstable();
|
||||
tools.dedup();
|
||||
|
||||
let tool = match cursor.tool.as_deref() {
|
||||
Some(current) if cursor.tool_credit > 0 && tools.contains(¤t) => current.to_owned(),
|
||||
_ => {
|
||||
let chosen = match cursor.tool.as_deref() {
|
||||
None => tools[0].to_owned(),
|
||||
Some(current) => tools
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| *candidate > current)
|
||||
.unwrap_or(tools[0])
|
||||
.to_owned(),
|
||||
};
|
||||
next.tool = Some(chosen.clone());
|
||||
next.tool_credit = tool_weight;
|
||||
chosen
|
||||
}
|
||||
};
|
||||
|
||||
let index = queue
|
||||
.iter()
|
||||
.position(|entry| entry.agent_key == agent && entry.operation_id == tool)?;
|
||||
next.agent_credit = next.agent_credit.saturating_sub(1);
|
||||
next.tool_credit = next.tool_credit.saturating_sub(1);
|
||||
Some(AdmissionSelection {
|
||||
index,
|
||||
cursor: next,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest,
|
||||
AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence,
|
||||
select_next_grant,
|
||||
};
|
||||
use crate::{NoTasksFallback, ToolExecutionMode};
|
||||
|
||||
fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry {
|
||||
super::AdmissionQueueEntry {
|
||||
ticket_id,
|
||||
sequence: ticket_id,
|
||||
agent_key: agent.to_owned(),
|
||||
operation_id: tool.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn request() -> AdmissionRequest {
|
||||
AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
agent_key: ADMIN_ORIGIN_KEY.to_owned(),
|
||||
operation_id: "op_leads".to_owned(),
|
||||
operation_version: 3,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_0123456789".to_owned(),
|
||||
expires_at_unix_ms: 1_790_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_keys_exclude_version_and_keep_agent_bucket() {
|
||||
let scope = request().scope;
|
||||
assert_eq!(scope.agent_bucket(), "ws_default:admin");
|
||||
assert_eq!(scope.tool_bucket(), "ws_default:op_leads");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() {
|
||||
let mut invalid = request();
|
||||
invalid.expires_at_unix_ms = 0;
|
||||
assert!(matches!(
|
||||
invalid.validate(),
|
||||
Err(AdmissionStoreError::Invalid { .. })
|
||||
));
|
||||
|
||||
let mut invalid = request();
|
||||
invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1);
|
||||
assert!(invalid.validate().is_err());
|
||||
|
||||
let mut invalid = request();
|
||||
invalid.scope.operation_version = 0;
|
||||
assert!(invalid.validate().is_err());
|
||||
|
||||
assert!(request().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weights_are_bounded_positive_configuration() {
|
||||
assert!(
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 0,
|
||||
tool_weight: 1
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 1,
|
||||
tool_weight: 65
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(AdmissionSchedulerConfig::default().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weighted_round_robin_serves_each_agent_its_quantum() {
|
||||
let mut queue = vec![
|
||||
entry(1, "agent_a", "op_1"),
|
||||
entry(2, "agent_a", "op_1"),
|
||||
entry(3, "agent_a", "op_1"),
|
||||
entry(4, "agent_b", "op_1"),
|
||||
entry(5, "agent_b", "op_1"),
|
||||
];
|
||||
let config = AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
};
|
||||
let mut cursor = AdmissionCursor::default();
|
||||
let mut order = Vec::new();
|
||||
while !queue.is_empty() {
|
||||
let selection = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
order.push(queue.remove(selection.index).agent_key.clone());
|
||||
cursor = selection.cursor;
|
||||
}
|
||||
assert_eq!(
|
||||
order,
|
||||
vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_inside_one_tool_and_round_robin_between_tools() {
|
||||
let mut queue = vec![
|
||||
entry(1, "agent_a", "op_1"),
|
||||
entry(2, "agent_a", "op_1"),
|
||||
entry(3, "agent_a", "op_2"),
|
||||
];
|
||||
let config = AdmissionSchedulerConfig::default();
|
||||
let mut cursor = AdmissionCursor::default();
|
||||
let mut order = Vec::new();
|
||||
while !queue.is_empty() {
|
||||
let selection = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
// A granted ticket leaves the waiting queue: granted entries are never re-granted.
|
||||
order.push(queue.remove(selection.index).ticket_id);
|
||||
cursor = selection.cursor;
|
||||
}
|
||||
// One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue.
|
||||
assert_eq!(order, vec![1, 3, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() {
|
||||
let queue = vec![entry(1, "agent_a", "op_1")];
|
||||
let config = AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
};
|
||||
let cursor = AdmissionCursor::default();
|
||||
let first = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
assert_eq!(first.cursor.agent.as_deref(), Some("agent_a"));
|
||||
assert_eq!(first.cursor.agent_credit, 1);
|
||||
|
||||
// A later Agent joins; the cursor keeps its remaining quantum for `agent_a`.
|
||||
let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")];
|
||||
let second = select_next_grant(&queue, &first.cursor, config).unwrap();
|
||||
assert_eq!(queue[second.index].agent_key, "agent_a");
|
||||
assert_eq!(second.cursor.agent_credit, 0);
|
||||
|
||||
let third = select_next_grant(&queue, &second.cursor, config).unwrap();
|
||||
assert_eq!(queue[third.index].agent_key, "agent_b");
|
||||
assert_eq!(third.cursor.agent_credit, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_queue_keeps_cursor_untouched() {
|
||||
let cursor = AdmissionCursor {
|
||||
agent: Some("agent_b".to_owned()),
|
||||
agent_credit: 1,
|
||||
tool: Some("op_1".to_owned()),
|
||||
tool_credit: 0,
|
||||
};
|
||||
assert_eq!(
|
||||
select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()),
|
||||
None
|
||||
);
|
||||
assert_eq!(cursor.agent.as_deref(), Some("agent_b"));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod access;
|
||||
pub mod admission;
|
||||
pub mod agent;
|
||||
pub mod approval;
|
||||
pub mod auth;
|
||||
@@ -60,10 +61,11 @@ pub mod domain {
|
||||
pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
|
||||
pub use crate::operation::{
|
||||
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
|
||||
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
|
||||
IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode,
|
||||
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
|
||||
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy,
|
||||
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState,
|
||||
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
|
||||
ToolExecutionPolicy, WizardState,
|
||||
};
|
||||
pub use crate::product_event::{
|
||||
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
|
||||
@@ -114,6 +116,13 @@ pub use access::{
|
||||
InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
|
||||
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
|
||||
};
|
||||
pub use admission::{
|
||||
ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
|
||||
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
|
||||
AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission,
|
||||
MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant,
|
||||
};
|
||||
pub use agent::{
|
||||
Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
|
||||
ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
|
||||
@@ -168,13 +177,14 @@ pub use observability::{
|
||||
pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
|
||||
pub use operation::{
|
||||
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
|
||||
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
|
||||
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
|
||||
OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError,
|
||||
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState,
|
||||
ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor,
|
||||
RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample,
|
||||
WizardState,
|
||||
IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation,
|
||||
OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
|
||||
OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle,
|
||||
OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy,
|
||||
OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract,
|
||||
RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget,
|
||||
RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
|
||||
ToolExecutionPolicy, ToolExecutionPolicyError, WizardState,
|
||||
};
|
||||
pub use product_event::{
|
||||
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
|
||||
|
||||
@@ -270,6 +270,72 @@ pub struct OperationApprovalPolicy {
|
||||
pub elicitation_message: Option<String>,
|
||||
}
|
||||
|
||||
/// Published execution mode of one Tool (Story 4.1, FR-21).
|
||||
///
|
||||
/// This is a scheduling policy of the Published Tool version. It is deliberately separate from
|
||||
/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolExecutionMode {
|
||||
#[default]
|
||||
Synchronous,
|
||||
Background,
|
||||
Adaptive,
|
||||
}
|
||||
|
||||
/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NoTasksFallback {
|
||||
/// Execute through the same synchronous admission path.
|
||||
#[default]
|
||||
Synchronous,
|
||||
/// Reject with a typed unsupported-execution-mode failure; never a Task ID.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000;
|
||||
|
||||
/// Execution policy pinned by a Published Tool version.
|
||||
///
|
||||
/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by
|
||||
/// `ExecutionConfig::timeout_ms`, with no background path.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ToolExecutionPolicy {
|
||||
#[serde(default)]
|
||||
pub mode: ToolExecutionMode,
|
||||
/// Absolute budget of one execution, counted from the start of admission.
|
||||
pub deadline_ms: u64,
|
||||
#[serde(default)]
|
||||
pub no_tasks_fallback: NoTasksFallback,
|
||||
}
|
||||
|
||||
impl Default for ToolExecutionPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolExecutionPolicy {
|
||||
pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> {
|
||||
if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) {
|
||||
return Err(ToolExecutionPolicyError::DeadlineOutOfRange);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ToolExecutionPolicyError {
|
||||
#[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")]
|
||||
DeadlineOutOfRange,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ExecutionConfig {
|
||||
@@ -285,6 +351,8 @@ pub struct ExecutionConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub approval_policy: Option<OperationApprovalPolicy>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub execution_policy: Option<ToolExecutionPolicy>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub auth_profile_ref: Option<AuthProfileId>,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub headers: BTreeMap<String, String>,
|
||||
@@ -444,9 +512,10 @@ mod tests {
|
||||
edition::OperationSecurityLevel,
|
||||
ids::{AuthProfileId, OperationId},
|
||||
operation::{
|
||||
ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle,
|
||||
OperationLifecycleAction, OperationLifecycleError, OperationStatus,
|
||||
ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability,
|
||||
OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus,
|
||||
OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample,
|
||||
ToolExecutionMode, ToolExecutionPolicy,
|
||||
},
|
||||
protocol::{AuthKind, ExportMode, HttpMethod, Protocol},
|
||||
};
|
||||
@@ -664,6 +733,7 @@ updated_at: 2026-03-25T08:10:00Z
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: Some(AuthProfileId::new("auth_01")),
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
@@ -684,4 +754,76 @@ updated_at: 2026-03-25T08:10:00Z
|
||||
published_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_round_trips_all_modes_and_fallbacks() {
|
||||
for (mode, fallback) in [
|
||||
(ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous),
|
||||
(ToolExecutionMode::Background, NoTasksFallback::Synchronous),
|
||||
(ToolExecutionMode::Background, NoTasksFallback::Unsupported),
|
||||
(ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported),
|
||||
] {
|
||||
let config = ExecutionConfig {
|
||||
execution_policy: Some(ToolExecutionPolicy {
|
||||
mode,
|
||||
deadline_ms: 45_000,
|
||||
no_tasks_fallback: fallback,
|
||||
}),
|
||||
..ExecutionConfig::default()
|
||||
};
|
||||
let encoded = serde_json::to_value(&config).unwrap();
|
||||
assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000));
|
||||
let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap();
|
||||
assert_eq!(decoded, config);
|
||||
assert_eq!(
|
||||
decoded.execution_policy.unwrap().mode,
|
||||
mode,
|
||||
"the published mode must survive the JSON round-trip"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_execution_config_without_policy_stays_synchronous() {
|
||||
let legacy: ExecutionConfig =
|
||||
serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap();
|
||||
assert_eq!(legacy.execution_policy, None);
|
||||
assert_eq!(legacy.timeout_ms, 12_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_rejects_explicit_invalid_deadline() {
|
||||
assert!(
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 0,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Adaptive,
|
||||
deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(ToolExecutionPolicy::default().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_rejects_unknown_fields() {
|
||||
assert!(
|
||||
serde_json::from_value::<ToolExecutionPolicy>(json!({
|
||||
"mode": "background",
|
||||
"deadline_ms": 1000,
|
||||
"no_tasks_fallback": "synchronous",
|
||||
"task_queue": "default"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -231,6 +231,7 @@ fn default_execution_config() -> ExecutionConfig {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
|
||||
@@ -564,6 +564,7 @@ fn default_execution_config() -> ExecutionConfig {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
|
||||
@@ -217,6 +217,34 @@ string_enum! {
|
||||
Concurrency => "concurrency",
|
||||
RateLimit => "rate_limit",
|
||||
McpStream => "mcp_stream",
|
||||
Admission => "admission",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum RejectionReason {
|
||||
Limit => "limit",
|
||||
Capacity => "capacity",
|
||||
QueueFull => "queue_full",
|
||||
Deadline => "deadline",
|
||||
Unsupported => "unsupported",
|
||||
DependencyUnavailable => "dependency_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum ScopeKind {
|
||||
Process => "process",
|
||||
Global => "global",
|
||||
Agent => "agent",
|
||||
Tool => "tool",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum AdmissionModeLabel {
|
||||
Synchronous => "synchronous",
|
||||
Background => "background",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,14 +16,15 @@ mod schema;
|
||||
|
||||
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
|
||||
pub use labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
|
||||
UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
pub use record::{
|
||||
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service,
|
||||
bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome,
|
||||
bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting,
|
||||
record_admission_queue_wait, record_admission_rejection, record_cache_outcome,
|
||||
record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
|
||||
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
|
||||
|
||||
@@ -5,10 +5,10 @@ use metrics::Gauge;
|
||||
|
||||
use crate::schema::{MetricDefinition, metric_schema};
|
||||
use crate::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter,
|
||||
HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage,
|
||||
McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason,
|
||||
ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
|
||||
static BINDING: OnceLock<MetricService> = OnceLock::new();
|
||||
@@ -352,11 +352,47 @@ pub fn record_limit_rejection(stage: LimitStage) {
|
||||
}
|
||||
metrics::counter!(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
"stage" => stage.as_str()
|
||||
"stage" => stage.as_str(),
|
||||
"reason" => RejectionReason::Limit.as_str(),
|
||||
"scope_kind" => ScopeKind::Process.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool,
|
||||
/// request id, fingerprint or arbitrary error text ever becomes a label.
|
||||
pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) {
|
||||
if bound_family("crank_runtime_limit_rejections_total").is_none() {
|
||||
return;
|
||||
}
|
||||
metrics::counter!(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
"stage" => LimitStage::Admission.as_str(),
|
||||
"reason" => reason.as_str(),
|
||||
"scope_kind" => scope_kind.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) {
|
||||
if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() {
|
||||
return;
|
||||
}
|
||||
metrics::gauge!(
|
||||
"crank_runtime_admission_queue_wait_seconds",
|
||||
"mode" => mode.as_str()
|
||||
)
|
||||
.set(waited_ms as f64 / 1000.0);
|
||||
}
|
||||
|
||||
pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option<u64>) {
|
||||
if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() {
|
||||
return;
|
||||
}
|
||||
let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0;
|
||||
metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds);
|
||||
}
|
||||
|
||||
pub fn record_cache_outcome(outcome: CacheOutcome) {
|
||||
if bound_family("crank_runtime_cache_total").is_none() {
|
||||
return;
|
||||
|
||||
@@ -3,16 +3,17 @@
|
||||
|
||||
use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP};
|
||||
use crate::labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
|
||||
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
|
||||
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome,
|
||||
UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
|
||||
pub const METRIC_SCHEMA_VERSION: u32 = 2;
|
||||
pub const MAX_METRIC_FAMILIES: usize = 128;
|
||||
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611;
|
||||
pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348;
|
||||
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
@@ -179,6 +180,9 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
|
||||
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
|
||||
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
|
||||
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
|
||||
const REJECTION_REASONS: &[&str] = RejectionReason::VALUES;
|
||||
const SCOPE_KINDS: &[&str] = ScopeKind::VALUES;
|
||||
const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES;
|
||||
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
|
||||
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
|
||||
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
|
||||
@@ -403,13 +407,39 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[
|
||||
"crank_runtime_limit_rejections_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["stage"],
|
||||
&[domain("stage", LIMIT_STAGES)],
|
||||
&["stage", "reason", "scope_kind"],
|
||||
&[
|
||||
domain("stage", LIMIT_STAGES),
|
||||
domain("reason", REJECTION_REASONS),
|
||||
domain("scope_kind", SCOPE_KINDS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
3,
|
||||
96,
|
||||
"Runtime executions rejected by a bounded limit.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_admission_queue_wait_seconds",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Seconds,
|
||||
&["mode"],
|
||||
&[domain("mode", ADMISSION_MODES)],
|
||||
&[],
|
||||
BOTH,
|
||||
2,
|
||||
"Wait of the most recently admitted execution before it received a scoped permit.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_admission_oldest_waiting_seconds",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Seconds,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"Age of the oldest waiting admission request.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_cache_total",
|
||||
MetricKind::Counter,
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
|
||||
use crank_metrics::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics,
|
||||
bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome,
|
||||
record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage,
|
||||
McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType,
|
||||
ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema,
|
||||
record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection,
|
||||
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
|
||||
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
|
||||
set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
|
||||
@@ -71,6 +72,9 @@ fn schema_names_are_unique_and_labels_are_closed() {
|
||||
"state",
|
||||
"signal_type",
|
||||
"exporter",
|
||||
"reason",
|
||||
"scope_kind",
|
||||
"mode",
|
||||
];
|
||||
for metric in schema {
|
||||
assert!(
|
||||
@@ -265,6 +269,9 @@ fn every_recording_api_matches_the_declared_schema() {
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
record_limit_rejection(LimitStage::Concurrency);
|
||||
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
|
||||
record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5);
|
||||
record_admission_oldest_waiting(Some(5));
|
||||
record_cache_outcome(CacheOutcome::Hit);
|
||||
record_idempotency_outcome(IdempotencyOutcome::Replay);
|
||||
record_confirmation_outcome(ConfirmationOutcome::Required);
|
||||
|
||||
@@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
|
||||
|
||||
#[test]
|
||||
fn frozen_budget_edges_are_exact() {
|
||||
assert!(validate_budget(6_284, 19_514).is_ok());
|
||||
assert!(validate_budget(6_285, 19_515).is_ok());
|
||||
assert!(validate_budget(6_286, 16_000).is_err());
|
||||
assert!(validate_budget(6_000, 19_516).is_err());
|
||||
assert!(validate_budget(6_380, 19_610).is_ok());
|
||||
assert!(validate_budget(6_381, 19_611).is_ok());
|
||||
assert!(validate_budget(6_382, 19_000).is_err());
|
||||
assert!(validate_budget(6_000, 19_612).is_err());
|
||||
assert_eq!(
|
||||
crank_metrics::max_exemplar_slots(MetricService::AdminApi),
|
||||
12_348
|
||||
@@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() {
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::AdminApi),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 6_285,
|
||||
rendered_series: 19_515
|
||||
logical_series: 6_381,
|
||||
rendered_series: 19_611
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::McpServer),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 1_083,
|
||||
rendered_series: 3_558
|
||||
logical_series: 1_179,
|
||||
rendered_series: 3_654
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() {
|
||||
"root обязан объявлять ровно один ключ budgets"
|
||||
);
|
||||
assert!(rendered.ends_with(
|
||||
",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\
|
||||
\"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
|
||||
\"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n"
|
||||
",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\
|
||||
\"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
|
||||
\"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n"
|
||||
));
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::AdminApi),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 6_285,
|
||||
rendered_series: 19_515
|
||||
logical_series: 6_381,
|
||||
rendered_series: 19_611
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::McpServer),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 1_083,
|
||||
rendered_series: 3_558
|
||||
logical_series: 1_179,
|
||||
rendered_series: 3_654
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() {
|
||||
/// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны.
|
||||
#[test]
|
||||
fn frozen_caps_and_edges_are_exact() {
|
||||
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285);
|
||||
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515);
|
||||
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381);
|
||||
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611);
|
||||
assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348);
|
||||
assert!(validate_budget(6_285, 19_515).is_ok());
|
||||
assert!(validate_budget(6_286, 19_515).is_err());
|
||||
assert!(validate_budget(6_381, 19_611).is_ok());
|
||||
assert!(validate_budget(6_382, 19_611).is_err());
|
||||
assert!(validate_exemplar_budget(12_347).is_ok());
|
||||
assert!(validate_exemplar_budget(12_348).is_ok());
|
||||
assert!(validate_exemplar_budget(12_349).is_err());
|
||||
|
||||
@@ -13,7 +13,7 @@ mod schema;
|
||||
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
|
||||
pub use crank_metrics::{
|
||||
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit,
|
||||
bind_metric_service, bound_metric_service, metric_schema,
|
||||
bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting,
|
||||
};
|
||||
pub use error_reporting::{
|
||||
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
|
||||
|
||||
@@ -7,6 +7,7 @@ publish.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1"
|
||||
crank-artifacts = { path = "../crank-artifacts" }
|
||||
crank-core = { path = "../crank-core" }
|
||||
crank-mapping = { path = "../crank-mapping" }
|
||||
|
||||
@@ -116,6 +116,7 @@ pub use model::{
|
||||
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
|
||||
};
|
||||
pub use postgres::{
|
||||
AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError,
|
||||
PostgresRegistry,
|
||||
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
|
||||
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, admission_health,
|
||||
admission_snapshot,
|
||||
};
|
||||
@@ -6,6 +6,7 @@ mod artifact_cleanup_indexes_v13;
|
||||
mod artifact_metadata_v12;
|
||||
mod authority;
|
||||
mod baseline_v1;
|
||||
mod execution_admission_v24;
|
||||
mod execution_outcome_v5;
|
||||
mod import_plan_v14;
|
||||
mod master_key_identity_v7;
|
||||
@@ -35,6 +36,7 @@ mod schema_guard_v19;
|
||||
mod schema_guard_v20;
|
||||
mod schema_guard_v22;
|
||||
mod schema_guard_v23;
|
||||
mod schema_guard_v24;
|
||||
mod schema_guard_v7;
|
||||
mod schema_guard_v8;
|
||||
mod schema_guard_v9;
|
||||
|
||||
@@ -4,6 +4,7 @@ use super::agent_resource_bindings_v18;
|
||||
use super::approval_side_effects_v10;
|
||||
use super::artifact_cleanup_indexes_v13;
|
||||
use super::artifact_metadata_v12;
|
||||
use super::execution_admission_v24;
|
||||
use super::execution_outcome_v5;
|
||||
use super::import_plan_v14;
|
||||
use super::master_key_identity_v7;
|
||||
@@ -39,9 +40,9 @@ use preflight::*;
|
||||
use contract::baseline_source_digest;
|
||||
use contract::validate_descriptors;
|
||||
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
|
||||
const CURRENT_VERSION: i64 = 23;
|
||||
const CURRENT_VERSION: i64 = 24;
|
||||
const IMPLEMENTED_VERSIONS: &[i64] = &[
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
||||
];
|
||||
const BASELINE_SOURCE_SHA256: &str =
|
||||
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
|
||||
@@ -330,6 +331,12 @@ impl MigrationAuthority {
|
||||
prompt_dependencies_v23::SOURCE_SHA256,
|
||||
22,
|
||||
),
|
||||
expand_descriptor(
|
||||
24,
|
||||
execution_admission_v24::NAME,
|
||||
execution_admission_v24::SOURCE_SHA256,
|
||||
23,
|
||||
),
|
||||
]
|
||||
}
|
||||
pub fn validate_sequence() -> Result<(), MigrationError> {
|
||||
@@ -489,6 +496,9 @@ impl MigrationAuthority {
|
||||
if from < 23 {
|
||||
prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?;
|
||||
}
|
||||
if from < 24 {
|
||||
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
|
||||
}
|
||||
match inspect(&mut transaction).await? {
|
||||
MigrationPreflight::Current {
|
||||
version: CURRENT_VERSION,
|
||||
|
||||
@@ -140,6 +140,8 @@ pub(super) fn validate_descriptors(
|
||||
!= agent_resource_bindings_v18::SOURCE_SHA256
|
||||
|| sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes())
|
||||
!= resource_security_revocations_v19::SOURCE_SHA256
|
||||
|| sha256_hex(execution_admission_v24::SOURCE.as_bytes())
|
||||
!= execution_admission_v24::SOURCE_SHA256
|
||||
{
|
||||
return Err(MigrationError::new(
|
||||
"invalid_contract",
|
||||
|
||||
@@ -148,6 +148,7 @@ pub(super) async fn inspect(
|
||||
owned_relations::PROMPTS,
|
||||
owned_relations::PROMPT_PUBLICATIONS,
|
||||
owned_relations::PROMPT_DEPENDENCIES,
|
||||
owned_relations::EXECUTION_ADMISSION,
|
||||
] {
|
||||
validate_required_relations(connection, relations, CURRENT_VERSION).await?;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "execution-admission-v24";
|
||||
pub(super) const SOURCE: &str = include_str!("execution_admission_v24.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.execution_admission_v24",
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.canonical_ledger",
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
-- V24: authoritative execution admission state (Story 4.1, FR-21, FR-24).
|
||||
--
|
||||
-- Один deployment-wide admission: PostgreSQL хранит cursor/fairness state, ожидающие тикеты,
|
||||
-- выданные permits и durable enqueue decisions, поэтому несколько процессов делят общий бюджет.
|
||||
--
|
||||
-- `execution_admission_entries` — bounded набор: waiting (ожидает capacity), executing (scoped
|
||||
-- permit выдан, держится до явного release) и reserved (durable enqueue decision с зарезервированным
|
||||
-- queue slot; не исполняется и не рекламируется как Task до Story 4.2).
|
||||
--
|
||||
-- Строка не хранит ни credentials, ни connection/session id, ни prepared headers, ни тело запроса:
|
||||
-- quota key — (workspace, agent) и (workspace, operation) без version. `operation_version` пишется
|
||||
-- только как pin исполняемых semantics и не участвует в квотах.
|
||||
--
|
||||
-- `expires_at` — абсолютный deadline. Queue decisions без dispatch можно expire по deadline; активный
|
||||
-- permit НЕ освобождается по TTL (иначе paused/crashed owner обойдёт global limit): его снимает только
|
||||
-- явный release/cancel владельца.
|
||||
|
||||
create table execution_admission_controllers (
|
||||
workspace_id text not null,
|
||||
cursor_agent text,
|
||||
cursor_agent_credit integer not null default 0,
|
||||
cursor_tool text,
|
||||
cursor_tool_credit integer not null default 0,
|
||||
updated_at timestamptz not null default now(),
|
||||
constraint execution_admission_controllers_pkey primary key (workspace_id),
|
||||
constraint execution_admission_controllers_workspace_check check (
|
||||
octet_length(workspace_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_controllers_agent_credit_check check (
|
||||
cursor_agent_credit between 0 and 64
|
||||
),
|
||||
constraint execution_admission_controllers_tool_credit_check check (
|
||||
cursor_tool_credit between 0 and 64
|
||||
)
|
||||
);
|
||||
|
||||
create table execution_admission_entries (
|
||||
ticket_id bigint generated always as identity,
|
||||
workspace_id text not null,
|
||||
agent_key text not null,
|
||||
operation_id text not null,
|
||||
operation_version integer not null,
|
||||
mode text not null,
|
||||
policy_mode text not null,
|
||||
state text not null,
|
||||
owner_token text not null,
|
||||
fingerprint text not null,
|
||||
created_at timestamptz not null default now(),
|
||||
expires_at timestamptz not null,
|
||||
granted_at timestamptz,
|
||||
constraint execution_admission_entries_pkey primary key (ticket_id),
|
||||
constraint execution_admission_entries_workspace_check check (
|
||||
octet_length(workspace_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_agent_check check (
|
||||
octet_length(agent_key) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_operation_check check (
|
||||
octet_length(operation_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_version_check check (operation_version >= 1),
|
||||
constraint execution_admission_entries_mode_check check (
|
||||
mode in ('synchronous', 'background')
|
||||
),
|
||||
constraint execution_admission_entries_policy_mode_check check (
|
||||
policy_mode in ('synchronous', 'background', 'adaptive')
|
||||
),
|
||||
constraint execution_admission_entries_state_check check (
|
||||
state in ('waiting', 'executing', 'reserved')
|
||||
),
|
||||
constraint execution_admission_entries_owner_check check (
|
||||
octet_length(owner_token) between 1 and 64
|
||||
),
|
||||
constraint execution_admission_entries_fingerprint_check check (
|
||||
octet_length(fingerprint) between 1 and 128
|
||||
),
|
||||
constraint execution_admission_entries_reserved_mode_check check (
|
||||
state <> 'reserved' or mode = 'background'
|
||||
)
|
||||
);
|
||||
|
||||
create unique index execution_admission_entries_owner_uidx
|
||||
on execution_admission_entries (owner_token);
|
||||
|
||||
create index execution_admission_entries_state_idx
|
||||
on execution_admission_entries (state, workspace_id, ticket_id);
|
||||
|
||||
create index execution_admission_entries_agent_idx
|
||||
on execution_admission_entries (state, workspace_id, agent_key);
|
||||
|
||||
create index execution_admission_entries_operation_idx
|
||||
on execution_admission_entries (state, workspace_id, operation_id);
|
||||
@@ -55,6 +55,11 @@ pub(super) const PROMPT_PUBLICATIONS: &[&str] = &["prompt_publications", "agent_
|
||||
pub(super) const PROMPT_DEPENDENCIES: &[&str] =
|
||||
&["prompt_dependencies", "prompt_message_resource_refs"];
|
||||
|
||||
pub(super) const EXECUTION_ADMISSION: &[&str] = &[
|
||||
"execution_admission_controllers",
|
||||
"execution_admission_entries",
|
||||
];
|
||||
|
||||
pub(super) const ARTIFACT_METADATA: &[&str] = &["artifact_blobs", "artifact_sources"];
|
||||
|
||||
pub(super) const IMPORT_PLAN: &[&str] = &["import_plans", "import_plan_applications"];
|
||||
|
||||
@@ -468,6 +468,13 @@ pub(super) async fn validate_schema_fingerprint(
|
||||
} else {
|
||||
super::schema_guard_v23::validate_v23_prompt_dependencies(connection).await?;
|
||||
}
|
||||
if current_version < 24 {
|
||||
if !super::schema_guard_v24::validate_v24_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v24::validate_v24_execution_admission(connection).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,8 @@ pub(crate) const OWNED_RELATIONS: &[&str] = &[
|
||||
"prompt_messages",
|
||||
"prompt_dependencies",
|
||||
"prompt_message_resource_refs",
|
||||
"execution_admission_controllers",
|
||||
"execution_admission_entries",
|
||||
];
|
||||
|
||||
pub(crate) const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
//! V24 schema guard: authoritative execution admission state (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Схема V24 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
|
||||
//! ожидаемой форме: controller-row с weighted round-robin cursor и bounded entries с закрытым
|
||||
//! словарём mode/policy_mode/state, уникальным owner token и индексами квот.
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const CONTROLLERS: &str = "execution_admission_controllers";
|
||||
const ENTRIES: &str = "execution_admission_entries";
|
||||
|
||||
const OWNED_RELATIONS: &[&str] = &[CONTROLLERS, ENTRIES];
|
||||
|
||||
const CONTROLLER_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("workspace_id", "text", false),
|
||||
("cursor_agent", "text", true),
|
||||
("cursor_agent_credit", "integer", false),
|
||||
("cursor_tool", "text", true),
|
||||
("cursor_tool_credit", "integer", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
];
|
||||
|
||||
const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("ticket_id", "bigint", false),
|
||||
("workspace_id", "text", false),
|
||||
("agent_key", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("mode", "text", false),
|
||||
("policy_mode", "text", false),
|
||||
("state", "text", false),
|
||||
("owner_token", "text", false),
|
||||
("fingerprint", "text", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("expires_at", "timestamp with time zone", false),
|
||||
("granted_at", "timestamp with time zone", true),
|
||||
];
|
||||
|
||||
const CONSTRAINT_COUNT: &[(&str, usize)] = &[(CONTROLLERS, 4), (ENTRIES, 11)];
|
||||
|
||||
const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"];
|
||||
|
||||
fn v24_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
/// Схема V24 отсутствует целиком (для current_version < 24).
|
||||
pub(super) async fn validate_v24_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
for relation in OWNED_RELATIONS {
|
||||
if relation_exists(connection, relation).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
for index in UNIQUE_INDEXES {
|
||||
if index_exists(connection, index).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24).
|
||||
pub(super) async fn validate_v24_execution_admission(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
for relation in OWNED_RELATIONS {
|
||||
if !relation_exists(connection, relation).await? {
|
||||
return Err(v24_error("v24.relations"));
|
||||
}
|
||||
}
|
||||
for (table, columns) in [(CONTROLLERS, CONTROLLER_COLUMNS), (ENTRIES, ENTRY_COLUMNS)] {
|
||||
validate_columns(connection, table, columns).await?;
|
||||
}
|
||||
for (table, expected) in CONSTRAINT_COUNT {
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1",
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(*expected).unwrap_or_default() {
|
||||
return Err(v24_error("v24.constraints"));
|
||||
}
|
||||
}
|
||||
for (table, constraint) in [
|
||||
(CONTROLLERS, "execution_admission_controllers_pkey"),
|
||||
(ENTRIES, "execution_admission_entries_pkey"),
|
||||
(ENTRIES, "execution_admission_entries_state_check"),
|
||||
] {
|
||||
if !constraint_exists(connection, table, constraint).await? {
|
||||
return Err(v24_error("v24.constraint_missing"));
|
||||
}
|
||||
}
|
||||
if !normalized_expression(
|
||||
connection,
|
||||
ENTRIES,
|
||||
"execution_admission_entries_state_check",
|
||||
)
|
||||
.await?
|
||||
.is_some_and(|expression| {
|
||||
super::schema_guard::enum_constraint_matches(
|
||||
&expression,
|
||||
"state",
|
||||
&["waiting", "executing", "reserved"],
|
||||
)
|
||||
}) {
|
||||
return Err(v24_error("v24.state_vocabulary"));
|
||||
}
|
||||
for index in UNIQUE_INDEXES {
|
||||
let unique = index_unique(connection, index).await?;
|
||||
if unique != Some(true) {
|
||||
return Err(v24_error("v24.indexes"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_columns(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
columns: &[(&str, &str, bool)],
|
||||
) -> Result<(), MigrationError> {
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(columns.len()).unwrap_or_default() {
|
||||
return Err(v24_error("v24.columns"));
|
||||
}
|
||||
for (column, data_type, nullable) in columns {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v24_error("v24.columns"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn constraint_exists(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
) as present",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn normalized_expression(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<Option<String>, MigrationError> {
|
||||
let row = query(
|
||||
"select pg_get_expr(c.conbin, c.conrelid) as expression
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
and c.contype = 'c'",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
|
||||
}
|
||||
|
||||
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_class idx
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1
|
||||
) as present",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn index_unique(
|
||||
connection: &mut PgConnection,
|
||||
index: &str,
|
||||
) -> Result<Option<bool>, MigrationError> {
|
||||
let row = query(
|
||||
"select i.indisunique, i.indisvalid
|
||||
from pg_catalog.pg_index i
|
||||
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| {
|
||||
let unique = row.try_get::<bool, _>("indisunique").ok()?;
|
||||
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
|
||||
Some(unique && valid)
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,618 @@
|
||||
//! PostgreSQL implementation of the mandatory execution-admission store (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Source of truth for capacity and durable enqueue decisions of every process: a short transaction
|
||||
//! locks one deployment controller row and one workspace controller row in a fixed order, then reads
|
||||
//! bounded counters, inserts/transitions at most one entry and persists the weighted round-robin
|
||||
//! cursor. No network call happens under the transaction and no connection is held while a waiter is
|
||||
//! idle between attempts.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
|
||||
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
|
||||
AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, select_next_grant,
|
||||
};
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEPLOYMENT_CONTROLLER: &str = "*";
|
||||
const OVERLOAD_RETRY_AFTER_MS: u64 = crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS;
|
||||
const MAX_WAIT_BACKOFF_MS: u64 = 250;
|
||||
const INITIAL_WAIT_BACKOFF_MS: u64 = 5;
|
||||
const EXPIRED_CLEANUP_BATCH: i64 = 128;
|
||||
const QUEUE_READ_LIMIT: i64 = 1024;
|
||||
|
||||
/// Deployment-wide bounds of one admission store. They come from validated deployment
|
||||
/// configuration, never from a request.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionLimits {
|
||||
pub global_limit: usize,
|
||||
pub agent_limit: usize,
|
||||
pub tool_limit: usize,
|
||||
pub queue_limit: usize,
|
||||
}
|
||||
|
||||
impl Default for AdmissionLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
global_limit: 64,
|
||||
agent_limit: 16,
|
||||
tool_limit: 8,
|
||||
queue_limit: 32,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdmissionLimits {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
for (field, value) in [
|
||||
("global_limit", self.global_limit),
|
||||
("agent_limit", self.agent_limit),
|
||||
("tool_limit", self.tool_limit),
|
||||
] {
|
||||
if !(1..=65_535).contains(&value) {
|
||||
return Err(AdmissionStoreError::invalid(format!(
|
||||
"admission {field} is outside its allowed bounds"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if !(1..=1_024).contains(&self.queue_limit) {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission queue_limit is outside its allowed bounds",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PostgresExecutionAdmission {
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
}
|
||||
|
||||
impl PostgresExecutionAdmission {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
) -> Result<Self, AdmissionStoreError> {
|
||||
limits.validate()?;
|
||||
scheduler.validate()?;
|
||||
Ok(Self {
|
||||
pool,
|
||||
limits,
|
||||
scheduler,
|
||||
})
|
||||
}
|
||||
|
||||
fn unavailable(error: impl std::fmt::Display) -> AdmissionStoreError {
|
||||
AdmissionStoreError::unavailable(format!(
|
||||
"execution admission store is unavailable: {error}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
struct AttemptContext<'a> {
|
||||
request: &'a AdmissionRequest,
|
||||
owner_token: &'a str,
|
||||
}
|
||||
|
||||
enum Attempt {
|
||||
Granted,
|
||||
Enqueued(u64),
|
||||
Waiting,
|
||||
Overloaded,
|
||||
Expired,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionAdmission for PostgresExecutionAdmission {
|
||||
async fn health(&self) -> Result<(), AdmissionStoreError> {
|
||||
admission_health(&self.pool).await
|
||||
}
|
||||
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<AdmissionOutcome, AdmissionStoreError> {
|
||||
request.validate()?;
|
||||
if request.approval == ApprovalEvidence::NotGranted {
|
||||
return Ok(AdmissionOutcome::ApprovalRequired);
|
||||
}
|
||||
if request.mode == AdmissionMode::Background
|
||||
&& request.no_tasks_fallback == NoTasksFallback::Unsupported
|
||||
{
|
||||
return Ok(AdmissionOutcome::Unsupported);
|
||||
}
|
||||
|
||||
let owner_token = format!("adm_{}", Uuid::now_v7().simple());
|
||||
let mut ticket: Option<u64> = None;
|
||||
let mut backoff = INITIAL_WAIT_BACKOFF_MS;
|
||||
loop {
|
||||
if crank_core::current_unix_ms() >= request.expires_at_unix_ms {
|
||||
if let Some(ticket_id) = ticket {
|
||||
self.cancel(ticket_id, &owner_token).await?;
|
||||
}
|
||||
return Ok(AdmissionOutcome::DeadlineElapsed);
|
||||
}
|
||||
let attempt = self
|
||||
.attempt(
|
||||
AttemptContext {
|
||||
request: &request,
|
||||
owner_token: &owner_token,
|
||||
},
|
||||
&mut ticket,
|
||||
)
|
||||
.await?;
|
||||
match attempt {
|
||||
Attempt::Granted => {
|
||||
return Ok(AdmissionOutcome::Permitted(AdmissionPermit {
|
||||
ticket_id: ticket.unwrap_or_default(),
|
||||
owner_token,
|
||||
scope: request.scope.clone(),
|
||||
mode: request.mode,
|
||||
policy_mode: request.policy_mode,
|
||||
}));
|
||||
}
|
||||
Attempt::Enqueued(ticket_id) => {
|
||||
return Ok(AdmissionOutcome::Enqueued(AdmissionDecision {
|
||||
ticket_id,
|
||||
scope: request.scope.clone(),
|
||||
expires_at_unix_ms: request.expires_at_unix_ms,
|
||||
policy_mode: request.policy_mode,
|
||||
}));
|
||||
}
|
||||
Attempt::Expired => {
|
||||
return Ok(AdmissionOutcome::DeadlineElapsed);
|
||||
}
|
||||
Attempt::Overloaded => {
|
||||
return Ok(AdmissionOutcome::Overloaded {
|
||||
retry_after_ms: OVERLOAD_RETRY_AFTER_MS,
|
||||
});
|
||||
}
|
||||
Attempt::Waiting => {
|
||||
let remaining = request
|
||||
.expires_at_unix_ms
|
||||
.saturating_sub(crank_core::current_unix_ms())
|
||||
.max(1);
|
||||
let sleep_ms = i64::try_from(backoff).unwrap_or(remaining).min(remaining);
|
||||
tokio::time::sleep(Duration::from_millis(
|
||||
u64::try_from(sleep_ms).unwrap_or(backoff),
|
||||
))
|
||||
.await;
|
||||
backoff = (backoff * 2).min(MAX_WAIT_BACKOFF_MS);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError> {
|
||||
let result = sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id = $1 and owner_token = $2 and state = 'executing'",
|
||||
)
|
||||
.bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX))
|
||||
.bind(&permit.owner_token)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError> {
|
||||
let result = sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')",
|
||||
)
|
||||
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
|
||||
.bind(owner_token)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
|
||||
admission_snapshot(&self.pool).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness probe of the mandatory admission state: both tables must be reachable and readable.
|
||||
pub async fn admission_health(pool: &PgPool) -> Result<(), AdmissionStoreError> {
|
||||
sqlx::query(
|
||||
"select count(*)::bigint from execution_admission_controllers where workspace_id = $1",
|
||||
)
|
||||
.bind(DEPLOYMENT_CONTROLLER)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
sqlx::query("select count(*)::bigint from execution_admission_entries where false")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Authoritative queue readings of the deployment-wide admission state.
|
||||
pub async fn admission_snapshot(
|
||||
pool: &PgPool,
|
||||
) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
|
||||
let row = sqlx::query(
|
||||
"select
|
||||
count(*) filter (where state in ('waiting', 'reserved')) as queued,
|
||||
count(*) filter (where state = 'executing') as executing,
|
||||
coalesce(
|
||||
(extract(epoch from (now() - min(created_at) filter (where state in ('waiting', 'reserved')))) * 1000)::bigint,
|
||||
-1
|
||||
) as oldest_waiting_ms
|
||||
from execution_admission_entries",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
let queued: i64 = row
|
||||
.try_get("queued")
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
let executing: i64 = row
|
||||
.try_get("executing")
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
let oldest: i64 = row
|
||||
.try_get("oldest_waiting_ms")
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
Ok(AdmissionQueueSnapshot {
|
||||
waiting: u64::try_from(queued).unwrap_or_default(),
|
||||
executing: u64::try_from(executing).unwrap_or_default(),
|
||||
oldest_waiting_ms: (oldest >= 0).then(|| u64::try_from(oldest).unwrap_or_default()),
|
||||
})
|
||||
}
|
||||
|
||||
impl PostgresExecutionAdmission {
|
||||
async fn attempt(
|
||||
&self,
|
||||
context: AttemptContext<'_>,
|
||||
ticket: &mut Option<u64>,
|
||||
) -> Result<Attempt, AdmissionStoreError> {
|
||||
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local lock_timeout = '3000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local statement_timeout = '5000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
|
||||
match self.attempt_locked(&mut transaction, context, ticket).await {
|
||||
Ok(attempt) => {
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(attempt)
|
||||
}
|
||||
Err(AttemptError::Contended) => {
|
||||
transaction.rollback().await.ok();
|
||||
Ok(Attempt::Waiting)
|
||||
}
|
||||
Err(AttemptError::Overloaded) => {
|
||||
transaction.rollback().await.ok();
|
||||
Ok(Attempt::Overloaded)
|
||||
}
|
||||
Err(AttemptError::Failure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn attempt_locked(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
context: AttemptContext<'_>,
|
||||
ticket: &mut Option<u64>,
|
||||
) -> Result<Attempt, AttemptError> {
|
||||
let workspace_id = context.request.scope.workspace_id.as_str();
|
||||
sqlx::query(
|
||||
"insert into execution_admission_controllers (workspace_id)
|
||||
values ($1), ($2)
|
||||
on conflict (workspace_id) do nothing",
|
||||
)
|
||||
.bind(DEPLOYMENT_CONTROLLER)
|
||||
.bind(workspace_id)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
// Fixed lock order: deployment controller first, then the workspace controller.
|
||||
lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?;
|
||||
let cursor = lock_controller(transaction, workspace_id).await?;
|
||||
|
||||
// Expired queue decisions and waiting tickets leave the bounded queue; held permits never do.
|
||||
sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id in (
|
||||
select ticket_id from execution_admission_entries
|
||||
where state in ('waiting', 'reserved') and expires_at <= now()
|
||||
order by ticket_id
|
||||
limit $1
|
||||
)",
|
||||
)
|
||||
.bind(EXPIRED_CLEANUP_BATCH)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
if ticket.is_none() {
|
||||
let queued: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries
|
||||
where state in ('waiting', 'reserved')",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
if u64::try_from(queued).unwrap_or(u64::MAX)
|
||||
>= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX)
|
||||
{
|
||||
return Err(AttemptError::Overloaded);
|
||||
}
|
||||
let state = match context.request.mode {
|
||||
AdmissionMode::Synchronous => "waiting",
|
||||
AdmissionMode::Background => "reserved",
|
||||
};
|
||||
let row = sqlx::query(
|
||||
"insert into execution_admission_entries (
|
||||
workspace_id, agent_key, operation_id, operation_version, mode, policy_mode,
|
||||
state, owner_token, fingerprint, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
to_timestamp($10::double precision / 1000.0))
|
||||
returning ticket_id",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(&context.request.scope.agent_key)
|
||||
.bind(&context.request.scope.operation_id)
|
||||
.bind(i32::try_from(context.request.scope.operation_version).unwrap_or(i32::MAX))
|
||||
.bind(context.request.mode.as_str())
|
||||
.bind(policy_mode_str(context.request))
|
||||
.bind(state)
|
||||
.bind(context.owner_token)
|
||||
.bind(&context.request.fingerprint)
|
||||
.bind(context.request.expires_at_unix_ms)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
|
||||
*ticket = Some(u64::try_from(ticket_id).unwrap_or_default());
|
||||
if context.request.mode == AdmissionMode::Background {
|
||||
return Ok(Attempt::Enqueued(
|
||||
u64::try_from(ticket_id).unwrap_or_default(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
self.grant_next(transaction, workspace_id, &cursor).await?;
|
||||
|
||||
let Some(ticket_id) = *ticket else {
|
||||
return Ok(Attempt::Waiting);
|
||||
};
|
||||
let state: Option<String> =
|
||||
sqlx::query("select state from execution_admission_entries where ticket_id = $1")
|
||||
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.map(|row| row.try_get("state"))
|
||||
.transpose()
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
match state.as_deref() {
|
||||
Some("executing") => Ok(Attempt::Granted),
|
||||
Some(_) => Ok(Attempt::Waiting),
|
||||
// The ticket expired inside the store (or was removed by another cleanup).
|
||||
None => Ok(Attempt::Expired),
|
||||
}
|
||||
}
|
||||
|
||||
/// Grants at most one waiting ticket per transaction, following the persisted weighted
|
||||
/// round-robin cursor. A blocked head keeps its turn: nothing later is granted in its place.
|
||||
async fn grant_next(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
cursor: &AdmissionCursor,
|
||||
) -> Result<(), AttemptError> {
|
||||
let rows = sqlx::query(
|
||||
"select ticket_id, agent_key, operation_id from execution_admission_entries
|
||||
where workspace_id = $1 and state = 'waiting'
|
||||
order by ticket_id
|
||||
limit $2",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(QUEUE_READ_LIMIT)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let queue = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
Ok(AdmissionQueueEntry {
|
||||
ticket_id: u64::try_from(
|
||||
row.try_get::<i64, _>("ticket_id")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
sequence: 0,
|
||||
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
|
||||
operation_id: row
|
||||
.try_get("operation_id")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, AttemptError>>()?;
|
||||
let Some(selection) = select_next_grant(&queue, cursor, self.scheduler) else {
|
||||
return Ok(());
|
||||
};
|
||||
let head = &queue[selection.index];
|
||||
|
||||
let global_executing: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries
|
||||
where state = 'executing'",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let agent_counts = scoped_counts(transaction, workspace_id, "agent_key").await?;
|
||||
let tool_counts = scoped_counts(transaction, workspace_id, "operation_id").await?;
|
||||
if u64::try_from(global_executing).unwrap_or(u64::MAX)
|
||||
>= u64::try_from(self.limits.global_limit).unwrap_or(u64::MAX)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let agent_executing = agent_counts
|
||||
.get(head.agent_key.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if agent_executing >= u64::try_from(self.limits.agent_limit).unwrap_or(u64::MAX) {
|
||||
return Ok(());
|
||||
}
|
||||
let tool_executing = tool_counts
|
||||
.get(head.operation_id.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if tool_executing >= u64::try_from(self.limits.tool_limit).unwrap_or(u64::MAX) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"update execution_admission_entries
|
||||
set state = 'executing', granted_at = now()
|
||||
where ticket_id = $1 and state = 'waiting'",
|
||||
)
|
||||
.bind(i64::try_from(head.ticket_id).unwrap_or(i64::MAX))
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
sqlx::query(
|
||||
"update execution_admission_controllers
|
||||
set cursor_agent = $2,
|
||||
cursor_agent_credit = $3,
|
||||
cursor_tool = $4,
|
||||
cursor_tool_credit = $5,
|
||||
updated_at = now()
|
||||
where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(selection.cursor.agent.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
|
||||
.bind(selection.cursor.tool.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
enum AttemptError {
|
||||
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
|
||||
Contended,
|
||||
/// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait.
|
||||
Overloaded,
|
||||
Failure(AdmissionStoreError),
|
||||
}
|
||||
|
||||
impl AttemptError {
|
||||
fn from_sql(error: sqlx::Error) -> Self {
|
||||
if let Some(database) = error.as_database_error()
|
||||
&& matches!(database.code().as_deref(), Some("55P03" | "57014"))
|
||||
{
|
||||
return Self::Contended;
|
||||
}
|
||||
Self::Failure(PostgresExecutionAdmission::unavailable(error))
|
||||
}
|
||||
}
|
||||
|
||||
async fn lock_controller(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
) -> Result<AdmissionCursor, AttemptError> {
|
||||
sqlx::query(
|
||||
"update execution_admission_controllers set updated_at = updated_at
|
||||
where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let row = sqlx::query(
|
||||
"select cursor_agent, cursor_agent_credit, cursor_tool, cursor_tool_credit
|
||||
from execution_admission_controllers where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
Ok(AdmissionCursor {
|
||||
agent: row
|
||||
.try_get("cursor_agent")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
agent_credit: u32::try_from(
|
||||
row.try_get::<i32, _>("cursor_agent_credit")
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.max(0),
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
tool: row.try_get("cursor_tool").map_err(AttemptError::from_sql)?,
|
||||
tool_credit: u32::try_from(
|
||||
row.try_get::<i32, _>("cursor_tool_credit")
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.max(0),
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn scoped_counts(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
column: &str,
|
||||
) -> Result<BTreeMap<String, u64>, AttemptError> {
|
||||
let statement = if column == "agent_key" {
|
||||
"select agent_key as key, count(*)::bigint as count
|
||||
from execution_admission_entries
|
||||
where workspace_id = $1 and state = 'executing'
|
||||
group by agent_key"
|
||||
} else {
|
||||
"select operation_id as key, count(*)::bigint as count
|
||||
from execution_admission_entries
|
||||
where workspace_id = $1 and state = 'executing'
|
||||
group by operation_id"
|
||||
};
|
||||
let rows = sqlx::query(statement)
|
||||
.bind(workspace_id)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let mut counts = BTreeMap::new();
|
||||
for row in rows {
|
||||
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
|
||||
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
|
||||
counts.insert(key, u64::try_from(count).unwrap_or_default());
|
||||
}
|
||||
Ok(counts)
|
||||
}
|
||||
|
||||
fn policy_mode_str(request: &AdmissionRequest) -> &'static str {
|
||||
match request.policy_mode {
|
||||
crank_core::ToolExecutionMode::Synchronous => "synchronous",
|
||||
crank_core::ToolExecutionMode::Background => "background",
|
||||
crank_core::ToolExecutionMode::Adaptive => "adaptive",
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,19 @@ impl PostgresRegistry {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Readiness probe of the mandatory execution-admission state (Story 4.1). A missing or
|
||||
/// unreadable admission schema is a dependency failure, never a degraded-but-ready state.
|
||||
pub async fn execution_admission_health(&self) -> Result<(), crank_core::AdmissionStoreError> {
|
||||
super::admission::admission_health(&self.pool).await
|
||||
}
|
||||
|
||||
/// Authoritative admission queue readings for low-cardinality metrics.
|
||||
pub async fn execution_admission_snapshot(
|
||||
&self,
|
||||
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
|
||||
super::admission::admission_snapshot(&self.pool).await
|
||||
}
|
||||
|
||||
async fn connect_in_schema(
|
||||
database_url: &str,
|
||||
schema: Option<&str>,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod admission;
|
||||
mod agent;
|
||||
mod agent_catalog;
|
||||
mod agent_prompt_bindings;
|
||||
@@ -36,6 +37,9 @@ use serde_json::Value;
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow, types::Json};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
pub use admission::{
|
||||
AdmissionLimits, PostgresExecutionAdmission, admission_health, admission_snapshot,
|
||||
};
|
||||
pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError};
|
||||
pub use resource::{AuthorizedResourceRead, BoundResourceAdmission};
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod integration {
|
||||
mod admission;
|
||||
mod agent_resource_bindings;
|
||||
mod agents_usage;
|
||||
mod approval;
|
||||
|
||||
@@ -0,0 +1,496 @@
|
||||
//! ATDD Story 4.1 (FR-21/FR-24): authoritative PostgreSQL execution admission.
|
||||
//!
|
||||
//! These oracles drive the real store against a real PostgreSQL schema: scoped permits, bounded
|
||||
//! queue, typed overload, durable enqueue decisions, deadline propagation, quota keys that ignore
|
||||
//! the operation version, fairness order and fail-closed health.
|
||||
|
||||
use super::common;
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
|
||||
AdmissionSchedulerConfig, AdmissionScope, ApprovalEvidence, ExecutionAdmission,
|
||||
NoTasksFallback, ToolExecutionMode, current_unix_ms,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
fn limits(global: usize, agent: usize, tool: usize, queue: usize) -> AdmissionLimits {
|
||||
AdmissionLimits {
|
||||
global_limit: global,
|
||||
agent_limit: agent,
|
||||
tool_limit: tool,
|
||||
queue_limit: queue,
|
||||
}
|
||||
}
|
||||
|
||||
fn store(
|
||||
pool: sqlx::PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
) -> PostgresExecutionAdmission {
|
||||
PostgresExecutionAdmission::new(pool, limits, scheduler).expect("valid admission store")
|
||||
}
|
||||
|
||||
fn scope(agent: &str, operation: &str, version: u32) -> AdmissionScope {
|
||||
AdmissionScope {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
agent_key: agent.to_owned(),
|
||||
operation_id: operation.to_owned(),
|
||||
operation_version: version,
|
||||
}
|
||||
}
|
||||
|
||||
fn request(scope: AdmissionScope, deadline_in_ms: i64) -> AdmissionRequest {
|
||||
AdmissionRequest {
|
||||
scope,
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: format!("fp_{}", current_unix_ms()),
|
||||
expires_at_unix_ms: current_unix_ms() + deadline_in_ms,
|
||||
}
|
||||
}
|
||||
|
||||
fn background(scope: AdmissionScope, fallback: NoTasksFallback) -> AdmissionRequest {
|
||||
AdmissionRequest {
|
||||
mode: AdmissionMode::Background,
|
||||
no_tasks_fallback: fallback,
|
||||
..request(scope, 30_000)
|
||||
}
|
||||
}
|
||||
|
||||
fn permit(outcome: AdmissionOutcome) -> AdmissionPermit {
|
||||
match outcome {
|
||||
AdmissionOutcome::Permitted(permit) => permit,
|
||||
other => panic!("expected a scoped permit, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_metric_poll() {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn permits_are_scoped_released_once_and_respect_the_agent_budget() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 1, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let first = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
assert_eq!(first.scope.agent_key, "agent_a");
|
||||
|
||||
// The same Agent is already at its scoped budget: the second request waits and expires instead of
|
||||
// over-admitting, and it never dispatches.
|
||||
let waiting = admission
|
||||
.admit(request(scope("agent_a", "op_leads", 2), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(waiting, AdmissionOutcome::DeadlineElapsed);
|
||||
|
||||
// Another Agent still has the global capacity it is entitled to.
|
||||
let other = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_b", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
assert_eq!(other.scope.agent_key, "agent_b");
|
||||
assert!(admission.release(&other).await.unwrap());
|
||||
// Repeated release is idempotent and never frees a foreign or already released permit.
|
||||
assert!(!admission.release(&other).await.unwrap());
|
||||
|
||||
assert!(admission.release(&first).await.unwrap());
|
||||
let snapshot = admission.snapshot().await.unwrap();
|
||||
assert_eq!(snapshot.executing, 0);
|
||||
assert_eq!(snapshot.waiting, 0);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_queue_returns_a_typed_overload_with_retry_guidance() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 1, 1, 2),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 5_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
|
||||
// Two bounded waiting entries fill the queue.
|
||||
let mut waiters = Vec::new();
|
||||
for version in 1..=2 {
|
||||
let admission = admission.clone();
|
||||
waiters.push(tokio::spawn(async move {
|
||||
admission
|
||||
.admit(request(scope("agent_b", "op_leads", version), 5_000))
|
||||
.await
|
||||
}));
|
||||
}
|
||||
while admission.snapshot().await.unwrap().waiting < 2 {
|
||||
wait_for_metric_poll().await;
|
||||
}
|
||||
|
||||
let rejected = admission
|
||||
.admit(request(scope("agent_c", "op_leads", 1), 5_000))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
match rejected {
|
||||
AdmissionOutcome::Overloaded { retry_after_ms } => {
|
||||
assert!((1..=crank_core::MAX_RETRY_AFTER_MS).contains(&retry_after_ms));
|
||||
}
|
||||
other => panic!("expected typed overload, got {other:?}"),
|
||||
}
|
||||
|
||||
// Cancelling the held permit lets the queued waiters make progress instead of leaking capacity.
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
for waiter in waiters.drain(..) {
|
||||
let outcome = waiter.await.expect("waiter task");
|
||||
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
|
||||
let _ = admission.release(&permit).await;
|
||||
}
|
||||
}
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_decisions_are_durable_pinned_and_released_once() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let decision = match admission
|
||||
.admit(background(
|
||||
scope("agent_a", "op_report", 3),
|
||||
NoTasksFallback::Synchronous,
|
||||
))
|
||||
.await
|
||||
.expect("admission store is reachable")
|
||||
{
|
||||
AdmissionOutcome::Enqueued(decision) => decision,
|
||||
other => panic!("expected a durable enqueue decision, got {other:?}"),
|
||||
};
|
||||
assert_eq!(decision.scope.operation_version, 3);
|
||||
assert_eq!(decision.policy_mode, ToolExecutionMode::Synchronous);
|
||||
|
||||
// A second process (fresh pool and store over the same schema) must read the committed decision:
|
||||
// it occupies bounded queue capacity and is not a runnable execution.
|
||||
let reconnected = store(
|
||||
database.concurrent_registry(2).await.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
let snapshot = reconnected.snapshot().await.unwrap();
|
||||
assert_eq!(snapshot.waiting, 1);
|
||||
assert_eq!(snapshot.executing, 0);
|
||||
|
||||
assert!(
|
||||
reconnected
|
||||
.cancel(decision.ticket_id, "not_the_owner")
|
||||
.await
|
||||
.unwrap()
|
||||
.eq(&false),
|
||||
"a foreign cancel must never release the decision"
|
||||
);
|
||||
assert!(
|
||||
reconnected.snapshot().await.unwrap().waiting.eq(&1),
|
||||
"the foreign cancel must leave the decision intact"
|
||||
);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_without_a_task_transport_is_typed_unsupported() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let outcome = admission
|
||||
.admit(background(
|
||||
scope("agent_a", "op_report", 1),
|
||||
NoTasksFallback::Unsupported,
|
||||
))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(outcome, AdmissionOutcome::Unsupported);
|
||||
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unverified_approval_never_produces_a_decision() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let mut denied = background(
|
||||
scope("agent_a", "op_report", 1),
|
||||
NoTasksFallback::Synchronous,
|
||||
);
|
||||
denied.approval = ApprovalEvidence::NotGranted;
|
||||
assert_eq!(
|
||||
admission.admit(denied).await.unwrap(),
|
||||
AdmissionOutcome::ApprovalRequired
|
||||
);
|
||||
|
||||
let mut denied_sync = request(scope("agent_a", "op_report", 1), 1_000);
|
||||
denied_sync.approval = ApprovalEvidence::NotGranted;
|
||||
assert_eq!(
|
||||
admission.admit(denied_sync).await.unwrap(),
|
||||
AdmissionOutcome::ApprovalRequired
|
||||
);
|
||||
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
|
||||
assert_eq!(admission.snapshot().await.unwrap().executing, 0);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_keys_ignore_the_operation_version() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 1, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
// A different Published Version of the same Tool shares the same per-Tool budget.
|
||||
let outcome = admission
|
||||
.admit(request(scope("agent_a", "op_leads", 2), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn weighted_agents_are_served_their_quantum_in_order() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 4, 4, 8),
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
},
|
||||
);
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_z", "op_leads", 1), 5_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
|
||||
let (sender, mut receiver) = mpsc::channel(3);
|
||||
let mut tasks = Vec::new();
|
||||
for (agent, version) in [("agent_a", 1), ("agent_a", 2), ("agent_b", 1)] {
|
||||
let admission = admission.clone();
|
||||
let sender = sender.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let outcome = admission
|
||||
.admit(request(scope(agent, "op_leads", version), 5_000))
|
||||
.await;
|
||||
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
|
||||
let _ = sender.send(permit.scope.agent_key.clone()).await;
|
||||
// Hold briefly so the grant order, not task scheduling, decides the sequence.
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
let _ = admission.release(&permit).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
drop(sender);
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
|
||||
let mut order = Vec::new();
|
||||
while let Some(agent) = receiver.recv().await {
|
||||
order.push(agent);
|
||||
if order.len() == 3 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for task in tasks {
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), task).await;
|
||||
}
|
||||
// agent_a carries weight 2 and therefore receives two consecutive grants before agent_b.
|
||||
assert_eq!(order, vec!["agent_a", "agent_a", "agent_b"]);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mandatory_state_failure_fails_closed_and_reports_not_ready() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// No migration on this schema: the mandatory admission state does not exist yet.
|
||||
let unmigrated = database.raw_pool().await;
|
||||
let admission = store(unmigrated, limits(4, 4, 4, 8), Default::default());
|
||||
assert!(admission.health().await.is_err());
|
||||
assert!(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 1_000))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn held_permits_survive_a_new_process_view() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
|
||||
let reconnected = store(
|
||||
database.concurrent_registry(2).await.pool().clone(),
|
||||
limits(1, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
assert_eq!(reconnected.snapshot().await.unwrap().executing, 1);
|
||||
let outcome = reconnected
|
||||
.admit(request(scope("agent_b", "op_leads", 1), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(
|
||||
outcome,
|
||||
AdmissionOutcome::DeadlineElapsed,
|
||||
"a held permit is not silently reused by another process"
|
||||
);
|
||||
assert!(reconnected.release(&held).await.unwrap());
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn published_execution_policy_is_pinned_across_a_new_draft_version() {
|
||||
use crank_core::{NoTasksFallback, OperationStatus, ToolExecutionMode, ToolExecutionPolicy};
|
||||
use crank_registry::{CreateVersionRequest, PublishRequest};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let workspace = common::test_workspace_id();
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 45_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
let mut version_one = common::test_operation("op_policy", 1, OperationStatus::Draft);
|
||||
version_one.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&workspace, &version_one, None)
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &workspace,
|
||||
operation_id: &version_one.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse("2026-03-25T12:10:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let published = registry
|
||||
.get_published_operation(&version_one.id)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("published operation");
|
||||
assert_eq!(published.execution_config.execution_policy, Some(policy));
|
||||
|
||||
// A later Draft with another policy never rewrites the already accepted publication.
|
||||
let mut version_two = common::test_operation("op_policy", 2, OperationStatus::Draft);
|
||||
version_two.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 1_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
});
|
||||
registry
|
||||
.create_version(CreateVersionRequest {
|
||||
workspace_id: &workspace,
|
||||
snapshot: &version_two,
|
||||
change_note: Some("next draft"),
|
||||
created_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let unchanged = registry
|
||||
.get_published_operation(&version_one.id)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("published operation");
|
||||
assert_eq!(unchanged.version, 1);
|
||||
assert_eq!(unchanged.execution_config.execution_policy, Some(policy));
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_origin_bucket_is_bounded_together_with_agents() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 1, 1, 4),
|
||||
Default::default(),
|
||||
);
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope(ADMIN_ORIGIN_KEY, "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
let started = Instant::now();
|
||||
let outcome = admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
|
||||
assert!(started.elapsed() >= Duration::from_millis(100));
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
database.cleanup().await;
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
//! Common execution admission seam of the runtime (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Every real execution — synchronous, or a published `background`/`adaptive` policy that falls back
|
||||
//! to synchronous because no durable Task transport exists yet — passes this one gate before any
|
||||
//! adapter work. The gate asks the authoritative store for a scoped permit bounded by the published
|
||||
//! Tool policy, the verified approval state of the exact Published Version/scope/input and the caller
|
||||
//! deadline, and it releases the permit explicitly when the local execution future ends.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
|
||||
AdmissionScope, CoordinationStateStore, ExecutionAdmission, NoTasksFallback, ToolExecutionMode,
|
||||
current_unix_ms,
|
||||
};
|
||||
use crank_metrics::{
|
||||
AdmissionModeLabel, RejectionReason, ScopeKind, record_admission_queue_wait,
|
||||
record_admission_rejection,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::{RuntimeError, RuntimeOperation, RuntimeRequestContext};
|
||||
|
||||
/// Effective absolute deadline of one execution: the published policy deadline never extends the
|
||||
/// caller deadline and always includes queue waiting time.
|
||||
pub(crate) fn effective_deadline(
|
||||
operation: &RuntimeOperation,
|
||||
caller_deadline: Instant,
|
||||
) -> Instant {
|
||||
match operation.execution_config.execution_policy {
|
||||
Some(policy) => {
|
||||
caller_deadline.min(Instant::now() + Duration::from_millis(policy.deadline_ms))
|
||||
}
|
||||
None => caller_deadline,
|
||||
}
|
||||
}
|
||||
|
||||
/// Effective execution path for this deployment.
|
||||
///
|
||||
/// Story 4.1 has no Task transport, so a `background` or `adaptive` policy resolves through its
|
||||
/// published `no_tasks_fallback`: either the same synchronous admission path, or a typed
|
||||
/// unsupported refusal with no admission entry, no Task id and no outbound call.
|
||||
pub(crate) fn effective_mode(
|
||||
operation: &RuntimeOperation,
|
||||
) -> Result<ToolExecutionMode, RuntimeError> {
|
||||
let Some(policy) = operation.execution_config.execution_policy else {
|
||||
return Ok(ToolExecutionMode::Synchronous);
|
||||
};
|
||||
match policy.mode {
|
||||
ToolExecutionMode::Synchronous => Ok(ToolExecutionMode::Synchronous),
|
||||
mode => match policy.no_tasks_fallback {
|
||||
NoTasksFallback::Synchronous => Ok(ToolExecutionMode::Synchronous),
|
||||
NoTasksFallback::Unsupported => Err(RuntimeError::ExecutionPolicyUnsupported {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
mode,
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct AdmissionLease {
|
||||
store: Arc<dyn ExecutionAdmission>,
|
||||
permit: Option<AdmissionPermit>,
|
||||
runtime: tokio::runtime::Handle,
|
||||
}
|
||||
|
||||
impl AdmissionLease {
|
||||
fn new(store: Arc<dyn ExecutionAdmission>, permit: AdmissionPermit) -> Self {
|
||||
Self {
|
||||
store,
|
||||
permit: Some(permit),
|
||||
runtime: tokio::runtime::Handle::current(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit release on the normal/error path of the local execution. The `Drop` cleanup stays as
|
||||
/// the cancellation path; both CAS on the same owner token, so only one of them can free it.
|
||||
pub(crate) async fn release(&mut self) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
if let Err(error) = self.store.release(&permit).await {
|
||||
warn!(
|
||||
name: "runtime.admission.release_failed",
|
||||
error = %error,
|
||||
"failed to release an admission permit; the reservation stays fail-closed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AdmissionLease {
|
||||
fn drop(&mut self) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
let store = Arc::clone(&self.store);
|
||||
// Bounded cleanup: the explicit release path already ran on success/error; cancellation
|
||||
// reaches here and releases by owner token without ever freeing a foreign permit. A failed
|
||||
// release keeps the reservation fail-closed (no TTL reuse of an active permit).
|
||||
self.runtime.spawn(async move {
|
||||
match store.release(&permit).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => warn!(
|
||||
name: "runtime.admission.release_skipped",
|
||||
"admission permit was already released"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
name: "runtime.admission.release_failed",
|
||||
error = %error,
|
||||
"failed to release an admission permit; the reservation stays fail-closed"
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquires a scoped permit from the authoritative store. Returns `None` when this deployment has no
|
||||
/// admission store wired (embedded/unit-test executors keep the pre-4.1 local limiter).
|
||||
pub(crate) async fn acquire(
|
||||
store: Option<&Arc<dyn ExecutionAdmission>>,
|
||||
coordination_store: Option<&dyn CoordinationStateStore>,
|
||||
request: &crate::RuntimeExecutionRequest<'_>,
|
||||
deadline: Instant,
|
||||
) -> Result<Option<AdmissionLease>, RuntimeError> {
|
||||
let Some(store) = store else {
|
||||
return Ok(None);
|
||||
};
|
||||
let operation = request.operation();
|
||||
let input = request.input();
|
||||
let request_context = request.request_context();
|
||||
let operation_id = operation.operation_id.as_str().to_owned();
|
||||
let policy_mode = effective_mode(operation)?;
|
||||
// The one admission snapshot carries the authoritative approval state of this exact Published
|
||||
// Version, scope, input and deadline. A confirmation-requiring request without a verified
|
||||
// approval never takes capacity and never dispatches: the pending human decision is answered
|
||||
// here with the existing challenge, and the single-use token is consumed only at dispatch.
|
||||
let approval = crate::confirmation::approval_evidence(
|
||||
coordination_store,
|
||||
operation,
|
||||
input,
|
||||
Some(request_context),
|
||||
)
|
||||
.await?;
|
||||
let agent_key = request.agent_id().map(|agent| agent.as_str());
|
||||
let workspace_id = request.workspace_id().as_str().to_owned();
|
||||
let request = AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id,
|
||||
agent_key: agent_key.unwrap_or(ADMIN_ORIGIN_KEY).to_owned(),
|
||||
operation_id: operation_id.clone(),
|
||||
operation_version: operation.operation_version,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode,
|
||||
no_tasks_fallback: operation
|
||||
.execution_config
|
||||
.execution_policy
|
||||
.map(|policy| policy.no_tasks_fallback)
|
||||
.unwrap_or(NoTasksFallback::Synchronous),
|
||||
approval,
|
||||
fingerprint: admission_fingerprint(request_context, operation),
|
||||
expires_at_unix_ms: current_unix_ms().saturating_add(
|
||||
deadline_remaining_ms(deadline)
|
||||
.try_into()
|
||||
.unwrap_or(i64::MAX),
|
||||
),
|
||||
};
|
||||
let started = Instant::now();
|
||||
let outcome = store.admit(request).await.map_err(|error| {
|
||||
record_admission_rejection(RejectionReason::DependencyUnavailable, ScopeKind::Global);
|
||||
RuntimeError::AdmissionStoreUnavailable {
|
||||
operation_id: operation_id.clone(),
|
||||
details: error.to_string(),
|
||||
}
|
||||
})?;
|
||||
let waited_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or_default();
|
||||
match outcome {
|
||||
AdmissionOutcome::Permitted(permit) => {
|
||||
record_admission_queue_wait(AdmissionModeLabel::Synchronous, waited_ms);
|
||||
Ok(Some(AdmissionLease::new(Arc::clone(store), permit)))
|
||||
}
|
||||
AdmissionOutcome::Overloaded { retry_after_ms } => {
|
||||
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
|
||||
Err(RuntimeError::AdmissionOverloaded {
|
||||
operation_id,
|
||||
retry_after_ms,
|
||||
})
|
||||
}
|
||||
AdmissionOutcome::DeadlineElapsed => {
|
||||
// Queue wait exhausted the absolute deadline before a permit was granted: nothing was
|
||||
// dispatched, so this is an admission refusal, not an upstream timeout.
|
||||
Err(deadline_refusal(operation))
|
||||
}
|
||||
AdmissionOutcome::ApprovalRequired => {
|
||||
// The store refused before reserving any capacity or queue entry. Answer with the
|
||||
// existing single-use challenge so the caller can confirm out of band.
|
||||
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
|
||||
Err(crate::confirmation::challenge(
|
||||
coordination_store,
|
||||
operation,
|
||||
input,
|
||||
Some(request_context),
|
||||
)
|
||||
.await)
|
||||
}
|
||||
AdmissionOutcome::Unsupported | AdmissionOutcome::Enqueued(_) => {
|
||||
// No Task transport exists in this deployment: an enqueue decision can never become a
|
||||
// runnable execution here, so it must not be reported as accepted work.
|
||||
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
|
||||
Err(RuntimeError::ExecutionPolicyUnsupported {
|
||||
operation_id,
|
||||
mode: policy_mode,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed refusal for an admission that never granted a permit within the absolute deadline. Reuses the
|
||||
/// existing `execution_overloaded` vocabulary (stage `admission`, retry guidance, certain outcome) so
|
||||
/// no new client error code or API shape appears, while reporting zero dispatch truthfully.
|
||||
pub(crate) fn deadline_refusal(operation: &RuntimeOperation) -> RuntimeError {
|
||||
record_admission_rejection(RejectionReason::Deadline, ScopeKind::Global);
|
||||
RuntimeError::AdmissionOverloaded {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
}
|
||||
}
|
||||
|
||||
fn deadline_remaining_ms(deadline: Instant) -> u64 {
|
||||
u64::try_from(
|
||||
deadline
|
||||
.saturating_duration_since(Instant::now())
|
||||
.as_millis(),
|
||||
)
|
||||
.unwrap_or_default()
|
||||
.max(1)
|
||||
}
|
||||
|
||||
/// Bounded request identity for correlating one admission entry. Never contains payload, headers or
|
||||
/// credentials.
|
||||
fn admission_fingerprint(
|
||||
request_context: &RuntimeRequestContext,
|
||||
operation: &RuntimeOperation,
|
||||
) -> String {
|
||||
let mut fingerprint = format!(
|
||||
"{}:{}:{}",
|
||||
operation.operation_id.as_str(),
|
||||
operation.operation_version,
|
||||
request_context.request_id
|
||||
);
|
||||
fingerprint.truncate(crank_core::ADMISSION_FINGERPRINT_MAX_BYTES);
|
||||
fingerprint
|
||||
}
|
||||
@@ -2,8 +2,8 @@ use std::time::Duration;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use crank_core::{
|
||||
CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod, OperationSafetyClass,
|
||||
OperationSafetyPolicy, Target,
|
||||
ApprovalEvidence, CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod,
|
||||
OperationSafetyClass, OperationSafetyPolicy, Target,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -26,32 +26,14 @@ pub async fn confirm_operation(
|
||||
}
|
||||
|
||||
let Some(store) = store else {
|
||||
return Err(RuntimeError::ConfirmationStoreUnavailable {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
});
|
||||
return Err(confirmation_store_unavailable(operation));
|
||||
};
|
||||
|
||||
let scope = confirmation_scope(operation, request_context)?;
|
||||
let input_hash = hash_json(input)?;
|
||||
let provided_token = request_context
|
||||
.and_then(RuntimeRequestContext::confirmation_token)
|
||||
.or_else(|| confirmation_token_from_input(input));
|
||||
|
||||
let Some(provided_token) = provided_token else {
|
||||
let token = issue_confirmation_token(
|
||||
store,
|
||||
operation.operation_id.as_str(),
|
||||
&scope,
|
||||
&input_hash,
|
||||
&safety,
|
||||
)
|
||||
.await?;
|
||||
return Err(RuntimeError::ConfirmationRequired {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
safety_class: safety.class,
|
||||
confirmation_token: token,
|
||||
expires_in_ms: confirmation_ttl_ms(&safety),
|
||||
});
|
||||
let Some(provided_token) = provided_confirmation_token(request_context, input) else {
|
||||
return Err(confirmation_required(store, operation, &scope, &input_hash, &safety).await);
|
||||
};
|
||||
|
||||
consume_confirmation_token(
|
||||
@@ -64,6 +46,121 @@ pub async fn confirm_operation(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Read-only approval proof for the shared admission gate (Story 4.1, AC2).
|
||||
///
|
||||
/// It reports the authoritative approval state of this exact Published Version, scope and input and
|
||||
/// never consumes the single-use confirmation token: the authoritative consumption stays in
|
||||
/// [`confirm_operation`] immediately before the adapter call. A missing approval is `NotGranted`, so
|
||||
/// admission refuses the request before taking execution capacity while the human decision is
|
||||
/// pending; an unusable token or an unreachable safety store is a typed refusal too, never a permit.
|
||||
pub(crate) async fn approval_evidence(
|
||||
store: Option<&dyn CoordinationStateStore>,
|
||||
operation: &RuntimeOperation,
|
||||
input: &Value,
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
) -> Result<ApprovalEvidence, RuntimeError> {
|
||||
let safety = effective_safety_policy(operation);
|
||||
if !safety.class.requires_confirmation() {
|
||||
return Ok(ApprovalEvidence::NotRequired);
|
||||
}
|
||||
if request_context.is_some_and(RuntimeRequestContext::approval_granted) {
|
||||
return Ok(ApprovalEvidence::Granted);
|
||||
}
|
||||
let Some(store) = store else {
|
||||
return Err(confirmation_store_unavailable(operation));
|
||||
};
|
||||
let scope = confirmation_scope(operation, request_context)?;
|
||||
let input_hash = hash_json(input)?;
|
||||
|
||||
let Some(token) = provided_confirmation_token(request_context, input) else {
|
||||
return Ok(ApprovalEvidence::NotGranted);
|
||||
};
|
||||
|
||||
let stored = store
|
||||
.get_value(
|
||||
CacheScope::Coordination,
|
||||
&confirmation_cache_key(&scope, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| confirmation_store_unavailable(operation))?;
|
||||
let verified = stored.is_some_and(|value| {
|
||||
value.payload.get("scope").and_then(Value::as_str) == Some(scope.as_str())
|
||||
&& value.payload.get("input_hash").and_then(Value::as_str) == Some(input_hash.as_str())
|
||||
});
|
||||
if verified {
|
||||
Ok(ApprovalEvidence::Granted)
|
||||
} else {
|
||||
Err(RuntimeError::InvalidConfirmationToken {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Issues the existing single-use confirmation challenge and returns the typed refusal that carries
|
||||
/// it. Admission calls this when no verified approval exists yet, so waiting for the human decision
|
||||
/// never holds execution capacity.
|
||||
pub(crate) async fn challenge(
|
||||
store: Option<&dyn CoordinationStateStore>,
|
||||
operation: &RuntimeOperation,
|
||||
input: &Value,
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
) -> RuntimeError {
|
||||
let safety = effective_safety_policy(operation);
|
||||
let Some(store) = store else {
|
||||
return confirmation_store_unavailable(operation);
|
||||
};
|
||||
let scope = match confirmation_scope(operation, request_context) {
|
||||
Ok(scope) => scope,
|
||||
Err(error) => return error,
|
||||
};
|
||||
let input_hash = match hash_json(input) {
|
||||
Ok(input_hash) => input_hash,
|
||||
Err(error) => return error,
|
||||
};
|
||||
confirmation_required(store, operation, &scope, &input_hash, &safety).await
|
||||
}
|
||||
|
||||
fn provided_confirmation_token<'a>(
|
||||
request_context: Option<&'a RuntimeRequestContext>,
|
||||
input: &'a Value,
|
||||
) -> Option<&'a str> {
|
||||
request_context
|
||||
.and_then(RuntimeRequestContext::confirmation_token)
|
||||
.or_else(|| confirmation_token_from_input(input))
|
||||
}
|
||||
|
||||
fn confirmation_store_unavailable(operation: &RuntimeOperation) -> RuntimeError {
|
||||
RuntimeError::ConfirmationStoreUnavailable {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn confirmation_required(
|
||||
store: &dyn CoordinationStateStore,
|
||||
operation: &RuntimeOperation,
|
||||
scope: &str,
|
||||
input_hash: &str,
|
||||
safety: &OperationSafetyPolicy,
|
||||
) -> RuntimeError {
|
||||
match issue_confirmation_token(
|
||||
store,
|
||||
operation.operation_id.as_str(),
|
||||
scope,
|
||||
input_hash,
|
||||
safety,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(token) => RuntimeError::ConfirmationRequired {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
safety_class: safety.class,
|
||||
confirmation_token: token,
|
||||
expires_in_ms: confirmation_ttl_ms(safety),
|
||||
},
|
||||
Err(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool {
|
||||
effective_safety_policy(operation)
|
||||
.class
|
||||
|
||||
@@ -43,6 +43,27 @@ pub enum RuntimeError {
|
||||
confirmation_token: String,
|
||||
expires_in_ms: u64,
|
||||
},
|
||||
#[error(
|
||||
"execution mode {mode:?} of operation {operation_id} is unavailable without a durable Task transport"
|
||||
)]
|
||||
ExecutionPolicyUnsupported {
|
||||
operation_id: String,
|
||||
mode: crank_core::ToolExecutionMode,
|
||||
},
|
||||
#[error(
|
||||
"execution admission is overloaded for operation {operation_id}; retry after {retry_after_ms} ms"
|
||||
)]
|
||||
AdmissionOverloaded {
|
||||
operation_id: String,
|
||||
retry_after_ms: u64,
|
||||
},
|
||||
#[error("execution admission requires a verified approval for operation {operation_id}")]
|
||||
AdmissionApprovalRequired { operation_id: String },
|
||||
#[error("execution admission store is unavailable for operation {operation_id}: {details}")]
|
||||
AdmissionStoreUnavailable {
|
||||
operation_id: String,
|
||||
details: String,
|
||||
},
|
||||
#[error("confirmation token for operation {operation_id} is invalid, expired, or already used")]
|
||||
InvalidConfirmationToken { operation_id: String },
|
||||
#[error("confirmation store is unavailable for operation {operation_id}")]
|
||||
|
||||
@@ -57,6 +57,11 @@ fn normalize_runtime_error_with_effect(
|
||||
| RuntimeError::IdempotencyCompletionUnknown { .. } => {
|
||||
failure = failure.with_dispatch_uncertainty();
|
||||
}
|
||||
RuntimeError::AdmissionOverloaded { retry_after_ms, .. } => {
|
||||
if let Ok(with_retry) = failure.clone().try_with_retry_after_ms(*retry_after_ms) {
|
||||
failure = with_retry;
|
||||
}
|
||||
}
|
||||
RuntimeError::ConfirmationRequired {
|
||||
confirmation_token,
|
||||
expires_in_ms,
|
||||
@@ -113,6 +118,14 @@ fn error_code(error: &RuntimeError) -> ExecutionErrorCode {
|
||||
RuntimeError::UnsupportedExecutionMode { .. } => {
|
||||
ExecutionErrorCode::ExecutionModeUnsupported
|
||||
}
|
||||
RuntimeError::ExecutionPolicyUnsupported { .. } => {
|
||||
ExecutionErrorCode::ExecutionModeUnsupported
|
||||
}
|
||||
RuntimeError::AdmissionOverloaded { .. } => ExecutionErrorCode::ExecutionOverloaded,
|
||||
RuntimeError::AdmissionApprovalRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
|
||||
RuntimeError::AdmissionStoreUnavailable { .. } => {
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
}
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => ExecutionErrorCode::ExecutionOverloaded,
|
||||
RuntimeError::InvalidPreparedRequest { .. } => ExecutionErrorCode::PreparedRequestInvalid,
|
||||
RuntimeError::ConfirmationRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
|
||||
|
||||
@@ -220,6 +220,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -7,9 +7,9 @@ use std::time::{Duration, Instant};
|
||||
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
use crank_core::{
|
||||
AdapterRegistry, CoordinationStateStore, ExecutionMode, HttpMethod, InvocationSource,
|
||||
InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
|
||||
Target,
|
||||
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
|
||||
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
|
||||
SharedProtocolAdapter, Target,
|
||||
};
|
||||
use crank_metrics::{
|
||||
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
|
||||
@@ -41,6 +41,7 @@ pub struct RuntimeExecutor {
|
||||
unary_limiter: Arc<Semaphore>,
|
||||
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
||||
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
||||
admission: Option<Arc<dyn ExecutionAdmission>>,
|
||||
metering_sink: SharedMeteringSink,
|
||||
}
|
||||
|
||||
@@ -139,6 +140,7 @@ impl RuntimeExecutor {
|
||||
adapters: AdapterRegistry,
|
||||
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
||||
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
||||
admission: Option<Arc<dyn ExecutionAdmission>>,
|
||||
metering_sink: SharedMeteringSink,
|
||||
) -> Self {
|
||||
Self {
|
||||
@@ -147,6 +149,7 @@ impl RuntimeExecutor {
|
||||
limits,
|
||||
response_cache,
|
||||
coordination_store,
|
||||
admission,
|
||||
metering_sink,
|
||||
}
|
||||
}
|
||||
@@ -167,6 +170,11 @@ impl RuntimeExecutor {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
|
||||
self.admission = Some(admission);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_request(
|
||||
&self,
|
||||
request: RuntimeExecutionRequest<'_>,
|
||||
@@ -188,11 +196,16 @@ impl RuntimeExecutor {
|
||||
request.agent_id()
|
||||
);
|
||||
let operation_may_mutate = operation_may_mutate(operation);
|
||||
let execution_deadline =
|
||||
crate::admission::effective_deadline(operation, request.deadline());
|
||||
let runtime_span = Stage::RuntimeExecute.span();
|
||||
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
|
||||
log_runtime_event("unary.execute", operation, Some(request_context));
|
||||
let started_at = Instant::now();
|
||||
let dispatch_started = Arc::new(AtomicBool::new(false));
|
||||
// True until the admission gate answered. A deadline that fires while the request is still
|
||||
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
|
||||
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
|
||||
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
|
||||
metric_invocation_source(Some(request_context)),
|
||||
request_context
|
||||
@@ -206,6 +219,17 @@ impl RuntimeExecutor {
|
||||
.flatten(),
|
||||
);
|
||||
let execution = async {
|
||||
// One admission gate for every executable path (Story 4.1): the published policy decides
|
||||
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
|
||||
// is held for the whole local execution future, including retries.
|
||||
let mut lease = crate::admission::acquire(
|
||||
self.admission.as_ref(),
|
||||
self.coordination_store.as_deref(),
|
||||
&request,
|
||||
execution_deadline,
|
||||
)
|
||||
.await?;
|
||||
admission_pending.store(false, Ordering::Release);
|
||||
let _permit = self.acquire_unary_permit(operation)?;
|
||||
let _inflight = InFlightGuard::runtime();
|
||||
let mapping_span = Stage::RuntimeArgumentsMap.span();
|
||||
@@ -215,7 +239,7 @@ impl RuntimeExecutor {
|
||||
let prepared_request = prepared_request?;
|
||||
let request_preview = sanitized_request_preview(&prepared_request);
|
||||
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
|
||||
let output = self
|
||||
let result = self
|
||||
.execute_prepared(
|
||||
operation,
|
||||
input,
|
||||
@@ -223,19 +247,27 @@ impl RuntimeExecutor {
|
||||
Some(request_context),
|
||||
Arc::clone(&dispatch_started),
|
||||
)
|
||||
.await?;
|
||||
.await;
|
||||
if let Some(lease) = lease.as_mut() {
|
||||
lease.release().await;
|
||||
}
|
||||
let output = result?;
|
||||
Ok((output, request_preview))
|
||||
};
|
||||
let result = tokio::time::timeout_at(
|
||||
request.deadline().into(),
|
||||
execution_deadline.into(),
|
||||
execution.instrument(runtime_span.clone()),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
Err(RuntimeError::ExecutionDeadlineElapsed {
|
||||
may_have_dispatched: operation_may_mutate
|
||||
&& dispatch_started.load(Ordering::Acquire),
|
||||
})
|
||||
if admission_pending.load(Ordering::Acquire) {
|
||||
Err(crate::admission::deadline_refusal(operation))
|
||||
} else {
|
||||
Err(RuntimeError::ExecutionDeadlineElapsed {
|
||||
may_have_dispatched: operation_may_mutate
|
||||
&& dispatch_started.load(Ordering::Acquire),
|
||||
})
|
||||
}
|
||||
});
|
||||
record_runtime_result(&runtime_span, &result);
|
||||
drop(runtime_span);
|
||||
@@ -952,6 +984,10 @@ fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
|
||||
| RuntimeError::UnsupportedProtocol { .. }
|
||||
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
|
||||
RuntimeError::AdmissionOverloaded { .. }
|
||||
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
|
||||
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
|
||||
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
|
||||
RuntimeError::ConfirmationRequired { .. }
|
||||
| RuntimeError::InvalidConfirmationToken { .. }
|
||||
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
|
||||
@@ -1008,6 +1044,13 @@ fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
|
||||
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
|
||||
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
|
||||
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
|
||||
// label domain stays fixed; the precise code lives in the execution failure/invocation log
|
||||
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
|
||||
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
|
||||
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
|
||||
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
|
||||
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
|
||||
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
|
||||
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
|
||||
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
|
||||
|
||||
@@ -2,8 +2,8 @@ use std::sync::Arc;
|
||||
|
||||
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter};
|
||||
use crank_core::{
|
||||
AdapterRegistry, CoordinationStateStore, MeteringSink, NoopMeteringSink, ResponseCacheStore,
|
||||
SharedMeteringSink, SharedProtocolAdapter,
|
||||
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, MeteringSink, NoopMeteringSink,
|
||||
ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
|
||||
};
|
||||
|
||||
use crate::{RuntimeExecutor, RuntimeLimits};
|
||||
@@ -13,6 +13,7 @@ pub struct RuntimeExecutorBuilder {
|
||||
adapters: AdapterRegistry,
|
||||
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
||||
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
||||
admission: Option<Arc<dyn ExecutionAdmission>>,
|
||||
metering_sink: SharedMeteringSink,
|
||||
}
|
||||
|
||||
@@ -23,6 +24,7 @@ impl RuntimeExecutorBuilder {
|
||||
adapters: AdapterRegistry::empty(),
|
||||
response_cache: None,
|
||||
coordination_store: None,
|
||||
admission: None,
|
||||
metering_sink: Arc::new(NoopMeteringSink) as Arc<dyn MeteringSink>,
|
||||
}
|
||||
}
|
||||
@@ -47,6 +49,13 @@ impl RuntimeExecutorBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Wires the mandatory deployment-wide admission store (Story 4.1). Production processes wire
|
||||
/// the PostgreSQL implementation; an executor without it keeps the pre-4.1 local limit.
|
||||
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
|
||||
self.admission = Some(admission);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_metering_sink(mut self, sink: SharedMeteringSink) -> Self {
|
||||
self.metering_sink = sink;
|
||||
self
|
||||
@@ -58,6 +67,7 @@ impl RuntimeExecutorBuilder {
|
||||
self.adapters,
|
||||
self.response_cache,
|
||||
self.coordination_store,
|
||||
self.admission,
|
||||
self.metering_sink,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -446,6 +446,7 @@ mod tests {
|
||||
}),
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod admission;
|
||||
mod auth;
|
||||
mod cache;
|
||||
mod cache_factory;
|
||||
|
||||
@@ -333,6 +333,7 @@ fn runtime_operation(draft: RestImportCandidate) -> RuntimeOperation {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -96,6 +96,7 @@ paths:
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -334,6 +334,7 @@ fn destructive_delete_operation() -> Operation<Schema, MappingSet> {
|
||||
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
|
||||
}),
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -340,6 +340,7 @@ fn idempotent_post_operation() -> Operation<Schema, MappingSet> {
|
||||
}),
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -109,6 +109,7 @@ fn no_input_get_operation() -> Operation<Schema, MappingSet> {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -573,6 +573,7 @@ fn operation(
|
||||
idempotency,
|
||||
safety,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -10,7 +10,7 @@ use std::{
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdapterResponse, ConfirmationPolicy, CorrelationContext, ExecutionConfig, ExecutionErrorCode,
|
||||
ExecutionMode, ExecutionOrigin, HttpMethod, IdempotencyMode, IdempotencyPolicy,
|
||||
ExecutionMode, ExecutionOrigin, ExecutionStage, HttpMethod, IdempotencyMode, IdempotencyPolicy,
|
||||
InvocationSource, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy,
|
||||
OperationSecurityLevel, OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter,
|
||||
ProtocolAdapterError, RestTarget, RetryPolicy, Retryability, Target, ToolDescription,
|
||||
@@ -64,6 +64,10 @@ async fn deadline_after_dispatch_is_normalized_as_unknown_outcome() {
|
||||
failure.outcome_certainty(),
|
||||
OutcomeCertainty::OutcomeUnknown
|
||||
);
|
||||
// A deadline that fires after the adapter was called stays an upstream timeout: the admission
|
||||
// refusal vocabulary (Story 4.1) only covers deadlines reached before any dispatch.
|
||||
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
|
||||
assert_eq!(failure.stage(), ExecutionStage::Upstream);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -98,6 +102,8 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
|
||||
let failure = executor.execute_outcome(request).await.unwrap_err();
|
||||
assert_eq!(failure.retryability(), Retryability::AfterDelay);
|
||||
assert_eq!(failure.outcome_certainty(), OutcomeCertainty::Certain);
|
||||
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
|
||||
assert_eq!(failure.stage(), ExecutionStage::Upstream);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -632,6 +638,7 @@ fn operation() -> Operation<Schema, MappingSet> {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -178,6 +178,8 @@ scrape_configs:
|
||||
- `crank_upstream_requests_total`,
|
||||
`crank_upstream_request_duration_seconds`;
|
||||
- `crank_runtime_inflight`, `crank_runtime_limit_rejections_total`,
|
||||
`crank_runtime_admission_queue_wait_seconds`,
|
||||
`crank_runtime_admission_oldest_waiting_seconds`,
|
||||
`crank_runtime_cache_total`, `crank_idempotency_total`,
|
||||
`crank_confirmation_total`;
|
||||
- `crank_db_pool_connections`;
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6285,"max_rendered_series":19515,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
|
||||
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6381,"max_rendered_series":19611,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
|
||||
@@ -1 +1 @@
|
||||
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
|
||||
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
|
||||
@@ -55,3 +55,15 @@ run_atdd "capability advertisement registry" -p crank-community-mcp --lib protoc
|
||||
# smoke) runs against the real endpoint, a real published catalog and a real counted upstream.
|
||||
run_atdd "tool call lifecycle bounds" -p crank-community-mcp --lib app::lifecycle -- --test-threads=1
|
||||
run_atdd "tool call progress and cancellation" -p mcp-server --test integration tools_lifecycle -- --test-threads=1
|
||||
|
||||
# Story 4.1 (FR-21, FR-24): a published Tool execution policy and one authoritative admission gate.
|
||||
# The policy/JSON contract and the fair-scheduler arithmetic are unit-level; the authoritative store
|
||||
# (scoped permits, bounded queue, typed overload, durable decisions, version pinning, fail-closed
|
||||
# health) and the runtime wiring (one shared global budget across executors, typed refusals with no
|
||||
# outbound call, adaptive/synchronous fallback) run against a real PostgreSQL schema.
|
||||
run_atdd "execution policy contract" -p crank-core --lib operation::tests::execution_policy -- --test-threads=1
|
||||
run_atdd "admission scheduler bounds" -p crank-core --lib admission:: -- --test-threads=1
|
||||
run_atdd "execution policy validation" -p admin-api --lib operation_validation::tests:: -- --test-threads=1
|
||||
run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1
|
||||
run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1
|
||||
run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1
|
||||
Reference in new issue
Block a user