Compare commits

..
Author SHA1 Message Date
Codex 4e7440f25e ci: separate protected source integration from explicit production deployment
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 12m54s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 3s
CI / Deploy (pull_request) Skipped
2026-10-27 00:21:37 +03:00
76 changed files with 392 additions and 4160 deletions

No files matched your search

+7 -1
View File
@@ -11,6 +11,12 @@ on:
options: [story, feature]
default: story
required: true
deploy_target:
description: Deployment target for this run
type: choice
options: [none, production]
default: none
required: true
push:
branches:
- main
@@ -499,7 +505,7 @@ jobs:
runs-on: ubuntu-latest
needs:
- required-checks
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
if: ${{ gitea.event_name == 'workflow_dispatch' && gitea.ref == 'refs/heads/main' && inputs.validation_level == 'feature' && inputs.deploy_target == 'production' }}
env:
REGISTRY: git.itexp.me
IMAGE_TAG: ${{ gitea.sha }}
Generated
-1
View File
@@ -826,7 +826,6 @@ dependencies = [
name = "crank-registry"
version = "0.3.1"
dependencies = [
"async-trait",
"crank-artifacts",
"crank-core",
"crank-mapping",
-1
View File
@@ -181,7 +181,6 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+1 -18
View File
@@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
use admin_api::{
app::build_app,
auth::{AuthSettings, BootstrapAdminConfig},
pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics},
pool_metrics::spawn_postgres_pool_metrics,
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
service::AdminServiceBuilder,
state::AppState,
@@ -169,7 +169,6 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let base_url = config
.runtime
@@ -215,16 +214,6 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(std::sync::Arc::new(
crank_registry::PostgresExecutionAdmission::new(
registry.pool().clone(),
crank_registry::AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
},
crank_core::AdmissionSchedulerConfig::default(),
)?,
))
.build();
let identity_provider =
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
@@ -332,12 +321,6 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
-14
View File
@@ -11,17 +11,3 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
+2 -27
View File
@@ -37,22 +37,13 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
} else {
"not_ready"
};
let execution_admission = if checks.execution_admission {
"ready"
} else {
"not_ready"
};
if checks.is_ready() {
(
StatusCode::OK,
Json(json!({
"service": "admin-api",
"status": "ready",
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
})),
)
} else {
@@ -61,11 +52,7 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
Json(json!({
"service": "admin-api",
"status": "not_ready",
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
})),
)
}
@@ -80,7 +67,6 @@ mod tests {
let checks = ReadinessChecks {
postgres: false,
artifact_storage: true,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -106,7 +92,6 @@ mod tests {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: false,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -126,14 +111,4 @@ mod tests {
"not_ready"
);
}
#[test]
fn readiness_checks_identify_an_admission_state_failure() {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: true,
execution_admission: false,
};
assert!(!checks.is_ready());
}
}
+3 -9
View File
@@ -55,8 +55,8 @@ use crate::{
storage::LocalArtifactStorage,
};
use operation_validation::{
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy,
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_protocol_target, validate_response_cache_policy,
};
#[derive(Clone)]
@@ -104,12 +104,11 @@ pub use crate::dto::*;
pub struct ReadinessChecks {
pub postgres: bool,
pub artifact_storage: bool,
pub execution_admission: bool,
}
impl ReadinessChecks {
pub const fn is_ready(self) -> bool {
self.postgres && self.artifact_storage && self.execution_admission
self.postgres && self.artifact_storage
}
}
@@ -151,8 +150,6 @@ pub(super) struct CredentialAuditRecord<'a> {
impl AdminService {
pub async fn readiness(&self) -> ReadinessChecks {
let postgres = self.registry.ping().await.is_ok();
// Story 4.1: the mandatory admission state is a readiness dependency of every execution.
let execution_admission = self.registry.execution_admission_health().await.is_ok();
let store = Arc::clone(&self.artifact_store);
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
.await
@@ -160,7 +157,6 @@ impl AdminService {
ReadinessChecks {
postgres,
artifact_storage,
execution_admission,
}
}
@@ -729,7 +725,6 @@ impl AdminService {
validate_response_cache_policy(&payload.target, &payload.execution_config)?;
validate_idempotency_policy(&payload.target, &payload.execution_config)?;
validate_approval_policy(&payload.execution_config)?;
validate_execution_policy(&payload.execution_config)?;
payload.input_mapping.validate_paths()?;
payload.output_mapping.validate_paths()?;
Ok(())
@@ -743,7 +738,6 @@ impl AdminService {
validate_response_cache_policy(&operation.target, &operation.execution_config)?;
validate_idempotency_policy(&operation.target, &operation.execution_config)?;
validate_approval_policy(&operation.execution_config)?;
validate_execution_policy(&operation.execution_config)?;
operation.input_mapping.validate_paths()?;
operation.output_mapping.validate_paths()?;
Ok(())
-1
View File
@@ -441,7 +441,6 @@ fn demo_rest_operation_payload() -> OperationPayload {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
-1
View File
@@ -635,7 +635,6 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -610,7 +610,6 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref,
headers: BTreeMap::new(),
},
@@ -160,23 +160,6 @@ pub(super) fn validate_approval_policy(
Ok(())
}
pub(super) fn validate_execution_policy(
execution_config: &crank_core::ExecutionConfig,
) -> Result<(), ApiError> {
let Some(policy) = execution_config.execution_policy.as_ref() else {
return Ok(());
};
if let Err(error) = policy.validate() {
return Err(ApiError::validation_with_context(
error.to_string(),
json!({
"field": "execution_config.execution_policy.deadline_ms",
}),
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
@@ -188,8 +171,8 @@ mod tests {
};
use super::{
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_response_cache_policy,
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_response_cache_policy,
};
fn cacheable_execution_config() -> ExecutionConfig {
@@ -200,7 +183,6 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -219,7 +201,6 @@ mod tests {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -384,36 +365,4 @@ mod tests {
"approval ttl must be between 1 and 300 seconds"
);
}
#[test]
fn accepts_published_execution_policy_and_absent_legacy_policy() {
let mut config = cacheable_execution_config();
validate_execution_policy(&config).unwrap();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Adaptive,
deadline_ms: 120_000,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
});
validate_execution_policy(&config).unwrap();
}
#[test]
fn rejects_zero_and_overflowing_execution_policy_deadline() {
let mut config = cacheable_execution_config();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Synchronous,
deadline_ms: 0,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
let error = validate_execution_policy(&config).unwrap_err();
assert!(matches!(error, crate::error::ApiError::Validation { .. }));
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Background,
deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
assert!(validate_execution_policy(&config).is_err());
}
}
@@ -495,7 +495,6 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+2 -29
View File
@@ -14,14 +14,13 @@ use crank_observability::{
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
};
use crank_registry::{
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry,
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics};
use mcp_server::pool_metrics::spawn_postgres_pool_metrics;
use sqlx::postgres::PgConnectOptions;
use tokio::net::TcpListener;
use tracing::info;
@@ -176,7 +175,6 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
let secret_crypto =
@@ -192,10 +190,6 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(execution_admission(
registry.pool().clone(),
runtime_limits,
)?)
.build();
// Published static Resources are read from the same immutable content-addressed root the
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
@@ -293,21 +287,6 @@ fn init_observability(
)?)
}
fn execution_admission(
pool: sqlx::PgPool,
runtime_limits: RuntimeLimits,
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
let store = PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..AdmissionLimits::default()
},
crank_core::AdmissionSchedulerConfig::default(),
)?;
Ok(std::sync::Arc::new(store))
}
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
database_options(&config.database).map_err(|_| invalid("database.source"))?;
@@ -328,12 +307,6 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
-14
View File
@@ -11,17 +11,3 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
-874
View File
@@ -1,874 +0,0 @@
//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission.
//!
//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a
//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported
//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal.
//!
//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this
//! app already wires both and this is the real deployment shape.
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
use crank_core::{
AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation,
OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget,
RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy,
WorkspaceId,
};
use crank_mapping::MappingSet;
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits};
use crank_schema::{Schema, SchemaKind};
use serde_json::json;
use sqlx::PgPool;
use time::OffsetDateTime;
struct CountingAdapter {
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
#[async_trait]
impl ProtocolAdapter for CountingAdapter {
fn protocol(&self) -> Protocol {
Protocol::Rest
}
fn supports_mode(&self, mode: ExecutionMode) -> bool {
mode == ExecutionMode::Unary
}
async fn invoke_unary(
&self,
_target: &Target,
_prepared: &crank_core::PreparedRequest,
context: &RuntimeRequestContext,
) -> Result<AdapterResponse, crank_core::ProtocolAdapterError> {
context.mark_dispatch_started();
self.calls.fetch_add(1, Ordering::SeqCst);
let current = self.current.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(current, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(150)).await;
self.current.fetch_sub(1, Ordering::SeqCst);
Ok(AdapterResponse {
status_code: 200,
headers: BTreeMap::new(),
body: json!({"ok": true}),
data: json!({"ok": true}),
})
}
}
fn schema(kind: SchemaKind) -> Schema {
Schema {
kind,
format: None,
description: None,
required: false,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
}
}
fn operation(policy: Option<ToolExecutionPolicy>) -> Operation<Schema, MappingSet> {
Operation {
id: OperationId::new("op_admission_budget"),
name: "admission_budget".to_owned(),
display_name: "Admission budget".to_owned(),
category: "general".to_owned(),
protocol: Protocol::Rest,
security_level: OperationSecurityLevel::Standard,
status: OperationStatus::Published,
version: 7,
target: Target::Rest(RestTarget {
base_url: "https://api.example.com".to_owned(),
method: HttpMethod::Get,
path_template: "/v1/items".to_owned(),
static_headers: BTreeMap::new(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: BTreeMap::new(),
}),
input_schema: schema(SchemaKind::Object),
output_schema: schema(SchemaKind::Object),
input_mapping: MappingSet::default(),
output_mapping: MappingSet::default(),
execution_config: ExecutionConfig {
timeout_ms: 5_000,
execution_policy: policy,
..ExecutionConfig::default()
},
tool_description: ToolDescription {
title: "Admission".to_owned(),
description: "Admission budget probe".to_owned(),
tags: Vec::new(),
examples: Vec::new(),
},
samples: None,
generated_draft: None,
config_export: None,
wizard_state: None,
created_at: OffsetDateTime::now_utc(),
updated_at: OffsetDateTime::now_utc(),
published_at: None,
}
}
async fn execute(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = WorkspaceId::new("ws_test");
let context = crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun);
execute_in(executor, operation, input, &context).await
}
async fn execute_in(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
context: &crank_runtime::RuntimeRequestContext,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = context
.metering_context()
.map(|metering| metering.workspace_id.clone())
.unwrap_or_else(|| WorkspaceId::new("ws_test"));
let request = crank_runtime::RuntimeExecutionRequest::try_new(
&workspace,
ExecutionOrigin::AdminDraft,
None,
operation,
input,
crank_runtime::ExecutionAuthorization::Authorized,
None,
context,
Instant::now() + Duration::from_secs(30),
)
.map_err(|_| {
crank_core::ExecutionFailure::new(
ExecutionErrorCode::RuntimeInternal,
crank_core::CorrelationContext::new(
context.request_id.clone(),
context.trace_context.clone(),
),
)
})?;
executor
.execute_outcome(request)
.await
.map(|success| success.output)
}
async fn schema_pool(schema_url: &str) -> PgPool {
let pool = PgPool::connect(schema_url)
.await
.expect("admission schema pool");
crank_registry::MigrationAuthority::apply(&pool)
.await
.expect("migration authority applies the admission schema");
pool
}
fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc<PostgresExecutionAdmission> {
Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: global,
agent_limit: 8,
tool_limit: 8,
queue_limit: queue,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)
}
fn executor(
store: Arc<PostgresExecutionAdmission>,
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
) -> RuntimeExecutor {
RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current,
peak,
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store)
.build()
}
#[tokio::test]
async fn two_executors_share_one_global_budget() {
let schema_url = crank_test_support::postgres_schema_url("admission_budget").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 8);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let first = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let second = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let mut tasks = Vec::new();
for index in 0..4 {
let executor = if index % 2 == 0 {
first.clone()
} else {
second.clone()
};
let operation = operation.clone();
tasks.push(tokio::spawn(async move {
execute(&executor, &operation, &json!({})).await
}));
}
let mut completed = 0;
for task in tasks {
match task.await.expect("execution task") {
Ok(_) => completed += 1,
Err(failure) => panic!("admission must grant every request: {failure:?}"),
}
}
assert_eq!(completed, 4);
assert_eq!(calls.load(Ordering::SeqCst), 4);
assert!(
peak.load(Ordering::SeqCst) <= 2,
"peak concurrency {} exceeded the shared global limit",
peak.load(Ordering::SeqCst)
);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn saturation_returns_typed_overload_without_an_outbound_call() {
let schema_url = crank_test_support::postgres_schema_url("admission_overload").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 1, 1);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// Another process already holds the only global permit.
let held = match store
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_test".to_owned(),
agent_key: "admin".to_owned(),
operation_id: "op_admission_budget".to_owned(),
operation_version: 7,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_holder".to_owned(),
expires_at_unix_ms: crank_core::current_unix_ms() + 10_000,
})
.await
.expect("admission store is reachable")
{
crank_core::AdmissionOutcome::Permitted(permit) => permit,
other => panic!("expected a permit, got {other:?}"),
};
let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 250,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let waiting_runtime = runtime.clone();
let first_waiting_operation = waiting_operation.clone();
let waiting = tokio::spawn(async move {
execute(&waiting_runtime, &first_waiting_operation, &json!({})).await
});
// Let the first request occupy the bounded queue before the next one arrives.
while store.snapshot().await.unwrap().waiting == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
let overloaded = execute(&runtime, &waiting_operation, &json!({})).await;
let failure = overloaded.expect_err("a full queue must refuse the request");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionOverloaded
);
assert_eq!(failure.stage(), ExecutionStage::Admission);
assert_eq!(failure.retry_after_ms(), Some(250));
// The queued request hits its absolute deadline instead of dispatching. That refusal happened
// inside admission (nothing was ever dispatched), so the client must see an admission-stage
// overload with retry guidance — not an `upstream_timeout`.
let expired = waiting
.await
.expect("waiting task")
.expect_err("deadline must refuse");
assert_eq!(
expired.error_code(),
ExecutionErrorCode::ExecutionOverloaded,
"queue waiting counts against the published deadline"
);
assert_eq!(expired.stage(), ExecutionStage::Admission);
assert_eq!(expired.retry_after_ms(), Some(250));
assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain);
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"no outbound call on refusal"
);
assert!(store.release(&held).await.unwrap());
}
#[tokio::test]
async fn background_policy_without_a_task_transport_is_typed_unsupported() {
let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
}))
.into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("no Task transport exists in Story 4.1");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionModeUnsupported
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let snapshot = store.snapshot().await.unwrap();
assert_eq!(snapshot.waiting, 0);
assert_eq!(snapshot.executing, 0);
}
#[tokio::test]
async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() {
let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let output = execute(&runtime, &operation, &json!({}))
.await
.expect("adaptive resolves through the published synchronous fallback");
// The response shape is unchanged by admission: an operation without output mapping still
// returns the same mapped output object it returned before Story 4.1.
assert_eq!(output, json!({}));
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn admission_store_outage_fails_closed_before_dispatch() {
let schema_url = crank_test_support::postgres_schema_url("admission_outage").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// The mandatory state disappears after the store was built: every execution must fail closed.
sqlx::query("drop table execution_admission_entries")
.execute(&pool)
.await
.expect("drop the mandatory admission state");
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("a missing mandatory store must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(store.health().await.is_err());
}
/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2).
struct RecordingAdmission {
inner: Arc<PostgresExecutionAdmission>,
requests: Mutex<Vec<AdmissionRequest>>,
permits: AtomicUsize,
releases: AtomicUsize,
}
impl RecordingAdmission {
fn new(inner: Arc<PostgresExecutionAdmission>) -> Self {
Self {
inner,
requests: Mutex::new(Vec::new()),
permits: AtomicUsize::new(0),
releases: AtomicUsize::new(0),
}
}
fn requests(&self) -> Vec<AdmissionRequest> {
self.requests.lock().expect("admission log").clone()
}
fn permits(&self) -> usize {
self.permits.load(Ordering::SeqCst)
}
fn releases(&self) -> usize {
self.releases.load(Ordering::SeqCst)
}
}
#[async_trait]
impl ExecutionAdmission for RecordingAdmission {
async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> {
self.inner.health().await
}
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<crank_core::AdmissionOutcome, crank_core::AdmissionStoreError> {
self.requests
.lock()
.expect("admission log")
.push(request.clone());
let outcome = self.inner.admit(request).await?;
if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) {
self.permits.fetch_add(1, Ordering::SeqCst);
}
Ok(outcome)
}
async fn release(
&self,
permit: &crank_core::AdmissionPermit,
) -> Result<bool, crank_core::AdmissionStoreError> {
let released = self.inner.release(permit).await?;
if released {
self.releases.fetch_add(1, Ordering::SeqCst);
}
Ok(released)
}
async fn cancel(
&self,
ticket_id: u64,
owner_token: &str,
) -> Result<bool, crank_core::AdmissionStoreError> {
self.inner.cancel(ticket_id, owner_token).await
}
async fn snapshot(
&self,
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
self.inner.snapshot().await
}
}
fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation {
let mut source = operation(None);
source.id = OperationId::new("op_admission_confirmation");
source.name = "admission_confirmation".to_owned();
source.version = version;
let Target::Rest(target) = &mut source.target;
target.method = HttpMethod::Delete;
source.execution_config.safety = Some(OperationSafetyPolicy {
class: OperationSafetyClass::Destructive,
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
});
source.into()
}
fn confirmation_runtime(
admission: Arc<RecordingAdmission>,
coordination: Option<Arc<dyn crank_core::CoordinationStateStore>>,
calls: Arc<AtomicUsize>,
) -> RuntimeExecutor {
let store: Arc<dyn ExecutionAdmission> = admission;
let builder = RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current: Arc::new(AtomicUsize::new(0)),
peak: Arc::new(AtomicUsize::new(0)),
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store);
match coordination {
Some(coordination) => builder.with_coordination_store(coordination).build(),
None => builder.build(),
}
}
fn confirmation_context() -> crank_runtime::RuntimeRequestContext {
crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_response_cache_scope("ws_test", "agent_a")
.with_metering_context(
WorkspaceId::new("ws_test"),
None,
InvocationSource::AdminTestRun,
)
}
#[tokio::test]
async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let coordination: Arc<dyn crank_core::CoordinationStateStore> =
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default());
let calls = Arc::new(AtomicUsize::new(0));
let runtime = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::clone(&coordination)),
Arc::clone(&calls),
);
let operation = confirmation_operation(3);
let context = confirmation_context();
let input = json!({ "reason": "remove the stale item" });
// 1. No approval yet: the existing challenge is returned and no execution capacity is taken.
let challenge = execute_in(&runtime, &operation, &input, &context)
.await
.expect_err("an unapproved destructive call must only stage confirmation");
assert_eq!(
challenge.error_code(),
ExecutionErrorCode::ConfirmationRequired
);
let token = challenge
.confirmation()
.expect("expected the confirmation challenge")
.token()
.to_owned();
assert_eq!(
recording.permits(),
0,
"an unapproved request must not take an execution permit"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let requests = recording.requests();
assert_eq!(
requests.len(),
1,
"the one admission snapshot is consulted once"
);
assert_eq!(
requests[0].approval,
ApprovalEvidence::NotGranted,
"admission must see the real approval state, not a fixed NotRequired"
);
assert_eq!(requests[0].scope.operation_version, 3);
let snapshot = recording.snapshot().await.expect("admission snapshot");
assert_eq!((snapshot.executing, snapshot.waiting), (0, 0));
// 2. An approval issued for another published version never opens this execution.
let other_version = confirmation_operation(4);
let foreign = execute_in(
&runtime,
&other_version,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a foreign approval must not authorise this version");
assert_eq!(
foreign.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(
recording.requests().len(),
1,
"an unusable approval is refused before any capacity request"
);
// 3. An approval is bound to its exact input as well.
let mismatched_input = json!({ "reason": "remove another item" });
let mismatched = execute_in(
&runtime,
&operation,
&mismatched_input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a confirmation token must not authorise different input");
assert_eq!(
mismatched.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(recording.requests().len(), 1);
// 4. The matching approval passes admission with truthful evidence and executes exactly once.
let executed = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect("a verified approval executes");
assert!(executed.is_object());
assert_eq!(calls.load(Ordering::SeqCst), 1);
let requests = recording.requests();
assert_eq!(requests.len(), 2);
assert_eq!(requests[1].approval, ApprovalEvidence::Granted);
assert_eq!(requests[1].scope.operation_version, 3);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.releases(),
1,
"the permit is released after execution"
);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
// 5. Replay of the consumed confirmation is refused before capacity and without a side effect.
let replay = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token),
)
.await
.expect_err("a confirmation token is single-use");
assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.requests().len(),
2,
"a replayed approval never reaches admission"
);
// 6. The trusted approved-recovery path still passes the same gate with real evidence.
let approved = execute_in(
&runtime,
&operation,
&json!({ "reason": "approved by an operator" }),
&context.with_approval_granted(),
)
.await;
assert!(approved.is_ok());
assert_eq!(calls.load(Ordering::SeqCst), 2);
let requests = recording.requests();
assert_eq!(requests.len(), 3);
assert_eq!(requests[2].approval, ApprovalEvidence::Granted);
assert_eq!(recording.permits(), 2);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
}
#[tokio::test]
async fn mandatory_safety_store_outage_fails_closed_before_admission() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let calls = Arc::new(AtomicUsize::new(0));
// Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail
// closed instead of reserving capacity and hoping for an in-memory fallback.
let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls));
let operation = confirmation_operation(3);
let failure = execute_in(
&runtime,
&operation,
&json!({ "reason": "no store" }),
&confirmation_context(),
)
.await
.expect_err("a mandatory safety store outage must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(recording.permits(), 0);
let after_missing_store = recording.requests().len();
// The same Tool with a claimed approval and a *failing* safety store never reaches the capacity
// store at all: the approval claim stays unverified, so no permit and no outbound call exist.
let failing = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::new(UnavailableSafetyStore)),
Arc::clone(&calls),
);
let outage = execute_in(
&failing,
&operation,
&json!({ "reason": "failing store" }),
&confirmation_context().with_confirmation_token("ct_unavailable"),
)
.await
.expect_err("an unreachable safety store must refuse the claimed approval");
assert_eq!(
outage.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(
recording.requests().len(),
after_missing_store,
"an unverified approval never becomes a capacity request"
);
assert_eq!(recording.permits(), 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
/// Coordination store whose mandatory backend is down: every operation fails closed.
struct UnavailableSafetyStore;
impl UnavailableSafetyStore {
fn error() -> crank_core::CacheStoreError {
crank_core::CacheStoreError::Unavailable {
message: "test backend unavailable".to_owned(),
}
}
}
#[async_trait]
impl crank_core::CoordinationStateStore for UnavailableSafetyStore {
async fn get_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn put_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn delete_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn take_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn reserve_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<crank_core::CoordinationStateReservation, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn compare_and_set_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_expected: &crank_core::CoordinationStateValue,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<bool, crank_core::CacheStoreError> {
Err(Self::error())
}
}
@@ -621,7 +621,6 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation<Schema, Ma
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
-2
View File
@@ -1036,7 +1036,6 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Secret name is required.',
'wizard.error.secret_value_required': 'Secret value is required.',
'wizard.error.save_upstream_first': 'Save the API host after creating a new authorization profile before continuing.',
'wizard.error.execution_policy_invalid': 'Execution policy is invalid: mode, deadline from 1 to 3600000 ms and no-Tasks fallback are required.',
'wizard.toast.auth_profile_created_title': 'Authorization profile created',
'wizard.toast.auth_profile_created_body': 'New authorization profile is ready and selected.',
'wizard.toast.auth_profile_error_title': 'Authorization profile creation failed',
@@ -2354,7 +2353,6 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Нужно указать имя секрета.',
'wizard.error.secret_value_required': 'Нужно указать значение секрета.',
'wizard.error.save_upstream_first': 'Сначала сохраните API-хост после создания нового профиля авторизации.',
'wizard.error.execution_policy_invalid': 'Политика выполнения некорректна: нужны режим, deadline от 1 до 3600000 мс и поведение без Tasks.',
'wizard.toast.auth_profile_created_title': 'Профиль авторизации создан',
'wizard.toast.auth_profile_created_body': 'Новый профиль авторизации готов и уже выбран.',
'wizard.toast.auth_profile_error_title': 'Не удалось создать профиль авторизации',
-30
View File
@@ -214,33 +214,10 @@ function parseExecutionConfig(text) {
auth_profile_ref: authProfileRef,
headers: headers,
};
var executionPolicy = parseExecutionPolicy(value.execution_policy);
if (executionPolicy) {
config.execution_policy = executionPolicy;
}
return config;
}
// Story 4.1: the published execution policy must survive a wizard save. The editor is a free-form
// document, so unknown/legacy shapes stay untouched: only a fully valid policy is projected, and an
// explicit invalid value is reported instead of being silently replaced by defaults.
function parseExecutionPolicy(raw) {
if (raw === undefined || raw === null) return null;
if (typeof raw !== 'object' || Array.isArray(raw)) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
var mode = raw.mode === undefined ? 'synchronous' : raw.mode;
var fallback = raw.no_tasks_fallback === undefined ? 'synchronous' : raw.no_tasks_fallback;
var deadline = Number(raw.deadline_ms);
if (['synchronous', 'background', 'adaptive'].indexOf(mode) < 0
|| ['synchronous', 'unsupported'].indexOf(fallback) < 0
|| !Number.isInteger(deadline) || deadline < 1 || deadline > 3600000) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback };
}
function currentUpstream() {
var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; });
if (selected) return selected;
@@ -438,13 +415,6 @@ function executionConfigToEditorValue(config) {
var value = {
timeout_ms: config.timeout_ms,
};
if (config.execution_policy) {
value.execution_policy = {
mode: config.execution_policy.mode,
deadline_ms: config.execution_policy.deadline_ms,
no_tasks_fallback: config.execution_policy.no_tasks_fallback,
};
}
if (config.retry_policy) {
value.retry = { max_attempts: config.retry_policy.max_attempts };
}
@@ -1,139 +0,0 @@
const { test, expect } = require('@playwright/test');
const { getCurrentWorkspace, login } = require('./helpers');
// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a
// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit
// invalid value must be reported instead of being silently replaced by defaults.
function draftSnapshot(operationId) {
return {
id: operationId,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
target: {
kind: 'rest',
base_url: 'https://api.example.test',
method: 'GET',
path_template: '/rates',
static_headers: {},
},
input_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
output_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
input_mapping: { rules: [] },
output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] },
execution_config: {
timeout_ms: 10000,
retry_policy: null,
auth_profile_ref: null,
headers: {},
approval_policy: null,
execution_policy: {
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
},
},
tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] },
wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] },
};
}
test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => {
await login(page);
const workspace = await getCurrentWorkspace(page);
const operationId = 'policy_rates_41';
const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`;
const savedPayloads = [];
await page.route(operationPath, async (route) => {
if (route.request().method() === 'PATCH') {
savedPayloads.push(route.request().postDataJSON());
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }),
});
return;
}
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
id: operationId,
workspace_id: workspace.id,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
status: 'draft',
current_draft_version: 1,
draft_version_ref: { version: 1, status: 'draft' },
published_version_ref: null,
agent_refs: [],
}),
});
});
await page.route(`${operationPath}/versions/*`, async (route) => {
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
operation_id: operationId,
workspace_id: workspace.id,
version: 1,
status: 'draft',
snapshot: draftSnapshot(operationId),
}),
});
});
await page.route('**/operations/analyze-quality', async (route) => {
await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) });
});
page.on('dialog', (dialog) => dialog.accept());
await page.goto(`/wizard/?mode=edit&operationId=${operationId}`);
await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates');
const editorValue = await page.locator('#tool-exec-config').inputValue();
expect(editorValue).toContain('execution_policy');
expect(editorValue).toContain('adaptive');
expect(editorValue).toContain('45000');
expect(editorValue).toContain('unsupported');
// A real edit still saves the pinned policy unchanged.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(4));
await page.locator('#tool-display-name').fill('Policy rates edited');
await page.locator('.btn-save-draft').click();
await expect.poll(() => savedPayloads.length).toBe(1);
expect(savedPayloads[0].execution_config.execution_policy).toEqual({
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
});
// An explicit invalid policy is refused instead of being silently rewritten to defaults.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
await expect(page.locator('#tool-exec-config')).toBeVisible();
await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n');
await page.locator('.btn-save-draft').click();
await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/);
expect(savedPayloads.length).toBe(1);
});
+14 -31
View File
@@ -311,41 +311,24 @@ async fn health() -> Json<Value> {
}
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
let postgres_ready = state.registry.ping().await.is_ok();
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
// reflect it rather than reporting ready on a bare `select 1`.
let admission_ready = state.registry.execution_admission_health().await.is_ok();
if postgres_ready && admission_ready {
return Json(json!({
match state.registry.ping().await {
Ok(()) => Json(json!({
"service": "mcp-server",
"status": "ready",
"checks": { "postgres": "ready", "execution_admission": "ready" }
"checks": { "postgres": "ready" }
}))
.into_response();
.into_response(),
Err(_) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": "database is unavailable"
})),
)
.into_response(),
}
let mut checks = serde_json::Map::new();
checks.insert(
"postgres".to_owned(),
json!(if postgres_ready { "ready" } else { "not_ready" }),
);
checks.insert(
"execution_admission".to_owned(),
json!(if admission_ready {
"ready"
} else {
"not_ready"
}),
);
(
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": checks,
"error": "a mandatory dependency is unavailable"
})),
)
.into_response()
}
#[cfg(test)]
@@ -550,7 +550,6 @@ pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -136,7 +136,6 @@ fn published_tool(name: &str, title: &str, description: &str, version: u32) -> P
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -284,7 +284,6 @@ fn operation() -> RegistryOperation {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
-531
View File
@@ -1,531 +0,0 @@
//! Typed execution-admission seam (Story 4.1, FR-21/FR-24).
//!
//! Admission is the single gate in front of every real Tool execution: it decides whether the
//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide
//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives
//! in PostgreSQL (`crank-registry`), so several processes share one budget.
//!
//! Deliberately absent from this contract: credentials, connection or protocol revision (they must
//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2).
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use crate::{NoTasksFallback, ToolExecutionMode};
pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128;
pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64;
pub const MAX_RETRY_AFTER_MS: u64 = 60_000;
/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for
/// the store and every runtime so the client contract stays stable.
pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250;
/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms).
pub fn current_unix_ms() -> i64 {
i64::try_from(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_millis())
.unwrap_or_default(),
)
.unwrap_or(i64::MAX)
}
/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin
/// bucket that is still bounded by the global limit.
pub const ADMIN_ORIGIN_KEY: &str = "admin";
/// Exact execution identity pinned at admission time.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AdmissionScope {
pub workspace_id: String,
/// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution.
pub agent_key: String,
pub operation_id: String,
/// Quota keys exclude the version; the version is pinned for execution semantics only.
pub operation_version: u32,
}
impl AdmissionScope {
/// Quota key of the Agent bucket: `(workspace, agent)`.
pub fn agent_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.agent_key)
}
/// Quota key of the Tool bucket: `(workspace, operation)` without version.
pub fn tool_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.operation_id)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AdmissionMode {
/// The caller waits for a scoped permit and dispatches inside it.
Synchronous,
/// The store commits a durable enqueue decision and reserves queue capacity.
Background,
}
impl AdmissionMode {
pub const fn as_str(self) -> &'static str {
match self {
Self::Synchronous => "synchronous",
Self::Background => "background",
}
}
}
/// Non-consuming approval evidence gathered by the runtime before admission.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ApprovalEvidence {
/// The operation does not require confirmation.
NotRequired,
/// A verified approval exists for this exact scope and input.
Granted,
/// Confirmation is required and no verified approval exists yet.
NotGranted,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionRequest {
pub scope: AdmissionScope,
pub mode: AdmissionMode,
/// Published policy mode that produced this request.
pub policy_mode: ToolExecutionMode,
pub no_tasks_fallback: NoTasksFallback,
pub approval: ApprovalEvidence,
/// Bounded request identity used only to correlate one admission entry.
pub fingerprint: String,
/// Absolute deadline (unix milliseconds), already including queue waiting time.
pub expires_at_unix_ms: i64,
}
impl AdmissionRequest {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if self.fingerprint.is_empty()
|| self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES
|| self
.fingerprint
.bytes()
.any(|byte| byte.is_ascii_control() || byte == b' ')
{
return Err(AdmissionStoreError::invalid(
"admission fingerprint is outside its allowed bounds",
));
}
if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission workspace is invalid",
));
}
if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 {
return Err(AdmissionStoreError::invalid("admission agent is invalid"));
}
if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission operation is invalid",
));
}
if self.scope.operation_version == 0 {
return Err(AdmissionStoreError::invalid(
"admission operation version must be positive",
));
}
if self.expires_at_unix_ms <= 0 {
return Err(AdmissionStoreError::invalid(
"admission deadline is invalid",
));
}
Ok(())
}
}
/// Scoped permit. Held for the whole local execution future (including retries) and released
/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionPermit {
pub ticket_id: u64,
pub owner_token: String,
pub scope: AdmissionScope,
pub mode: AdmissionMode,
pub policy_mode: ToolExecutionMode,
}
/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics.
/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionDecision {
pub ticket_id: u64,
pub scope: AdmissionScope,
pub expires_at_unix_ms: i64,
pub policy_mode: ToolExecutionMode,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AdmissionOutcome {
Permitted(AdmissionPermit),
Enqueued(AdmissionDecision),
/// Capacity or queue bounds were exhausted before any unbounded allocation.
Overloaded {
retry_after_ms: u64,
},
/// The queue wait reached the absolute deadline; nothing was dispatched.
DeadlineElapsed,
/// Confirmation is mandatory for this operation and no verified approval exists.
ApprovalRequired,
/// The published policy does not advertise durable Tasks and refuses the fallback.
Unsupported,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionQueueSnapshot {
pub waiting: u64,
pub executing: u64,
/// Age of the oldest waiting request in the authoritative queue, in milliseconds.
pub oldest_waiting_ms: Option<u64>,
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum AdmissionStoreError {
/// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed.
#[error("execution admission store is unavailable: {message}")]
Unavailable { message: String },
/// The store rejected a malformed admission request.
#[error("execution admission request is invalid: {message}")]
Invalid { message: String },
}
impl AdmissionStoreError {
pub fn unavailable(message: impl Into<String>) -> Self {
Self::Unavailable {
message: message.into(),
}
}
pub fn invalid(message: impl Into<String>) -> Self {
Self::Invalid {
message: message.into(),
}
}
}
#[async_trait]
pub trait ExecutionAdmission: Send + Sync {
/// Proves the mandatory admission state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), AdmissionStoreError>;
/// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue
/// decision for a background request, or returns a typed refusal.
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<AdmissionOutcome, AdmissionStoreError>;
/// Releases the caller's own permit. Idempotent: a repeated release returns `false`.
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError>;
/// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket.
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError>;
/// Bounded authoritative queue readings for low-cardinality metrics.
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError>;
}
/// One waiting entry as seen by the fairness scheduler.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionQueueEntry {
pub ticket_id: u64,
/// Authoritative insertion order inside `(Agent, Tool)`; never rewritten.
pub sequence: u64,
pub agent_key: String,
pub operation_id: String,
}
/// Weighted round-robin state persisted with the authoritative queue.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionCursor {
pub agent: Option<String>,
pub agent_credit: u32,
pub tool: Option<String>,
pub tool_credit: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionSchedulerConfig {
pub agent_weight: u32,
pub tool_weight: u32,
}
impl Default for AdmissionSchedulerConfig {
fn default() -> Self {
Self {
agent_weight: 1,
tool_weight: 1,
}
}
}
impl AdmissionSchedulerConfig {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) {
return Err(AdmissionStoreError::invalid(
"admission weights must be between 1 and 64",
));
}
Ok(())
}
}
/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the
/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its
/// turn instead of being skipped by newer arrivals.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionSelection {
pub index: usize,
pub cursor: AdmissionCursor,
}
/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside
/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit.
pub fn select_next_grant(
queue: &[AdmissionQueueEntry],
cursor: &AdmissionCursor,
config: AdmissionSchedulerConfig,
) -> Option<AdmissionSelection> {
if queue.is_empty() {
return None;
}
let agent_weight = config.agent_weight.max(1);
let tool_weight = config.tool_weight.max(1);
let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect();
agents.sort_unstable();
agents.dedup();
let mut next = cursor.clone();
let agent = match cursor.agent.as_deref() {
Some(current) if cursor.agent_credit > 0 && agents.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.agent.as_deref() {
None => agents[0].to_owned(),
Some(current) => agents
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(agents[0])
.to_owned(),
};
if cursor.agent.as_deref() != Some(chosen.as_str()) {
next.tool = None;
next.tool_credit = 0;
}
next.agent = Some(chosen.clone());
next.agent_credit = agent_weight;
chosen
}
};
let mut tools: Vec<&str> = queue
.iter()
.filter(|entry| entry.agent_key == agent)
.map(|entry| entry.operation_id.as_str())
.collect();
tools.sort_unstable();
tools.dedup();
let tool = match cursor.tool.as_deref() {
Some(current) if cursor.tool_credit > 0 && tools.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.tool.as_deref() {
None => tools[0].to_owned(),
Some(current) => tools
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(tools[0])
.to_owned(),
};
next.tool = Some(chosen.clone());
next.tool_credit = tool_weight;
chosen
}
};
let index = queue
.iter()
.position(|entry| entry.agent_key == agent && entry.operation_id == tool)?;
next.agent_credit = next.agent_credit.saturating_sub(1);
next.tool_credit = next.tool_credit.saturating_sub(1);
Some(AdmissionSelection {
index,
cursor: next,
})
}
#[cfg(test)]
mod tests {
use super::{
ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest,
AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence,
select_next_grant,
};
use crate::{NoTasksFallback, ToolExecutionMode};
fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry {
super::AdmissionQueueEntry {
ticket_id,
sequence: ticket_id,
agent_key: agent.to_owned(),
operation_id: tool.to_owned(),
}
}
fn request() -> AdmissionRequest {
AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_default".to_owned(),
agent_key: ADMIN_ORIGIN_KEY.to_owned(),
operation_id: "op_leads".to_owned(),
operation_version: 3,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_0123456789".to_owned(),
expires_at_unix_ms: 1_790_000_000_000,
}
}
#[test]
fn quota_keys_exclude_version_and_keep_agent_bucket() {
let scope = request().scope;
assert_eq!(scope.agent_bucket(), "ws_default:admin");
assert_eq!(scope.tool_bucket(), "ws_default:op_leads");
}
#[test]
fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() {
let mut invalid = request();
invalid.expires_at_unix_ms = 0;
assert!(matches!(
invalid.validate(),
Err(AdmissionStoreError::Invalid { .. })
));
let mut invalid = request();
invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1);
assert!(invalid.validate().is_err());
let mut invalid = request();
invalid.scope.operation_version = 0;
assert!(invalid.validate().is_err());
assert!(request().validate().is_ok());
}
#[test]
fn weights_are_bounded_positive_configuration() {
assert!(
AdmissionSchedulerConfig {
agent_weight: 0,
tool_weight: 1
}
.validate()
.is_err()
);
assert!(
AdmissionSchedulerConfig {
agent_weight: 1,
tool_weight: 65
}
.validate()
.is_err()
);
assert!(AdmissionSchedulerConfig::default().validate().is_ok());
}
#[test]
fn weighted_round_robin_serves_each_agent_its_quantum() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_1"),
entry(4, "agent_b", "op_1"),
entry(5, "agent_b", "op_1"),
];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
order.push(queue.remove(selection.index).agent_key.clone());
cursor = selection.cursor;
}
assert_eq!(
order,
vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"]
);
}
#[test]
fn fifo_inside_one_tool_and_round_robin_between_tools() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_2"),
];
let config = AdmissionSchedulerConfig::default();
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
// A granted ticket leaves the waiting queue: granted entries are never re-granted.
order.push(queue.remove(selection.index).ticket_id);
cursor = selection.cursor;
}
// One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue.
assert_eq!(order, vec![1, 3, 2]);
}
#[test]
fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() {
let queue = vec![entry(1, "agent_a", "op_1")];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let cursor = AdmissionCursor::default();
let first = select_next_grant(&queue, &cursor, config).unwrap();
assert_eq!(first.cursor.agent.as_deref(), Some("agent_a"));
assert_eq!(first.cursor.agent_credit, 1);
// A later Agent joins; the cursor keeps its remaining quantum for `agent_a`.
let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")];
let second = select_next_grant(&queue, &first.cursor, config).unwrap();
assert_eq!(queue[second.index].agent_key, "agent_a");
assert_eq!(second.cursor.agent_credit, 0);
let third = select_next_grant(&queue, &second.cursor, config).unwrap();
assert_eq!(queue[third.index].agent_key, "agent_b");
assert_eq!(third.cursor.agent_credit, 1);
}
#[test]
fn empty_queue_keeps_cursor_untouched() {
let cursor = AdmissionCursor {
agent: Some("agent_b".to_owned()),
agent_credit: 1,
tool: Some("op_1".to_owned()),
tool_credit: 0,
};
assert_eq!(
select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()),
None
);
assert_eq!(cursor.agent.as_deref(), Some("agent_b"));
}
}
+9 -19
View File
@@ -1,5 +1,4 @@
pub mod access;
pub mod admission;
pub mod agent;
pub mod approval;
pub mod auth;
@@ -61,11 +60,10 @@ pub mod domain {
pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use crate::operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, WizardState,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState,
};
pub use crate::product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
@@ -116,13 +114,6 @@ pub use access::{
InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
};
pub use admission::{
ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS,
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission,
MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant,
};
pub use agent::{
Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
@@ -177,14 +168,13 @@ pub use observability::{
pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation,
OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy,
OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract,
RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget,
RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, ToolExecutionPolicyError, WizardState,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState,
ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor,
RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample,
WizardState,
};
pub use product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
+2 -144
View File
@@ -270,72 +270,6 @@ pub struct OperationApprovalPolicy {
pub elicitation_message: Option<String>,
}
/// Published execution mode of one Tool (Story 4.1, FR-21).
///
/// This is a scheduling policy of the Published Tool version. It is deliberately separate from
/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolExecutionMode {
#[default]
Synchronous,
Background,
Adaptive,
}
/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NoTasksFallback {
/// Execute through the same synchronous admission path.
#[default]
Synchronous,
/// Reject with a typed unsupported-execution-mode failure; never a Task ID.
Unsupported,
}
pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000;
/// Execution policy pinned by a Published Tool version.
///
/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by
/// `ExecutionConfig::timeout_ms`, with no background path.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ToolExecutionPolicy {
#[serde(default)]
pub mode: ToolExecutionMode,
/// Absolute budget of one execution, counted from the start of admission.
pub deadline_ms: u64,
#[serde(default)]
pub no_tasks_fallback: NoTasksFallback,
}
impl Default for ToolExecutionPolicy {
fn default() -> Self {
Self {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
}
}
impl ToolExecutionPolicy {
pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> {
if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) {
return Err(ToolExecutionPolicyError::DeadlineOutOfRange);
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ToolExecutionPolicyError {
#[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")]
DeadlineOutOfRange,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExecutionConfig {
@@ -351,8 +285,6 @@ pub struct ExecutionConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub approval_policy: Option<OperationApprovalPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub execution_policy: Option<ToolExecutionPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auth_profile_ref: Option<AuthProfileId>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub headers: BTreeMap<String, String>,
@@ -512,10 +444,9 @@ mod tests {
edition::OperationSecurityLevel,
ids::{AuthProfileId, OperationId},
operation::{
ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability,
OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus,
ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationStatus,
OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample,
ToolExecutionMode, ToolExecutionPolicy,
},
protocol::{AuthKind, ExportMode, HttpMethod, Protocol},
};
@@ -733,7 +664,6 @@ updated_at: 2026-03-25T08:10:00Z
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: Some(AuthProfileId::new("auth_01")),
headers: BTreeMap::new(),
},
@@ -754,76 +684,4 @@ updated_at: 2026-03-25T08:10:00Z
published_at: None,
}
}
#[test]
fn execution_policy_round_trips_all_modes_and_fallbacks() {
for (mode, fallback) in [
(ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Unsupported),
(ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported),
] {
let config = ExecutionConfig {
execution_policy: Some(ToolExecutionPolicy {
mode,
deadline_ms: 45_000,
no_tasks_fallback: fallback,
}),
..ExecutionConfig::default()
};
let encoded = serde_json::to_value(&config).unwrap();
assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000));
let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap();
assert_eq!(decoded, config);
assert_eq!(
decoded.execution_policy.unwrap().mode,
mode,
"the published mode must survive the JSON round-trip"
);
}
}
#[test]
fn legacy_execution_config_without_policy_stays_synchronous() {
let legacy: ExecutionConfig =
serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap();
assert_eq!(legacy.execution_policy, None);
assert_eq!(legacy.timeout_ms, 12_000);
}
#[test]
fn execution_policy_rejects_explicit_invalid_deadline() {
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 0,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
.validate()
.is_err()
);
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: NoTasksFallback::Unsupported,
}
.validate()
.is_err()
);
assert!(ToolExecutionPolicy::default().validate().is_ok());
}
#[test]
fn execution_policy_rejects_unknown_fields() {
assert!(
serde_json::from_value::<ToolExecutionPolicy>(json!({
"mode": "background",
"deadline_ms": 1000,
"no_tasks_fallback": "synchronous",
"task_queue": "default"
}))
.is_err()
);
}
}
@@ -231,7 +231,6 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
-1
View File
@@ -564,7 +564,6 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
-28
View File
@@ -217,34 +217,6 @@ string_enum! {
Concurrency => "concurrency",
RateLimit => "rate_limit",
McpStream => "mcp_stream",
Admission => "admission",
}
}
string_enum! {
pub enum RejectionReason {
Limit => "limit",
Capacity => "capacity",
QueueFull => "queue_full",
Deadline => "deadline",
Unsupported => "unsupported",
DependencyUnavailable => "dependency_unavailable",
}
}
string_enum! {
pub enum ScopeKind {
Process => "process",
Global => "global",
Agent => "agent",
Tool => "tool",
}
}
string_enum! {
pub enum AdmissionModeLabel {
Synchronous => "synchronous",
Background => "background",
}
}
+5 -6
View File
@@ -16,15 +16,14 @@ mod schema;
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
pub use labels::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
UpstreamOutcome,
};
pub use record::{
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service,
bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting,
record_admission_queue_wait, record_admission_rejection, record_cache_outcome,
bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
+5 -41
View File
@@ -5,10 +5,10 @@ use metrics::Gauge;
use crate::schema::{MetricDefinition, metric_schema};
use crate::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter,
HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason,
ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
static BINDING: OnceLock<MetricService> = OnceLock::new();
@@ -352,47 +352,11 @@ pub fn record_limit_rejection(stage: LimitStage) {
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => stage.as_str(),
"reason" => RejectionReason::Limit.as_str(),
"scope_kind" => ScopeKind::Process.as_str()
"stage" => stage.as_str()
)
.increment(1);
}
/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool,
/// request id, fingerprint or arbitrary error text ever becomes a label.
pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) {
if bound_family("crank_runtime_limit_rejections_total").is_none() {
return;
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => LimitStage::Admission.as_str(),
"reason" => reason.as_str(),
"scope_kind" => scope_kind.as_str()
)
.increment(1);
}
pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) {
if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() {
return;
}
metrics::gauge!(
"crank_runtime_admission_queue_wait_seconds",
"mode" => mode.as_str()
)
.set(waited_ms as f64 / 1000.0);
}
pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option<u64>) {
if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() {
return;
}
let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0;
metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds);
}
pub fn record_cache_outcome(outcome: CacheOutcome) {
if bound_family("crank_runtime_cache_total").is_none() {
return;
+8 -38
View File
@@ -3,17 +3,16 @@
use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP};
use crate::labels::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome,
UpstreamOperationKind, UpstreamOutcome,
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
pub const METRIC_SCHEMA_VERSION: u32 = 2;
pub const MAX_METRIC_FAMILIES: usize = 128;
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515;
pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348;
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
@@ -180,9 +179,6 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
const REJECTION_REASONS: &[&str] = RejectionReason::VALUES;
const SCOPE_KINDS: &[&str] = ScopeKind::VALUES;
const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES;
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
@@ -407,39 +403,13 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[
"crank_runtime_limit_rejections_total",
MetricKind::Counter,
MetricUnit::Count,
&["stage", "reason", "scope_kind"],
&[
domain("stage", LIMIT_STAGES),
domain("reason", REJECTION_REASONS),
domain("scope_kind", SCOPE_KINDS),
],
&["stage"],
&[domain("stage", LIMIT_STAGES)],
&[],
BOTH,
96,
3,
"Runtime executions rejected by a bounded limit.",
),
metric(
"crank_runtime_admission_queue_wait_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&["mode"],
&[domain("mode", ADMISSION_MODES)],
&[],
BOTH,
2,
"Wait of the most recently admitted execution before it received a scoped permit.",
),
metric(
"crank_runtime_admission_oldest_waiting_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&[],
&[],
&[],
BOTH,
1,
"Age of the oldest waiting admission request.",
),
metric(
"crank_runtime_cache_total",
MetricKind::Counter,
+6 -13
View File
@@ -1,13 +1,12 @@
use std::{collections::BTreeSet, time::Duration};
use crank_metrics::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType,
ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema,
record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection,
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics,
bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
@@ -72,9 +71,6 @@ fn schema_names_are_unique_and_labels_are_closed() {
"state",
"signal_type",
"exporter",
"reason",
"scope_kind",
"mode",
];
for metric in schema {
assert!(
@@ -269,9 +265,6 @@ fn every_recording_api_matches_the_declared_schema() {
Duration::from_millis(1),
);
record_limit_rejection(LimitStage::Concurrency);
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5);
record_admission_oldest_waiting(Some(5));
record_cache_outcome(CacheOutcome::Hit);
record_idempotency_outcome(IdempotencyOutcome::Replay);
record_confirmation_outcome(ConfirmationOutcome::Required);
+8 -8
View File
@@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
#[test]
fn frozen_budget_edges_are_exact() {
assert!(validate_budget(6_380, 19_610).is_ok());
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_000).is_err());
assert!(validate_budget(6_000, 19_612).is_err());
assert!(validate_budget(6_284, 19_514).is_ok());
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 16_000).is_err());
assert!(validate_budget(6_000, 19_516).is_err());
assert_eq!(
crank_metrics::max_exemplar_slots(MetricService::AdminApi),
12_348
@@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() {
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_381,
rendered_series: 19_611
logical_series: 6_285,
rendered_series: 19_515
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_179,
rendered_series: 3_654
logical_series: 1_083,
rendered_series: 3_558
})
);
}
+11 -11
View File
@@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() {
"root обязан объявлять ровно один ключ budgets"
);
assert!(rendered.ends_with(
",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\
\"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n"
",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\
\"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n"
));
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_381,
rendered_series: 19_611
logical_series: 6_285,
rendered_series: 19_515
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_179,
rendered_series: 3_654
logical_series: 1_083,
rendered_series: 3_558
})
);
}
@@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() {
/// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны.
#[test]
fn frozen_caps_and_edges_are_exact() {
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611);
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515);
assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348);
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_611).is_err());
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 19_515).is_err());
assert!(validate_exemplar_budget(12_347).is_ok());
assert!(validate_exemplar_budget(12_348).is_ok());
assert!(validate_exemplar_budget(12_349).is_err());
+1 -1
View File
@@ -13,7 +13,7 @@ mod schema;
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
pub use crank_metrics::{
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit,
bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting,
bind_metric_service, bound_metric_service, metric_schema,
};
pub use error_reporting::{
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
-1
View File
@@ -7,7 +7,6 @@ publish.workspace = true
version.workspace = true
[dependencies]
async-trait = "0.1"
crank-artifacts = { path = "../crank-artifacts" }
crank-core = { path = "../crank-core" }
crank-mapping = { path = "../crank-mapping" }
+2 -3
View File
@@ -116,7 +116,6 @@ pub use model::{
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
};
pub use postgres::{
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, admission_health,
admission_snapshot,
AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError,
PostgresRegistry,
};
-2
View File
@@ -6,7 +6,6 @@ mod artifact_cleanup_indexes_v13;
mod artifact_metadata_v12;
mod authority;
mod baseline_v1;
mod execution_admission_v24;
mod execution_outcome_v5;
mod import_plan_v14;
mod master_key_identity_v7;
@@ -36,7 +35,6 @@ mod schema_guard_v19;
mod schema_guard_v20;
mod schema_guard_v22;
mod schema_guard_v23;
mod schema_guard_v24;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -4,7 +4,6 @@ use super::agent_resource_bindings_v18;
use super::approval_side_effects_v10;
use super::artifact_cleanup_indexes_v13;
use super::artifact_metadata_v12;
use super::execution_admission_v24;
use super::execution_outcome_v5;
use super::import_plan_v14;
use super::master_key_identity_v7;
@@ -40,9 +39,9 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 24;
const CURRENT_VERSION: i64 = 23;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -331,12 +330,6 @@ impl MigrationAuthority {
prompt_dependencies_v23::SOURCE_SHA256,
22,
),
expand_descriptor(
24,
execution_admission_v24::NAME,
execution_admission_v24::SOURCE_SHA256,
23,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -496,9 +489,6 @@ impl MigrationAuthority {
if from < 23 {
prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?;
}
if from < 24 {
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -140,8 +140,6 @@ pub(super) fn validate_descriptors(
!= agent_resource_bindings_v18::SOURCE_SHA256
|| sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes())
!= resource_security_revocations_v19::SOURCE_SHA256
|| sha256_hex(execution_admission_v24::SOURCE.as_bytes())
!= execution_admission_v24::SOURCE_SHA256
{
return Err(MigrationError::new(
"invalid_contract",
@@ -148,7 +148,6 @@ pub(super) async fn inspect(
owned_relations::PROMPTS,
owned_relations::PROMPT_PUBLICATIONS,
owned_relations::PROMPT_DEPENDENCIES,
owned_relations::EXECUTION_ADMISSION,
] {
validate_required_relations(connection, relations, CURRENT_VERSION).await?;
}
@@ -1,45 +0,0 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "execution-admission-v24";
pub(super) const SOURCE: &str = include_str!("execution_admission_v24.sql");
pub(super) const SOURCE_SHA256: &str =
"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.execution_admission_v24",
Some(24),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.canonical_ledger",
Some(24),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -1,92 +0,0 @@
-- V24: authoritative execution admission state (Story 4.1, FR-21, FR-24).
--
-- Один deployment-wide admission: PostgreSQL хранит cursor/fairness state, ожидающие тикеты,
-- выданные permits и durable enqueue decisions, поэтому несколько процессов делят общий бюджет.
--
-- `execution_admission_entries` — bounded набор: waiting (ожидает capacity), executing (scoped
-- permit выдан, держится до явного release) и reserved (durable enqueue decision с зарезервированным
-- queue slot; не исполняется и не рекламируется как Task до Story 4.2).
--
-- Строка не хранит ни credentials, ни connection/session id, ни prepared headers, ни тело запроса:
-- quota key — (workspace, agent) и (workspace, operation) без version. `operation_version` пишется
-- только как pin исполняемых semantics и не участвует в квотах.
--
-- `expires_at` — абсолютный deadline. Queue decisions без dispatch можно expire по deadline; активный
-- permit НЕ освобождается по TTL (иначе paused/crashed owner обойдёт global limit): его снимает только
-- явный release/cancel владельца.
create table execution_admission_controllers (
workspace_id text not null,
cursor_agent text,
cursor_agent_credit integer not null default 0,
cursor_tool text,
cursor_tool_credit integer not null default 0,
updated_at timestamptz not null default now(),
constraint execution_admission_controllers_pkey primary key (workspace_id),
constraint execution_admission_controllers_workspace_check check (
octet_length(workspace_id) between 1 and 160
),
constraint execution_admission_controllers_agent_credit_check check (
cursor_agent_credit between 0 and 64
),
constraint execution_admission_controllers_tool_credit_check check (
cursor_tool_credit between 0 and 64
)
);
create table execution_admission_entries (
ticket_id bigint generated always as identity,
workspace_id text not null,
agent_key text not null,
operation_id text not null,
operation_version integer not null,
mode text not null,
policy_mode text not null,
state text not null,
owner_token text not null,
fingerprint text not null,
created_at timestamptz not null default now(),
expires_at timestamptz not null,
granted_at timestamptz,
constraint execution_admission_entries_pkey primary key (ticket_id),
constraint execution_admission_entries_workspace_check check (
octet_length(workspace_id) between 1 and 160
),
constraint execution_admission_entries_agent_check check (
octet_length(agent_key) between 1 and 160
),
constraint execution_admission_entries_operation_check check (
octet_length(operation_id) between 1 and 160
),
constraint execution_admission_entries_version_check check (operation_version >= 1),
constraint execution_admission_entries_mode_check check (
mode in ('synchronous', 'background')
),
constraint execution_admission_entries_policy_mode_check check (
policy_mode in ('synchronous', 'background', 'adaptive')
),
constraint execution_admission_entries_state_check check (
state in ('waiting', 'executing', 'reserved')
),
constraint execution_admission_entries_owner_check check (
octet_length(owner_token) between 1 and 64
),
constraint execution_admission_entries_fingerprint_check check (
octet_length(fingerprint) between 1 and 128
),
constraint execution_admission_entries_reserved_mode_check check (
state <> 'reserved' or mode = 'background'
)
);
create unique index execution_admission_entries_owner_uidx
on execution_admission_entries (owner_token);
create index execution_admission_entries_state_idx
on execution_admission_entries (state, workspace_id, ticket_id);
create index execution_admission_entries_agent_idx
on execution_admission_entries (state, workspace_id, agent_key);
create index execution_admission_entries_operation_idx
on execution_admission_entries (state, workspace_id, operation_id);
@@ -55,11 +55,6 @@ pub(super) const PROMPT_PUBLICATIONS: &[&str] = &["prompt_publications", "agent_
pub(super) const PROMPT_DEPENDENCIES: &[&str] =
&["prompt_dependencies", "prompt_message_resource_refs"];
pub(super) const EXECUTION_ADMISSION: &[&str] = &[
"execution_admission_controllers",
"execution_admission_entries",
];
pub(super) const ARTIFACT_METADATA: &[&str] = &["artifact_blobs", "artifact_sources"];
pub(super) const IMPORT_PLAN: &[&str] = &["import_plans", "import_plan_applications"];
@@ -468,13 +468,6 @@ pub(super) async fn validate_schema_fingerprint(
} else {
super::schema_guard_v23::validate_v23_prompt_dependencies(connection).await?;
}
if current_version < 24 {
if !super::schema_guard_v24::validate_v24_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v24::validate_v24_execution_admission(connection).await?;
}
Ok(())
}
@@ -51,8 +51,6 @@ pub(crate) const OWNED_RELATIONS: &[&str] = &[
"prompt_messages",
"prompt_dependencies",
"prompt_message_resource_refs",
"execution_admission_controllers",
"execution_admission_entries",
];
pub(crate) const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[
@@ -1,254 +0,0 @@
//! V24 schema guard: authoritative execution admission state (Story 4.1, FR-21/FR-24).
//!
//! Схема V24 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
//! ожидаемой форме: controller-row с weighted round-robin cursor и bounded entries с закрытым
//! словарём mode/policy_mode/state, уникальным owner token и индексами квот.
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const CONTROLLERS: &str = "execution_admission_controllers";
const ENTRIES: &str = "execution_admission_entries";
const OWNED_RELATIONS: &[&str] = &[CONTROLLERS, ENTRIES];
const CONTROLLER_COLUMNS: &[(&str, &str, bool)] = &[
("workspace_id", "text", false),
("cursor_agent", "text", true),
("cursor_agent_credit", "integer", false),
("cursor_tool", "text", true),
("cursor_tool_credit", "integer", false),
("updated_at", "timestamp with time zone", false),
];
const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[
("ticket_id", "bigint", false),
("workspace_id", "text", false),
("agent_key", "text", false),
("operation_id", "text", false),
("operation_version", "integer", false),
("mode", "text", false),
("policy_mode", "text", false),
("state", "text", false),
("owner_token", "text", false),
("fingerprint", "text", false),
("created_at", "timestamp with time zone", false),
("expires_at", "timestamp with time zone", false),
("granted_at", "timestamp with time zone", true),
];
const CONSTRAINT_COUNT: &[(&str, usize)] = &[(CONTROLLERS, 4), (ENTRIES, 11)];
const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"];
fn v24_error(stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(24),
"restore_known_good_backup",
)
}
/// Схема V24 отсутствует целиком (для current_version < 24).
pub(super) async fn validate_v24_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
for relation in OWNED_RELATIONS {
if relation_exists(connection, relation).await? {
return Ok(false);
}
}
for index in UNIQUE_INDEXES {
if index_exists(connection, index).await? {
return Ok(false);
}
}
Ok(true)
}
/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24).
pub(super) async fn validate_v24_execution_admission(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
for relation in OWNED_RELATIONS {
if !relation_exists(connection, relation).await? {
return Err(v24_error("v24.relations"));
}
}
for (table, columns) in [(CONTROLLERS, CONTROLLER_COLUMNS), (ENTRIES, ENTRY_COLUMNS)] {
validate_columns(connection, table, columns).await?;
}
for (table, expected) in CONSTRAINT_COUNT {
let count: i64 = query(
"select count(*)::bigint as count
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1",
)
.bind(table)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(*expected).unwrap_or_default() {
return Err(v24_error("v24.constraints"));
}
}
for (table, constraint) in [
(CONTROLLERS, "execution_admission_controllers_pkey"),
(ENTRIES, "execution_admission_entries_pkey"),
(ENTRIES, "execution_admission_entries_state_check"),
] {
if !constraint_exists(connection, table, constraint).await? {
return Err(v24_error("v24.constraint_missing"));
}
}
if !normalized_expression(
connection,
ENTRIES,
"execution_admission_entries_state_check",
)
.await?
.is_some_and(|expression| {
super::schema_guard::enum_constraint_matches(
&expression,
"state",
&["waiting", "executing", "reserved"],
)
}) {
return Err(v24_error("v24.state_vocabulary"));
}
for index in UNIQUE_INDEXES {
let unique = index_unique(connection, index).await?;
if unique != Some(true) {
return Err(v24_error("v24.indexes"));
}
}
Ok(())
}
async fn validate_columns(
connection: &mut PgConnection,
table: &str,
columns: &[(&str, &str, bool)],
) -> Result<(), MigrationError> {
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(table)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(columns.len()).unwrap_or_default() {
return Err(v24_error("v24.columns"));
}
for (column, data_type, nullable) in columns {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(table)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if *nullable { "YES" } else { "NO" })
});
if !valid {
return Err(v24_error("v24.columns"));
}
}
Ok(())
}
async fn constraint_exists(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
) as present",
)
.bind(table)
.bind(constraint)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn normalized_expression(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<Option<String>, MigrationError> {
let row = query(
"select pg_get_expr(c.conbin, c.conrelid) as expression
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
and c.contype = 'c'",
)
.bind(table)
.bind(constraint)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
}
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_class idx
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1
) as present",
)
.bind(index)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn index_unique(
connection: &mut PgConnection,
index: &str,
) -> Result<Option<bool>, MigrationError> {
let row = query(
"select i.indisunique, i.indisvalid
from pg_catalog.pg_index i
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1",
)
.bind(index)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| {
let unique = row.try_get::<bool, _>("indisunique").ok()?;
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
Some(unique && valid)
}))
}
@@ -1,618 +0,0 @@
//! PostgreSQL implementation of the mandatory execution-admission store (Story 4.1, FR-21/FR-24).
//!
//! Source of truth for capacity and durable enqueue decisions of every process: a short transaction
//! locks one deployment controller row and one workspace controller row in a fixed order, then reads
//! bounded counters, inserts/transitions at most one entry and persists the weighted round-robin
//! cursor. No network call happens under the transaction and no connection is held while a waiter is
//! idle between attempts.
use std::collections::BTreeMap;
use std::time::Duration;
use async_trait::async_trait;
use crank_core::{
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, select_next_grant,
};
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
const DEPLOYMENT_CONTROLLER: &str = "*";
const OVERLOAD_RETRY_AFTER_MS: u64 = crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS;
const MAX_WAIT_BACKOFF_MS: u64 = 250;
const INITIAL_WAIT_BACKOFF_MS: u64 = 5;
const EXPIRED_CLEANUP_BATCH: i64 = 128;
const QUEUE_READ_LIMIT: i64 = 1024;
/// Deployment-wide bounds of one admission store. They come from validated deployment
/// configuration, never from a request.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionLimits {
pub global_limit: usize,
pub agent_limit: usize,
pub tool_limit: usize,
pub queue_limit: usize,
}
impl Default for AdmissionLimits {
fn default() -> Self {
Self {
global_limit: 64,
agent_limit: 16,
tool_limit: 8,
queue_limit: 32,
}
}
}
impl AdmissionLimits {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
for (field, value) in [
("global_limit", self.global_limit),
("agent_limit", self.agent_limit),
("tool_limit", self.tool_limit),
] {
if !(1..=65_535).contains(&value) {
return Err(AdmissionStoreError::invalid(format!(
"admission {field} is outside its allowed bounds"
)));
}
}
if !(1..=1_024).contains(&self.queue_limit) {
return Err(AdmissionStoreError::invalid(
"admission queue_limit is outside its allowed bounds",
));
}
Ok(())
}
}
#[derive(Clone)]
pub struct PostgresExecutionAdmission {
pool: PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
}
impl PostgresExecutionAdmission {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> Result<Self, AdmissionStoreError> {
limits.validate()?;
scheduler.validate()?;
Ok(Self {
pool,
limits,
scheduler,
})
}
fn unavailable(error: impl std::fmt::Display) -> AdmissionStoreError {
AdmissionStoreError::unavailable(format!(
"execution admission store is unavailable: {error}"
))
}
}
struct AttemptContext<'a> {
request: &'a AdmissionRequest,
owner_token: &'a str,
}
enum Attempt {
Granted,
Enqueued(u64),
Waiting,
Overloaded,
Expired,
}
#[async_trait]
impl ExecutionAdmission for PostgresExecutionAdmission {
async fn health(&self) -> Result<(), AdmissionStoreError> {
admission_health(&self.pool).await
}
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<AdmissionOutcome, AdmissionStoreError> {
request.validate()?;
if request.approval == ApprovalEvidence::NotGranted {
return Ok(AdmissionOutcome::ApprovalRequired);
}
if request.mode == AdmissionMode::Background
&& request.no_tasks_fallback == NoTasksFallback::Unsupported
{
return Ok(AdmissionOutcome::Unsupported);
}
let owner_token = format!("adm_{}", Uuid::now_v7().simple());
let mut ticket: Option<u64> = None;
let mut backoff = INITIAL_WAIT_BACKOFF_MS;
loop {
if crank_core::current_unix_ms() >= request.expires_at_unix_ms {
if let Some(ticket_id) = ticket {
self.cancel(ticket_id, &owner_token).await?;
}
return Ok(AdmissionOutcome::DeadlineElapsed);
}
let attempt = self
.attempt(
AttemptContext {
request: &request,
owner_token: &owner_token,
},
&mut ticket,
)
.await?;
match attempt {
Attempt::Granted => {
return Ok(AdmissionOutcome::Permitted(AdmissionPermit {
ticket_id: ticket.unwrap_or_default(),
owner_token,
scope: request.scope.clone(),
mode: request.mode,
policy_mode: request.policy_mode,
}));
}
Attempt::Enqueued(ticket_id) => {
return Ok(AdmissionOutcome::Enqueued(AdmissionDecision {
ticket_id,
scope: request.scope.clone(),
expires_at_unix_ms: request.expires_at_unix_ms,
policy_mode: request.policy_mode,
}));
}
Attempt::Expired => {
return Ok(AdmissionOutcome::DeadlineElapsed);
}
Attempt::Overloaded => {
return Ok(AdmissionOutcome::Overloaded {
retry_after_ms: OVERLOAD_RETRY_AFTER_MS,
});
}
Attempt::Waiting => {
let remaining = request
.expires_at_unix_ms
.saturating_sub(crank_core::current_unix_ms())
.max(1);
let sleep_ms = i64::try_from(backoff).unwrap_or(remaining).min(remaining);
tokio::time::sleep(Duration::from_millis(
u64::try_from(sleep_ms).unwrap_or(backoff),
))
.await;
backoff = (backoff * 2).min(MAX_WAIT_BACKOFF_MS);
}
}
}
}
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError> {
let result = sqlx::query(
"delete from execution_admission_entries
where ticket_id = $1 and owner_token = $2 and state = 'executing'",
)
.bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX))
.bind(&permit.owner_token)
.execute(&self.pool)
.await
.map_err(Self::unavailable)?;
Ok(result.rows_affected() > 0)
}
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError> {
let result = sqlx::query(
"delete from execution_admission_entries
where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')",
)
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
.bind(owner_token)
.execute(&self.pool)
.await
.map_err(Self::unavailable)?;
Ok(result.rows_affected() > 0)
}
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
admission_snapshot(&self.pool).await
}
}
/// Readiness probe of the mandatory admission state: both tables must be reachable and readable.
pub async fn admission_health(pool: &PgPool) -> Result<(), AdmissionStoreError> {
sqlx::query(
"select count(*)::bigint from execution_admission_controllers where workspace_id = $1",
)
.bind(DEPLOYMENT_CONTROLLER)
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
sqlx::query("select count(*)::bigint from execution_admission_entries where false")
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
Ok(())
}
/// Authoritative queue readings of the deployment-wide admission state.
pub async fn admission_snapshot(
pool: &PgPool,
) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
let row = sqlx::query(
"select
count(*) filter (where state in ('waiting', 'reserved')) as queued,
count(*) filter (where state = 'executing') as executing,
coalesce(
(extract(epoch from (now() - min(created_at) filter (where state in ('waiting', 'reserved')))) * 1000)::bigint,
-1
) as oldest_waiting_ms
from execution_admission_entries",
)
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
let queued: i64 = row
.try_get("queued")
.map_err(PostgresExecutionAdmission::unavailable)?;
let executing: i64 = row
.try_get("executing")
.map_err(PostgresExecutionAdmission::unavailable)?;
let oldest: i64 = row
.try_get("oldest_waiting_ms")
.map_err(PostgresExecutionAdmission::unavailable)?;
Ok(AdmissionQueueSnapshot {
waiting: u64::try_from(queued).unwrap_or_default(),
executing: u64::try_from(executing).unwrap_or_default(),
oldest_waiting_ms: (oldest >= 0).then(|| u64::try_from(oldest).unwrap_or_default()),
})
}
impl PostgresExecutionAdmission {
async fn attempt(
&self,
context: AttemptContext<'_>,
ticket: &mut Option<u64>,
) -> Result<Attempt, AdmissionStoreError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
match self.attempt_locked(&mut transaction, context, ticket).await {
Ok(attempt) => {
transaction.commit().await.map_err(Self::unavailable)?;
Ok(attempt)
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Ok(Attempt::Waiting)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Ok(Attempt::Overloaded)
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
}
}
async fn attempt_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
context: AttemptContext<'_>,
ticket: &mut Option<u64>,
) -> Result<Attempt, AttemptError> {
let workspace_id = context.request.scope.workspace_id.as_str();
sqlx::query(
"insert into execution_admission_controllers (workspace_id)
values ($1), ($2)
on conflict (workspace_id) do nothing",
)
.bind(DEPLOYMENT_CONTROLLER)
.bind(workspace_id)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
// Fixed lock order: deployment controller first, then the workspace controller.
lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?;
let cursor = lock_controller(transaction, workspace_id).await?;
// Expired queue decisions and waiting tickets leave the bounded queue; held permits never do.
sqlx::query(
"delete from execution_admission_entries
where ticket_id in (
select ticket_id from execution_admission_entries
where state in ('waiting', 'reserved') and expires_at <= now()
order by ticket_id
limit $1
)",
)
.bind(EXPIRED_CLEANUP_BATCH)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
if ticket.is_none() {
let queued: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries
where state in ('waiting', 'reserved')",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
if u64::try_from(queued).unwrap_or(u64::MAX)
>= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX)
{
return Err(AttemptError::Overloaded);
}
let state = match context.request.mode {
AdmissionMode::Synchronous => "waiting",
AdmissionMode::Background => "reserved",
};
let row = sqlx::query(
"insert into execution_admission_entries (
workspace_id, agent_key, operation_id, operation_version, mode, policy_mode,
state, owner_token, fingerprint, expires_at
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9,
to_timestamp($10::double precision / 1000.0))
returning ticket_id",
)
.bind(workspace_id)
.bind(&context.request.scope.agent_key)
.bind(&context.request.scope.operation_id)
.bind(i32::try_from(context.request.scope.operation_version).unwrap_or(i32::MAX))
.bind(context.request.mode.as_str())
.bind(policy_mode_str(context.request))
.bind(state)
.bind(context.owner_token)
.bind(&context.request.fingerprint)
.bind(context.request.expires_at_unix_ms)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
*ticket = Some(u64::try_from(ticket_id).unwrap_or_default());
if context.request.mode == AdmissionMode::Background {
return Ok(Attempt::Enqueued(
u64::try_from(ticket_id).unwrap_or_default(),
));
}
}
self.grant_next(transaction, workspace_id, &cursor).await?;
let Some(ticket_id) = *ticket else {
return Ok(Attempt::Waiting);
};
let state: Option<String> =
sqlx::query("select state from execution_admission_entries where ticket_id = $1")
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.map(|row| row.try_get("state"))
.transpose()
.map_err(AttemptError::from_sql)?;
match state.as_deref() {
Some("executing") => Ok(Attempt::Granted),
Some(_) => Ok(Attempt::Waiting),
// The ticket expired inside the store (or was removed by another cleanup).
None => Ok(Attempt::Expired),
}
}
/// Grants at most one waiting ticket per transaction, following the persisted weighted
/// round-robin cursor. A blocked head keeps its turn: nothing later is granted in its place.
async fn grant_next(
&self,
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
cursor: &AdmissionCursor,
) -> Result<(), AttemptError> {
let rows = sqlx::query(
"select ticket_id, agent_key, operation_id from execution_admission_entries
where workspace_id = $1 and state = 'waiting'
order by ticket_id
limit $2",
)
.bind(workspace_id)
.bind(QUEUE_READ_LIMIT)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
if rows.is_empty() {
return Ok(());
}
let queue = rows
.iter()
.map(|row| {
Ok(AdmissionQueueEntry {
ticket_id: u64::try_from(
row.try_get::<i64, _>("ticket_id")
.map_err(AttemptError::from_sql)?,
)
.unwrap_or_default(),
sequence: 0,
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
operation_id: row
.try_get("operation_id")
.map_err(AttemptError::from_sql)?,
})
})
.collect::<Result<Vec<_>, AttemptError>>()?;
let Some(selection) = select_next_grant(&queue, cursor, self.scheduler) else {
return Ok(());
};
let head = &queue[selection.index];
let global_executing: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries
where state = 'executing'",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
let agent_counts = scoped_counts(transaction, workspace_id, "agent_key").await?;
let tool_counts = scoped_counts(transaction, workspace_id, "operation_id").await?;
if u64::try_from(global_executing).unwrap_or(u64::MAX)
>= u64::try_from(self.limits.global_limit).unwrap_or(u64::MAX)
{
return Ok(());
}
let agent_executing = agent_counts
.get(head.agent_key.as_str())
.copied()
.unwrap_or_default();
if agent_executing >= u64::try_from(self.limits.agent_limit).unwrap_or(u64::MAX) {
return Ok(());
}
let tool_executing = tool_counts
.get(head.operation_id.as_str())
.copied()
.unwrap_or_default();
if tool_executing >= u64::try_from(self.limits.tool_limit).unwrap_or(u64::MAX) {
return Ok(());
}
sqlx::query(
"update execution_admission_entries
set state = 'executing', granted_at = now()
where ticket_id = $1 and state = 'waiting'",
)
.bind(i64::try_from(head.ticket_id).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
sqlx::query(
"update execution_admission_controllers
set cursor_agent = $2,
cursor_agent_credit = $3,
cursor_tool = $4,
cursor_tool_credit = $5,
updated_at = now()
where workspace_id = $1",
)
.bind(workspace_id)
.bind(selection.cursor.agent.as_deref())
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
.bind(selection.cursor.tool.as_deref())
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(())
}
}
enum AttemptError {
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
Contended,
/// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait.
Overloaded,
Failure(AdmissionStoreError),
}
impl AttemptError {
fn from_sql(error: sqlx::Error) -> Self {
if let Some(database) = error.as_database_error()
&& matches!(database.code().as_deref(), Some("55P03" | "57014"))
{
return Self::Contended;
}
Self::Failure(PostgresExecutionAdmission::unavailable(error))
}
}
async fn lock_controller(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
) -> Result<AdmissionCursor, AttemptError> {
sqlx::query(
"update execution_admission_controllers set updated_at = updated_at
where workspace_id = $1",
)
.bind(workspace_id)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let row = sqlx::query(
"select cursor_agent, cursor_agent_credit, cursor_tool, cursor_tool_credit
from execution_admission_controllers where workspace_id = $1",
)
.bind(workspace_id)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(AdmissionCursor {
agent: row
.try_get("cursor_agent")
.map_err(AttemptError::from_sql)?,
agent_credit: u32::try_from(
row.try_get::<i32, _>("cursor_agent_credit")
.map_err(AttemptError::from_sql)?
.max(0),
)
.unwrap_or_default(),
tool: row.try_get("cursor_tool").map_err(AttemptError::from_sql)?,
tool_credit: u32::try_from(
row.try_get::<i32, _>("cursor_tool_credit")
.map_err(AttemptError::from_sql)?
.max(0),
)
.unwrap_or_default(),
})
}
async fn scoped_counts(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
column: &str,
) -> Result<BTreeMap<String, u64>, AttemptError> {
let statement = if column == "agent_key" {
"select agent_key as key, count(*)::bigint as count
from execution_admission_entries
where workspace_id = $1 and state = 'executing'
group by agent_key"
} else {
"select operation_id as key, count(*)::bigint as count
from execution_admission_entries
where workspace_id = $1 and state = 'executing'
group by operation_id"
};
let rows = sqlx::query(statement)
.bind(workspace_id)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let mut counts = BTreeMap::new();
for row in rows {
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
counts.insert(key, u64::try_from(count).unwrap_or_default());
}
Ok(counts)
}
fn policy_mode_str(request: &AdmissionRequest) -> &'static str {
match request.policy_mode {
crank_core::ToolExecutionMode::Synchronous => "synchronous",
crank_core::ToolExecutionMode::Background => "background",
crank_core::ToolExecutionMode::Adaptive => "adaptive",
}
}
@@ -46,19 +46,6 @@ impl PostgresRegistry {
Ok(())
}
/// Readiness probe of the mandatory execution-admission state (Story 4.1). A missing or
/// unreadable admission schema is a dependency failure, never a degraded-but-ready state.
pub async fn execution_admission_health(&self) -> Result<(), crank_core::AdmissionStoreError> {
super::admission::admission_health(&self.pool).await
}
/// Authoritative admission queue readings for low-cardinality metrics.
pub async fn execution_admission_snapshot(
&self,
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
super::admission::admission_snapshot(&self.pool).await
}
async fn connect_in_schema(
database_url: &str,
schema: Option<&str>,
@@ -1,4 +1,3 @@
mod admission;
mod agent;
mod agent_catalog;
mod agent_prompt_bindings;
@@ -37,9 +36,6 @@ use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow, types::Json};
use time::OffsetDateTime;
pub use admission::{
AdmissionLimits, PostgresExecutionAdmission, admission_health, admission_snapshot,
};
pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError};
pub use resource::{AuthorizedResourceRead, BoundResourceAdmission};
@@ -1,5 +1,4 @@
mod integration {
mod admission;
mod agent_resource_bindings;
mod agents_usage;
mod approval;
@@ -1,496 +0,0 @@
//! ATDD Story 4.1 (FR-21/FR-24): authoritative PostgreSQL execution admission.
//!
//! These oracles drive the real store against a real PostgreSQL schema: scoped permits, bounded
//! queue, typed overload, durable enqueue decisions, deadline propagation, quota keys that ignore
//! the operation version, fairness order and fail-closed health.
use super::common;
use std::time::{Duration, Instant};
use crank_core::{
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
AdmissionSchedulerConfig, AdmissionScope, ApprovalEvidence, ExecutionAdmission,
NoTasksFallback, ToolExecutionMode, current_unix_ms,
};
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
use tokio::sync::mpsc;
fn limits(global: usize, agent: usize, tool: usize, queue: usize) -> AdmissionLimits {
AdmissionLimits {
global_limit: global,
agent_limit: agent,
tool_limit: tool,
queue_limit: queue,
}
}
fn store(
pool: sqlx::PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> PostgresExecutionAdmission {
PostgresExecutionAdmission::new(pool, limits, scheduler).expect("valid admission store")
}
fn scope(agent: &str, operation: &str, version: u32) -> AdmissionScope {
AdmissionScope {
workspace_id: "ws_default".to_owned(),
agent_key: agent.to_owned(),
operation_id: operation.to_owned(),
operation_version: version,
}
}
fn request(scope: AdmissionScope, deadline_in_ms: i64) -> AdmissionRequest {
AdmissionRequest {
scope,
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: format!("fp_{}", current_unix_ms()),
expires_at_unix_ms: current_unix_ms() + deadline_in_ms,
}
}
fn background(scope: AdmissionScope, fallback: NoTasksFallback) -> AdmissionRequest {
AdmissionRequest {
mode: AdmissionMode::Background,
no_tasks_fallback: fallback,
..request(scope, 30_000)
}
}
fn permit(outcome: AdmissionOutcome) -> AdmissionPermit {
match outcome {
AdmissionOutcome::Permitted(permit) => permit,
other => panic!("expected a scoped permit, got {other:?}"),
}
}
async fn wait_for_metric_poll() {
tokio::time::sleep(Duration::from_millis(20)).await;
}
#[tokio::test]
async fn permits_are_scoped_released_once_and_respect_the_agent_budget() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 1, 4, 8),
Default::default(),
);
let first = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
assert_eq!(first.scope.agent_key, "agent_a");
// The same Agent is already at its scoped budget: the second request waits and expires instead of
// over-admitting, and it never dispatches.
let waiting = admission
.admit(request(scope("agent_a", "op_leads", 2), 120))
.await
.expect("admission store is reachable");
assert_eq!(waiting, AdmissionOutcome::DeadlineElapsed);
// Another Agent still has the global capacity it is entitled to.
let other = permit(
admission
.admit(request(scope("agent_b", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
assert_eq!(other.scope.agent_key, "agent_b");
assert!(admission.release(&other).await.unwrap());
// Repeated release is idempotent and never frees a foreign or already released permit.
assert!(!admission.release(&other).await.unwrap());
assert!(admission.release(&first).await.unwrap());
let snapshot = admission.snapshot().await.unwrap();
assert_eq!(snapshot.executing, 0);
assert_eq!(snapshot.waiting, 0);
database.cleanup().await;
}
#[tokio::test]
async fn full_queue_returns_a_typed_overload_with_retry_guidance() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 1, 1, 2),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable"),
);
// Two bounded waiting entries fill the queue.
let mut waiters = Vec::new();
for version in 1..=2 {
let admission = admission.clone();
waiters.push(tokio::spawn(async move {
admission
.admit(request(scope("agent_b", "op_leads", version), 5_000))
.await
}));
}
while admission.snapshot().await.unwrap().waiting < 2 {
wait_for_metric_poll().await;
}
let rejected = admission
.admit(request(scope("agent_c", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable");
match rejected {
AdmissionOutcome::Overloaded { retry_after_ms } => {
assert!((1..=crank_core::MAX_RETRY_AFTER_MS).contains(&retry_after_ms));
}
other => panic!("expected typed overload, got {other:?}"),
}
// Cancelling the held permit lets the queued waiters make progress instead of leaking capacity.
assert!(admission.release(&held).await.unwrap());
for waiter in waiters.drain(..) {
let outcome = waiter.await.expect("waiter task");
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
let _ = admission.release(&permit).await;
}
}
database.cleanup().await;
}
#[tokio::test]
async fn background_decisions_are_durable_pinned_and_released_once() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let decision = match admission
.admit(background(
scope("agent_a", "op_report", 3),
NoTasksFallback::Synchronous,
))
.await
.expect("admission store is reachable")
{
AdmissionOutcome::Enqueued(decision) => decision,
other => panic!("expected a durable enqueue decision, got {other:?}"),
};
assert_eq!(decision.scope.operation_version, 3);
assert_eq!(decision.policy_mode, ToolExecutionMode::Synchronous);
// A second process (fresh pool and store over the same schema) must read the committed decision:
// it occupies bounded queue capacity and is not a runnable execution.
let reconnected = store(
database.concurrent_registry(2).await.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let snapshot = reconnected.snapshot().await.unwrap();
assert_eq!(snapshot.waiting, 1);
assert_eq!(snapshot.executing, 0);
assert!(
reconnected
.cancel(decision.ticket_id, "not_the_owner")
.await
.unwrap()
.eq(&false),
"a foreign cancel must never release the decision"
);
assert!(
reconnected.snapshot().await.unwrap().waiting.eq(&1),
"the foreign cancel must leave the decision intact"
);
database.cleanup().await;
}
#[tokio::test]
async fn background_without_a_task_transport_is_typed_unsupported() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let outcome = admission
.admit(background(
scope("agent_a", "op_report", 1),
NoTasksFallback::Unsupported,
))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::Unsupported);
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
database.cleanup().await;
}
#[tokio::test]
async fn unverified_approval_never_produces_a_decision() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let mut denied = background(
scope("agent_a", "op_report", 1),
NoTasksFallback::Synchronous,
);
denied.approval = ApprovalEvidence::NotGranted;
assert_eq!(
admission.admit(denied).await.unwrap(),
AdmissionOutcome::ApprovalRequired
);
let mut denied_sync = request(scope("agent_a", "op_report", 1), 1_000);
denied_sync.approval = ApprovalEvidence::NotGranted;
assert_eq!(
admission.admit(denied_sync).await.unwrap(),
AdmissionOutcome::ApprovalRequired
);
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
assert_eq!(admission.snapshot().await.unwrap().executing, 0);
database.cleanup().await;
}
#[tokio::test]
async fn quota_keys_ignore_the_operation_version() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 1, 8),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
// A different Published Version of the same Tool shares the same per-Tool budget.
let outcome = admission
.admit(request(scope("agent_a", "op_leads", 2), 120))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
assert!(admission.release(&held).await.unwrap());
database.cleanup().await;
}
#[tokio::test]
async fn weighted_agents_are_served_their_quantum_in_order() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 4, 4, 8),
AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
},
);
let held = permit(
admission
.admit(request(scope("agent_z", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable"),
);
let (sender, mut receiver) = mpsc::channel(3);
let mut tasks = Vec::new();
for (agent, version) in [("agent_a", 1), ("agent_a", 2), ("agent_b", 1)] {
let admission = admission.clone();
let sender = sender.clone();
tasks.push(tokio::spawn(async move {
let outcome = admission
.admit(request(scope(agent, "op_leads", version), 5_000))
.await;
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
let _ = sender.send(permit.scope.agent_key.clone()).await;
// Hold briefly so the grant order, not task scheduling, decides the sequence.
tokio::time::sleep(Duration::from_millis(30)).await;
let _ = admission.release(&permit).await;
}
}));
}
drop(sender);
assert!(admission.release(&held).await.unwrap());
let mut order = Vec::new();
while let Some(agent) = receiver.recv().await {
order.push(agent);
if order.len() == 3 {
break;
}
}
for task in tasks {
let _ = tokio::time::timeout(Duration::from_secs(5), task).await;
}
// agent_a carries weight 2 and therefore receives two consecutive grants before agent_b.
assert_eq!(order, vec!["agent_a", "agent_a", "agent_b"]);
database.cleanup().await;
}
#[tokio::test]
async fn mandatory_state_failure_fails_closed_and_reports_not_ready() {
let database = common::TestDatabase::new().await;
// No migration on this schema: the mandatory admission state does not exist yet.
let unmigrated = database.raw_pool().await;
let admission = store(unmigrated, limits(4, 4, 4, 8), Default::default());
assert!(admission.health().await.is_err());
assert!(
admission
.admit(request(scope("agent_a", "op_leads", 1), 1_000))
.await
.is_err()
);
database.cleanup().await;
}
#[tokio::test]
async fn held_permits_survive_a_new_process_view() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 4, 4, 8),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
let reconnected = store(
database.concurrent_registry(2).await.pool().clone(),
limits(1, 4, 4, 8),
Default::default(),
);
assert_eq!(reconnected.snapshot().await.unwrap().executing, 1);
let outcome = reconnected
.admit(request(scope("agent_b", "op_leads", 1), 120))
.await
.expect("admission store is reachable");
assert_eq!(
outcome,
AdmissionOutcome::DeadlineElapsed,
"a held permit is not silently reused by another process"
);
assert!(reconnected.release(&held).await.unwrap());
database.cleanup().await;
}
#[tokio::test]
async fn published_execution_policy_is_pinned_across_a_new_draft_version() {
use crank_core::{NoTasksFallback, OperationStatus, ToolExecutionMode, ToolExecutionPolicy};
use crank_registry::{CreateVersionRequest, PublishRequest};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let workspace = common::test_workspace_id();
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 45_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
let mut version_one = common::test_operation("op_policy", 1, OperationStatus::Draft);
version_one.execution_config.execution_policy = Some(policy);
registry
.create_operation(&workspace, &version_one, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &workspace,
operation_id: &version_one.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-25T12:10:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
let published = registry
.get_published_operation(&version_one.id)
.await
.unwrap()
.expect("published operation");
assert_eq!(published.execution_config.execution_policy, Some(policy));
// A later Draft with another policy never rewrites the already accepted publication.
let mut version_two = common::test_operation("op_policy", 2, OperationStatus::Draft);
version_two.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 1_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
});
registry
.create_version(CreateVersionRequest {
workspace_id: &workspace,
snapshot: &version_two,
change_note: Some("next draft"),
created_by: Some("alice"),
})
.await
.unwrap();
let unchanged = registry
.get_published_operation(&version_one.id)
.await
.unwrap()
.expect("published operation");
assert_eq!(unchanged.version, 1);
assert_eq!(unchanged.execution_config.execution_policy, Some(policy));
database.cleanup().await;
}
#[tokio::test]
async fn admin_origin_bucket_is_bounded_together_with_agents() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 1, 1, 4),
Default::default(),
);
let held = permit(
admission
.admit(request(scope(ADMIN_ORIGIN_KEY, "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
let started = Instant::now();
let outcome = admission
.admit(request(scope("agent_a", "op_leads", 1), 120))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
assert!(started.elapsed() >= Duration::from_millis(100));
assert!(admission.release(&held).await.unwrap());
database.cleanup().await;
}
-255
View File
@@ -1,255 +0,0 @@
//! Common execution admission seam of the runtime (Story 4.1, FR-21/FR-24).
//!
//! Every real execution — synchronous, or a published `background`/`adaptive` policy that falls back
//! to synchronous because no durable Task transport exists yet — passes this one gate before any
//! adapter work. The gate asks the authoritative store for a scoped permit bounded by the published
//! Tool policy, the verified approval state of the exact Published Version/scope/input and the caller
//! deadline, and it releases the permit explicitly when the local execution future ends.
use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
AdmissionScope, CoordinationStateStore, ExecutionAdmission, NoTasksFallback, ToolExecutionMode,
current_unix_ms,
};
use crank_metrics::{
AdmissionModeLabel, RejectionReason, ScopeKind, record_admission_queue_wait,
record_admission_rejection,
};
use tracing::warn;
use crate::{RuntimeError, RuntimeOperation, RuntimeRequestContext};
/// Effective absolute deadline of one execution: the published policy deadline never extends the
/// caller deadline and always includes queue waiting time.
pub(crate) fn effective_deadline(
operation: &RuntimeOperation,
caller_deadline: Instant,
) -> Instant {
match operation.execution_config.execution_policy {
Some(policy) => {
caller_deadline.min(Instant::now() + Duration::from_millis(policy.deadline_ms))
}
None => caller_deadline,
}
}
/// Effective execution path for this deployment.
///
/// Story 4.1 has no Task transport, so a `background` or `adaptive` policy resolves through its
/// published `no_tasks_fallback`: either the same synchronous admission path, or a typed
/// unsupported refusal with no admission entry, no Task id and no outbound call.
pub(crate) fn effective_mode(
operation: &RuntimeOperation,
) -> Result<ToolExecutionMode, RuntimeError> {
let Some(policy) = operation.execution_config.execution_policy else {
return Ok(ToolExecutionMode::Synchronous);
};
match policy.mode {
ToolExecutionMode::Synchronous => Ok(ToolExecutionMode::Synchronous),
mode => match policy.no_tasks_fallback {
NoTasksFallback::Synchronous => Ok(ToolExecutionMode::Synchronous),
NoTasksFallback::Unsupported => Err(RuntimeError::ExecutionPolicyUnsupported {
operation_id: operation.operation_id.as_str().to_owned(),
mode,
}),
},
}
}
pub(crate) struct AdmissionLease {
store: Arc<dyn ExecutionAdmission>,
permit: Option<AdmissionPermit>,
runtime: tokio::runtime::Handle,
}
impl AdmissionLease {
fn new(store: Arc<dyn ExecutionAdmission>, permit: AdmissionPermit) -> Self {
Self {
store,
permit: Some(permit),
runtime: tokio::runtime::Handle::current(),
}
}
/// Explicit release on the normal/error path of the local execution. The `Drop` cleanup stays as
/// the cancellation path; both CAS on the same owner token, so only one of them can free it.
pub(crate) async fn release(&mut self) {
let Some(permit) = self.permit.take() else {
return;
};
if let Err(error) = self.store.release(&permit).await {
warn!(
name: "runtime.admission.release_failed",
error = %error,
"failed to release an admission permit; the reservation stays fail-closed"
);
}
}
}
impl Drop for AdmissionLease {
fn drop(&mut self) {
let Some(permit) = self.permit.take() else {
return;
};
let store = Arc::clone(&self.store);
// Bounded cleanup: the explicit release path already ran on success/error; cancellation
// reaches here and releases by owner token without ever freeing a foreign permit. A failed
// release keeps the reservation fail-closed (no TTL reuse of an active permit).
self.runtime.spawn(async move {
match store.release(&permit).await {
Ok(true) => {}
Ok(false) => warn!(
name: "runtime.admission.release_skipped",
"admission permit was already released"
),
Err(error) => warn!(
name: "runtime.admission.release_failed",
error = %error,
"failed to release an admission permit; the reservation stays fail-closed"
),
}
});
}
}
/// Acquires a scoped permit from the authoritative store. Returns `None` when this deployment has no
/// admission store wired (embedded/unit-test executors keep the pre-4.1 local limiter).
pub(crate) async fn acquire(
store: Option<&Arc<dyn ExecutionAdmission>>,
coordination_store: Option<&dyn CoordinationStateStore>,
request: &crate::RuntimeExecutionRequest<'_>,
deadline: Instant,
) -> Result<Option<AdmissionLease>, RuntimeError> {
let Some(store) = store else {
return Ok(None);
};
let operation = request.operation();
let input = request.input();
let request_context = request.request_context();
let operation_id = operation.operation_id.as_str().to_owned();
let policy_mode = effective_mode(operation)?;
// The one admission snapshot carries the authoritative approval state of this exact Published
// Version, scope, input and deadline. A confirmation-requiring request without a verified
// approval never takes capacity and never dispatches: the pending human decision is answered
// here with the existing challenge, and the single-use token is consumed only at dispatch.
let approval = crate::confirmation::approval_evidence(
coordination_store,
operation,
input,
Some(request_context),
)
.await?;
let agent_key = request.agent_id().map(|agent| agent.as_str());
let workspace_id = request.workspace_id().as_str().to_owned();
let request = AdmissionRequest {
scope: AdmissionScope {
workspace_id,
agent_key: agent_key.unwrap_or(ADMIN_ORIGIN_KEY).to_owned(),
operation_id: operation_id.clone(),
operation_version: operation.operation_version,
},
mode: AdmissionMode::Synchronous,
policy_mode,
no_tasks_fallback: operation
.execution_config
.execution_policy
.map(|policy| policy.no_tasks_fallback)
.unwrap_or(NoTasksFallback::Synchronous),
approval,
fingerprint: admission_fingerprint(request_context, operation),
expires_at_unix_ms: current_unix_ms().saturating_add(
deadline_remaining_ms(deadline)
.try_into()
.unwrap_or(i64::MAX),
),
};
let started = Instant::now();
let outcome = store.admit(request).await.map_err(|error| {
record_admission_rejection(RejectionReason::DependencyUnavailable, ScopeKind::Global);
RuntimeError::AdmissionStoreUnavailable {
operation_id: operation_id.clone(),
details: error.to_string(),
}
})?;
let waited_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or_default();
match outcome {
AdmissionOutcome::Permitted(permit) => {
record_admission_queue_wait(AdmissionModeLabel::Synchronous, waited_ms);
Ok(Some(AdmissionLease::new(Arc::clone(store), permit)))
}
AdmissionOutcome::Overloaded { retry_after_ms } => {
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
Err(RuntimeError::AdmissionOverloaded {
operation_id,
retry_after_ms,
})
}
AdmissionOutcome::DeadlineElapsed => {
// Queue wait exhausted the absolute deadline before a permit was granted: nothing was
// dispatched, so this is an admission refusal, not an upstream timeout.
Err(deadline_refusal(operation))
}
AdmissionOutcome::ApprovalRequired => {
// The store refused before reserving any capacity or queue entry. Answer with the
// existing single-use challenge so the caller can confirm out of band.
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
Err(crate::confirmation::challenge(
coordination_store,
operation,
input,
Some(request_context),
)
.await)
}
AdmissionOutcome::Unsupported | AdmissionOutcome::Enqueued(_) => {
// No Task transport exists in this deployment: an enqueue decision can never become a
// runnable execution here, so it must not be reported as accepted work.
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
Err(RuntimeError::ExecutionPolicyUnsupported {
operation_id,
mode: policy_mode,
})
}
}
}
/// Typed refusal for an admission that never granted a permit within the absolute deadline. Reuses the
/// existing `execution_overloaded` vocabulary (stage `admission`, retry guidance, certain outcome) so
/// no new client error code or API shape appears, while reporting zero dispatch truthfully.
pub(crate) fn deadline_refusal(operation: &RuntimeOperation) -> RuntimeError {
record_admission_rejection(RejectionReason::Deadline, ScopeKind::Global);
RuntimeError::AdmissionOverloaded {
operation_id: operation.operation_id.as_str().to_owned(),
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
}
}
fn deadline_remaining_ms(deadline: Instant) -> u64 {
u64::try_from(
deadline
.saturating_duration_since(Instant::now())
.as_millis(),
)
.unwrap_or_default()
.max(1)
}
/// Bounded request identity for correlating one admission entry. Never contains payload, headers or
/// credentials.
fn admission_fingerprint(
request_context: &RuntimeRequestContext,
operation: &RuntimeOperation,
) -> String {
let mut fingerprint = format!(
"{}:{}:{}",
operation.operation_id.as_str(),
operation.operation_version,
request_context.request_id
);
fingerprint.truncate(crank_core::ADMISSION_FINGERPRINT_MAX_BYTES);
fingerprint
}
+23 -120
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
ApprovalEvidence, CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod,
OperationSafetyClass, OperationSafetyPolicy, Target,
CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod, OperationSafetyClass,
OperationSafetyPolicy, Target,
};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
@@ -26,14 +26,32 @@ pub async fn confirm_operation(
}
let Some(store) = store else {
return Err(confirmation_store_unavailable(operation));
return Err(RuntimeError::ConfirmationStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
});
};
let scope = confirmation_scope(operation, request_context)?;
let input_hash = hash_json(input)?;
let provided_token = request_context
.and_then(RuntimeRequestContext::confirmation_token)
.or_else(|| confirmation_token_from_input(input));
let Some(provided_token) = provided_confirmation_token(request_context, input) else {
return Err(confirmation_required(store, operation, &scope, &input_hash, &safety).await);
let Some(provided_token) = provided_token else {
let token = issue_confirmation_token(
store,
operation.operation_id.as_str(),
&scope,
&input_hash,
&safety,
)
.await?;
return Err(RuntimeError::ConfirmationRequired {
operation_id: operation.operation_id.as_str().to_owned(),
safety_class: safety.class,
confirmation_token: token,
expires_in_ms: confirmation_ttl_ms(&safety),
});
};
consume_confirmation_token(
@@ -46,121 +64,6 @@ pub async fn confirm_operation(
.await
}
/// Read-only approval proof for the shared admission gate (Story 4.1, AC2).
///
/// It reports the authoritative approval state of this exact Published Version, scope and input and
/// never consumes the single-use confirmation token: the authoritative consumption stays in
/// [`confirm_operation`] immediately before the adapter call. A missing approval is `NotGranted`, so
/// admission refuses the request before taking execution capacity while the human decision is
/// pending; an unusable token or an unreachable safety store is a typed refusal too, never a permit.
pub(crate) async fn approval_evidence(
store: Option<&dyn CoordinationStateStore>,
operation: &RuntimeOperation,
input: &Value,
request_context: Option<&RuntimeRequestContext>,
) -> Result<ApprovalEvidence, RuntimeError> {
let safety = effective_safety_policy(operation);
if !safety.class.requires_confirmation() {
return Ok(ApprovalEvidence::NotRequired);
}
if request_context.is_some_and(RuntimeRequestContext::approval_granted) {
return Ok(ApprovalEvidence::Granted);
}
let Some(store) = store else {
return Err(confirmation_store_unavailable(operation));
};
let scope = confirmation_scope(operation, request_context)?;
let input_hash = hash_json(input)?;
let Some(token) = provided_confirmation_token(request_context, input) else {
return Ok(ApprovalEvidence::NotGranted);
};
let stored = store
.get_value(
CacheScope::Coordination,
&confirmation_cache_key(&scope, token),
)
.await
.map_err(|_| confirmation_store_unavailable(operation))?;
let verified = stored.is_some_and(|value| {
value.payload.get("scope").and_then(Value::as_str) == Some(scope.as_str())
&& value.payload.get("input_hash").and_then(Value::as_str) == Some(input_hash.as_str())
});
if verified {
Ok(ApprovalEvidence::Granted)
} else {
Err(RuntimeError::InvalidConfirmationToken {
operation_id: operation.operation_id.as_str().to_owned(),
})
}
}
/// Issues the existing single-use confirmation challenge and returns the typed refusal that carries
/// it. Admission calls this when no verified approval exists yet, so waiting for the human decision
/// never holds execution capacity.
pub(crate) async fn challenge(
store: Option<&dyn CoordinationStateStore>,
operation: &RuntimeOperation,
input: &Value,
request_context: Option<&RuntimeRequestContext>,
) -> RuntimeError {
let safety = effective_safety_policy(operation);
let Some(store) = store else {
return confirmation_store_unavailable(operation);
};
let scope = match confirmation_scope(operation, request_context) {
Ok(scope) => scope,
Err(error) => return error,
};
let input_hash = match hash_json(input) {
Ok(input_hash) => input_hash,
Err(error) => return error,
};
confirmation_required(store, operation, &scope, &input_hash, &safety).await
}
fn provided_confirmation_token<'a>(
request_context: Option<&'a RuntimeRequestContext>,
input: &'a Value,
) -> Option<&'a str> {
request_context
.and_then(RuntimeRequestContext::confirmation_token)
.or_else(|| confirmation_token_from_input(input))
}
fn confirmation_store_unavailable(operation: &RuntimeOperation) -> RuntimeError {
RuntimeError::ConfirmationStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
}
}
async fn confirmation_required(
store: &dyn CoordinationStateStore,
operation: &RuntimeOperation,
scope: &str,
input_hash: &str,
safety: &OperationSafetyPolicy,
) -> RuntimeError {
match issue_confirmation_token(
store,
operation.operation_id.as_str(),
scope,
input_hash,
safety,
)
.await
{
Ok(token) => RuntimeError::ConfirmationRequired {
operation_id: operation.operation_id.as_str().to_owned(),
safety_class: safety.class,
confirmation_token: token,
expires_in_ms: confirmation_ttl_ms(safety),
},
Err(error) => error,
}
}
pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool {
effective_safety_policy(operation)
.class
-21
View File
@@ -43,27 +43,6 @@ pub enum RuntimeError {
confirmation_token: String,
expires_in_ms: u64,
},
#[error(
"execution mode {mode:?} of operation {operation_id} is unavailable without a durable Task transport"
)]
ExecutionPolicyUnsupported {
operation_id: String,
mode: crank_core::ToolExecutionMode,
},
#[error(
"execution admission is overloaded for operation {operation_id}; retry after {retry_after_ms} ms"
)]
AdmissionOverloaded {
operation_id: String,
retry_after_ms: u64,
},
#[error("execution admission requires a verified approval for operation {operation_id}")]
AdmissionApprovalRequired { operation_id: String },
#[error("execution admission store is unavailable for operation {operation_id}: {details}")]
AdmissionStoreUnavailable {
operation_id: String,
details: String,
},
#[error("confirmation token for operation {operation_id} is invalid, expired, or already used")]
InvalidConfirmationToken { operation_id: String },
#[error("confirmation store is unavailable for operation {operation_id}")]
@@ -57,11 +57,6 @@ fn normalize_runtime_error_with_effect(
| RuntimeError::IdempotencyCompletionUnknown { .. } => {
failure = failure.with_dispatch_uncertainty();
}
RuntimeError::AdmissionOverloaded { retry_after_ms, .. } => {
if let Ok(with_retry) = failure.clone().try_with_retry_after_ms(*retry_after_ms) {
failure = with_retry;
}
}
RuntimeError::ConfirmationRequired {
confirmation_token,
expires_in_ms,
@@ -118,14 +113,6 @@ fn error_code(error: &RuntimeError) -> ExecutionErrorCode {
RuntimeError::UnsupportedExecutionMode { .. } => {
ExecutionErrorCode::ExecutionModeUnsupported
}
RuntimeError::ExecutionPolicyUnsupported { .. } => {
ExecutionErrorCode::ExecutionModeUnsupported
}
RuntimeError::AdmissionOverloaded { .. } => ExecutionErrorCode::ExecutionOverloaded,
RuntimeError::AdmissionApprovalRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
RuntimeError::AdmissionStoreUnavailable { .. } => {
ExecutionErrorCode::SafetyStoreUnavailable
}
RuntimeError::ConcurrencyLimitExceeded { .. } => ExecutionErrorCode::ExecutionOverloaded,
RuntimeError::InvalidPreparedRequest { .. } => ExecutionErrorCode::PreparedRequestInvalid,
RuntimeError::ConfirmationRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
@@ -220,7 +220,6 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+10 -53
View File
@@ -7,9 +7,9 @@ use std::time::{Duration, Instant};
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
use crank_core::{
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
SharedProtocolAdapter, Target,
AdapterRegistry, CoordinationStateStore, ExecutionMode, HttpMethod, InvocationSource,
InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
Target,
};
use crank_metrics::{
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
@@ -41,7 +41,6 @@ pub struct RuntimeExecutor {
unary_limiter: Arc<Semaphore>,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
@@ -140,7 +139,6 @@ impl RuntimeExecutor {
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
) -> Self {
Self {
@@ -149,7 +147,6 @@ impl RuntimeExecutor {
limits,
response_cache,
coordination_store,
admission,
metering_sink,
}
}
@@ -170,11 +167,6 @@ impl RuntimeExecutor {
self
}
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub(crate) async fn execute_request(
&self,
request: RuntimeExecutionRequest<'_>,
@@ -196,16 +188,11 @@ impl RuntimeExecutor {
request.agent_id()
);
let operation_may_mutate = operation_may_mutate(operation);
let execution_deadline =
crate::admission::effective_deadline(operation, request.deadline());
let runtime_span = Stage::RuntimeExecute.span();
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
log_runtime_event("unary.execute", operation, Some(request_context));
let started_at = Instant::now();
let dispatch_started = Arc::new(AtomicBool::new(false));
// True until the admission gate answered. A deadline that fires while the request is still
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
metric_invocation_source(Some(request_context)),
request_context
@@ -219,17 +206,6 @@ impl RuntimeExecutor {
.flatten(),
);
let execution = async {
// One admission gate for every executable path (Story 4.1): the published policy decides
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
// is held for the whole local execution future, including retries.
let mut lease = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
let _permit = self.acquire_unary_permit(operation)?;
let _inflight = InFlightGuard::runtime();
let mapping_span = Stage::RuntimeArgumentsMap.span();
@@ -239,7 +215,7 @@ impl RuntimeExecutor {
let prepared_request = prepared_request?;
let request_preview = sanitized_request_preview(&prepared_request);
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
let result = self
let output = self
.execute_prepared(
operation,
input,
@@ -247,27 +223,19 @@ impl RuntimeExecutor {
Some(request_context),
Arc::clone(&dispatch_started),
)
.await;
if let Some(lease) = lease.as_mut() {
lease.release().await;
}
let output = result?;
.await?;
Ok((output, request_preview))
};
let result = tokio::time::timeout_at(
execution_deadline.into(),
request.deadline().into(),
execution.instrument(runtime_span.clone()),
)
.await
.unwrap_or_else(|_| {
if admission_pending.load(Ordering::Acquire) {
Err(crate::admission::deadline_refusal(operation))
} else {
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
}
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
});
record_runtime_result(&runtime_span, &result);
drop(runtime_span);
@@ -984,10 +952,6 @@ fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
| RuntimeError::UnsupportedProtocol { .. }
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
RuntimeError::AdmissionOverloaded { .. }
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
RuntimeError::ConfirmationRequired { .. }
| RuntimeError::InvalidConfirmationToken { .. }
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
@@ -1044,13 +1008,6 @@ fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
// label domain stays fixed; the precise code lives in the execution failure/invocation log
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
+2 -12
View File
@@ -2,8 +2,8 @@ use std::sync::Arc;
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter};
use crank_core::{
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, MeteringSink, NoopMeteringSink,
ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
AdapterRegistry, CoordinationStateStore, MeteringSink, NoopMeteringSink, ResponseCacheStore,
SharedMeteringSink, SharedProtocolAdapter,
};
use crate::{RuntimeExecutor, RuntimeLimits};
@@ -13,7 +13,6 @@ pub struct RuntimeExecutorBuilder {
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
@@ -24,7 +23,6 @@ impl RuntimeExecutorBuilder {
adapters: AdapterRegistry::empty(),
response_cache: None,
coordination_store: None,
admission: None,
metering_sink: Arc::new(NoopMeteringSink) as Arc<dyn MeteringSink>,
}
}
@@ -49,13 +47,6 @@ impl RuntimeExecutorBuilder {
self
}
/// Wires the mandatory deployment-wide admission store (Story 4.1). Production processes wire
/// the PostgreSQL implementation; an executor without it keeps the pre-4.1 local limit.
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub fn with_metering_sink(mut self, sink: SharedMeteringSink) -> Self {
self.metering_sink = sink;
self
@@ -67,7 +58,6 @@ impl RuntimeExecutorBuilder {
self.adapters,
self.response_cache,
self.coordination_store,
self.admission,
self.metering_sink,
)
}
-1
View File
@@ -446,7 +446,6 @@ mod tests {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
-1
View File
@@ -1,4 +1,3 @@
mod admission;
mod auth;
mod cache;
mod cache_factory;
@@ -333,7 +333,6 @@ fn runtime_operation(draft: RestImportCandidate) -> RuntimeOperation {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -96,7 +96,6 @@ paths:
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -334,7 +334,6 @@ fn destructive_delete_operation() -> Operation<Schema, MappingSet> {
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
}),
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -340,7 +340,6 @@ fn idempotent_post_operation() -> Operation<Schema, MappingSet> {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -109,7 +109,6 @@ fn no_input_get_operation() -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -573,7 +573,6 @@ fn operation(
idempotency,
safety,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+1 -8
View File
@@ -10,7 +10,7 @@ use std::{
use async_trait::async_trait;
use crank_core::{
AdapterResponse, ConfirmationPolicy, CorrelationContext, ExecutionConfig, ExecutionErrorCode,
ExecutionMode, ExecutionOrigin, ExecutionStage, HttpMethod, IdempotencyMode, IdempotencyPolicy,
ExecutionMode, ExecutionOrigin, HttpMethod, IdempotencyMode, IdempotencyPolicy,
InvocationSource, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy,
OperationSecurityLevel, OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter,
ProtocolAdapterError, RestTarget, RetryPolicy, Retryability, Target, ToolDescription,
@@ -64,10 +64,6 @@ async fn deadline_after_dispatch_is_normalized_as_unknown_outcome() {
failure.outcome_certainty(),
OutcomeCertainty::OutcomeUnknown
);
// A deadline that fires after the adapter was called stays an upstream timeout: the admission
// refusal vocabulary (Story 4.1) only covers deadlines reached before any dispatch.
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
@@ -102,8 +98,6 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
let failure = executor.execute_outcome(request).await.unwrap_err();
assert_eq!(failure.retryability(), Retryability::AfterDelay);
assert_eq!(failure.outcome_certainty(), OutcomeCertainty::Certain);
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
@@ -638,7 +632,6 @@ fn operation() -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+3 -2
View File
@@ -18,8 +18,9 @@ feature jobs должны быть `skipped` для story и `success` для fe
У Crank и FreeHub полный Rust clippy/workspace test и тяжёлые feature/security
проверки выполняются только на feature. UI build и быстрые UI unit tests остаются
на story, Playwright E2E — на feature. Crank image build/scan/compose smoke — на
feature; фактический Deploy остаётся отдельным push-main job со своими build,
scan, health и требованием успешного SHA. Тег релиза по умолчанию запускает story
feature; фактический Deploy запускается только явным workflow_dispatch на main
с validation_level=feature и deploy_target=production после успешных feature gates.
Обычный merge/push не выкатывает production. Порядок: delivery-and-deploy.md. Тег релиза по умолчанию запускает story
ATDD, сборку бинарных/UI артефактов, security audit и image scan. Полный release
clippy/workspace/E2E доступен через ручной dispatch с `validation_level: feature`;
отчёт evidence прикладывается только если такой E2E был выполнен. FreeHub
+60
View File
@@ -0,0 +1,60 @@
# Delivery и production deploy
## Разделение merge и выкатки
Push в `main` запускает только source CI: story-проверки, UI build и
`Required Checks`. Он не запускает production deploy и не загружает секреты
OpenBao. PR, метка `feature-ready`, story dispatch и dispatch без явно выбранного
production также не выкатывают production.
Production deploy выполняет только job `Deploy` workflow `CI` и только при
одновременном выполнении условий:
- событие `workflow_dispatch`;
- ref `refs/heads/main`;
- `validation_level: feature`;
- `deploy_target: production`;
- успешный `Required Checks`, включая feature jobs `Rust Feature Validation`,
`Frontend E2E` и `Community Image Smoke`.
Любая другая комбинация оставляет deploy пропущенным, поэтому принятый merge в
`main` больше не блокируется ожиданием production-выкатки.
## Ручной запуск
В Gitea UI: `Actions` -> workflow `CI` -> `Run workflow`.
Указать точные входные значения:
| Input | Значение | Обязателен |
| --- | --- | --- |
| Branch / ref | `main` | да |
| `validation_level` | `feature` | да |
| `deploy_target` | `production` | да |
`validation_level: story`, `deploy_target: none` или отсутствие `deploy_target`
дают только source CI либо feature-проверки без deploy. Нельзя запускать
production deploy из PR, с другой ветки или через push в `main`.
## Approval и runbook
Сам по себе ручной dispatch не является автоматическим разрешением на
production. Перед запуском release owner или DevOps явно подтверждает:
1. SHA совпадает с принятым и проверенным коммитом в `main`.
2. `Required Checks` для этого SHA успешны, включая feature jobs.
3. Пройдены [production checklist](../production-checklist.md) и актуальный
[deploy and staging smoke](../deploy-and-staging-smoke.md); известен порядок
rollback и проверено наличие backup.
4. Нет незапланированных изменений секретов, миграций или инфраструктуры.
После deploy проверить health endpoints и authenticated product smoke, затем
зафиксировать результат в staging/production regression notes. Секреты OpenBao
загружаются только внутри уже разрешённого job `Deploy`; не выводить их в логи и
не передавать в PR или dispatch story.
## Ограничения
Workflow `Community Release` для тегов остаётся отдельным процессом публикации
release-образов и не является заменой этому ручному production deploy. Изменение
`ci.yml` не даёт постоянного разрешения на выкатку и не меняет приложение.
-2
View File
@@ -178,8 +178,6 @@ scrape_configs:
- `crank_upstream_requests_total`,
`crank_upstream_request_duration_seconds`;
- `crank_runtime_inflight`, `crank_runtime_limit_rejections_total`,
`crank_runtime_admission_queue_wait_seconds`,
`crank_runtime_admission_oldest_waiting_seconds`,
`crank_runtime_cache_total`, `crank_idempotency_total`,
`crank_confirmation_total`;
- `crank_db_pool_connections`;
+1 -1
View File
@@ -1 +1 @@
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6381,"max_rendered_series":19611,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6285,"max_rendered_series":19515,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
+1 -1
View File
@@ -1 +1 @@
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
-12
View File
@@ -55,15 +55,3 @@ run_atdd "capability advertisement registry" -p crank-community-mcp --lib protoc
# smoke) runs against the real endpoint, a real published catalog and a real counted upstream.
run_atdd "tool call lifecycle bounds" -p crank-community-mcp --lib app::lifecycle -- --test-threads=1
run_atdd "tool call progress and cancellation" -p mcp-server --test integration tools_lifecycle -- --test-threads=1
# Story 4.1 (FR-21, FR-24): a published Tool execution policy and one authoritative admission gate.
# The policy/JSON contract and the fair-scheduler arithmetic are unit-level; the authoritative store
# (scoped permits, bounded queue, typed overload, durable decisions, version pinning, fail-closed
# health) and the runtime wiring (one shared global budget across executors, typed refusals with no
# outbound call, adaptive/synchronous fallback) run against a real PostgreSQL schema.
run_atdd "execution policy contract" -p crank-core --lib operation::tests::execution_policy -- --test-threads=1
run_atdd "admission scheduler bounds" -p crank-core --lib admission:: -- --test-threads=1
run_atdd "execution policy validation" -p admin-api --lib operation_validation::tests:: -- --test-threads=1
run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1
run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1
run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1
+201
View File
@@ -0,0 +1,201 @@
from __future__ import annotations
import re
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
def workflow_text() -> str:
return WORKFLOW.read_text(encoding="utf-8")
def job_block(name: str) -> str:
lines = workflow_text().splitlines()
start = None
for index, line in enumerate(lines):
if line == f" {name}:":
start = index
break
if start is None:
raise AssertionError(f"job {name!r} not found")
end = len(lines)
for index in range(start + 1, len(lines)):
if re.match(r"^ [A-Za-z0-9_-]+:\s*$", lines[index]):
end = index
break
return "\n".join(lines[start:end])
def deploy_guard() -> str:
match = re.search(r"(?m)^ if:\s*(.+)$", job_block("deploy"))
if match is None:
raise AssertionError("deploy job has no guard")
return match.group(1).strip()
def guard_conditions() -> dict[str, str]:
expression = deploy_guard()
if expression.startswith("${{") and expression.endswith("}}"):
expression = expression[3:-2].strip()
conditions: dict[str, str] = {}
for part in expression.split("&&"):
match = re.fullmatch(
r"([A-Za-z0-9_.]+)\s*==\s*['\"]([^'\"]+)['\"]", part.strip()
)
if match is None:
raise AssertionError(f"unsupported deploy guard condition: {part!r}")
conditions[match.group(1)] = match.group(2)
return conditions
def deploys(context: dict[str, str | None]) -> bool:
return all(
context.get(condition) == expected
for condition, expected in guard_conditions().items()
)
def dispatch_input(name: str) -> str:
match = re.search(
rf"(?m)^ {re.escape(name)}:\n((?: .*\n)+)", workflow_text()
)
if match is None:
raise AssertionError(f"workflow_dispatch input {name!r} not found")
return match.group(0)
class DeployPolicyTests(unittest.TestCase):
def test_guard_allows_only_manual_production_feature_on_main(self):
self.assertEqual(
guard_conditions(),
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
},
)
def test_manual_production_feature_on_main_deploys(self):
self.assertTrue(
deploys(
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
}
)
)
def test_push_to_main_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "push",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
}
)
)
def test_pull_request_with_feature_label_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "pull_request",
"gitea.ref": "refs/pull/42/head",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
"gitea.event.pull_request.labels.*.name": "feature-ready",
}
)
)
def test_missing_or_none_deploy_target_does_not_deploy(self):
base = {
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
}
self.assertFalse(deploys(base))
self.assertFalse(deploys({**base, "inputs.deploy_target": "none"}))
def test_story_dispatch_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "story",
"inputs.deploy_target": "production",
}
)
)
def test_dispatch_on_other_ref_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/release",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
}
)
)
def test_deploy_target_is_explicit_choice_defaulting_to_none(self):
block = dispatch_input("deploy_target")
self.assertIn("type: choice", block)
self.assertIn("options: [none, production]", block)
self.assertIn("default: none", block)
self.assertIn("required: true", block)
def test_push_main_source_ci_stays_enabled(self):
self.assertRegex(
workflow_text(),
r"(?m)^ push:\n branches:\n - main$",
)
def test_deploy_requires_successful_required_checks(self):
deploy = job_block("deploy")
self.assertRegex(deploy, r"(?ms)^ needs:\n - required-checks$")
def test_required_checks_gate_feature_jobs(self):
block = job_block("required-checks")
for job in ("rust-feature", "frontend-e2e", "deployment"):
self.assertIn(f" - {job}", block)
for variable in ("RUST_FEATURE_RESULT", "E2E_RESULT", "IMAGE_RESULT"):
self.assertIn(variable, block)
def test_only_deploy_loads_production_secrets(self):
text = workflow_text()
self.assertEqual(text.count("scripts/load-openbao-env.sh"), 1)
self.assertIn("scripts/load-openbao-env.sh", job_block("deploy"))
self.assertNotIn("openbao", job_block("deployment").lower())
def test_deploy_keeps_build_scan_backup_health_and_smoke_steps(self):
deploy = job_block("deploy")
for step in (
"Load deployment secrets from OpenBao",
"Build and push images",
"Save deploy image scan evidence",
"Sync deployment files to server",
"Write environment file",
"Validate required environment variables",
"Deploy with Docker Compose",
"Verify health endpoints",
"Run authenticated product smoke",
):
self.assertIn(f"name: {step}", deploy)
self.assertIn("scripts/scan-images.sh", deploy)
self.assertIn("deploy-community.sh", deploy)
if __name__ == "__main__":
unittest.main()