From e80d30ddb0c1f05b073520e8c2525a986ead01b4 Mon Sep 17 00:00:00 2001 From: bsodfather Date: Mon, 26 Oct 2026 21:08:08 +0000 Subject: [PATCH] =?UTF-8?q?feat(epic4):=20=D0=BF=D1=83=D0=B1=D0=BB=D0=B8?= =?UTF-8?q?=D0=BA=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20execution=20mode=20?= =?UTF-8?q?=D0=B8=20=D0=BF=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=8F=D1=82=D1=8C?= =?UTF-8?q?=20=D0=BE=D0=B1=D1=89=D0=B8=D0=B9=20admission=20(Story=204.1,?= =?UTF-8?q?=20FR-21,=20FR-24)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool Draft фиксирует synchronous/background/adaptive, deadline и fallback; runtime применяет единый снимок политики (concurrency, очередь, approval, deadline) до внешнего действия. Насыщение отклоняется typed overload с retry guidance, mandatory safety store fail closed, background/adaptive используют опубликованный synchronous/unsupported fallback. Refs: #29 --- Cargo.lock | 1 + apps/admin-api/src/import_guidance.rs | 1 + apps/admin-api/src/main.rs | 19 +- apps/admin-api/src/pool_metrics.rs | 14 + apps/admin-api/src/routes.rs | 29 +- apps/admin-api/src/service.rs | 12 +- apps/admin-api/src/service/demo.rs | 1 + apps/admin-api/src/service/imports.rs | 1 + .../src/service/imports/import_plan.rs | 1 + .../src/service/operation_validation.rs | 55 +- apps/admin-api/tests/integration/common.rs | 1 + apps/mcp-server/src/main.rs | 31 +- apps/mcp-server/src/pool_metrics.rs | 14 + apps/mcp-server/tests/admission_budget.rs | 874 ++++++++++++++++++ apps/mcp-server/tests/integration/common.rs | 1 + apps/ui/js/i18n.js | 2 + apps/ui/js/wizard-model.js | 30 + .../tests/e2e/wizard-execution-policy.spec.js | 139 +++ crates/crank-community-mcp/src/app.rs | 45 +- .../src/dispatch_observer_support.rs | 1 + .../src/protocol/signals_tests.rs | 1 + .../tests/unit/manifest.rs | 1 + crates/crank-core/src/admission.rs | 531 +++++++++++ crates/crank-core/src/lib.rs | 28 +- crates/crank-core/src/operation.rs | 146 ++- .../crank-import/src/rest/legacy_payload.rs | 1 + crates/crank-import/src/rest/payload.rs | 1 + crates/crank-metrics/src/labels.rs | 28 + crates/crank-metrics/src/lib.rs | 11 +- crates/crank-metrics/src/record.rs | 46 +- crates/crank-metrics/src/schema/descriptor.rs | 46 +- crates/crank-metrics/tests/contract.rs | 19 +- crates/crank-metrics/tests/schema_v1.rs | 16 +- crates/crank-metrics/tests/schema_v2.rs | 22 +- crates/crank-observability/src/lib.rs | 2 +- crates/crank-registry/Cargo.toml | 1 + crates/crank-registry/src/lib.rs | 5 +- crates/crank-registry/src/migrations.rs | 2 + .../src/migrations/authority.rs | 14 +- .../src/migrations/authority/contract.rs | 2 + .../src/migrations/authority/preflight.rs | 1 + .../src/migrations/execution_admission_v24.rs | 45 + .../migrations/execution_admission_v24.sql | 92 ++ .../src/migrations/owned_relations.rs | 5 + .../src/migrations/schema_guard.rs | 7 + .../schema_guard/required_tables.rs | 2 + .../src/migrations/schema_guard_v24.rs | 254 +++++ .../crank-registry/src/postgres/admission.rs | 618 +++++++++++++ .../crank-registry/src/postgres/connection.rs | 13 + crates/crank-registry/src/postgres/mod.rs | 4 + crates/crank-registry/tests/integration.rs | 1 + .../tests/integration/admission.rs | 496 ++++++++++ crates/crank-runtime/src/admission.rs | 255 +++++ crates/crank-runtime/src/confirmation.rs | 143 ++- crates/crank-runtime/src/error.rs | 21 + crates/crank-runtime/src/execution_failure.rs | 13 + crates/crank-runtime/src/execution_request.rs | 1 + crates/crank-runtime/src/executor.rs | 63 +- crates/crank-runtime/src/executor_builder.rs | 14 +- crates/crank-runtime/src/idempotency.rs | 1 + crates/crank-runtime/src/lib.rs | 1 + crates/crank-runtime/tests/http_contract.rs | 1 + .../crank-runtime/tests/http_contract_wire.rs | 1 + .../tests/integration/confirmation.rs | 1 + .../tests/integration/idempotency.rs | 1 + .../tests/integration/no_input_get.rs | 1 + crates/crank-runtime/tests/product_metrics.rs | 1 + crates/crank-runtime/tests/stages.rs | 9 +- docs/observability.md | 2 + docs/schemas/metrics-registry-v2.json | 2 +- docs/schemas/migration-sequence.json | 2 +- scripts/ci-story-checks.sh | 12 + 72 files changed, 4157 insertions(+), 121 deletions(-) create mode 100644 apps/mcp-server/tests/admission_budget.rs create mode 100644 apps/ui/tests/e2e/wizard-execution-policy.spec.js create mode 100644 crates/crank-core/src/admission.rs create mode 100644 crates/crank-registry/src/migrations/execution_admission_v24.rs create mode 100644 crates/crank-registry/src/migrations/execution_admission_v24.sql create mode 100644 crates/crank-registry/src/migrations/schema_guard_v24.rs create mode 100644 crates/crank-registry/src/postgres/admission.rs create mode 100644 crates/crank-registry/tests/integration/admission.rs create mode 100644 crates/crank-runtime/src/admission.rs diff --git a/Cargo.lock b/Cargo.lock index 94c43eb..debfca6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -826,6 +826,7 @@ dependencies = [ name = "crank-registry" version = "0.3.1" dependencies = [ + "async-trait", "crank-artifacts", "crank-core", "crank-mapping", diff --git a/apps/admin-api/src/import_guidance.rs b/apps/admin-api/src/import_guidance.rs index 5b0a948..b5b4b97 100644 --- a/apps/admin-api/src/import_guidance.rs +++ b/apps/admin-api/src/import_guidance.rs @@ -181,6 +181,7 @@ mod tests { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/apps/admin-api/src/main.rs b/apps/admin-api/src/main.rs index 1bb35b6..e7e4fdb 100644 --- a/apps/admin-api/src/main.rs +++ b/apps/admin-api/src/main.rs @@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration}; use admin_api::{ app::build_app, auth::{AuthSettings, BootstrapAdminConfig}, - pool_metrics::spawn_postgres_pool_metrics, + pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics}, reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation}, service::AdminServiceBuilder, state::AppState, @@ -169,6 +169,7 @@ async fn run( }; if metrics_enabled { spawn_postgres_pool_metrics(registry.pool().clone()); + spawn_admission_metrics(registry.pool().clone()); } let base_url = config .runtime @@ -214,6 +215,16 @@ async fn run( .with_limits(runtime_limits) .with_response_cache(cache_stores.response.clone()) .with_coordination_store(cache_stores.coordination.clone()) + .with_admission(std::sync::Arc::new( + crank_registry::PostgresExecutionAdmission::new( + registry.pool().clone(), + crank_registry::AdmissionLimits { + global_limit: runtime_limits.max_concurrent_unary, + ..crank_registry::AdmissionLimits::default() + }, + crank_core::AdmissionSchedulerConfig::default(), + )?, + )) .build(); let identity_provider = PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone()); @@ -321,6 +332,12 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con config.runtime.max_concurrent_sessions, ) .map_err(|_| invalid("runtime.limits"))?; + crank_registry::AdmissionLimits { + global_limit: config.runtime.max_concurrent_unary, + ..crank_registry::AdmissionLimits::default() + } + .validate() + .map_err(|_| invalid("runtime.admission"))?; runtime_cache_config(config).map_err(|_| invalid("cache"))?; RequestRateLimitConfig::new( config.rate_limit.requests_per_second, diff --git a/apps/admin-api/src/pool_metrics.rs b/apps/admin-api/src/pool_metrics.rs index ba08ea8..860cadc 100644 --- a/apps/admin-api/src/pool_metrics.rs +++ b/apps/admin-api/src/pool_metrics.rs @@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> } }) } + +/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns +/// to zero) without waiting for a new request. +pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + loop { + interval.tick().await; + if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await { + crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms); + } + } + }) +} diff --git a/apps/admin-api/src/routes.rs b/apps/admin-api/src/routes.rs index 83db24b..205cc78 100644 --- a/apps/admin-api/src/routes.rs +++ b/apps/admin-api/src/routes.rs @@ -37,13 +37,22 @@ pub async fn readiness(State(state): State) -> impl IntoResponse { } else { "not_ready" }; + let execution_admission = if checks.execution_admission { + "ready" + } else { + "not_ready" + }; if checks.is_ready() { ( StatusCode::OK, Json(json!({ "service": "admin-api", "status": "ready", - "checks": { "postgres": postgres, "artifact_storage": artifact_storage } + "checks": { + "postgres": postgres, + "artifact_storage": artifact_storage, + "execution_admission": execution_admission + } })), ) } else { @@ -52,7 +61,11 @@ pub async fn readiness(State(state): State) -> impl IntoResponse { Json(json!({ "service": "admin-api", "status": "not_ready", - "checks": { "postgres": postgres, "artifact_storage": artifact_storage } + "checks": { + "postgres": postgres, + "artifact_storage": artifact_storage, + "execution_admission": execution_admission + } })), ) } @@ -67,6 +80,7 @@ mod tests { let checks = ReadinessChecks { postgres: false, artifact_storage: true, + execution_admission: true, }; assert!(!checks.is_ready()); assert_eq!( @@ -92,6 +106,7 @@ mod tests { let checks = ReadinessChecks { postgres: true, artifact_storage: false, + execution_admission: true, }; assert!(!checks.is_ready()); assert_eq!( @@ -111,4 +126,14 @@ mod tests { "not_ready" ); } + + #[test] + fn readiness_checks_identify_an_admission_state_failure() { + let checks = ReadinessChecks { + postgres: true, + artifact_storage: true, + execution_admission: false, + }; + assert!(!checks.is_ready()); + } } diff --git a/apps/admin-api/src/service.rs b/apps/admin-api/src/service.rs index 79b59bd..7b2b754 100644 --- a/apps/admin-api/src/service.rs +++ b/apps/admin-api/src/service.rs @@ -55,8 +55,8 @@ use crate::{ storage::LocalArtifactStorage, }; use operation_validation::{ - validate_approval_policy, validate_execution_timeout, validate_idempotency_policy, - validate_protocol_target, validate_response_cache_policy, + validate_approval_policy, validate_execution_policy, validate_execution_timeout, + validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy, }; #[derive(Clone)] @@ -104,11 +104,12 @@ pub use crate::dto::*; pub struct ReadinessChecks { pub postgres: bool, pub artifact_storage: bool, + pub execution_admission: bool, } impl ReadinessChecks { pub const fn is_ready(self) -> bool { - self.postgres && self.artifact_storage + self.postgres && self.artifact_storage && self.execution_admission } } @@ -150,6 +151,8 @@ pub(super) struct CredentialAuditRecord<'a> { impl AdminService { pub async fn readiness(&self) -> ReadinessChecks { let postgres = self.registry.ping().await.is_ok(); + // Story 4.1: the mandatory admission state is a readiness dependency of every execution. + let execution_admission = self.registry.execution_admission_health().await.is_ok(); let store = Arc::clone(&self.artifact_store); let artifact_storage = tokio::task::spawn_blocking(move || store.check_health()) .await @@ -157,6 +160,7 @@ impl AdminService { ReadinessChecks { postgres, artifact_storage, + execution_admission, } } @@ -725,6 +729,7 @@ impl AdminService { validate_response_cache_policy(&payload.target, &payload.execution_config)?; validate_idempotency_policy(&payload.target, &payload.execution_config)?; validate_approval_policy(&payload.execution_config)?; + validate_execution_policy(&payload.execution_config)?; payload.input_mapping.validate_paths()?; payload.output_mapping.validate_paths()?; Ok(()) @@ -738,6 +743,7 @@ impl AdminService { validate_response_cache_policy(&operation.target, &operation.execution_config)?; validate_idempotency_policy(&operation.target, &operation.execution_config)?; validate_approval_policy(&operation.execution_config)?; + validate_execution_policy(&operation.execution_config)?; operation.input_mapping.validate_paths()?; operation.output_mapping.validate_paths()?; Ok(()) diff --git a/apps/admin-api/src/service/demo.rs b/apps/admin-api/src/service/demo.rs index 3b71562..920c1e2 100644 --- a/apps/admin-api/src/service/demo.rs +++ b/apps/admin-api/src/service/demo.rs @@ -441,6 +441,7 @@ fn demo_rest_operation_payload() -> OperationPayload { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/apps/admin-api/src/service/imports.rs b/apps/admin-api/src/service/imports.rs index 4b10d54..0e8aaae 100644 --- a/apps/admin-api/src/service/imports.rs +++ b/apps/admin-api/src/service/imports.rs @@ -635,6 +635,7 @@ impl AdminService { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/apps/admin-api/src/service/imports/import_plan.rs b/apps/admin-api/src/service/imports/import_plan.rs index 5b77212..38747d6 100644 --- a/apps/admin-api/src/service/imports/import_plan.rs +++ b/apps/admin-api/src/service/imports/import_plan.rs @@ -610,6 +610,7 @@ impl AdminService { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref, headers: BTreeMap::new(), }, diff --git a/apps/admin-api/src/service/operation_validation.rs b/apps/admin-api/src/service/operation_validation.rs index a535186..6c52a8d 100644 --- a/apps/admin-api/src/service/operation_validation.rs +++ b/apps/admin-api/src/service/operation_validation.rs @@ -160,6 +160,23 @@ pub(super) fn validate_approval_policy( Ok(()) } +pub(super) fn validate_execution_policy( + execution_config: &crank_core::ExecutionConfig, +) -> Result<(), ApiError> { + let Some(policy) = execution_config.execution_policy.as_ref() else { + return Ok(()); + }; + if let Err(error) = policy.validate() { + return Err(ApiError::validation_with_context( + error.to_string(), + json!({ + "field": "execution_config.execution_policy.deadline_ms", + }), + )); + } + Ok(()) +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -171,8 +188,8 @@ mod tests { }; use super::{ - validate_approval_policy, validate_execution_timeout, validate_idempotency_policy, - validate_response_cache_policy, + validate_approval_policy, validate_execution_policy, validate_execution_timeout, + validate_idempotency_policy, validate_response_cache_policy, }; fn cacheable_execution_config() -> ExecutionConfig { @@ -183,6 +200,7 @@ mod tests { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), } @@ -201,6 +219,7 @@ mod tests { }), safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), } @@ -365,4 +384,36 @@ mod tests { "approval ttl must be between 1 and 300 seconds" ); } + + #[test] + fn accepts_published_execution_policy_and_absent_legacy_policy() { + let mut config = cacheable_execution_config(); + validate_execution_policy(&config).unwrap(); + + config.execution_policy = Some(crank_core::ToolExecutionPolicy { + mode: crank_core::ToolExecutionMode::Adaptive, + deadline_ms: 120_000, + no_tasks_fallback: crank_core::NoTasksFallback::Unsupported, + }); + validate_execution_policy(&config).unwrap(); + } + + #[test] + fn rejects_zero_and_overflowing_execution_policy_deadline() { + let mut config = cacheable_execution_config(); + config.execution_policy = Some(crank_core::ToolExecutionPolicy { + mode: crank_core::ToolExecutionMode::Synchronous, + deadline_ms: 0, + no_tasks_fallback: crank_core::NoTasksFallback::Synchronous, + }); + let error = validate_execution_policy(&config).unwrap_err(); + assert!(matches!(error, crate::error::ApiError::Validation { .. })); + + config.execution_policy = Some(crank_core::ToolExecutionPolicy { + mode: crank_core::ToolExecutionMode::Background, + deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1, + no_tasks_fallback: crank_core::NoTasksFallback::Synchronous, + }); + assert!(validate_execution_policy(&config).is_err()); + } } diff --git a/apps/admin-api/tests/integration/common.rs b/apps/admin-api/tests/integration/common.rs index 5fb655c..91d03a9 100644 --- a/apps/admin-api/tests/integration/common.rs +++ b/apps/admin-api/tests/integration/common.rs @@ -495,6 +495,7 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/apps/mcp-server/src/main.rs b/apps/mcp-server/src/main.rs index 0aca5c8..805bcd9 100644 --- a/apps/mcp-server/src/main.rs +++ b/apps/mcp-server/src/main.rs @@ -14,13 +14,14 @@ use crank_observability::{ OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error, }; use crank_registry::{ - MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry, + AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, + PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, }; use crank_runtime::{ RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores, RuntimeLimits, SecretCrypto, }; -use mcp_server::pool_metrics::spawn_postgres_pool_metrics; +use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics}; use sqlx::postgres::PgConnectOptions; use tokio::net::TcpListener; use tracing::info; @@ -175,6 +176,7 @@ async fn run( }; if metrics_enabled { spawn_postgres_pool_metrics(registry.pool().clone()); + spawn_admission_metrics(registry.pool().clone()); } let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?; let secret_crypto = @@ -190,6 +192,10 @@ async fn run( .with_limits(runtime_limits) .with_response_cache(cache_stores.response.clone()) .with_coordination_store(cache_stores.coordination.clone()) + .with_admission(execution_admission( + registry.pool().clone(), + runtime_limits, + )?) .build(); // Published static Resources are read from the same immutable content-addressed root the // admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root @@ -287,6 +293,21 @@ fn init_observability( )?) } +fn execution_admission( + pool: sqlx::PgPool, + runtime_limits: RuntimeLimits, +) -> Result, Box> { + let store = PostgresExecutionAdmission::new( + pool, + AdmissionLimits { + global_limit: runtime_limits.max_concurrent_unary, + ..AdmissionLimits::default() + }, + crank_core::AdmissionSchedulerConfig::default(), + )?; + Ok(std::sync::Arc::new(store)) +} + fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> { let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field); database_options(&config.database).map_err(|_| invalid("database.source"))?; @@ -307,6 +328,12 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi config.runtime.max_concurrent_sessions, ) .map_err(|_| invalid("runtime.limits"))?; + crank_registry::AdmissionLimits { + global_limit: config.runtime.max_concurrent_unary, + ..crank_registry::AdmissionLimits::default() + } + .validate() + .map_err(|_| invalid("runtime.admission"))?; runtime_cache_config(config).map_err(|_| invalid("cache"))?; RequestRateLimitConfig::new( config.rate_limit.requests_per_second, diff --git a/apps/mcp-server/src/pool_metrics.rs b/apps/mcp-server/src/pool_metrics.rs index ba08ea8..860cadc 100644 --- a/apps/mcp-server/src/pool_metrics.rs +++ b/apps/mcp-server/src/pool_metrics.rs @@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> } }) } + +/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns +/// to zero) without waiting for a new request. +pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + loop { + interval.tick().await; + if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await { + crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms); + } + } + }) +} diff --git a/apps/mcp-server/tests/admission_budget.rs b/apps/mcp-server/tests/admission_budget.rs new file mode 100644 index 0000000..2070804 --- /dev/null +++ b/apps/mcp-server/tests/admission_budget.rs @@ -0,0 +1,874 @@ +//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission. +//! +//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a +//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported +//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal. +//! +//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this +//! app already wires both and this is the real deployment shape. + +use std::{ + collections::BTreeMap, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use crank_core::{ + AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, + ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode, + ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation, + OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel, + OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget, + RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy, + WorkspaceId, +}; +use crank_mapping::MappingSet; +use crank_registry::{AdmissionLimits, PostgresExecutionAdmission}; +use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits}; +use crank_schema::{Schema, SchemaKind}; +use serde_json::json; +use sqlx::PgPool; +use time::OffsetDateTime; + +struct CountingAdapter { + calls: Arc, + current: Arc, + peak: Arc, +} + +#[async_trait] +impl ProtocolAdapter for CountingAdapter { + fn protocol(&self) -> Protocol { + Protocol::Rest + } + + fn supports_mode(&self, mode: ExecutionMode) -> bool { + mode == ExecutionMode::Unary + } + + async fn invoke_unary( + &self, + _target: &Target, + _prepared: &crank_core::PreparedRequest, + context: &RuntimeRequestContext, + ) -> Result { + context.mark_dispatch_started(); + self.calls.fetch_add(1, Ordering::SeqCst); + let current = self.current.fetch_add(1, Ordering::SeqCst) + 1; + self.peak.fetch_max(current, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(150)).await; + self.current.fetch_sub(1, Ordering::SeqCst); + Ok(AdapterResponse { + status_code: 200, + headers: BTreeMap::new(), + body: json!({"ok": true}), + data: json!({"ok": true}), + }) + } +} + +fn schema(kind: SchemaKind) -> Schema { + Schema { + kind, + format: None, + description: None, + required: false, + nullable: false, + default_value: None, + fields: BTreeMap::new(), + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} + +fn operation(policy: Option) -> Operation { + Operation { + id: OperationId::new("op_admission_budget"), + name: "admission_budget".to_owned(), + display_name: "Admission budget".to_owned(), + category: "general".to_owned(), + protocol: Protocol::Rest, + security_level: OperationSecurityLevel::Standard, + status: OperationStatus::Published, + version: 7, + target: Target::Rest(RestTarget { + base_url: "https://api.example.com".to_owned(), + method: HttpMethod::Get, + path_template: "/v1/items".to_owned(), + static_headers: BTreeMap::new(), + request_contract: None, + response_contract: Vec::new(), + multipart_field_mime_types: BTreeMap::new(), + }), + input_schema: schema(SchemaKind::Object), + output_schema: schema(SchemaKind::Object), + input_mapping: MappingSet::default(), + output_mapping: MappingSet::default(), + execution_config: ExecutionConfig { + timeout_ms: 5_000, + execution_policy: policy, + ..ExecutionConfig::default() + }, + tool_description: ToolDescription { + title: "Admission".to_owned(), + description: "Admission budget probe".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + samples: None, + generated_draft: None, + config_export: None, + wizard_state: None, + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + published_at: None, + } +} + +async fn execute( + executor: &RuntimeExecutor, + operation: &crank_runtime::RuntimeOperation, + input: &serde_json::Value, +) -> Result { + let workspace = WorkspaceId::new("ws_test"); + let context = crank_runtime::RuntimeRequestContext::from_correlation( + &crank_core::CorrelationContext::generate(), + ) + .with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun); + execute_in(executor, operation, input, &context).await +} + +async fn execute_in( + executor: &RuntimeExecutor, + operation: &crank_runtime::RuntimeOperation, + input: &serde_json::Value, + context: &crank_runtime::RuntimeRequestContext, +) -> Result { + let workspace = context + .metering_context() + .map(|metering| metering.workspace_id.clone()) + .unwrap_or_else(|| WorkspaceId::new("ws_test")); + let request = crank_runtime::RuntimeExecutionRequest::try_new( + &workspace, + ExecutionOrigin::AdminDraft, + None, + operation, + input, + crank_runtime::ExecutionAuthorization::Authorized, + None, + context, + Instant::now() + Duration::from_secs(30), + ) + .map_err(|_| { + crank_core::ExecutionFailure::new( + ExecutionErrorCode::RuntimeInternal, + crank_core::CorrelationContext::new( + context.request_id.clone(), + context.trace_context.clone(), + ), + ) + })?; + executor + .execute_outcome(request) + .await + .map(|success| success.output) +} + +async fn schema_pool(schema_url: &str) -> PgPool { + let pool = PgPool::connect(schema_url) + .await + .expect("admission schema pool"); + crank_registry::MigrationAuthority::apply(&pool) + .await + .expect("migration authority applies the admission schema"); + pool +} + +fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc { + Arc::new( + PostgresExecutionAdmission::new( + pool, + AdmissionLimits { + global_limit: global, + agent_limit: 8, + tool_limit: 8, + queue_limit: queue, + }, + crank_core::AdmissionSchedulerConfig::default(), + ) + .expect("valid admission limits"), + ) +} + +fn executor( + store: Arc, + calls: Arc, + current: Arc, + peak: Arc, +) -> RuntimeExecutor { + RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(CountingAdapter { + calls, + current, + peak, + })) + .with_limits(RuntimeLimits::try_new(8, 2).expect("local limits")) + .with_admission(store) + .build() +} + +#[tokio::test] +async fn two_executors_share_one_global_budget() { + let schema_url = crank_test_support::postgres_schema_url("admission_budget").await; + let pool = schema_pool(&schema_url).await; + let store = admission_store(pool.clone(), 2, 8); + let calls = Arc::new(AtomicUsize::new(0)); + let current = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let first = executor( + Arc::clone(&store), + Arc::clone(&calls), + Arc::clone(¤t), + Arc::clone(&peak), + ); + let second = executor( + Arc::clone(&store), + Arc::clone(&calls), + Arc::clone(¤t), + Arc::clone(&peak), + ); + + let operation: crank_runtime::RuntimeOperation = operation(None).into(); + let mut tasks = Vec::new(); + for index in 0..4 { + let executor = if index % 2 == 0 { + first.clone() + } else { + second.clone() + }; + let operation = operation.clone(); + tasks.push(tokio::spawn(async move { + execute(&executor, &operation, &json!({})).await + })); + } + let mut completed = 0; + for task in tasks { + match task.await.expect("execution task") { + Ok(_) => completed += 1, + Err(failure) => panic!("admission must grant every request: {failure:?}"), + } + } + assert_eq!(completed, 4); + assert_eq!(calls.load(Ordering::SeqCst), 4); + assert!( + peak.load(Ordering::SeqCst) <= 2, + "peak concurrency {} exceeded the shared global limit", + peak.load(Ordering::SeqCst) + ); + assert_eq!(store.snapshot().await.unwrap().executing, 0); +} + +#[tokio::test] +async fn saturation_returns_typed_overload_without_an_outbound_call() { + let schema_url = crank_test_support::postgres_schema_url("admission_overload").await; + let pool = schema_pool(&schema_url).await; + let store = admission_store(pool, 1, 1); + let calls = Arc::new(AtomicUsize::new(0)); + let current = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let runtime = executor( + Arc::clone(&store), + Arc::clone(&calls), + Arc::clone(¤t), + Arc::clone(&peak), + ); + + // Another process already holds the only global permit. + let held = match store + .admit(AdmissionRequest { + scope: AdmissionScope { + workspace_id: "ws_test".to_owned(), + agent_key: "admin".to_owned(), + operation_id: "op_admission_budget".to_owned(), + operation_version: 7, + }, + mode: AdmissionMode::Synchronous, + policy_mode: ToolExecutionMode::Synchronous, + no_tasks_fallback: NoTasksFallback::Synchronous, + approval: ApprovalEvidence::NotRequired, + fingerprint: "fp_holder".to_owned(), + expires_at_unix_ms: crank_core::current_unix_ms() + 10_000, + }) + .await + .expect("admission store is reachable") + { + crank_core::AdmissionOutcome::Permitted(permit) => permit, + other => panic!("expected a permit, got {other:?}"), + }; + + let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy { + mode: ToolExecutionMode::Synchronous, + deadline_ms: 250, + no_tasks_fallback: NoTasksFallback::Synchronous, + })) + .into(); + let waiting_runtime = runtime.clone(); + let first_waiting_operation = waiting_operation.clone(); + let waiting = tokio::spawn(async move { + execute(&waiting_runtime, &first_waiting_operation, &json!({})).await + }); + // Let the first request occupy the bounded queue before the next one arrives. + while store.snapshot().await.unwrap().waiting == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + let overloaded = execute(&runtime, &waiting_operation, &json!({})).await; + let failure = overloaded.expect_err("a full queue must refuse the request"); + assert_eq!( + failure.error_code(), + ExecutionErrorCode::ExecutionOverloaded + ); + assert_eq!(failure.stage(), ExecutionStage::Admission); + assert_eq!(failure.retry_after_ms(), Some(250)); + + // The queued request hits its absolute deadline instead of dispatching. That refusal happened + // inside admission (nothing was ever dispatched), so the client must see an admission-stage + // overload with retry guidance — not an `upstream_timeout`. + let expired = waiting + .await + .expect("waiting task") + .expect_err("deadline must refuse"); + assert_eq!( + expired.error_code(), + ExecutionErrorCode::ExecutionOverloaded, + "queue waiting counts against the published deadline" + ); + assert_eq!(expired.stage(), ExecutionStage::Admission); + assert_eq!(expired.retry_after_ms(), Some(250)); + assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "no outbound call on refusal" + ); + assert!(store.release(&held).await.unwrap()); +} + +#[tokio::test] +async fn background_policy_without_a_task_transport_is_typed_unsupported() { + let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await; + let pool = schema_pool(&schema_url).await; + let store = admission_store(pool, 2, 4); + let calls = Arc::new(AtomicUsize::new(0)); + let current = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let runtime = executor( + Arc::clone(&store), + Arc::clone(&calls), + Arc::clone(¤t), + Arc::clone(&peak), + ); + + let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 2_000, + no_tasks_fallback: NoTasksFallback::Unsupported, + })) + .into(); + let failure = execute(&runtime, &operation, &json!({})) + .await + .expect_err("no Task transport exists in Story 4.1"); + assert_eq!( + failure.error_code(), + ExecutionErrorCode::ExecutionModeUnsupported + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + let snapshot = store.snapshot().await.unwrap(); + assert_eq!(snapshot.waiting, 0); + assert_eq!(snapshot.executing, 0); +} + +#[tokio::test] +async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() { + let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await; + let pool = schema_pool(&schema_url).await; + let store = admission_store(pool, 2, 4); + let calls = Arc::new(AtomicUsize::new(0)); + let current = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let runtime = executor( + Arc::clone(&store), + Arc::clone(&calls), + Arc::clone(¤t), + Arc::clone(&peak), + ); + + let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy { + mode: ToolExecutionMode::Adaptive, + deadline_ms: 2_000, + no_tasks_fallback: NoTasksFallback::Synchronous, + })) + .into(); + let output = execute(&runtime, &operation, &json!({})) + .await + .expect("adaptive resolves through the published synchronous fallback"); + // The response shape is unchanged by admission: an operation without output mapping still + // returns the same mapped output object it returned before Story 4.1. + assert_eq!(output, json!({})); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(store.snapshot().await.unwrap().executing, 0); +} + +#[tokio::test] +async fn admission_store_outage_fails_closed_before_dispatch() { + let schema_url = crank_test_support::postgres_schema_url("admission_outage").await; + let pool = schema_pool(&schema_url).await; + let store = admission_store(pool.clone(), 2, 4); + let calls = Arc::new(AtomicUsize::new(0)); + let current = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let runtime = executor( + Arc::clone(&store), + Arc::clone(&calls), + Arc::clone(¤t), + Arc::clone(&peak), + ); + // The mandatory state disappears after the store was built: every execution must fail closed. + sqlx::query("drop table execution_admission_entries") + .execute(&pool) + .await + .expect("drop the mandatory admission state"); + let operation: crank_runtime::RuntimeOperation = operation(None).into(); + let failure = execute(&runtime, &operation, &json!({})) + .await + .expect_err("a missing mandatory store must fail closed"); + assert_eq!( + failure.error_code(), + ExecutionErrorCode::SafetyStoreUnavailable + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert!(store.health().await.is_err()); +} + +/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2). +struct RecordingAdmission { + inner: Arc, + requests: Mutex>, + permits: AtomicUsize, + releases: AtomicUsize, +} + +impl RecordingAdmission { + fn new(inner: Arc) -> Self { + Self { + inner, + requests: Mutex::new(Vec::new()), + permits: AtomicUsize::new(0), + releases: AtomicUsize::new(0), + } + } + + fn requests(&self) -> Vec { + self.requests.lock().expect("admission log").clone() + } + + fn permits(&self) -> usize { + self.permits.load(Ordering::SeqCst) + } + + fn releases(&self) -> usize { + self.releases.load(Ordering::SeqCst) + } +} + +#[async_trait] +impl ExecutionAdmission for RecordingAdmission { + async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> { + self.inner.health().await + } + + async fn admit( + &self, + request: AdmissionRequest, + ) -> Result { + self.requests + .lock() + .expect("admission log") + .push(request.clone()); + let outcome = self.inner.admit(request).await?; + if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) { + self.permits.fetch_add(1, Ordering::SeqCst); + } + Ok(outcome) + } + + async fn release( + &self, + permit: &crank_core::AdmissionPermit, + ) -> Result { + let released = self.inner.release(permit).await?; + if released { + self.releases.fetch_add(1, Ordering::SeqCst); + } + Ok(released) + } + + async fn cancel( + &self, + ticket_id: u64, + owner_token: &str, + ) -> Result { + self.inner.cancel(ticket_id, owner_token).await + } + + async fn snapshot( + &self, + ) -> Result { + self.inner.snapshot().await + } +} + +fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation { + let mut source = operation(None); + source.id = OperationId::new("op_admission_confirmation"); + source.name = "admission_confirmation".to_owned(); + source.version = version; + let Target::Rest(target) = &mut source.target; + target.method = HttpMethod::Delete; + source.execution_config.safety = Some(OperationSafetyPolicy { + class: OperationSafetyClass::Destructive, + confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }), + }); + source.into() +} + +fn confirmation_runtime( + admission: Arc, + coordination: Option>, + calls: Arc, +) -> RuntimeExecutor { + let store: Arc = admission; + let builder = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(CountingAdapter { + calls, + current: Arc::new(AtomicUsize::new(0)), + peak: Arc::new(AtomicUsize::new(0)), + })) + .with_limits(RuntimeLimits::try_new(8, 2).expect("local limits")) + .with_admission(store); + match coordination { + Some(coordination) => builder.with_coordination_store(coordination).build(), + None => builder.build(), + } +} + +fn confirmation_context() -> crank_runtime::RuntimeRequestContext { + crank_runtime::RuntimeRequestContext::from_correlation( + &crank_core::CorrelationContext::generate(), + ) + .with_response_cache_scope("ws_test", "agent_a") + .with_metering_context( + WorkspaceId::new("ws_test"), + None, + InvocationSource::AdminTestRun, + ) +} + +#[tokio::test] +async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() { + let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await; + let pool = schema_pool(&schema_url).await; + let recording = Arc::new(RecordingAdmission::new(Arc::new( + PostgresExecutionAdmission::new( + pool, + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + crank_core::AdmissionSchedulerConfig::default(), + ) + .expect("valid admission limits"), + ))); + let coordination: Arc = + Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()); + let calls = Arc::new(AtomicUsize::new(0)); + let runtime = confirmation_runtime( + Arc::clone(&recording), + Some(Arc::clone(&coordination)), + Arc::clone(&calls), + ); + let operation = confirmation_operation(3); + let context = confirmation_context(); + let input = json!({ "reason": "remove the stale item" }); + + // 1. No approval yet: the existing challenge is returned and no execution capacity is taken. + let challenge = execute_in(&runtime, &operation, &input, &context) + .await + .expect_err("an unapproved destructive call must only stage confirmation"); + assert_eq!( + challenge.error_code(), + ExecutionErrorCode::ConfirmationRequired + ); + let token = challenge + .confirmation() + .expect("expected the confirmation challenge") + .token() + .to_owned(); + assert_eq!( + recording.permits(), + 0, + "an unapproved request must not take an execution permit" + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + let requests = recording.requests(); + assert_eq!( + requests.len(), + 1, + "the one admission snapshot is consulted once" + ); + assert_eq!( + requests[0].approval, + ApprovalEvidence::NotGranted, + "admission must see the real approval state, not a fixed NotRequired" + ); + assert_eq!(requests[0].scope.operation_version, 3); + let snapshot = recording.snapshot().await.expect("admission snapshot"); + assert_eq!((snapshot.executing, snapshot.waiting), (0, 0)); + + // 2. An approval issued for another published version never opens this execution. + let other_version = confirmation_operation(4); + let foreign = execute_in( + &runtime, + &other_version, + &input, + &context.clone().with_confirmation_token(token.clone()), + ) + .await + .expect_err("a foreign approval must not authorise this version"); + assert_eq!( + foreign.error_code(), + ExecutionErrorCode::ConfirmationInvalid + ); + assert_eq!(recording.permits(), 0); + assert_eq!( + recording.requests().len(), + 1, + "an unusable approval is refused before any capacity request" + ); + + // 3. An approval is bound to its exact input as well. + let mismatched_input = json!({ "reason": "remove another item" }); + let mismatched = execute_in( + &runtime, + &operation, + &mismatched_input, + &context.clone().with_confirmation_token(token.clone()), + ) + .await + .expect_err("a confirmation token must not authorise different input"); + assert_eq!( + mismatched.error_code(), + ExecutionErrorCode::ConfirmationInvalid + ); + assert_eq!(recording.permits(), 0); + assert_eq!(recording.requests().len(), 1); + + // 4. The matching approval passes admission with truthful evidence and executes exactly once. + let executed = execute_in( + &runtime, + &operation, + &input, + &context.clone().with_confirmation_token(token.clone()), + ) + .await + .expect("a verified approval executes"); + assert!(executed.is_object()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + let requests = recording.requests(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[1].approval, ApprovalEvidence::Granted); + assert_eq!(requests[1].scope.operation_version, 3); + assert_eq!(recording.permits(), 1); + assert_eq!( + recording.releases(), + 1, + "the permit is released after execution" + ); + assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0); + + // 5. Replay of the consumed confirmation is refused before capacity and without a side effect. + let replay = execute_in( + &runtime, + &operation, + &input, + &context.clone().with_confirmation_token(token), + ) + .await + .expect_err("a confirmation token is single-use"); + assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(recording.permits(), 1); + assert_eq!( + recording.requests().len(), + 2, + "a replayed approval never reaches admission" + ); + + // 6. The trusted approved-recovery path still passes the same gate with real evidence. + let approved = execute_in( + &runtime, + &operation, + &json!({ "reason": "approved by an operator" }), + &context.with_approval_granted(), + ) + .await; + assert!(approved.is_ok()); + assert_eq!(calls.load(Ordering::SeqCst), 2); + let requests = recording.requests(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[2].approval, ApprovalEvidence::Granted); + assert_eq!(recording.permits(), 2); + assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0); +} + +#[tokio::test] +async fn mandatory_safety_store_outage_fails_closed_before_admission() { + let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await; + let pool = schema_pool(&schema_url).await; + let recording = Arc::new(RecordingAdmission::new(Arc::new( + PostgresExecutionAdmission::new( + pool, + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + crank_core::AdmissionSchedulerConfig::default(), + ) + .expect("valid admission limits"), + ))); + let calls = Arc::new(AtomicUsize::new(0)); + // Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail + // closed instead of reserving capacity and hoping for an in-memory fallback. + let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls)); + let operation = confirmation_operation(3); + + let failure = execute_in( + &runtime, + &operation, + &json!({ "reason": "no store" }), + &confirmation_context(), + ) + .await + .expect_err("a mandatory safety store outage must fail closed"); + assert_eq!( + failure.error_code(), + ExecutionErrorCode::SafetyStoreUnavailable + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert_eq!(recording.permits(), 0); + let after_missing_store = recording.requests().len(); + + // The same Tool with a claimed approval and a *failing* safety store never reaches the capacity + // store at all: the approval claim stays unverified, so no permit and no outbound call exist. + let failing = confirmation_runtime( + Arc::clone(&recording), + Some(Arc::new(UnavailableSafetyStore)), + Arc::clone(&calls), + ); + let outage = execute_in( + &failing, + &operation, + &json!({ "reason": "failing store" }), + &confirmation_context().with_confirmation_token("ct_unavailable"), + ) + .await + .expect_err("an unreachable safety store must refuse the claimed approval"); + assert_eq!( + outage.error_code(), + ExecutionErrorCode::SafetyStoreUnavailable + ); + assert_eq!( + recording.requests().len(), + after_missing_store, + "an unverified approval never becomes a capacity request" + ); + assert_eq!(recording.permits(), 0); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +/// Coordination store whose mandatory backend is down: every operation fails closed. +struct UnavailableSafetyStore; + +impl UnavailableSafetyStore { + fn error() -> crank_core::CacheStoreError { + crank_core::CacheStoreError::Unavailable { + message: "test backend unavailable".to_owned(), + } + } +} + +#[async_trait] +impl crank_core::CoordinationStateStore for UnavailableSafetyStore { + async fn get_value( + &self, + _scope: crank_core::CacheScope, + _key: &str, + ) -> Result, crank_core::CacheStoreError> { + Err(Self::error()) + } + + async fn put_value( + &self, + _scope: crank_core::CacheScope, + _key: &str, + _value: crank_core::CoordinationStateValue, + _ttl: Duration, + ) -> Result<(), crank_core::CacheStoreError> { + Err(Self::error()) + } + + async fn delete_value( + &self, + _scope: crank_core::CacheScope, + _key: &str, + ) -> Result<(), crank_core::CacheStoreError> { + Err(Self::error()) + } + + async fn take_value( + &self, + _scope: crank_core::CacheScope, + _key: &str, + ) -> Result, crank_core::CacheStoreError> { + Err(Self::error()) + } + + async fn reserve_value( + &self, + _scope: crank_core::CacheScope, + _key: &str, + _value: crank_core::CoordinationStateValue, + _ttl: Duration, + ) -> Result { + Err(Self::error()) + } + + async fn compare_and_set_value( + &self, + _scope: crank_core::CacheScope, + _key: &str, + _expected: &crank_core::CoordinationStateValue, + _value: crank_core::CoordinationStateValue, + _ttl: Duration, + ) -> Result { + Err(Self::error()) + } +} diff --git a/apps/mcp-server/tests/integration/common.rs b/apps/mcp-server/tests/integration/common.rs index 964acb7..c92e649 100644 --- a/apps/mcp-server/tests/integration/common.rs +++ b/apps/mcp-server/tests/integration/common.rs @@ -621,6 +621,7 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation 3600000) { + throw new Error(tKey('wizard.error.execution_policy_invalid')); + } + return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback }; +} + function currentUpstream() { var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; }); if (selected) return selected; @@ -415,6 +438,13 @@ function executionConfigToEditorValue(config) { var value = { timeout_ms: config.timeout_ms, }; + if (config.execution_policy) { + value.execution_policy = { + mode: config.execution_policy.mode, + deadline_ms: config.execution_policy.deadline_ms, + no_tasks_fallback: config.execution_policy.no_tasks_fallback, + }; + } if (config.retry_policy) { value.retry = { max_attempts: config.retry_policy.max_attempts }; } diff --git a/apps/ui/tests/e2e/wizard-execution-policy.spec.js b/apps/ui/tests/e2e/wizard-execution-policy.spec.js new file mode 100644 index 0000000..9c6dbdf --- /dev/null +++ b/apps/ui/tests/e2e/wizard-execution-policy.spec.js @@ -0,0 +1,139 @@ +const { test, expect } = require('@playwright/test'); +const { getCurrentWorkspace, login } = require('./helpers'); + +// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a +// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit +// invalid value must be reported instead of being silently replaced by defaults. +function draftSnapshot(operationId) { + return { + id: operationId, + name: operationId, + display_name: 'Policy rates', + category: 'general', + protocol: 'rest', + target: { + kind: 'rest', + base_url: 'https://api.example.test', + method: 'GET', + path_template: '/rates', + static_headers: {}, + }, + input_schema: { + type: 'object', + description: null, + required: false, + nullable: false, + default_value: null, + fields: {}, + items: null, + enum_values: [], + variants: [], + }, + output_schema: { + type: 'object', + description: null, + required: false, + nullable: false, + default_value: null, + fields: {}, + items: null, + enum_values: [], + variants: [], + }, + input_mapping: { rules: [] }, + output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] }, + execution_config: { + timeout_ms: 10000, + retry_policy: null, + auth_profile_ref: null, + headers: {}, + approval_policy: null, + execution_policy: { + mode: 'adaptive', + deadline_ms: 45000, + no_tasks_fallback: 'unsupported', + }, + }, + tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] }, + wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] }, + }; +} + +test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => { + await login(page); + const workspace = await getCurrentWorkspace(page); + const operationId = 'policy_rates_41'; + const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`; + const savedPayloads = []; + + await page.route(operationPath, async (route) => { + if (route.request().method() === 'PATCH') { + savedPayloads.push(route.request().postDataJSON()); + await route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }), + }); + return; + } + await route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + id: operationId, + workspace_id: workspace.id, + name: operationId, + display_name: 'Policy rates', + category: 'general', + protocol: 'rest', + status: 'draft', + current_draft_version: 1, + draft_version_ref: { version: 1, status: 'draft' }, + published_version_ref: null, + agent_refs: [], + }), + }); + }); + await page.route(`${operationPath}/versions/*`, async (route) => { + await route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + operation_id: operationId, + workspace_id: workspace.id, + version: 1, + status: 'draft', + snapshot: draftSnapshot(operationId), + }), + }); + }); + await page.route('**/operations/analyze-quality', async (route) => { + await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) }); + }); + page.on('dialog', (dialog) => dialog.accept()); + + await page.goto(`/wizard/?mode=edit&operationId=${operationId}`); + await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates'); + + const editorValue = await page.locator('#tool-exec-config').inputValue(); + expect(editorValue).toContain('execution_policy'); + expect(editorValue).toContain('adaptive'); + expect(editorValue).toContain('45000'); + expect(editorValue).toContain('unsupported'); + + // A real edit still saves the pinned policy unchanged. + await page.evaluate(() => window.CrankWizardShell.doGoToStep(4)); + await page.locator('#tool-display-name').fill('Policy rates edited'); + await page.locator('.btn-save-draft').click(); + await expect.poll(() => savedPayloads.length).toBe(1); + expect(savedPayloads[0].execution_config.execution_policy).toEqual({ + mode: 'adaptive', + deadline_ms: 45000, + no_tasks_fallback: 'unsupported', + }); + + // An explicit invalid policy is refused instead of being silently rewritten to defaults. + await page.evaluate(() => window.CrankWizardShell.doGoToStep(5)); + await expect(page.locator('#tool-exec-config')).toBeVisible(); + await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n'); + await page.locator('.btn-save-draft').click(); + await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/); + expect(savedPayloads.length).toBe(1); +}); diff --git a/crates/crank-community-mcp/src/app.rs b/crates/crank-community-mcp/src/app.rs index 0874620..a76511e 100644 --- a/crates/crank-community-mcp/src/app.rs +++ b/crates/crank-community-mcp/src/app.rs @@ -311,24 +311,41 @@ async fn health() -> Json { } async fn readiness(State(state): State>) -> Response { - match state.registry.ping().await { - Ok(()) => Json(json!({ + let postgres_ready = state.registry.ping().await.is_ok(); + // Story 4.1: admission state is a mandatory dependency of every execution, so readiness must + // reflect it rather than reporting ready on a bare `select 1`. + let admission_ready = state.registry.execution_admission_health().await.is_ok(); + if postgres_ready && admission_ready { + return Json(json!({ "service": "mcp-server", "status": "ready", - "checks": { "postgres": "ready" } + "checks": { "postgres": "ready", "execution_admission": "ready" } })) - .into_response(), - Err(_) => ( - StatusCode::SERVICE_UNAVAILABLE, - Json(json!({ - "service": "mcp-server", - "status": "not_ready", - "checks": { "postgres": "not_ready" }, - "error": "database is unavailable" - })), - ) - .into_response(), + .into_response(); } + let mut checks = serde_json::Map::new(); + checks.insert( + "postgres".to_owned(), + json!(if postgres_ready { "ready" } else { "not_ready" }), + ); + checks.insert( + "execution_admission".to_owned(), + json!(if admission_ready { + "ready" + } else { + "not_ready" + }), + ); + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "service": "mcp-server", + "status": "not_ready", + "checks": checks, + "error": "a mandatory dependency is unavailable" + })), + ) + .into_response() } #[cfg(test)] diff --git a/crates/crank-community-mcp/src/dispatch_observer_support.rs b/crates/crank-community-mcp/src/dispatch_observer_support.rs index f105b01..f7f01b9 100644 --- a/crates/crank-community-mcp/src/dispatch_observer_support.rs +++ b/crates/crank-community-mcp/src/dispatch_observer_support.rs @@ -550,6 +550,7 @@ pub(super) fn test_operation(base_url: &str) -> Operation { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-community-mcp/src/protocol/signals_tests.rs b/crates/crank-community-mcp/src/protocol/signals_tests.rs index 0674af6..4cbba67 100644 --- a/crates/crank-community-mcp/src/protocol/signals_tests.rs +++ b/crates/crank-community-mcp/src/protocol/signals_tests.rs @@ -136,6 +136,7 @@ fn published_tool(name: &str, title: &str, description: &str, version: u32) -> P idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-community-mcp/tests/unit/manifest.rs b/crates/crank-community-mcp/tests/unit/manifest.rs index eceb4eb..b876f47 100644 --- a/crates/crank-community-mcp/tests/unit/manifest.rs +++ b/crates/crank-community-mcp/tests/unit/manifest.rs @@ -284,6 +284,7 @@ fn operation() -> RegistryOperation { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-core/src/admission.rs b/crates/crank-core/src/admission.rs new file mode 100644 index 0000000..d5e0156 --- /dev/null +++ b/crates/crank-core/src/admission.rs @@ -0,0 +1,531 @@ +//! Typed execution-admission seam (Story 4.1, FR-21/FR-24). +//! +//! Admission is the single gate in front of every real Tool execution: it decides whether the +//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide +//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives +//! in PostgreSQL (`crank-registry`), so several processes share one budget. +//! +//! Deliberately absent from this contract: credentials, connection or protocol revision (they must +//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2). + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::{NoTasksFallback, ToolExecutionMode}; + +pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128; +pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64; +pub const MAX_RETRY_AFTER_MS: u64 = 60_000; +/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for +/// the store and every runtime so the client contract stays stable. +pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250; + +/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms). +pub fn current_unix_ms() -> i64 { + i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis()) + .unwrap_or_default(), + ) + .unwrap_or(i64::MAX) +} + +/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin +/// bucket that is still bounded by the global limit. +pub const ADMIN_ORIGIN_KEY: &str = "admin"; + +/// Exact execution identity pinned at admission time. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AdmissionScope { + pub workspace_id: String, + /// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution. + pub agent_key: String, + pub operation_id: String, + /// Quota keys exclude the version; the version is pinned for execution semantics only. + pub operation_version: u32, +} + +impl AdmissionScope { + /// Quota key of the Agent bucket: `(workspace, agent)`. + pub fn agent_bucket(&self) -> String { + format!("{}:{}", self.workspace_id, self.agent_key) + } + + /// Quota key of the Tool bucket: `(workspace, operation)` without version. + pub fn tool_bucket(&self) -> String { + format!("{}:{}", self.workspace_id, self.operation_id) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AdmissionMode { + /// The caller waits for a scoped permit and dispatches inside it. + Synchronous, + /// The store commits a durable enqueue decision and reserves queue capacity. + Background, +} + +impl AdmissionMode { + pub const fn as_str(self) -> &'static str { + match self { + Self::Synchronous => "synchronous", + Self::Background => "background", + } + } +} + +/// Non-consuming approval evidence gathered by the runtime before admission. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ApprovalEvidence { + /// The operation does not require confirmation. + NotRequired, + /// A verified approval exists for this exact scope and input. + Granted, + /// Confirmation is required and no verified approval exists yet. + NotGranted, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmissionRequest { + pub scope: AdmissionScope, + pub mode: AdmissionMode, + /// Published policy mode that produced this request. + pub policy_mode: ToolExecutionMode, + pub no_tasks_fallback: NoTasksFallback, + pub approval: ApprovalEvidence, + /// Bounded request identity used only to correlate one admission entry. + pub fingerprint: String, + /// Absolute deadline (unix milliseconds), already including queue waiting time. + pub expires_at_unix_ms: i64, +} + +impl AdmissionRequest { + pub fn validate(&self) -> Result<(), AdmissionStoreError> { + if self.fingerprint.is_empty() + || self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES + || self + .fingerprint + .bytes() + .any(|byte| byte.is_ascii_control() || byte == b' ') + { + return Err(AdmissionStoreError::invalid( + "admission fingerprint is outside its allowed bounds", + )); + } + if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 { + return Err(AdmissionStoreError::invalid( + "admission workspace is invalid", + )); + } + if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 { + return Err(AdmissionStoreError::invalid("admission agent is invalid")); + } + if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 { + return Err(AdmissionStoreError::invalid( + "admission operation is invalid", + )); + } + if self.scope.operation_version == 0 { + return Err(AdmissionStoreError::invalid( + "admission operation version must be positive", + )); + } + if self.expires_at_unix_ms <= 0 { + return Err(AdmissionStoreError::invalid( + "admission deadline is invalid", + )); + } + Ok(()) + } +} + +/// Scoped permit. Held for the whole local execution future (including retries) and released +/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmissionPermit { + pub ticket_id: u64, + pub owner_token: String, + pub scope: AdmissionScope, + pub mode: AdmissionMode, + pub policy_mode: ToolExecutionMode, +} + +/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics. +/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmissionDecision { + pub ticket_id: u64, + pub scope: AdmissionScope, + pub expires_at_unix_ms: i64, + pub policy_mode: ToolExecutionMode, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AdmissionOutcome { + Permitted(AdmissionPermit), + Enqueued(AdmissionDecision), + /// Capacity or queue bounds were exhausted before any unbounded allocation. + Overloaded { + retry_after_ms: u64, + }, + /// The queue wait reached the absolute deadline; nothing was dispatched. + DeadlineElapsed, + /// Confirmation is mandatory for this operation and no verified approval exists. + ApprovalRequired, + /// The published policy does not advertise durable Tasks and refuses the fallback. + Unsupported, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AdmissionQueueSnapshot { + pub waiting: u64, + pub executing: u64, + /// Age of the oldest waiting request in the authoritative queue, in milliseconds. + pub oldest_waiting_ms: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum AdmissionStoreError { + /// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed. + #[error("execution admission store is unavailable: {message}")] + Unavailable { message: String }, + /// The store rejected a malformed admission request. + #[error("execution admission request is invalid: {message}")] + Invalid { message: String }, +} + +impl AdmissionStoreError { + pub fn unavailable(message: impl Into) -> Self { + Self::Unavailable { + message: message.into(), + } + } + + pub fn invalid(message: impl Into) -> Self { + Self::Invalid { + message: message.into(), + } + } +} + +#[async_trait] +pub trait ExecutionAdmission: Send + Sync { + /// Proves the mandatory admission state is usable (schema and operations reachable). + async fn health(&self) -> Result<(), AdmissionStoreError>; + + /// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue + /// decision for a background request, or returns a typed refusal. + async fn admit( + &self, + request: AdmissionRequest, + ) -> Result; + + /// Releases the caller's own permit. Idempotent: a repeated release returns `false`. + async fn release(&self, permit: &AdmissionPermit) -> Result; + + /// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket. + async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result; + + /// Bounded authoritative queue readings for low-cardinality metrics. + async fn snapshot(&self) -> Result; +} + +/// One waiting entry as seen by the fairness scheduler. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmissionQueueEntry { + pub ticket_id: u64, + /// Authoritative insertion order inside `(Agent, Tool)`; never rewritten. + pub sequence: u64, + pub agent_key: String, + pub operation_id: String, +} + +/// Weighted round-robin state persisted with the authoritative queue. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AdmissionCursor { + pub agent: Option, + pub agent_credit: u32, + pub tool: Option, + pub tool_credit: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AdmissionSchedulerConfig { + pub agent_weight: u32, + pub tool_weight: u32, +} + +impl Default for AdmissionSchedulerConfig { + fn default() -> Self { + Self { + agent_weight: 1, + tool_weight: 1, + } + } +} + +impl AdmissionSchedulerConfig { + pub fn validate(&self) -> Result<(), AdmissionStoreError> { + if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) { + return Err(AdmissionStoreError::invalid( + "admission weights must be between 1 and 64", + )); + } + Ok(()) + } +} + +/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the +/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its +/// turn instead of being skipped by newer arrivals. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AdmissionSelection { + pub index: usize, + pub cursor: AdmissionCursor, +} + +/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside +/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit. +pub fn select_next_grant( + queue: &[AdmissionQueueEntry], + cursor: &AdmissionCursor, + config: AdmissionSchedulerConfig, +) -> Option { + if queue.is_empty() { + return None; + } + let agent_weight = config.agent_weight.max(1); + let tool_weight = config.tool_weight.max(1); + let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect(); + agents.sort_unstable(); + agents.dedup(); + + let mut next = cursor.clone(); + let agent = match cursor.agent.as_deref() { + Some(current) if cursor.agent_credit > 0 && agents.contains(¤t) => current.to_owned(), + _ => { + let chosen = match cursor.agent.as_deref() { + None => agents[0].to_owned(), + Some(current) => agents + .iter() + .copied() + .find(|candidate| *candidate > current) + .unwrap_or(agents[0]) + .to_owned(), + }; + if cursor.agent.as_deref() != Some(chosen.as_str()) { + next.tool = None; + next.tool_credit = 0; + } + next.agent = Some(chosen.clone()); + next.agent_credit = agent_weight; + chosen + } + }; + + let mut tools: Vec<&str> = queue + .iter() + .filter(|entry| entry.agent_key == agent) + .map(|entry| entry.operation_id.as_str()) + .collect(); + tools.sort_unstable(); + tools.dedup(); + + let tool = match cursor.tool.as_deref() { + Some(current) if cursor.tool_credit > 0 && tools.contains(¤t) => current.to_owned(), + _ => { + let chosen = match cursor.tool.as_deref() { + None => tools[0].to_owned(), + Some(current) => tools + .iter() + .copied() + .find(|candidate| *candidate > current) + .unwrap_or(tools[0]) + .to_owned(), + }; + next.tool = Some(chosen.clone()); + next.tool_credit = tool_weight; + chosen + } + }; + + let index = queue + .iter() + .position(|entry| entry.agent_key == agent && entry.operation_id == tool)?; + next.agent_credit = next.agent_credit.saturating_sub(1); + next.tool_credit = next.tool_credit.saturating_sub(1); + Some(AdmissionSelection { + index, + cursor: next, + }) +} + +#[cfg(test)] +mod tests { + use super::{ + ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest, + AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence, + select_next_grant, + }; + use crate::{NoTasksFallback, ToolExecutionMode}; + + fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry { + super::AdmissionQueueEntry { + ticket_id, + sequence: ticket_id, + agent_key: agent.to_owned(), + operation_id: tool.to_owned(), + } + } + + fn request() -> AdmissionRequest { + AdmissionRequest { + scope: AdmissionScope { + workspace_id: "ws_default".to_owned(), + agent_key: ADMIN_ORIGIN_KEY.to_owned(), + operation_id: "op_leads".to_owned(), + operation_version: 3, + }, + mode: AdmissionMode::Synchronous, + policy_mode: ToolExecutionMode::Synchronous, + no_tasks_fallback: NoTasksFallback::Synchronous, + approval: ApprovalEvidence::NotRequired, + fingerprint: "fp_0123456789".to_owned(), + expires_at_unix_ms: 1_790_000_000_000, + } + } + + #[test] + fn quota_keys_exclude_version_and_keep_agent_bucket() { + let scope = request().scope; + assert_eq!(scope.agent_bucket(), "ws_default:admin"); + assert_eq!(scope.tool_bucket(), "ws_default:op_leads"); + } + + #[test] + fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() { + let mut invalid = request(); + invalid.expires_at_unix_ms = 0; + assert!(matches!( + invalid.validate(), + Err(AdmissionStoreError::Invalid { .. }) + )); + + let mut invalid = request(); + invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1); + assert!(invalid.validate().is_err()); + + let mut invalid = request(); + invalid.scope.operation_version = 0; + assert!(invalid.validate().is_err()); + + assert!(request().validate().is_ok()); + } + + #[test] + fn weights_are_bounded_positive_configuration() { + assert!( + AdmissionSchedulerConfig { + agent_weight: 0, + tool_weight: 1 + } + .validate() + .is_err() + ); + assert!( + AdmissionSchedulerConfig { + agent_weight: 1, + tool_weight: 65 + } + .validate() + .is_err() + ); + assert!(AdmissionSchedulerConfig::default().validate().is_ok()); + } + + #[test] + fn weighted_round_robin_serves_each_agent_its_quantum() { + let mut queue = vec![ + entry(1, "agent_a", "op_1"), + entry(2, "agent_a", "op_1"), + entry(3, "agent_a", "op_1"), + entry(4, "agent_b", "op_1"), + entry(5, "agent_b", "op_1"), + ]; + let config = AdmissionSchedulerConfig { + agent_weight: 2, + tool_weight: 1, + }; + let mut cursor = AdmissionCursor::default(); + let mut order = Vec::new(); + while !queue.is_empty() { + let selection = select_next_grant(&queue, &cursor, config).unwrap(); + order.push(queue.remove(selection.index).agent_key.clone()); + cursor = selection.cursor; + } + assert_eq!( + order, + vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"] + ); + } + + #[test] + fn fifo_inside_one_tool_and_round_robin_between_tools() { + let mut queue = vec![ + entry(1, "agent_a", "op_1"), + entry(2, "agent_a", "op_1"), + entry(3, "agent_a", "op_2"), + ]; + let config = AdmissionSchedulerConfig::default(); + let mut cursor = AdmissionCursor::default(); + let mut order = Vec::new(); + while !queue.is_empty() { + let selection = select_next_grant(&queue, &cursor, config).unwrap(); + // A granted ticket leaves the waiting queue: granted entries are never re-granted. + order.push(queue.remove(selection.index).ticket_id); + cursor = selection.cursor; + } + // One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue. + assert_eq!(order, vec![1, 3, 2]); + } + + #[test] + fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() { + let queue = vec![entry(1, "agent_a", "op_1")]; + let config = AdmissionSchedulerConfig { + agent_weight: 2, + tool_weight: 1, + }; + let cursor = AdmissionCursor::default(); + let first = select_next_grant(&queue, &cursor, config).unwrap(); + assert_eq!(first.cursor.agent.as_deref(), Some("agent_a")); + assert_eq!(first.cursor.agent_credit, 1); + + // A later Agent joins; the cursor keeps its remaining quantum for `agent_a`. + let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")]; + let second = select_next_grant(&queue, &first.cursor, config).unwrap(); + assert_eq!(queue[second.index].agent_key, "agent_a"); + assert_eq!(second.cursor.agent_credit, 0); + + let third = select_next_grant(&queue, &second.cursor, config).unwrap(); + assert_eq!(queue[third.index].agent_key, "agent_b"); + assert_eq!(third.cursor.agent_credit, 1); + } + + #[test] + fn empty_queue_keeps_cursor_untouched() { + let cursor = AdmissionCursor { + agent: Some("agent_b".to_owned()), + agent_credit: 1, + tool: Some("op_1".to_owned()), + tool_credit: 0, + }; + assert_eq!( + select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()), + None + ); + assert_eq!(cursor.agent.as_deref(), Some("agent_b")); + } +} diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index 4aa255f..5f8a2b1 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -1,4 +1,5 @@ pub mod access; +pub mod admission; pub mod agent; pub mod approval; pub mod auth; @@ -60,10 +61,11 @@ pub mod domain { pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId}; pub use crate::operation::{ ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus, - IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode, + IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel, OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy, - RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState, + RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode, + ToolExecutionPolicy, WizardState, }; pub use crate::product_event::{ OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION, @@ -114,6 +116,13 @@ pub use access::{ InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey, PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus, }; +pub use admission::{ + ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS, + AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit, + AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig, + AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, + MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant, +}; pub use agent::{ Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup, ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError, @@ -168,13 +177,14 @@ pub use observability::{ pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId}; pub use operation::{ ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus, - IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode, - OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel, - OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, - OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState, - ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor, - RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, - WizardState, + IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation, + OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, + OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle, + OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy, + OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract, + RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget, + RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode, + ToolExecutionPolicy, ToolExecutionPolicyError, WizardState, }; pub use product_event::{ OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION, diff --git a/crates/crank-core/src/operation.rs b/crates/crank-core/src/operation.rs index 33c4680..693203f 100644 --- a/crates/crank-core/src/operation.rs +++ b/crates/crank-core/src/operation.rs @@ -270,6 +270,72 @@ pub struct OperationApprovalPolicy { pub elicitation_message: Option, } +/// Published execution mode of one Tool (Story 4.1, FR-21). +/// +/// This is a scheduling policy of the Published Tool version. It is deliberately separate from +/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolExecutionMode { + #[default] + Synchronous, + Background, + Adaptive, +} + +/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NoTasksFallback { + /// Execute through the same synchronous admission path. + #[default] + Synchronous, + /// Reject with a typed unsupported-execution-mode failure; never a Task ID. + Unsupported, +} + +pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000; + +/// Execution policy pinned by a Published Tool version. +/// +/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by +/// `ExecutionConfig::timeout_ms`, with no background path. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ToolExecutionPolicy { + #[serde(default)] + pub mode: ToolExecutionMode, + /// Absolute budget of one execution, counted from the start of admission. + pub deadline_ms: u64, + #[serde(default)] + pub no_tasks_fallback: NoTasksFallback, +} + +impl Default for ToolExecutionPolicy { + fn default() -> Self { + Self { + mode: ToolExecutionMode::Synchronous, + deadline_ms: 30_000, + no_tasks_fallback: NoTasksFallback::Synchronous, + } + } +} + +impl ToolExecutionPolicy { + pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> { + if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) { + return Err(ToolExecutionPolicyError::DeadlineOutOfRange); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ToolExecutionPolicyError { + #[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")] + DeadlineOutOfRange, +} + #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ExecutionConfig { @@ -285,6 +351,8 @@ pub struct ExecutionConfig { #[serde(skip_serializing_if = "Option::is_none")] pub approval_policy: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub execution_policy: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auth_profile_ref: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub headers: BTreeMap, @@ -444,9 +512,10 @@ mod tests { edition::OperationSecurityLevel, ids::{AuthProfileId, OperationId}, operation::{ - ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle, - OperationLifecycleAction, OperationLifecycleError, OperationStatus, + ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability, + OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus, OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample, + ToolExecutionMode, ToolExecutionPolicy, }, protocol::{AuthKind, ExportMode, HttpMethod, Protocol}, }; @@ -664,6 +733,7 @@ updated_at: 2026-03-25T08:10:00Z idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: Some(AuthProfileId::new("auth_01")), headers: BTreeMap::new(), }, @@ -684,4 +754,76 @@ updated_at: 2026-03-25T08:10:00Z published_at: None, } } + + #[test] + fn execution_policy_round_trips_all_modes_and_fallbacks() { + for (mode, fallback) in [ + (ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous), + (ToolExecutionMode::Background, NoTasksFallback::Synchronous), + (ToolExecutionMode::Background, NoTasksFallback::Unsupported), + (ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported), + ] { + let config = ExecutionConfig { + execution_policy: Some(ToolExecutionPolicy { + mode, + deadline_ms: 45_000, + no_tasks_fallback: fallback, + }), + ..ExecutionConfig::default() + }; + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000)); + let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded, config); + assert_eq!( + decoded.execution_policy.unwrap().mode, + mode, + "the published mode must survive the JSON round-trip" + ); + } + } + + #[test] + fn legacy_execution_config_without_policy_stays_synchronous() { + let legacy: ExecutionConfig = + serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap(); + assert_eq!(legacy.execution_policy, None); + assert_eq!(legacy.timeout_ms, 12_000); + } + + #[test] + fn execution_policy_rejects_explicit_invalid_deadline() { + assert!( + ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 0, + no_tasks_fallback: NoTasksFallback::Synchronous, + } + .validate() + .is_err() + ); + assert!( + ToolExecutionPolicy { + mode: ToolExecutionMode::Adaptive, + deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1, + no_tasks_fallback: NoTasksFallback::Unsupported, + } + .validate() + .is_err() + ); + assert!(ToolExecutionPolicy::default().validate().is_ok()); + } + + #[test] + fn execution_policy_rejects_unknown_fields() { + assert!( + serde_json::from_value::(json!({ + "mode": "background", + "deadline_ms": 1000, + "no_tasks_fallback": "synchronous", + "task_queue": "default" + })) + .is_err() + ); + } } diff --git a/crates/crank-import/src/rest/legacy_payload.rs b/crates/crank-import/src/rest/legacy_payload.rs index 7c168da..041afa0 100644 --- a/crates/crank-import/src/rest/legacy_payload.rs +++ b/crates/crank-import/src/rest/legacy_payload.rs @@ -231,6 +231,7 @@ fn default_execution_config() -> ExecutionConfig { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), } diff --git a/crates/crank-import/src/rest/payload.rs b/crates/crank-import/src/rest/payload.rs index 0c8da65..9e5632a 100644 --- a/crates/crank-import/src/rest/payload.rs +++ b/crates/crank-import/src/rest/payload.rs @@ -564,6 +564,7 @@ fn default_execution_config() -> ExecutionConfig { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), } diff --git a/crates/crank-metrics/src/labels.rs b/crates/crank-metrics/src/labels.rs index 07d9381..4ed1de4 100644 --- a/crates/crank-metrics/src/labels.rs +++ b/crates/crank-metrics/src/labels.rs @@ -217,6 +217,34 @@ string_enum! { Concurrency => "concurrency", RateLimit => "rate_limit", McpStream => "mcp_stream", + Admission => "admission", + } +} + +string_enum! { + pub enum RejectionReason { + Limit => "limit", + Capacity => "capacity", + QueueFull => "queue_full", + Deadline => "deadline", + Unsupported => "unsupported", + DependencyUnavailable => "dependency_unavailable", + } +} + +string_enum! { + pub enum ScopeKind { + Process => "process", + Global => "global", + Agent => "agent", + Tool => "tool", + } +} + +string_enum! { + pub enum AdmissionModeLabel { + Synchronous => "synchronous", + Background => "background", } } diff --git a/crates/crank-metrics/src/lib.rs b/crates/crank-metrics/src/lib.rs index 70c4744..5e0aa7e 100644 --- a/crates/crank-metrics/src/lib.rs +++ b/crates/crank-metrics/src/lib.rs @@ -16,14 +16,15 @@ mod schema; pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot}; pub use labels::{ - CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute, - HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, - McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, - UpstreamOutcome, + AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, + HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, + McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, + ToolOutcome, UpstreamOperationKind, UpstreamOutcome, }; pub use record::{ InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service, - bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome, + bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting, + record_admission_queue_wait, record_admission_rejection, record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request, record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar, diff --git a/crates/crank-metrics/src/record.rs b/crates/crank-metrics/src/record.rs index 97f4ff0..310b4ea 100644 --- a/crates/crank-metrics/src/record.rs +++ b/crates/crank-metrics/src/record.rs @@ -5,10 +5,10 @@ use metrics::Gauge; use crate::schema::{MetricDefinition, metric_schema}; use crate::{ - CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod, - HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, - McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind, - ToolOutcome, UpstreamOperationKind, UpstreamOutcome, + AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, + HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, + McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason, + ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome, }; static BINDING: OnceLock = OnceLock::new(); @@ -352,11 +352,47 @@ pub fn record_limit_rejection(stage: LimitStage) { } metrics::counter!( "crank_runtime_limit_rejections_total", - "stage" => stage.as_str() + "stage" => stage.as_str(), + "reason" => RejectionReason::Limit.as_str(), + "scope_kind" => ScopeKind::Process.as_str() ) .increment(1); } +/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool, +/// request id, fingerprint or arbitrary error text ever becomes a label. +pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) { + if bound_family("crank_runtime_limit_rejections_total").is_none() { + return; + } + metrics::counter!( + "crank_runtime_limit_rejections_total", + "stage" => LimitStage::Admission.as_str(), + "reason" => reason.as_str(), + "scope_kind" => scope_kind.as_str() + ) + .increment(1); +} + +pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) { + if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() { + return; + } + metrics::gauge!( + "crank_runtime_admission_queue_wait_seconds", + "mode" => mode.as_str() + ) + .set(waited_ms as f64 / 1000.0); +} + +pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option) { + if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() { + return; + } + let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0; + metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds); +} + pub fn record_cache_outcome(outcome: CacheOutcome) { if bound_family("crank_runtime_cache_total").is_none() { return; diff --git a/crates/crank-metrics/src/schema/descriptor.rs b/crates/crank-metrics/src/schema/descriptor.rs index 4bfef96..5c01be5 100644 --- a/crates/crank-metrics/src/schema/descriptor.rs +++ b/crates/crank-metrics/src/schema/descriptor.rs @@ -3,16 +3,17 @@ use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP}; use crate::labels::{ - CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass, - IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode, - SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome, + AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, + HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, + McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome, + UpstreamOperationKind, UpstreamOutcome, }; pub const METRIC_SCHEMA_VERSION: u32 = 2; pub const MAX_METRIC_FAMILIES: usize = 128; pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4; -pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285; -pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515; +pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381; +pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611; pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348; pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024; @@ -179,6 +180,9 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES; const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES; const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES; const LIMIT_STAGES: &[&str] = LimitStage::VALUES; +const REJECTION_REASONS: &[&str] = RejectionReason::VALUES; +const SCOPE_KINDS: &[&str] = ScopeKind::VALUES; +const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES; const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES; const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES; const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES; @@ -403,13 +407,39 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[ "crank_runtime_limit_rejections_total", MetricKind::Counter, MetricUnit::Count, - &["stage"], - &[domain("stage", LIMIT_STAGES)], + &["stage", "reason", "scope_kind"], + &[ + domain("stage", LIMIT_STAGES), + domain("reason", REJECTION_REASONS), + domain("scope_kind", SCOPE_KINDS), + ], &[], BOTH, - 3, + 96, "Runtime executions rejected by a bounded limit.", ), + metric( + "crank_runtime_admission_queue_wait_seconds", + MetricKind::Gauge, + MetricUnit::Seconds, + &["mode"], + &[domain("mode", ADMISSION_MODES)], + &[], + BOTH, + 2, + "Wait of the most recently admitted execution before it received a scoped permit.", + ), + metric( + "crank_runtime_admission_oldest_waiting_seconds", + MetricKind::Gauge, + MetricUnit::Seconds, + &[], + &[], + &[], + BOTH, + 1, + "Age of the oldest waiting admission request.", + ), metric( "crank_runtime_cache_total", MetricKind::Counter, diff --git a/crates/crank-metrics/tests/contract.rs b/crates/crank-metrics/tests/contract.rs index 430fe41..c3fc7be 100644 --- a/crates/crank-metrics/tests/contract.rs +++ b/crates/crank-metrics/tests/contract.rs @@ -1,12 +1,13 @@ use std::{collections::BTreeSet, time::Duration}; use crank_metrics::{ - CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute, - HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod, - McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics, - ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics, - bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome, - record_confirmation_outcome, record_export_failure, record_http_request, + AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, + HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, + McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType, + ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome, + UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema, + record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection, + record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request, record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection, record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog, set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh, @@ -71,6 +72,9 @@ fn schema_names_are_unique_and_labels_are_closed() { "state", "signal_type", "exporter", + "reason", + "scope_kind", + "mode", ]; for metric in schema { assert!( @@ -265,6 +269,9 @@ fn every_recording_api_matches_the_declared_schema() { Duration::from_millis(1), ); record_limit_rejection(LimitStage::Concurrency); + record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global); + record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5); + record_admission_oldest_waiting(Some(5)); record_cache_outcome(CacheOutcome::Hit); record_idempotency_outcome(IdempotencyOutcome::Replay); record_confirmation_outcome(ConfirmationOutcome::Required); diff --git a/crates/crank-metrics/tests/schema_v1.rs b/crates/crank-metrics/tests/schema_v1.rs index f1690f2..ff71668 100644 --- a/crates/crank-metrics/tests/schema_v1.rs +++ b/crates/crank-metrics/tests/schema_v1.rs @@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() { #[test] fn frozen_budget_edges_are_exact() { - assert!(validate_budget(6_284, 19_514).is_ok()); - assert!(validate_budget(6_285, 19_515).is_ok()); - assert!(validate_budget(6_286, 16_000).is_err()); - assert!(validate_budget(6_000, 19_516).is_err()); + assert!(validate_budget(6_380, 19_610).is_ok()); + assert!(validate_budget(6_381, 19_611).is_ok()); + assert!(validate_budget(6_382, 19_000).is_err()); + assert!(validate_budget(6_000, 19_612).is_err()); assert_eq!( crank_metrics::max_exemplar_slots(MetricService::AdminApi), 12_348 @@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() { assert_eq!( schema_budget(MetricService::AdminApi), Ok(crank_metrics::SchemaBudget { - logical_series: 6_285, - rendered_series: 19_515 + logical_series: 6_381, + rendered_series: 19_611 }) ); assert_eq!( schema_budget(MetricService::McpServer), Ok(crank_metrics::SchemaBudget { - logical_series: 1_083, - rendered_series: 3_558 + logical_series: 1_179, + rendered_series: 3_654 }) ); } diff --git a/crates/crank-metrics/tests/schema_v2.rs b/crates/crank-metrics/tests/schema_v2.rs index 33eb3ca..e4aea13 100644 --- a/crates/crank-metrics/tests/schema_v2.rs +++ b/crates/crank-metrics/tests/schema_v2.rs @@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() { "root обязан объявлять ровно один ключ budgets" ); assert!(rendered.ends_with( - ",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\ - \"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\ - \"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n" + ",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\ + \"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\ + \"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n" )); assert_eq!( schema_budget(MetricService::AdminApi), Ok(crank_metrics::SchemaBudget { - logical_series: 6_285, - rendered_series: 19_515 + logical_series: 6_381, + rendered_series: 19_611 }) ); assert_eq!( schema_budget(MetricService::McpServer), Ok(crank_metrics::SchemaBudget { - logical_series: 1_083, - rendered_series: 3_558 + logical_series: 1_179, + rendered_series: 3_654 }) ); } @@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() { /// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны. #[test] fn frozen_caps_and_edges_are_exact() { - assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285); - assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515); + assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381); + assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611); assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348); - assert!(validate_budget(6_285, 19_515).is_ok()); - assert!(validate_budget(6_286, 19_515).is_err()); + assert!(validate_budget(6_381, 19_611).is_ok()); + assert!(validate_budget(6_382, 19_611).is_err()); assert!(validate_exemplar_budget(12_347).is_ok()); assert!(validate_exemplar_budget(12_348).is_ok()); assert!(validate_exemplar_budget(12_349).is_err()); diff --git a/crates/crank-observability/src/lib.rs b/crates/crank-observability/src/lib.rs index 66ccb72..9f8747b 100644 --- a/crates/crank-observability/src/lib.rs +++ b/crates/crank-observability/src/lib.rs @@ -13,7 +13,7 @@ mod schema; pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity}; pub use crank_metrics::{ DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit, - bind_metric_service, bound_metric_service, metric_schema, + bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting, }; pub use error_reporting::{ CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error, diff --git a/crates/crank-registry/Cargo.toml b/crates/crank-registry/Cargo.toml index 7ced393..a3a3cdd 100644 --- a/crates/crank-registry/Cargo.toml +++ b/crates/crank-registry/Cargo.toml @@ -7,6 +7,7 @@ publish.workspace = true version.workspace = true [dependencies] +async-trait = "0.1" crank-artifacts = { path = "../crank-artifacts" } crank-core = { path = "../crank-core" } crank-mapping = { path = "../crank-mapping" } diff --git a/crates/crank-registry/src/lib.rs b/crates/crank-registry/src/lib.rs index 74affab..7647058 100644 --- a/crates/crank-registry/src/lib.rs +++ b/crates/crank-registry/src/lib.rs @@ -116,6 +116,7 @@ pub use model::{ YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus, }; pub use postgres::{ - AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError, - PostgresRegistry, + AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission, + PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, admission_health, + admission_snapshot, }; diff --git a/crates/crank-registry/src/migrations.rs b/crates/crank-registry/src/migrations.rs index aa9df66..c25d66d 100644 --- a/crates/crank-registry/src/migrations.rs +++ b/crates/crank-registry/src/migrations.rs @@ -6,6 +6,7 @@ mod artifact_cleanup_indexes_v13; mod artifact_metadata_v12; mod authority; mod baseline_v1; +mod execution_admission_v24; mod execution_outcome_v5; mod import_plan_v14; mod master_key_identity_v7; @@ -35,6 +36,7 @@ mod schema_guard_v19; mod schema_guard_v20; mod schema_guard_v22; mod schema_guard_v23; +mod schema_guard_v24; mod schema_guard_v7; mod schema_guard_v8; mod schema_guard_v9; diff --git a/crates/crank-registry/src/migrations/authority.rs b/crates/crank-registry/src/migrations/authority.rs index 828f779..4bde7fa 100644 --- a/crates/crank-registry/src/migrations/authority.rs +++ b/crates/crank-registry/src/migrations/authority.rs @@ -4,6 +4,7 @@ use super::agent_resource_bindings_v18; use super::approval_side_effects_v10; use super::artifact_cleanup_indexes_v13; use super::artifact_metadata_v12; +use super::execution_admission_v24; use super::execution_outcome_v5; use super::import_plan_v14; use super::master_key_identity_v7; @@ -39,9 +40,9 @@ use preflight::*; use contract::baseline_source_digest; use contract::validate_descriptors; const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947; -const CURRENT_VERSION: i64 = 23; +const CURRENT_VERSION: i64 = 24; const IMPLEMENTED_VERSIONS: &[i64] = &[ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, ]; const BASELINE_SOURCE_SHA256: &str = "eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675"; @@ -330,6 +331,12 @@ impl MigrationAuthority { prompt_dependencies_v23::SOURCE_SHA256, 22, ), + expand_descriptor( + 24, + execution_admission_v24::NAME, + execution_admission_v24::SOURCE_SHA256, + 23, + ), ] } pub fn validate_sequence() -> Result<(), MigrationError> { @@ -489,6 +496,9 @@ impl MigrationAuthority { if from < 23 { prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?; } + if from < 24 { + execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?; + } match inspect(&mut transaction).await? { MigrationPreflight::Current { version: CURRENT_VERSION, diff --git a/crates/crank-registry/src/migrations/authority/contract.rs b/crates/crank-registry/src/migrations/authority/contract.rs index c9afe59..9457d27 100644 --- a/crates/crank-registry/src/migrations/authority/contract.rs +++ b/crates/crank-registry/src/migrations/authority/contract.rs @@ -140,6 +140,8 @@ pub(super) fn validate_descriptors( != agent_resource_bindings_v18::SOURCE_SHA256 || sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes()) != resource_security_revocations_v19::SOURCE_SHA256 + || sha256_hex(execution_admission_v24::SOURCE.as_bytes()) + != execution_admission_v24::SOURCE_SHA256 { return Err(MigrationError::new( "invalid_contract", diff --git a/crates/crank-registry/src/migrations/authority/preflight.rs b/crates/crank-registry/src/migrations/authority/preflight.rs index c23f6c8..f5e74b9 100644 --- a/crates/crank-registry/src/migrations/authority/preflight.rs +++ b/crates/crank-registry/src/migrations/authority/preflight.rs @@ -148,6 +148,7 @@ pub(super) async fn inspect( owned_relations::PROMPTS, owned_relations::PROMPT_PUBLICATIONS, owned_relations::PROMPT_DEPENDENCIES, + owned_relations::EXECUTION_ADMISSION, ] { validate_required_relations(connection, relations, CURRENT_VERSION).await?; } diff --git a/crates/crank-registry/src/migrations/execution_admission_v24.rs b/crates/crank-registry/src/migrations/execution_admission_v24.rs new file mode 100644 index 0000000..fd20778 --- /dev/null +++ b/crates/crank-registry/src/migrations/execution_admission_v24.rs @@ -0,0 +1,45 @@ +use sqlx::{Postgres, Transaction, query}; + +use super::authority::{MigrationDescriptor, MigrationError}; + +pub(super) const NAME: &str = "execution-admission-v24"; +pub(super) const SOURCE: &str = include_str!("execution_admission_v24.sql"); +pub(super) const SOURCE_SHA256: &str = + "efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681"; + +pub(super) async fn apply( + transaction: &mut Transaction<'_, Postgres>, + descriptor: &MigrationDescriptor, +) -> Result<(), MigrationError> { + sqlx::raw_sql(SOURCE) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.execution_admission_v24", + Some(24), + "restore_known_good_backup", + ) + })?; + query( + "insert into __crank_migrations (version, name, checksum, phase, compatibility) + values ($1, $2, $3, $4, $5)", + ) + .bind(descriptor.version) + .bind(descriptor.name) + .bind(&descriptor.checksum) + .bind(descriptor.phase) + .bind(descriptor.compatibility) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.canonical_ledger", + Some(24), + "restore_known_good_backup", + ) + })?; + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/execution_admission_v24.sql b/crates/crank-registry/src/migrations/execution_admission_v24.sql new file mode 100644 index 0000000..e485b8e --- /dev/null +++ b/crates/crank-registry/src/migrations/execution_admission_v24.sql @@ -0,0 +1,92 @@ +-- V24: authoritative execution admission state (Story 4.1, FR-21, FR-24). +-- +-- Один deployment-wide admission: PostgreSQL хранит cursor/fairness state, ожидающие тикеты, +-- выданные permits и durable enqueue decisions, поэтому несколько процессов делят общий бюджет. +-- +-- `execution_admission_entries` — bounded набор: waiting (ожидает capacity), executing (scoped +-- permit выдан, держится до явного release) и reserved (durable enqueue decision с зарезервированным +-- queue slot; не исполняется и не рекламируется как Task до Story 4.2). +-- +-- Строка не хранит ни credentials, ни connection/session id, ни prepared headers, ни тело запроса: +-- quota key — (workspace, agent) и (workspace, operation) без version. `operation_version` пишется +-- только как pin исполняемых semantics и не участвует в квотах. +-- +-- `expires_at` — абсолютный deadline. Queue decisions без dispatch можно expire по deadline; активный +-- permit НЕ освобождается по TTL (иначе paused/crashed owner обойдёт global limit): его снимает только +-- явный release/cancel владельца. + +create table execution_admission_controllers ( + workspace_id text not null, + cursor_agent text, + cursor_agent_credit integer not null default 0, + cursor_tool text, + cursor_tool_credit integer not null default 0, + updated_at timestamptz not null default now(), + constraint execution_admission_controllers_pkey primary key (workspace_id), + constraint execution_admission_controllers_workspace_check check ( + octet_length(workspace_id) between 1 and 160 + ), + constraint execution_admission_controllers_agent_credit_check check ( + cursor_agent_credit between 0 and 64 + ), + constraint execution_admission_controllers_tool_credit_check check ( + cursor_tool_credit between 0 and 64 + ) +); + +create table execution_admission_entries ( + ticket_id bigint generated always as identity, + workspace_id text not null, + agent_key text not null, + operation_id text not null, + operation_version integer not null, + mode text not null, + policy_mode text not null, + state text not null, + owner_token text not null, + fingerprint text not null, + created_at timestamptz not null default now(), + expires_at timestamptz not null, + granted_at timestamptz, + constraint execution_admission_entries_pkey primary key (ticket_id), + constraint execution_admission_entries_workspace_check check ( + octet_length(workspace_id) between 1 and 160 + ), + constraint execution_admission_entries_agent_check check ( + octet_length(agent_key) between 1 and 160 + ), + constraint execution_admission_entries_operation_check check ( + octet_length(operation_id) between 1 and 160 + ), + constraint execution_admission_entries_version_check check (operation_version >= 1), + constraint execution_admission_entries_mode_check check ( + mode in ('synchronous', 'background') + ), + constraint execution_admission_entries_policy_mode_check check ( + policy_mode in ('synchronous', 'background', 'adaptive') + ), + constraint execution_admission_entries_state_check check ( + state in ('waiting', 'executing', 'reserved') + ), + constraint execution_admission_entries_owner_check check ( + octet_length(owner_token) between 1 and 64 + ), + constraint execution_admission_entries_fingerprint_check check ( + octet_length(fingerprint) between 1 and 128 + ), + constraint execution_admission_entries_reserved_mode_check check ( + state <> 'reserved' or mode = 'background' + ) +); + +create unique index execution_admission_entries_owner_uidx + on execution_admission_entries (owner_token); + +create index execution_admission_entries_state_idx + on execution_admission_entries (state, workspace_id, ticket_id); + +create index execution_admission_entries_agent_idx + on execution_admission_entries (state, workspace_id, agent_key); + +create index execution_admission_entries_operation_idx + on execution_admission_entries (state, workspace_id, operation_id); diff --git a/crates/crank-registry/src/migrations/owned_relations.rs b/crates/crank-registry/src/migrations/owned_relations.rs index 93dea83..c10c06b 100644 --- a/crates/crank-registry/src/migrations/owned_relations.rs +++ b/crates/crank-registry/src/migrations/owned_relations.rs @@ -55,6 +55,11 @@ pub(super) const PROMPT_PUBLICATIONS: &[&str] = &["prompt_publications", "agent_ pub(super) const PROMPT_DEPENDENCIES: &[&str] = &["prompt_dependencies", "prompt_message_resource_refs"]; +pub(super) const EXECUTION_ADMISSION: &[&str] = &[ + "execution_admission_controllers", + "execution_admission_entries", +]; + pub(super) const ARTIFACT_METADATA: &[&str] = &["artifact_blobs", "artifact_sources"]; pub(super) const IMPORT_PLAN: &[&str] = &["import_plans", "import_plan_applications"]; diff --git a/crates/crank-registry/src/migrations/schema_guard.rs b/crates/crank-registry/src/migrations/schema_guard.rs index c37dcfd..d668351 100644 --- a/crates/crank-registry/src/migrations/schema_guard.rs +++ b/crates/crank-registry/src/migrations/schema_guard.rs @@ -468,6 +468,13 @@ pub(super) async fn validate_schema_fingerprint( } else { super::schema_guard_v23::validate_v23_prompt_dependencies(connection).await?; } + if current_version < 24 { + if !super::schema_guard_v24::validate_v24_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else { + super::schema_guard_v24::validate_v24_execution_admission(connection).await?; + } Ok(()) } diff --git a/crates/crank-registry/src/migrations/schema_guard/required_tables.rs b/crates/crank-registry/src/migrations/schema_guard/required_tables.rs index 9653397..d20471f 100644 --- a/crates/crank-registry/src/migrations/schema_guard/required_tables.rs +++ b/crates/crank-registry/src/migrations/schema_guard/required_tables.rs @@ -51,6 +51,8 @@ pub(crate) const OWNED_RELATIONS: &[&str] = &[ "prompt_messages", "prompt_dependencies", "prompt_message_resource_refs", + "execution_admission_controllers", + "execution_admission_entries", ]; pub(crate) const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[ diff --git a/crates/crank-registry/src/migrations/schema_guard_v24.rs b/crates/crank-registry/src/migrations/schema_guard_v24.rs new file mode 100644 index 0000000..aad6bf1 --- /dev/null +++ b/crates/crank-registry/src/migrations/schema_guard_v24.rs @@ -0,0 +1,254 @@ +//! V24 schema guard: authoritative execution admission state (Story 4.1, FR-21/FR-24). +//! +//! Схема V24 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в +//! ожидаемой форме: controller-row с weighted round-robin cursor и bounded entries с закрытым +//! словарём mode/policy_mode/state, уникальным owner token и индексами квот. + +use sqlx::{PgConnection, Row, query}; + +use super::authority::MigrationError; +use super::schema_guard::relation_exists; + +const CONTROLLERS: &str = "execution_admission_controllers"; +const ENTRIES: &str = "execution_admission_entries"; + +const OWNED_RELATIONS: &[&str] = &[CONTROLLERS, ENTRIES]; + +const CONTROLLER_COLUMNS: &[(&str, &str, bool)] = &[ + ("workspace_id", "text", false), + ("cursor_agent", "text", true), + ("cursor_agent_credit", "integer", false), + ("cursor_tool", "text", true), + ("cursor_tool_credit", "integer", false), + ("updated_at", "timestamp with time zone", false), +]; + +const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[ + ("ticket_id", "bigint", false), + ("workspace_id", "text", false), + ("agent_key", "text", false), + ("operation_id", "text", false), + ("operation_version", "integer", false), + ("mode", "text", false), + ("policy_mode", "text", false), + ("state", "text", false), + ("owner_token", "text", false), + ("fingerprint", "text", false), + ("created_at", "timestamp with time zone", false), + ("expires_at", "timestamp with time zone", false), + ("granted_at", "timestamp with time zone", true), +]; + +const CONSTRAINT_COUNT: &[(&str, usize)] = &[(CONTROLLERS, 4), (ENTRIES, 11)]; + +const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"]; + +fn v24_error(stage: &'static str) -> MigrationError { + MigrationError::new( + "partial_sequence", + stage, + Some(24), + "restore_known_good_backup", + ) +} + +/// Схема V24 отсутствует целиком (для current_version < 24). +pub(super) async fn validate_v24_absent( + connection: &mut PgConnection, +) -> Result { + for relation in OWNED_RELATIONS { + if relation_exists(connection, relation).await? { + return Ok(false); + } + } + for index in UNIQUE_INDEXES { + if index_exists(connection, index).await? { + return Ok(false); + } + } + Ok(true) +} + +/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24). +pub(super) async fn validate_v24_execution_admission( + connection: &mut PgConnection, +) -> Result<(), MigrationError> { + for relation in OWNED_RELATIONS { + if !relation_exists(connection, relation).await? { + return Err(v24_error("v24.relations")); + } + } + for (table, columns) in [(CONTROLLERS, CONTROLLER_COLUMNS), (ENTRIES, ENTRY_COLUMNS)] { + validate_columns(connection, table, columns).await?; + } + for (table, expected) in CONSTRAINT_COUNT { + let count: i64 = query( + "select count(*)::bigint as count + from pg_catalog.pg_constraint c + join pg_catalog.pg_class t on t.oid = c.conrelid + join pg_catalog.pg_namespace n on n.oid = t.relnamespace + where n.nspname = current_schema() and t.relname = $1", + ) + .bind(table) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get("count") + .map_err(|_| MigrationError::storage("preflight.schema"))?; + if count != i64::try_from(*expected).unwrap_or_default() { + return Err(v24_error("v24.constraints")); + } + } + for (table, constraint) in [ + (CONTROLLERS, "execution_admission_controllers_pkey"), + (ENTRIES, "execution_admission_entries_pkey"), + (ENTRIES, "execution_admission_entries_state_check"), + ] { + if !constraint_exists(connection, table, constraint).await? { + return Err(v24_error("v24.constraint_missing")); + } + } + if !normalized_expression( + connection, + ENTRIES, + "execution_admission_entries_state_check", + ) + .await? + .is_some_and(|expression| { + super::schema_guard::enum_constraint_matches( + &expression, + "state", + &["waiting", "executing", "reserved"], + ) + }) { + return Err(v24_error("v24.state_vocabulary")); + } + for index in UNIQUE_INDEXES { + let unique = index_unique(connection, index).await?; + if unique != Some(true) { + return Err(v24_error("v24.indexes")); + } + } + Ok(()) +} + +async fn validate_columns( + connection: &mut PgConnection, + table: &str, + columns: &[(&str, &str, bool)], +) -> Result<(), MigrationError> { + let count: i64 = query( + "select count(*)::bigint as count from information_schema.columns + where table_schema = current_schema() and table_name = $1", + ) + .bind(table) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get("count") + .map_err(|_| MigrationError::storage("preflight.schema"))?; + if count != i64::try_from(columns.len()).unwrap_or_default() { + return Err(v24_error("v24.columns")); + } + for (column, data_type, nullable) in columns { + let row = query( + "select data_type, is_nullable from information_schema.columns + where table_schema = current_schema() and table_name = $1 and column_name = $2", + ) + .bind(table) + .bind(column) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + let valid = row.is_some_and(|row| { + row.try_get::("data_type").ok().as_deref() == Some(*data_type) + && row.try_get::("is_nullable").ok().as_deref() + == Some(if *nullable { "YES" } else { "NO" }) + }); + if !valid { + return Err(v24_error("v24.columns")); + } + } + Ok(()) +} + +async fn constraint_exists( + connection: &mut PgConnection, + table: &str, + constraint: &str, +) -> Result { + query( + "select exists ( + select 1 from pg_catalog.pg_constraint c + join pg_catalog.pg_class t on t.oid = c.conrelid + join pg_catalog.pg_namespace n on n.oid = t.relnamespace + where n.nspname = current_schema() and t.relname = $1 and c.conname = $2 + ) as present", + ) + .bind(table) + .bind(constraint) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get::("present") + .map_err(|_| MigrationError::storage("preflight.schema")) +} + +async fn normalized_expression( + connection: &mut PgConnection, + table: &str, + constraint: &str, +) -> Result, MigrationError> { + let row = query( + "select pg_get_expr(c.conbin, c.conrelid) as expression + from pg_catalog.pg_constraint c + join pg_catalog.pg_class t on t.oid = c.conrelid + join pg_catalog.pg_namespace n on n.oid = t.relnamespace + where n.nspname = current_schema() and t.relname = $1 and c.conname = $2 + and c.contype = 'c'", + ) + .bind(table) + .bind(constraint) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + Ok(row.and_then(|row| row.try_get::("expression").ok())) +} + +async fn index_exists(connection: &mut PgConnection, index: &str) -> Result { + query( + "select exists ( + select 1 from pg_catalog.pg_class idx + join pg_catalog.pg_namespace n on n.oid = idx.relnamespace + where n.nspname = current_schema() and idx.relname = $1 + ) as present", + ) + .bind(index) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get::("present") + .map_err(|_| MigrationError::storage("preflight.schema")) +} + +async fn index_unique( + connection: &mut PgConnection, + index: &str, +) -> Result, MigrationError> { + let row = query( + "select i.indisunique, i.indisvalid + from pg_catalog.pg_index i + join pg_catalog.pg_class idx on idx.oid = i.indexrelid + join pg_catalog.pg_namespace n on n.oid = idx.relnamespace + where n.nspname = current_schema() and idx.relname = $1", + ) + .bind(index) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + Ok(row.and_then(|row| { + let unique = row.try_get::("indisunique").ok()?; + let valid = row.try_get::("indisvalid").ok()?; + Some(unique && valid) + })) +} diff --git a/crates/crank-registry/src/postgres/admission.rs b/crates/crank-registry/src/postgres/admission.rs new file mode 100644 index 0000000..e9a5f59 --- /dev/null +++ b/crates/crank-registry/src/postgres/admission.rs @@ -0,0 +1,618 @@ +//! PostgreSQL implementation of the mandatory execution-admission store (Story 4.1, FR-21/FR-24). +//! +//! Source of truth for capacity and durable enqueue decisions of every process: a short transaction +//! locks one deployment controller row and one workspace controller row in a fixed order, then reads +//! bounded counters, inserts/transitions at most one entry and persists the weighted round-robin +//! cursor. No network call happens under the transaction and no connection is held while a waiter is +//! idle between attempts. + +use std::collections::BTreeMap; +use std::time::Duration; + +use async_trait::async_trait; +use crank_core::{ + AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit, + AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig, + AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, select_next_grant, +}; +use sqlx::{PgPool, Postgres, Row, Transaction}; +use uuid::Uuid; + +const DEPLOYMENT_CONTROLLER: &str = "*"; +const OVERLOAD_RETRY_AFTER_MS: u64 = crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS; +const MAX_WAIT_BACKOFF_MS: u64 = 250; +const INITIAL_WAIT_BACKOFF_MS: u64 = 5; +const EXPIRED_CLEANUP_BATCH: i64 = 128; +const QUEUE_READ_LIMIT: i64 = 1024; + +/// Deployment-wide bounds of one admission store. They come from validated deployment +/// configuration, never from a request. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AdmissionLimits { + pub global_limit: usize, + pub agent_limit: usize, + pub tool_limit: usize, + pub queue_limit: usize, +} + +impl Default for AdmissionLimits { + fn default() -> Self { + Self { + global_limit: 64, + agent_limit: 16, + tool_limit: 8, + queue_limit: 32, + } + } +} + +impl AdmissionLimits { + pub fn validate(&self) -> Result<(), AdmissionStoreError> { + for (field, value) in [ + ("global_limit", self.global_limit), + ("agent_limit", self.agent_limit), + ("tool_limit", self.tool_limit), + ] { + if !(1..=65_535).contains(&value) { + return Err(AdmissionStoreError::invalid(format!( + "admission {field} is outside its allowed bounds" + ))); + } + } + if !(1..=1_024).contains(&self.queue_limit) { + return Err(AdmissionStoreError::invalid( + "admission queue_limit is outside its allowed bounds", + )); + } + Ok(()) + } +} + +#[derive(Clone)] +pub struct PostgresExecutionAdmission { + pool: PgPool, + limits: AdmissionLimits, + scheduler: AdmissionSchedulerConfig, +} + +impl PostgresExecutionAdmission { + pub fn new( + pool: PgPool, + limits: AdmissionLimits, + scheduler: AdmissionSchedulerConfig, + ) -> Result { + limits.validate()?; + scheduler.validate()?; + Ok(Self { + pool, + limits, + scheduler, + }) + } + + fn unavailable(error: impl std::fmt::Display) -> AdmissionStoreError { + AdmissionStoreError::unavailable(format!( + "execution admission store is unavailable: {error}" + )) + } +} + +struct AttemptContext<'a> { + request: &'a AdmissionRequest, + owner_token: &'a str, +} + +enum Attempt { + Granted, + Enqueued(u64), + Waiting, + Overloaded, + Expired, +} + +#[async_trait] +impl ExecutionAdmission for PostgresExecutionAdmission { + async fn health(&self) -> Result<(), AdmissionStoreError> { + admission_health(&self.pool).await + } + + async fn admit( + &self, + request: AdmissionRequest, + ) -> Result { + request.validate()?; + if request.approval == ApprovalEvidence::NotGranted { + return Ok(AdmissionOutcome::ApprovalRequired); + } + if request.mode == AdmissionMode::Background + && request.no_tasks_fallback == NoTasksFallback::Unsupported + { + return Ok(AdmissionOutcome::Unsupported); + } + + let owner_token = format!("adm_{}", Uuid::now_v7().simple()); + let mut ticket: Option = None; + let mut backoff = INITIAL_WAIT_BACKOFF_MS; + loop { + if crank_core::current_unix_ms() >= request.expires_at_unix_ms { + if let Some(ticket_id) = ticket { + self.cancel(ticket_id, &owner_token).await?; + } + return Ok(AdmissionOutcome::DeadlineElapsed); + } + let attempt = self + .attempt( + AttemptContext { + request: &request, + owner_token: &owner_token, + }, + &mut ticket, + ) + .await?; + match attempt { + Attempt::Granted => { + return Ok(AdmissionOutcome::Permitted(AdmissionPermit { + ticket_id: ticket.unwrap_or_default(), + owner_token, + scope: request.scope.clone(), + mode: request.mode, + policy_mode: request.policy_mode, + })); + } + Attempt::Enqueued(ticket_id) => { + return Ok(AdmissionOutcome::Enqueued(AdmissionDecision { + ticket_id, + scope: request.scope.clone(), + expires_at_unix_ms: request.expires_at_unix_ms, + policy_mode: request.policy_mode, + })); + } + Attempt::Expired => { + return Ok(AdmissionOutcome::DeadlineElapsed); + } + Attempt::Overloaded => { + return Ok(AdmissionOutcome::Overloaded { + retry_after_ms: OVERLOAD_RETRY_AFTER_MS, + }); + } + Attempt::Waiting => { + let remaining = request + .expires_at_unix_ms + .saturating_sub(crank_core::current_unix_ms()) + .max(1); + let sleep_ms = i64::try_from(backoff).unwrap_or(remaining).min(remaining); + tokio::time::sleep(Duration::from_millis( + u64::try_from(sleep_ms).unwrap_or(backoff), + )) + .await; + backoff = (backoff * 2).min(MAX_WAIT_BACKOFF_MS); + } + } + } + } + + async fn release(&self, permit: &AdmissionPermit) -> Result { + let result = sqlx::query( + "delete from execution_admission_entries + where ticket_id = $1 and owner_token = $2 and state = 'executing'", + ) + .bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX)) + .bind(&permit.owner_token) + .execute(&self.pool) + .await + .map_err(Self::unavailable)?; + Ok(result.rows_affected() > 0) + } + + async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result { + let result = sqlx::query( + "delete from execution_admission_entries + where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')", + ) + .bind(i64::try_from(ticket_id).unwrap_or(i64::MAX)) + .bind(owner_token) + .execute(&self.pool) + .await + .map_err(Self::unavailable)?; + Ok(result.rows_affected() > 0) + } + + async fn snapshot(&self) -> Result { + admission_snapshot(&self.pool).await + } +} + +/// Readiness probe of the mandatory admission state: both tables must be reachable and readable. +pub async fn admission_health(pool: &PgPool) -> Result<(), AdmissionStoreError> { + sqlx::query( + "select count(*)::bigint from execution_admission_controllers where workspace_id = $1", + ) + .bind(DEPLOYMENT_CONTROLLER) + .fetch_one(pool) + .await + .map_err(PostgresExecutionAdmission::unavailable)?; + sqlx::query("select count(*)::bigint from execution_admission_entries where false") + .fetch_one(pool) + .await + .map_err(PostgresExecutionAdmission::unavailable)?; + Ok(()) +} + +/// Authoritative queue readings of the deployment-wide admission state. +pub async fn admission_snapshot( + pool: &PgPool, +) -> Result { + let row = sqlx::query( + "select + count(*) filter (where state in ('waiting', 'reserved')) as queued, + count(*) filter (where state = 'executing') as executing, + coalesce( + (extract(epoch from (now() - min(created_at) filter (where state in ('waiting', 'reserved')))) * 1000)::bigint, + -1 + ) as oldest_waiting_ms + from execution_admission_entries", + ) + .fetch_one(pool) + .await + .map_err(PostgresExecutionAdmission::unavailable)?; + let queued: i64 = row + .try_get("queued") + .map_err(PostgresExecutionAdmission::unavailable)?; + let executing: i64 = row + .try_get("executing") + .map_err(PostgresExecutionAdmission::unavailable)?; + let oldest: i64 = row + .try_get("oldest_waiting_ms") + .map_err(PostgresExecutionAdmission::unavailable)?; + Ok(AdmissionQueueSnapshot { + waiting: u64::try_from(queued).unwrap_or_default(), + executing: u64::try_from(executing).unwrap_or_default(), + oldest_waiting_ms: (oldest >= 0).then(|| u64::try_from(oldest).unwrap_or_default()), + }) +} + +impl PostgresExecutionAdmission { + async fn attempt( + &self, + context: AttemptContext<'_>, + ticket: &mut Option, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?; + sqlx::query("set local lock_timeout = '3000ms'") + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + sqlx::query("set local statement_timeout = '5000ms'") + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + + match self.attempt_locked(&mut transaction, context, ticket).await { + Ok(attempt) => { + transaction.commit().await.map_err(Self::unavailable)?; + Ok(attempt) + } + Err(AttemptError::Contended) => { + transaction.rollback().await.ok(); + Ok(Attempt::Waiting) + } + Err(AttemptError::Overloaded) => { + transaction.rollback().await.ok(); + Ok(Attempt::Overloaded) + } + Err(AttemptError::Failure(error)) => { + transaction.rollback().await.ok(); + Err(error) + } + } + } + + async fn attempt_locked( + &self, + transaction: &mut Transaction<'_, Postgres>, + context: AttemptContext<'_>, + ticket: &mut Option, + ) -> Result { + let workspace_id = context.request.scope.workspace_id.as_str(); + sqlx::query( + "insert into execution_admission_controllers (workspace_id) + values ($1), ($2) + on conflict (workspace_id) do nothing", + ) + .bind(DEPLOYMENT_CONTROLLER) + .bind(workspace_id) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + + // Fixed lock order: deployment controller first, then the workspace controller. + lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?; + let cursor = lock_controller(transaction, workspace_id).await?; + + // Expired queue decisions and waiting tickets leave the bounded queue; held permits never do. + sqlx::query( + "delete from execution_admission_entries + where ticket_id in ( + select ticket_id from execution_admission_entries + where state in ('waiting', 'reserved') and expires_at <= now() + order by ticket_id + limit $1 + )", + ) + .bind(EXPIRED_CLEANUP_BATCH) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + + if ticket.is_none() { + let queued: i64 = sqlx::query( + "select count(*)::bigint as count from execution_admission_entries + where state in ('waiting', 'reserved')", + ) + .fetch_one(&mut **transaction) + .await + .map_err(AttemptError::from_sql)? + .try_get("count") + .map_err(AttemptError::from_sql)?; + if u64::try_from(queued).unwrap_or(u64::MAX) + >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) + { + return Err(AttemptError::Overloaded); + } + let state = match context.request.mode { + AdmissionMode::Synchronous => "waiting", + AdmissionMode::Background => "reserved", + }; + let row = sqlx::query( + "insert into execution_admission_entries ( + workspace_id, agent_key, operation_id, operation_version, mode, policy_mode, + state, owner_token, fingerprint, expires_at + ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, + to_timestamp($10::double precision / 1000.0)) + returning ticket_id", + ) + .bind(workspace_id) + .bind(&context.request.scope.agent_key) + .bind(&context.request.scope.operation_id) + .bind(i32::try_from(context.request.scope.operation_version).unwrap_or(i32::MAX)) + .bind(context.request.mode.as_str()) + .bind(policy_mode_str(context.request)) + .bind(state) + .bind(context.owner_token) + .bind(&context.request.fingerprint) + .bind(context.request.expires_at_unix_ms) + .fetch_one(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?; + *ticket = Some(u64::try_from(ticket_id).unwrap_or_default()); + if context.request.mode == AdmissionMode::Background { + return Ok(Attempt::Enqueued( + u64::try_from(ticket_id).unwrap_or_default(), + )); + } + } + + self.grant_next(transaction, workspace_id, &cursor).await?; + + let Some(ticket_id) = *ticket else { + return Ok(Attempt::Waiting); + }; + let state: Option = + sqlx::query("select state from execution_admission_entries where ticket_id = $1") + .bind(i64::try_from(ticket_id).unwrap_or(i64::MAX)) + .fetch_optional(&mut **transaction) + .await + .map_err(AttemptError::from_sql)? + .map(|row| row.try_get("state")) + .transpose() + .map_err(AttemptError::from_sql)?; + match state.as_deref() { + Some("executing") => Ok(Attempt::Granted), + Some(_) => Ok(Attempt::Waiting), + // The ticket expired inside the store (or was removed by another cleanup). + None => Ok(Attempt::Expired), + } + } + + /// Grants at most one waiting ticket per transaction, following the persisted weighted + /// round-robin cursor. A blocked head keeps its turn: nothing later is granted in its place. + async fn grant_next( + &self, + transaction: &mut Transaction<'_, Postgres>, + workspace_id: &str, + cursor: &AdmissionCursor, + ) -> Result<(), AttemptError> { + let rows = sqlx::query( + "select ticket_id, agent_key, operation_id from execution_admission_entries + where workspace_id = $1 and state = 'waiting' + order by ticket_id + limit $2", + ) + .bind(workspace_id) + .bind(QUEUE_READ_LIMIT) + .fetch_all(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + if rows.is_empty() { + return Ok(()); + } + let queue = rows + .iter() + .map(|row| { + Ok(AdmissionQueueEntry { + ticket_id: u64::try_from( + row.try_get::("ticket_id") + .map_err(AttemptError::from_sql)?, + ) + .unwrap_or_default(), + sequence: 0, + agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?, + operation_id: row + .try_get("operation_id") + .map_err(AttemptError::from_sql)?, + }) + }) + .collect::, AttemptError>>()?; + let Some(selection) = select_next_grant(&queue, cursor, self.scheduler) else { + return Ok(()); + }; + let head = &queue[selection.index]; + + let global_executing: i64 = sqlx::query( + "select count(*)::bigint as count from execution_admission_entries + where state = 'executing'", + ) + .fetch_one(&mut **transaction) + .await + .map_err(AttemptError::from_sql)? + .try_get("count") + .map_err(AttemptError::from_sql)?; + let agent_counts = scoped_counts(transaction, workspace_id, "agent_key").await?; + let tool_counts = scoped_counts(transaction, workspace_id, "operation_id").await?; + if u64::try_from(global_executing).unwrap_or(u64::MAX) + >= u64::try_from(self.limits.global_limit).unwrap_or(u64::MAX) + { + return Ok(()); + } + let agent_executing = agent_counts + .get(head.agent_key.as_str()) + .copied() + .unwrap_or_default(); + if agent_executing >= u64::try_from(self.limits.agent_limit).unwrap_or(u64::MAX) { + return Ok(()); + } + let tool_executing = tool_counts + .get(head.operation_id.as_str()) + .copied() + .unwrap_or_default(); + if tool_executing >= u64::try_from(self.limits.tool_limit).unwrap_or(u64::MAX) { + return Ok(()); + } + + sqlx::query( + "update execution_admission_entries + set state = 'executing', granted_at = now() + where ticket_id = $1 and state = 'waiting'", + ) + .bind(i64::try_from(head.ticket_id).unwrap_or(i64::MAX)) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + sqlx::query( + "update execution_admission_controllers + set cursor_agent = $2, + cursor_agent_credit = $3, + cursor_tool = $4, + cursor_tool_credit = $5, + updated_at = now() + where workspace_id = $1", + ) + .bind(workspace_id) + .bind(selection.cursor.agent.as_deref()) + .bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default()) + .bind(selection.cursor.tool.as_deref()) + .bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default()) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + Ok(()) + } +} + +enum AttemptError { + /// Lock or statement timeout: the transaction made no decision and the waiter retries. + Contended, + /// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait. + Overloaded, + Failure(AdmissionStoreError), +} + +impl AttemptError { + fn from_sql(error: sqlx::Error) -> Self { + if let Some(database) = error.as_database_error() + && matches!(database.code().as_deref(), Some("55P03" | "57014")) + { + return Self::Contended; + } + Self::Failure(PostgresExecutionAdmission::unavailable(error)) + } +} + +async fn lock_controller( + transaction: &mut Transaction<'_, Postgres>, + workspace_id: &str, +) -> Result { + sqlx::query( + "update execution_admission_controllers set updated_at = updated_at + where workspace_id = $1", + ) + .bind(workspace_id) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + let row = sqlx::query( + "select cursor_agent, cursor_agent_credit, cursor_tool, cursor_tool_credit + from execution_admission_controllers where workspace_id = $1", + ) + .bind(workspace_id) + .fetch_one(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + Ok(AdmissionCursor { + agent: row + .try_get("cursor_agent") + .map_err(AttemptError::from_sql)?, + agent_credit: u32::try_from( + row.try_get::("cursor_agent_credit") + .map_err(AttemptError::from_sql)? + .max(0), + ) + .unwrap_or_default(), + tool: row.try_get("cursor_tool").map_err(AttemptError::from_sql)?, + tool_credit: u32::try_from( + row.try_get::("cursor_tool_credit") + .map_err(AttemptError::from_sql)? + .max(0), + ) + .unwrap_or_default(), + }) +} + +async fn scoped_counts( + transaction: &mut Transaction<'_, Postgres>, + workspace_id: &str, + column: &str, +) -> Result, AttemptError> { + let statement = if column == "agent_key" { + "select agent_key as key, count(*)::bigint as count + from execution_admission_entries + where workspace_id = $1 and state = 'executing' + group by agent_key" + } else { + "select operation_id as key, count(*)::bigint as count + from execution_admission_entries + where workspace_id = $1 and state = 'executing' + group by operation_id" + }; + let rows = sqlx::query(statement) + .bind(workspace_id) + .fetch_all(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + let mut counts = BTreeMap::new(); + for row in rows { + let key: String = row.try_get("key").map_err(AttemptError::from_sql)?; + let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?; + counts.insert(key, u64::try_from(count).unwrap_or_default()); + } + Ok(counts) +} + +fn policy_mode_str(request: &AdmissionRequest) -> &'static str { + match request.policy_mode { + crank_core::ToolExecutionMode::Synchronous => "synchronous", + crank_core::ToolExecutionMode::Background => "background", + crank_core::ToolExecutionMode::Adaptive => "adaptive", + } +} diff --git a/crates/crank-registry/src/postgres/connection.rs b/crates/crank-registry/src/postgres/connection.rs index ecb9048..675bc3b 100644 --- a/crates/crank-registry/src/postgres/connection.rs +++ b/crates/crank-registry/src/postgres/connection.rs @@ -46,6 +46,19 @@ impl PostgresRegistry { Ok(()) } + /// Readiness probe of the mandatory execution-admission state (Story 4.1). A missing or + /// unreadable admission schema is a dependency failure, never a degraded-but-ready state. + pub async fn execution_admission_health(&self) -> Result<(), crank_core::AdmissionStoreError> { + super::admission::admission_health(&self.pool).await + } + + /// Authoritative admission queue readings for low-cardinality metrics. + pub async fn execution_admission_snapshot( + &self, + ) -> Result { + super::admission::admission_snapshot(&self.pool).await + } + async fn connect_in_schema( database_url: &str, schema: Option<&str>, diff --git a/crates/crank-registry/src/postgres/mod.rs b/crates/crank-registry/src/postgres/mod.rs index 16f7db4..c53b767 100644 --- a/crates/crank-registry/src/postgres/mod.rs +++ b/crates/crank-registry/src/postgres/mod.rs @@ -1,3 +1,4 @@ +mod admission; mod agent; mod agent_catalog; mod agent_prompt_bindings; @@ -36,6 +37,9 @@ use serde_json::Value; use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow, types::Json}; use time::OffsetDateTime; +pub use admission::{ + AdmissionLimits, PostgresExecutionAdmission, admission_health, admission_snapshot, +}; pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError}; pub use resource::{AuthorizedResourceRead, BoundResourceAdmission}; diff --git a/crates/crank-registry/tests/integration.rs b/crates/crank-registry/tests/integration.rs index b91fad1..1de7ee3 100644 --- a/crates/crank-registry/tests/integration.rs +++ b/crates/crank-registry/tests/integration.rs @@ -1,4 +1,5 @@ mod integration { + mod admission; mod agent_resource_bindings; mod agents_usage; mod approval; diff --git a/crates/crank-registry/tests/integration/admission.rs b/crates/crank-registry/tests/integration/admission.rs new file mode 100644 index 0000000..159ecf5 --- /dev/null +++ b/crates/crank-registry/tests/integration/admission.rs @@ -0,0 +1,496 @@ +//! ATDD Story 4.1 (FR-21/FR-24): authoritative PostgreSQL execution admission. +//! +//! These oracles drive the real store against a real PostgreSQL schema: scoped permits, bounded +//! queue, typed overload, durable enqueue decisions, deadline propagation, quota keys that ignore +//! the operation version, fairness order and fail-closed health. + +use super::common; + +use std::time::{Duration, Instant}; + +use crank_core::{ + ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest, + AdmissionSchedulerConfig, AdmissionScope, ApprovalEvidence, ExecutionAdmission, + NoTasksFallback, ToolExecutionMode, current_unix_ms, +}; +use crank_registry::{AdmissionLimits, PostgresExecutionAdmission}; +use tokio::sync::mpsc; + +fn limits(global: usize, agent: usize, tool: usize, queue: usize) -> AdmissionLimits { + AdmissionLimits { + global_limit: global, + agent_limit: agent, + tool_limit: tool, + queue_limit: queue, + } +} + +fn store( + pool: sqlx::PgPool, + limits: AdmissionLimits, + scheduler: AdmissionSchedulerConfig, +) -> PostgresExecutionAdmission { + PostgresExecutionAdmission::new(pool, limits, scheduler).expect("valid admission store") +} + +fn scope(agent: &str, operation: &str, version: u32) -> AdmissionScope { + AdmissionScope { + workspace_id: "ws_default".to_owned(), + agent_key: agent.to_owned(), + operation_id: operation.to_owned(), + operation_version: version, + } +} + +fn request(scope: AdmissionScope, deadline_in_ms: i64) -> AdmissionRequest { + AdmissionRequest { + scope, + mode: AdmissionMode::Synchronous, + policy_mode: ToolExecutionMode::Synchronous, + no_tasks_fallback: NoTasksFallback::Synchronous, + approval: ApprovalEvidence::NotRequired, + fingerprint: format!("fp_{}", current_unix_ms()), + expires_at_unix_ms: current_unix_ms() + deadline_in_ms, + } +} + +fn background(scope: AdmissionScope, fallback: NoTasksFallback) -> AdmissionRequest { + AdmissionRequest { + mode: AdmissionMode::Background, + no_tasks_fallback: fallback, + ..request(scope, 30_000) + } +} + +fn permit(outcome: AdmissionOutcome) -> AdmissionPermit { + match outcome { + AdmissionOutcome::Permitted(permit) => permit, + other => panic!("expected a scoped permit, got {other:?}"), + } +} + +async fn wait_for_metric_poll() { + tokio::time::sleep(Duration::from_millis(20)).await; +} + +#[tokio::test] +async fn permits_are_scoped_released_once_and_respect_the_agent_budget() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(4, 1, 4, 8), + Default::default(), + ); + + let first = permit( + admission + .admit(request(scope("agent_a", "op_leads", 1), 2_000)) + .await + .expect("admission store is reachable"), + ); + assert_eq!(first.scope.agent_key, "agent_a"); + + // The same Agent is already at its scoped budget: the second request waits and expires instead of + // over-admitting, and it never dispatches. + let waiting = admission + .admit(request(scope("agent_a", "op_leads", 2), 120)) + .await + .expect("admission store is reachable"); + assert_eq!(waiting, AdmissionOutcome::DeadlineElapsed); + + // Another Agent still has the global capacity it is entitled to. + let other = permit( + admission + .admit(request(scope("agent_b", "op_leads", 1), 2_000)) + .await + .expect("admission store is reachable"), + ); + assert_eq!(other.scope.agent_key, "agent_b"); + assert!(admission.release(&other).await.unwrap()); + // Repeated release is idempotent and never frees a foreign or already released permit. + assert!(!admission.release(&other).await.unwrap()); + + assert!(admission.release(&first).await.unwrap()); + let snapshot = admission.snapshot().await.unwrap(); + assert_eq!(snapshot.executing, 0); + assert_eq!(snapshot.waiting, 0); + database.cleanup().await; +} + +#[tokio::test] +async fn full_queue_returns_a_typed_overload_with_retry_guidance() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(1, 1, 1, 2), + Default::default(), + ); + + let held = permit( + admission + .admit(request(scope("agent_a", "op_leads", 1), 5_000)) + .await + .expect("admission store is reachable"), + ); + + // Two bounded waiting entries fill the queue. + let mut waiters = Vec::new(); + for version in 1..=2 { + let admission = admission.clone(); + waiters.push(tokio::spawn(async move { + admission + .admit(request(scope("agent_b", "op_leads", version), 5_000)) + .await + })); + } + while admission.snapshot().await.unwrap().waiting < 2 { + wait_for_metric_poll().await; + } + + let rejected = admission + .admit(request(scope("agent_c", "op_leads", 1), 5_000)) + .await + .expect("admission store is reachable"); + match rejected { + AdmissionOutcome::Overloaded { retry_after_ms } => { + assert!((1..=crank_core::MAX_RETRY_AFTER_MS).contains(&retry_after_ms)); + } + other => panic!("expected typed overload, got {other:?}"), + } + + // Cancelling the held permit lets the queued waiters make progress instead of leaking capacity. + assert!(admission.release(&held).await.unwrap()); + for waiter in waiters.drain(..) { + let outcome = waiter.await.expect("waiter task"); + if let Ok(AdmissionOutcome::Permitted(permit)) = outcome { + let _ = admission.release(&permit).await; + } + } + database.cleanup().await; +} + +#[tokio::test] +async fn background_decisions_are_durable_pinned_and_released_once() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(4, 4, 4, 8), + Default::default(), + ); + + let decision = match admission + .admit(background( + scope("agent_a", "op_report", 3), + NoTasksFallback::Synchronous, + )) + .await + .expect("admission store is reachable") + { + AdmissionOutcome::Enqueued(decision) => decision, + other => panic!("expected a durable enqueue decision, got {other:?}"), + }; + assert_eq!(decision.scope.operation_version, 3); + assert_eq!(decision.policy_mode, ToolExecutionMode::Synchronous); + + // A second process (fresh pool and store over the same schema) must read the committed decision: + // it occupies bounded queue capacity and is not a runnable execution. + let reconnected = store( + database.concurrent_registry(2).await.pool().clone(), + limits(4, 4, 4, 8), + Default::default(), + ); + let snapshot = reconnected.snapshot().await.unwrap(); + assert_eq!(snapshot.waiting, 1); + assert_eq!(snapshot.executing, 0); + + assert!( + reconnected + .cancel(decision.ticket_id, "not_the_owner") + .await + .unwrap() + .eq(&false), + "a foreign cancel must never release the decision" + ); + assert!( + reconnected.snapshot().await.unwrap().waiting.eq(&1), + "the foreign cancel must leave the decision intact" + ); + database.cleanup().await; +} + +#[tokio::test] +async fn background_without_a_task_transport_is_typed_unsupported() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(4, 4, 4, 8), + Default::default(), + ); + + let outcome = admission + .admit(background( + scope("agent_a", "op_report", 1), + NoTasksFallback::Unsupported, + )) + .await + .expect("admission store is reachable"); + assert_eq!(outcome, AdmissionOutcome::Unsupported); + assert_eq!(admission.snapshot().await.unwrap().waiting, 0); + database.cleanup().await; +} + +#[tokio::test] +async fn unverified_approval_never_produces_a_decision() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(4, 4, 4, 8), + Default::default(), + ); + + let mut denied = background( + scope("agent_a", "op_report", 1), + NoTasksFallback::Synchronous, + ); + denied.approval = ApprovalEvidence::NotGranted; + assert_eq!( + admission.admit(denied).await.unwrap(), + AdmissionOutcome::ApprovalRequired + ); + + let mut denied_sync = request(scope("agent_a", "op_report", 1), 1_000); + denied_sync.approval = ApprovalEvidence::NotGranted; + assert_eq!( + admission.admit(denied_sync).await.unwrap(), + AdmissionOutcome::ApprovalRequired + ); + assert_eq!(admission.snapshot().await.unwrap().waiting, 0); + assert_eq!(admission.snapshot().await.unwrap().executing, 0); + database.cleanup().await; +} + +#[tokio::test] +async fn quota_keys_ignore_the_operation_version() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(4, 4, 1, 8), + Default::default(), + ); + + let held = permit( + admission + .admit(request(scope("agent_a", "op_leads", 1), 2_000)) + .await + .expect("admission store is reachable"), + ); + // A different Published Version of the same Tool shares the same per-Tool budget. + let outcome = admission + .admit(request(scope("agent_a", "op_leads", 2), 120)) + .await + .expect("admission store is reachable"); + assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed); + assert!(admission.release(&held).await.unwrap()); + database.cleanup().await; +} + +#[tokio::test] +async fn weighted_agents_are_served_their_quantum_in_order() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(1, 4, 4, 8), + AdmissionSchedulerConfig { + agent_weight: 2, + tool_weight: 1, + }, + ); + let held = permit( + admission + .admit(request(scope("agent_z", "op_leads", 1), 5_000)) + .await + .expect("admission store is reachable"), + ); + + let (sender, mut receiver) = mpsc::channel(3); + let mut tasks = Vec::new(); + for (agent, version) in [("agent_a", 1), ("agent_a", 2), ("agent_b", 1)] { + let admission = admission.clone(); + let sender = sender.clone(); + tasks.push(tokio::spawn(async move { + let outcome = admission + .admit(request(scope(agent, "op_leads", version), 5_000)) + .await; + if let Ok(AdmissionOutcome::Permitted(permit)) = outcome { + let _ = sender.send(permit.scope.agent_key.clone()).await; + // Hold briefly so the grant order, not task scheduling, decides the sequence. + tokio::time::sleep(Duration::from_millis(30)).await; + let _ = admission.release(&permit).await; + } + })); + } + drop(sender); + assert!(admission.release(&held).await.unwrap()); + + let mut order = Vec::new(); + while let Some(agent) = receiver.recv().await { + order.push(agent); + if order.len() == 3 { + break; + } + } + for task in tasks { + let _ = tokio::time::timeout(Duration::from_secs(5), task).await; + } + // agent_a carries weight 2 and therefore receives two consecutive grants before agent_b. + assert_eq!(order, vec!["agent_a", "agent_a", "agent_b"]); + database.cleanup().await; +} + +#[tokio::test] +async fn mandatory_state_failure_fails_closed_and_reports_not_ready() { + let database = common::TestDatabase::new().await; + // No migration on this schema: the mandatory admission state does not exist yet. + let unmigrated = database.raw_pool().await; + let admission = store(unmigrated, limits(4, 4, 4, 8), Default::default()); + assert!(admission.health().await.is_err()); + assert!( + admission + .admit(request(scope("agent_a", "op_leads", 1), 1_000)) + .await + .is_err() + ); + database.cleanup().await; +} + +#[tokio::test] +async fn held_permits_survive_a_new_process_view() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(1, 4, 4, 8), + Default::default(), + ); + let held = permit( + admission + .admit(request(scope("agent_a", "op_leads", 1), 2_000)) + .await + .expect("admission store is reachable"), + ); + + let reconnected = store( + database.concurrent_registry(2).await.pool().clone(), + limits(1, 4, 4, 8), + Default::default(), + ); + assert_eq!(reconnected.snapshot().await.unwrap().executing, 1); + let outcome = reconnected + .admit(request(scope("agent_b", "op_leads", 1), 120)) + .await + .expect("admission store is reachable"); + assert_eq!( + outcome, + AdmissionOutcome::DeadlineElapsed, + "a held permit is not silently reused by another process" + ); + assert!(reconnected.release(&held).await.unwrap()); + database.cleanup().await; +} + +#[tokio::test] +async fn published_execution_policy_is_pinned_across_a_new_draft_version() { + use crank_core::{NoTasksFallback, OperationStatus, ToolExecutionMode, ToolExecutionPolicy}; + use crank_registry::{CreateVersionRequest, PublishRequest}; + use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let workspace = common::test_workspace_id(); + let policy = ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 45_000, + no_tasks_fallback: NoTasksFallback::Unsupported, + }; + let mut version_one = common::test_operation("op_policy", 1, OperationStatus::Draft); + version_one.execution_config.execution_policy = Some(policy); + registry + .create_operation(&workspace, &version_one, None) + .await + .unwrap(); + registry + .publish_operation(PublishRequest { + workspace_id: &workspace, + operation_id: &version_one.id, + version: 1, + published_at: &OffsetDateTime::parse("2026-03-25T12:10:00Z", &Rfc3339).unwrap(), + published_by: Some("alice"), + }) + .await + .unwrap(); + + let published = registry + .get_published_operation(&version_one.id) + .await + .unwrap() + .expect("published operation"); + assert_eq!(published.execution_config.execution_policy, Some(policy)); + + // A later Draft with another policy never rewrites the already accepted publication. + let mut version_two = common::test_operation("op_policy", 2, OperationStatus::Draft); + version_two.execution_config.execution_policy = Some(ToolExecutionPolicy { + mode: ToolExecutionMode::Synchronous, + deadline_ms: 1_000, + no_tasks_fallback: NoTasksFallback::Synchronous, + }); + registry + .create_version(CreateVersionRequest { + workspace_id: &workspace, + snapshot: &version_two, + change_note: Some("next draft"), + created_by: Some("alice"), + }) + .await + .unwrap(); + let unchanged = registry + .get_published_operation(&version_one.id) + .await + .unwrap() + .expect("published operation"); + assert_eq!(unchanged.version, 1); + assert_eq!(unchanged.execution_config.execution_policy, Some(policy)); + database.cleanup().await; +} + +#[tokio::test] +async fn admin_origin_bucket_is_bounded_together_with_agents() { + let database = common::TestDatabase::new().await; + let registry = database.registry().await; + let admission = store( + registry.pool().clone(), + limits(1, 1, 1, 4), + Default::default(), + ); + let held = permit( + admission + .admit(request(scope(ADMIN_ORIGIN_KEY, "op_leads", 1), 2_000)) + .await + .expect("admission store is reachable"), + ); + let started = Instant::now(); + let outcome = admission + .admit(request(scope("agent_a", "op_leads", 1), 120)) + .await + .expect("admission store is reachable"); + assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed); + assert!(started.elapsed() >= Duration::from_millis(100)); + assert!(admission.release(&held).await.unwrap()); + database.cleanup().await; +} diff --git a/crates/crank-runtime/src/admission.rs b/crates/crank-runtime/src/admission.rs new file mode 100644 index 0000000..6f23cca --- /dev/null +++ b/crates/crank-runtime/src/admission.rs @@ -0,0 +1,255 @@ +//! Common execution admission seam of the runtime (Story 4.1, FR-21/FR-24). +//! +//! Every real execution — synchronous, or a published `background`/`adaptive` policy that falls back +//! to synchronous because no durable Task transport exists yet — passes this one gate before any +//! adapter work. The gate asks the authoritative store for a scoped permit bounded by the published +//! Tool policy, the verified approval state of the exact Published Version/scope/input and the caller +//! deadline, and it releases the permit explicitly when the local execution future ends. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crank_core::{ + ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest, + AdmissionScope, CoordinationStateStore, ExecutionAdmission, NoTasksFallback, ToolExecutionMode, + current_unix_ms, +}; +use crank_metrics::{ + AdmissionModeLabel, RejectionReason, ScopeKind, record_admission_queue_wait, + record_admission_rejection, +}; +use tracing::warn; + +use crate::{RuntimeError, RuntimeOperation, RuntimeRequestContext}; + +/// Effective absolute deadline of one execution: the published policy deadline never extends the +/// caller deadline and always includes queue waiting time. +pub(crate) fn effective_deadline( + operation: &RuntimeOperation, + caller_deadline: Instant, +) -> Instant { + match operation.execution_config.execution_policy { + Some(policy) => { + caller_deadline.min(Instant::now() + Duration::from_millis(policy.deadline_ms)) + } + None => caller_deadline, + } +} + +/// Effective execution path for this deployment. +/// +/// Story 4.1 has no Task transport, so a `background` or `adaptive` policy resolves through its +/// published `no_tasks_fallback`: either the same synchronous admission path, or a typed +/// unsupported refusal with no admission entry, no Task id and no outbound call. +pub(crate) fn effective_mode( + operation: &RuntimeOperation, +) -> Result { + let Some(policy) = operation.execution_config.execution_policy else { + return Ok(ToolExecutionMode::Synchronous); + }; + match policy.mode { + ToolExecutionMode::Synchronous => Ok(ToolExecutionMode::Synchronous), + mode => match policy.no_tasks_fallback { + NoTasksFallback::Synchronous => Ok(ToolExecutionMode::Synchronous), + NoTasksFallback::Unsupported => Err(RuntimeError::ExecutionPolicyUnsupported { + operation_id: operation.operation_id.as_str().to_owned(), + mode, + }), + }, + } +} + +pub(crate) struct AdmissionLease { + store: Arc, + permit: Option, + runtime: tokio::runtime::Handle, +} + +impl AdmissionLease { + fn new(store: Arc, permit: AdmissionPermit) -> Self { + Self { + store, + permit: Some(permit), + runtime: tokio::runtime::Handle::current(), + } + } + + /// Explicit release on the normal/error path of the local execution. The `Drop` cleanup stays as + /// the cancellation path; both CAS on the same owner token, so only one of them can free it. + pub(crate) async fn release(&mut self) { + let Some(permit) = self.permit.take() else { + return; + }; + if let Err(error) = self.store.release(&permit).await { + warn!( + name: "runtime.admission.release_failed", + error = %error, + "failed to release an admission permit; the reservation stays fail-closed" + ); + } + } +} + +impl Drop for AdmissionLease { + fn drop(&mut self) { + let Some(permit) = self.permit.take() else { + return; + }; + let store = Arc::clone(&self.store); + // Bounded cleanup: the explicit release path already ran on success/error; cancellation + // reaches here and releases by owner token without ever freeing a foreign permit. A failed + // release keeps the reservation fail-closed (no TTL reuse of an active permit). + self.runtime.spawn(async move { + match store.release(&permit).await { + Ok(true) => {} + Ok(false) => warn!( + name: "runtime.admission.release_skipped", + "admission permit was already released" + ), + Err(error) => warn!( + name: "runtime.admission.release_failed", + error = %error, + "failed to release an admission permit; the reservation stays fail-closed" + ), + } + }); + } +} + +/// Acquires a scoped permit from the authoritative store. Returns `None` when this deployment has no +/// admission store wired (embedded/unit-test executors keep the pre-4.1 local limiter). +pub(crate) async fn acquire( + store: Option<&Arc>, + coordination_store: Option<&dyn CoordinationStateStore>, + request: &crate::RuntimeExecutionRequest<'_>, + deadline: Instant, +) -> Result, RuntimeError> { + let Some(store) = store else { + return Ok(None); + }; + let operation = request.operation(); + let input = request.input(); + let request_context = request.request_context(); + let operation_id = operation.operation_id.as_str().to_owned(); + let policy_mode = effective_mode(operation)?; + // The one admission snapshot carries the authoritative approval state of this exact Published + // Version, scope, input and deadline. A confirmation-requiring request without a verified + // approval never takes capacity and never dispatches: the pending human decision is answered + // here with the existing challenge, and the single-use token is consumed only at dispatch. + let approval = crate::confirmation::approval_evidence( + coordination_store, + operation, + input, + Some(request_context), + ) + .await?; + let agent_key = request.agent_id().map(|agent| agent.as_str()); + let workspace_id = request.workspace_id().as_str().to_owned(); + let request = AdmissionRequest { + scope: AdmissionScope { + workspace_id, + agent_key: agent_key.unwrap_or(ADMIN_ORIGIN_KEY).to_owned(), + operation_id: operation_id.clone(), + operation_version: operation.operation_version, + }, + mode: AdmissionMode::Synchronous, + policy_mode, + no_tasks_fallback: operation + .execution_config + .execution_policy + .map(|policy| policy.no_tasks_fallback) + .unwrap_or(NoTasksFallback::Synchronous), + approval, + fingerprint: admission_fingerprint(request_context, operation), + expires_at_unix_ms: current_unix_ms().saturating_add( + deadline_remaining_ms(deadline) + .try_into() + .unwrap_or(i64::MAX), + ), + }; + let started = Instant::now(); + let outcome = store.admit(request).await.map_err(|error| { + record_admission_rejection(RejectionReason::DependencyUnavailable, ScopeKind::Global); + RuntimeError::AdmissionStoreUnavailable { + operation_id: operation_id.clone(), + details: error.to_string(), + } + })?; + let waited_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or_default(); + match outcome { + AdmissionOutcome::Permitted(permit) => { + record_admission_queue_wait(AdmissionModeLabel::Synchronous, waited_ms); + Ok(Some(AdmissionLease::new(Arc::clone(store), permit))) + } + AdmissionOutcome::Overloaded { retry_after_ms } => { + record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global); + Err(RuntimeError::AdmissionOverloaded { + operation_id, + retry_after_ms, + }) + } + AdmissionOutcome::DeadlineElapsed => { + // Queue wait exhausted the absolute deadline before a permit was granted: nothing was + // dispatched, so this is an admission refusal, not an upstream timeout. + Err(deadline_refusal(operation)) + } + AdmissionOutcome::ApprovalRequired => { + // The store refused before reserving any capacity or queue entry. Answer with the + // existing single-use challenge so the caller can confirm out of band. + record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global); + Err(crate::confirmation::challenge( + coordination_store, + operation, + input, + Some(request_context), + ) + .await) + } + AdmissionOutcome::Unsupported | AdmissionOutcome::Enqueued(_) => { + // No Task transport exists in this deployment: an enqueue decision can never become a + // runnable execution here, so it must not be reported as accepted work. + record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global); + Err(RuntimeError::ExecutionPolicyUnsupported { + operation_id, + mode: policy_mode, + }) + } + } +} + +/// Typed refusal for an admission that never granted a permit within the absolute deadline. Reuses the +/// existing `execution_overloaded` vocabulary (stage `admission`, retry guidance, certain outcome) so +/// no new client error code or API shape appears, while reporting zero dispatch truthfully. +pub(crate) fn deadline_refusal(operation: &RuntimeOperation) -> RuntimeError { + record_admission_rejection(RejectionReason::Deadline, ScopeKind::Global); + RuntimeError::AdmissionOverloaded { + operation_id: operation.operation_id.as_str().to_owned(), + retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS, + } +} + +fn deadline_remaining_ms(deadline: Instant) -> u64 { + u64::try_from( + deadline + .saturating_duration_since(Instant::now()) + .as_millis(), + ) + .unwrap_or_default() + .max(1) +} + +/// Bounded request identity for correlating one admission entry. Never contains payload, headers or +/// credentials. +fn admission_fingerprint( + request_context: &RuntimeRequestContext, + operation: &RuntimeOperation, +) -> String { + let mut fingerprint = format!( + "{}:{}:{}", + operation.operation_id.as_str(), + operation.operation_version, + request_context.request_id + ); + fingerprint.truncate(crank_core::ADMISSION_FINGERPRINT_MAX_BYTES); + fingerprint +} diff --git a/crates/crank-runtime/src/confirmation.rs b/crates/crank-runtime/src/confirmation.rs index decf4c4..18a5313 100644 --- a/crates/crank-runtime/src/confirmation.rs +++ b/crates/crank-runtime/src/confirmation.rs @@ -2,8 +2,8 @@ use std::time::Duration; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use crank_core::{ - CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod, OperationSafetyClass, - OperationSafetyPolicy, Target, + ApprovalEvidence, CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod, + OperationSafetyClass, OperationSafetyPolicy, Target, }; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; @@ -26,32 +26,14 @@ pub async fn confirm_operation( } let Some(store) = store else { - return Err(RuntimeError::ConfirmationStoreUnavailable { - operation_id: operation.operation_id.as_str().to_owned(), - }); + return Err(confirmation_store_unavailable(operation)); }; let scope = confirmation_scope(operation, request_context)?; let input_hash = hash_json(input)?; - let provided_token = request_context - .and_then(RuntimeRequestContext::confirmation_token) - .or_else(|| confirmation_token_from_input(input)); - let Some(provided_token) = provided_token else { - let token = issue_confirmation_token( - store, - operation.operation_id.as_str(), - &scope, - &input_hash, - &safety, - ) - .await?; - return Err(RuntimeError::ConfirmationRequired { - operation_id: operation.operation_id.as_str().to_owned(), - safety_class: safety.class, - confirmation_token: token, - expires_in_ms: confirmation_ttl_ms(&safety), - }); + let Some(provided_token) = provided_confirmation_token(request_context, input) else { + return Err(confirmation_required(store, operation, &scope, &input_hash, &safety).await); }; consume_confirmation_token( @@ -64,6 +46,121 @@ pub async fn confirm_operation( .await } +/// Read-only approval proof for the shared admission gate (Story 4.1, AC2). +/// +/// It reports the authoritative approval state of this exact Published Version, scope and input and +/// never consumes the single-use confirmation token: the authoritative consumption stays in +/// [`confirm_operation`] immediately before the adapter call. A missing approval is `NotGranted`, so +/// admission refuses the request before taking execution capacity while the human decision is +/// pending; an unusable token or an unreachable safety store is a typed refusal too, never a permit. +pub(crate) async fn approval_evidence( + store: Option<&dyn CoordinationStateStore>, + operation: &RuntimeOperation, + input: &Value, + request_context: Option<&RuntimeRequestContext>, +) -> Result { + let safety = effective_safety_policy(operation); + if !safety.class.requires_confirmation() { + return Ok(ApprovalEvidence::NotRequired); + } + if request_context.is_some_and(RuntimeRequestContext::approval_granted) { + return Ok(ApprovalEvidence::Granted); + } + let Some(store) = store else { + return Err(confirmation_store_unavailable(operation)); + }; + let scope = confirmation_scope(operation, request_context)?; + let input_hash = hash_json(input)?; + + let Some(token) = provided_confirmation_token(request_context, input) else { + return Ok(ApprovalEvidence::NotGranted); + }; + + let stored = store + .get_value( + CacheScope::Coordination, + &confirmation_cache_key(&scope, token), + ) + .await + .map_err(|_| confirmation_store_unavailable(operation))?; + let verified = stored.is_some_and(|value| { + value.payload.get("scope").and_then(Value::as_str) == Some(scope.as_str()) + && value.payload.get("input_hash").and_then(Value::as_str) == Some(input_hash.as_str()) + }); + if verified { + Ok(ApprovalEvidence::Granted) + } else { + Err(RuntimeError::InvalidConfirmationToken { + operation_id: operation.operation_id.as_str().to_owned(), + }) + } +} + +/// Issues the existing single-use confirmation challenge and returns the typed refusal that carries +/// it. Admission calls this when no verified approval exists yet, so waiting for the human decision +/// never holds execution capacity. +pub(crate) async fn challenge( + store: Option<&dyn CoordinationStateStore>, + operation: &RuntimeOperation, + input: &Value, + request_context: Option<&RuntimeRequestContext>, +) -> RuntimeError { + let safety = effective_safety_policy(operation); + let Some(store) = store else { + return confirmation_store_unavailable(operation); + }; + let scope = match confirmation_scope(operation, request_context) { + Ok(scope) => scope, + Err(error) => return error, + }; + let input_hash = match hash_json(input) { + Ok(input_hash) => input_hash, + Err(error) => return error, + }; + confirmation_required(store, operation, &scope, &input_hash, &safety).await +} + +fn provided_confirmation_token<'a>( + request_context: Option<&'a RuntimeRequestContext>, + input: &'a Value, +) -> Option<&'a str> { + request_context + .and_then(RuntimeRequestContext::confirmation_token) + .or_else(|| confirmation_token_from_input(input)) +} + +fn confirmation_store_unavailable(operation: &RuntimeOperation) -> RuntimeError { + RuntimeError::ConfirmationStoreUnavailable { + operation_id: operation.operation_id.as_str().to_owned(), + } +} + +async fn confirmation_required( + store: &dyn CoordinationStateStore, + operation: &RuntimeOperation, + scope: &str, + input_hash: &str, + safety: &OperationSafetyPolicy, +) -> RuntimeError { + match issue_confirmation_token( + store, + operation.operation_id.as_str(), + scope, + input_hash, + safety, + ) + .await + { + Ok(token) => RuntimeError::ConfirmationRequired { + operation_id: operation.operation_id.as_str().to_owned(), + safety_class: safety.class, + confirmation_token: token, + expires_in_ms: confirmation_ttl_ms(safety), + }, + Err(error) => error, + } +} + pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool { effective_safety_policy(operation) .class diff --git a/crates/crank-runtime/src/error.rs b/crates/crank-runtime/src/error.rs index 1337e53..9ca27b5 100644 --- a/crates/crank-runtime/src/error.rs +++ b/crates/crank-runtime/src/error.rs @@ -43,6 +43,27 @@ pub enum RuntimeError { confirmation_token: String, expires_in_ms: u64, }, + #[error( + "execution mode {mode:?} of operation {operation_id} is unavailable without a durable Task transport" + )] + ExecutionPolicyUnsupported { + operation_id: String, + mode: crank_core::ToolExecutionMode, + }, + #[error( + "execution admission is overloaded for operation {operation_id}; retry after {retry_after_ms} ms" + )] + AdmissionOverloaded { + operation_id: String, + retry_after_ms: u64, + }, + #[error("execution admission requires a verified approval for operation {operation_id}")] + AdmissionApprovalRequired { operation_id: String }, + #[error("execution admission store is unavailable for operation {operation_id}: {details}")] + AdmissionStoreUnavailable { + operation_id: String, + details: String, + }, #[error("confirmation token for operation {operation_id} is invalid, expired, or already used")] InvalidConfirmationToken { operation_id: String }, #[error("confirmation store is unavailable for operation {operation_id}")] diff --git a/crates/crank-runtime/src/execution_failure.rs b/crates/crank-runtime/src/execution_failure.rs index e747726..1caa0f9 100644 --- a/crates/crank-runtime/src/execution_failure.rs +++ b/crates/crank-runtime/src/execution_failure.rs @@ -57,6 +57,11 @@ fn normalize_runtime_error_with_effect( | RuntimeError::IdempotencyCompletionUnknown { .. } => { failure = failure.with_dispatch_uncertainty(); } + RuntimeError::AdmissionOverloaded { retry_after_ms, .. } => { + if let Ok(with_retry) = failure.clone().try_with_retry_after_ms(*retry_after_ms) { + failure = with_retry; + } + } RuntimeError::ConfirmationRequired { confirmation_token, expires_in_ms, @@ -113,6 +118,14 @@ fn error_code(error: &RuntimeError) -> ExecutionErrorCode { RuntimeError::UnsupportedExecutionMode { .. } => { ExecutionErrorCode::ExecutionModeUnsupported } + RuntimeError::ExecutionPolicyUnsupported { .. } => { + ExecutionErrorCode::ExecutionModeUnsupported + } + RuntimeError::AdmissionOverloaded { .. } => ExecutionErrorCode::ExecutionOverloaded, + RuntimeError::AdmissionApprovalRequired { .. } => ExecutionErrorCode::ConfirmationRequired, + RuntimeError::AdmissionStoreUnavailable { .. } => { + ExecutionErrorCode::SafetyStoreUnavailable + } RuntimeError::ConcurrencyLimitExceeded { .. } => ExecutionErrorCode::ExecutionOverloaded, RuntimeError::InvalidPreparedRequest { .. } => ExecutionErrorCode::PreparedRequestInvalid, RuntimeError::ConfirmationRequired { .. } => ExecutionErrorCode::ConfirmationRequired, diff --git a/crates/crank-runtime/src/execution_request.rs b/crates/crank-runtime/src/execution_request.rs index 3962376..472da9b 100644 --- a/crates/crank-runtime/src/execution_request.rs +++ b/crates/crank-runtime/src/execution_request.rs @@ -220,6 +220,7 @@ mod tests { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/src/executor.rs b/crates/crank-runtime/src/executor.rs index 064de73..e3e6cd5 100644 --- a/crates/crank-runtime/src/executor.rs +++ b/crates/crank-runtime/src/executor.rs @@ -7,9 +7,9 @@ use std::time::{Duration, Instant}; const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100); use crank_core::{ - AdapterRegistry, CoordinationStateStore, ExecutionMode, HttpMethod, InvocationSource, - InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter, - Target, + AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod, + InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, + SharedProtocolAdapter, Target, }; use crank_metrics::{ CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard, @@ -41,6 +41,7 @@ pub struct RuntimeExecutor { unary_limiter: Arc, response_cache: Option>, coordination_store: Option>, + admission: Option>, metering_sink: SharedMeteringSink, } @@ -139,6 +140,7 @@ impl RuntimeExecutor { adapters: AdapterRegistry, response_cache: Option>, coordination_store: Option>, + admission: Option>, metering_sink: SharedMeteringSink, ) -> Self { Self { @@ -147,6 +149,7 @@ impl RuntimeExecutor { limits, response_cache, coordination_store, + admission, metering_sink, } } @@ -167,6 +170,11 @@ impl RuntimeExecutor { self } + pub fn with_admission(mut self, admission: Arc) -> Self { + self.admission = Some(admission); + self + } + pub(crate) async fn execute_request( &self, request: RuntimeExecutionRequest<'_>, @@ -188,11 +196,16 @@ impl RuntimeExecutor { request.agent_id() ); let operation_may_mutate = operation_may_mutate(operation); + let execution_deadline = + crate::admission::effective_deadline(operation, request.deadline()); let runtime_span = Stage::RuntimeExecute.span(); set_parent_from_trace_context(&runtime_span, &request_context.trace_context); log_runtime_event("unary.execute", operation, Some(request_context)); let started_at = Instant::now(); let dispatch_started = Arc::new(AtomicBool::new(false)); + // True until the admission gate answered. A deadline that fires while the request is still + // waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout. + let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some())); let invocation_metrics = ToolInvocationMetrics::start_with_exemplar( metric_invocation_source(Some(request_context)), request_context @@ -206,6 +219,17 @@ impl RuntimeExecutor { .flatten(), ); let execution = async { + // One admission gate for every executable path (Story 4.1): the published policy decides + // the effective mode, the absolute deadline includes queue waiting and the scoped permit + // is held for the whole local execution future, including retries. + let mut lease = crate::admission::acquire( + self.admission.as_ref(), + self.coordination_store.as_deref(), + &request, + execution_deadline, + ) + .await?; + admission_pending.store(false, Ordering::Release); let _permit = self.acquire_unary_permit(operation)?; let _inflight = InFlightGuard::runtime(); let mapping_span = Stage::RuntimeArgumentsMap.span(); @@ -215,7 +239,7 @@ impl RuntimeExecutor { let prepared_request = prepared_request?; let request_preview = sanitized_request_preview(&prepared_request); let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth()); - let output = self + let result = self .execute_prepared( operation, input, @@ -223,19 +247,27 @@ impl RuntimeExecutor { Some(request_context), Arc::clone(&dispatch_started), ) - .await?; + .await; + if let Some(lease) = lease.as_mut() { + lease.release().await; + } + let output = result?; Ok((output, request_preview)) }; let result = tokio::time::timeout_at( - request.deadline().into(), + execution_deadline.into(), execution.instrument(runtime_span.clone()), ) .await .unwrap_or_else(|_| { - Err(RuntimeError::ExecutionDeadlineElapsed { - may_have_dispatched: operation_may_mutate - && dispatch_started.load(Ordering::Acquire), - }) + if admission_pending.load(Ordering::Acquire) { + Err(crate::admission::deadline_refusal(operation)) + } else { + Err(RuntimeError::ExecutionDeadlineElapsed { + may_have_dispatched: operation_may_mutate + && dispatch_started.load(Ordering::Acquire), + }) + } }); record_runtime_result(&runtime_span, &result); drop(runtime_span); @@ -952,6 +984,10 @@ fn runtime_error_category(error: &RuntimeError) -> ErrorCategory { | RuntimeError::UnsupportedProtocol { .. } | RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream, RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency, + RuntimeError::AdmissionOverloaded { .. } + | RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency, + RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration, + RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval, RuntimeError::ConfirmationRequired { .. } | RuntimeError::InvalidConfirmationToken { .. } | RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval, @@ -1008,6 +1044,13 @@ fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind { RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol, RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode, RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit, + // Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation + // label domain stays fixed; the precise code lives in the execution failure/invocation log + // (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`). + RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit, + RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore, + RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired, + RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode, RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest, RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired, RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken, diff --git a/crates/crank-runtime/src/executor_builder.rs b/crates/crank-runtime/src/executor_builder.rs index eb043d6..97f981d 100644 --- a/crates/crank-runtime/src/executor_builder.rs +++ b/crates/crank-runtime/src/executor_builder.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter}; use crank_core::{ - AdapterRegistry, CoordinationStateStore, MeteringSink, NoopMeteringSink, ResponseCacheStore, - SharedMeteringSink, SharedProtocolAdapter, + AdapterRegistry, CoordinationStateStore, ExecutionAdmission, MeteringSink, NoopMeteringSink, + ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter, }; use crate::{RuntimeExecutor, RuntimeLimits}; @@ -13,6 +13,7 @@ pub struct RuntimeExecutorBuilder { adapters: AdapterRegistry, response_cache: Option>, coordination_store: Option>, + admission: Option>, metering_sink: SharedMeteringSink, } @@ -23,6 +24,7 @@ impl RuntimeExecutorBuilder { adapters: AdapterRegistry::empty(), response_cache: None, coordination_store: None, + admission: None, metering_sink: Arc::new(NoopMeteringSink) as Arc, } } @@ -47,6 +49,13 @@ impl RuntimeExecutorBuilder { self } + /// Wires the mandatory deployment-wide admission store (Story 4.1). Production processes wire + /// the PostgreSQL implementation; an executor without it keeps the pre-4.1 local limit. + pub fn with_admission(mut self, admission: Arc) -> Self { + self.admission = Some(admission); + self + } + pub fn with_metering_sink(mut self, sink: SharedMeteringSink) -> Self { self.metering_sink = sink; self @@ -58,6 +67,7 @@ impl RuntimeExecutorBuilder { self.adapters, self.response_cache, self.coordination_store, + self.admission, self.metering_sink, ) } diff --git a/crates/crank-runtime/src/idempotency.rs b/crates/crank-runtime/src/idempotency.rs index eb2fd3e..c3434c1 100644 --- a/crates/crank-runtime/src/idempotency.rs +++ b/crates/crank-runtime/src/idempotency.rs @@ -446,6 +446,7 @@ mod tests { }), safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/src/lib.rs b/crates/crank-runtime/src/lib.rs index 488b472..12a0e0a 100644 --- a/crates/crank-runtime/src/lib.rs +++ b/crates/crank-runtime/src/lib.rs @@ -1,3 +1,4 @@ +mod admission; mod auth; mod cache; mod cache_factory; diff --git a/crates/crank-runtime/tests/http_contract.rs b/crates/crank-runtime/tests/http_contract.rs index 747c5a1..7be0c41 100644 --- a/crates/crank-runtime/tests/http_contract.rs +++ b/crates/crank-runtime/tests/http_contract.rs @@ -333,6 +333,7 @@ fn runtime_operation(draft: RestImportCandidate) -> RuntimeOperation { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/tests/http_contract_wire.rs b/crates/crank-runtime/tests/http_contract_wire.rs index f56b289..ecc33a7 100644 --- a/crates/crank-runtime/tests/http_contract_wire.rs +++ b/crates/crank-runtime/tests/http_contract_wire.rs @@ -96,6 +96,7 @@ paths: idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/tests/integration/confirmation.rs b/crates/crank-runtime/tests/integration/confirmation.rs index c764aa1..28e279a 100644 --- a/crates/crank-runtime/tests/integration/confirmation.rs +++ b/crates/crank-runtime/tests/integration/confirmation.rs @@ -334,6 +334,7 @@ fn destructive_delete_operation() -> Operation { confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }), }), approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/tests/integration/idempotency.rs b/crates/crank-runtime/tests/integration/idempotency.rs index 11886ea..1c7f3ef 100644 --- a/crates/crank-runtime/tests/integration/idempotency.rs +++ b/crates/crank-runtime/tests/integration/idempotency.rs @@ -340,6 +340,7 @@ fn idempotent_post_operation() -> Operation { }), safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/tests/integration/no_input_get.rs b/crates/crank-runtime/tests/integration/no_input_get.rs index 0954e38..08c41e7 100644 --- a/crates/crank-runtime/tests/integration/no_input_get.rs +++ b/crates/crank-runtime/tests/integration/no_input_get.rs @@ -109,6 +109,7 @@ fn no_input_get_operation() -> Operation { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/tests/product_metrics.rs b/crates/crank-runtime/tests/product_metrics.rs index 4a8f159..fde13b9 100644 --- a/crates/crank-runtime/tests/product_metrics.rs +++ b/crates/crank-runtime/tests/product_metrics.rs @@ -573,6 +573,7 @@ fn operation( idempotency, safety, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/crates/crank-runtime/tests/stages.rs b/crates/crank-runtime/tests/stages.rs index 5450939..ee4858d 100644 --- a/crates/crank-runtime/tests/stages.rs +++ b/crates/crank-runtime/tests/stages.rs @@ -10,7 +10,7 @@ use std::{ use async_trait::async_trait; use crank_core::{ AdapterResponse, ConfirmationPolicy, CorrelationContext, ExecutionConfig, ExecutionErrorCode, - ExecutionMode, ExecutionOrigin, HttpMethod, IdempotencyMode, IdempotencyPolicy, + ExecutionMode, ExecutionOrigin, ExecutionStage, HttpMethod, IdempotencyMode, IdempotencyPolicy, InvocationSource, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, ProtocolAdapterError, RestTarget, RetryPolicy, Retryability, Target, ToolDescription, @@ -64,6 +64,10 @@ async fn deadline_after_dispatch_is_normalized_as_unknown_outcome() { failure.outcome_certainty(), OutcomeCertainty::OutcomeUnknown ); + // A deadline that fires after the adapter was called stays an upstream timeout: the admission + // refusal vocabulary (Story 4.1) only covers deadlines reached before any dispatch. + assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout); + assert_eq!(failure.stage(), ExecutionStage::Upstream); } #[tokio::test] @@ -98,6 +102,8 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() { let failure = executor.execute_outcome(request).await.unwrap_err(); assert_eq!(failure.retryability(), Retryability::AfterDelay); assert_eq!(failure.outcome_certainty(), OutcomeCertainty::Certain); + assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout); + assert_eq!(failure.stage(), ExecutionStage::Upstream); } #[tokio::test] @@ -632,6 +638,7 @@ fn operation() -> Operation { idempotency: None, safety: None, approval_policy: None, + execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, diff --git a/docs/observability.md b/docs/observability.md index 913a805..d0718df 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -178,6 +178,8 @@ scrape_configs: - `crank_upstream_requests_total`, `crank_upstream_request_duration_seconds`; - `crank_runtime_inflight`, `crank_runtime_limit_rejections_total`, + `crank_runtime_admission_queue_wait_seconds`, + `crank_runtime_admission_oldest_waiting_seconds`, `crank_runtime_cache_total`, `crank_idempotency_total`, `crank_confirmation_total`; - `crank_db_pool_connections`; diff --git a/docs/schemas/metrics-registry-v2.json b/docs/schemas/metrics-registry-v2.json index 7b6d4f6..9847450 100644 --- a/docs/schemas/metrics-registry-v2.json +++ b/docs/schemas/metrics-registry-v2.json @@ -1 +1 @@ -{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6285,"max_rendered_series":19515,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}/versions/{version}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/content","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/versions/{version}/content","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/versions/{version}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/secrets","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/upstreams","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/auth/bootstrap/complete","processes":["admin_api"]},{"route":"/api/auth/bootstrap/status","processes":["admin_api"]},{"route":"/api/auth/login","processes":["admin_api"]},{"route":"/api/auth/logout","processes":["admin_api"]},{"route":"/api/auth/password","processes":["admin_api"]},{"route":"/api/auth/profile","processes":["admin_api"]},{"route":"/api/auth/session","processes":["admin_api"]},{"route":"/api/auth/session/csrf","processes":["admin_api"]},{"route":"/health","processes":["admin_api","mcp_server"]},{"route":"/ready","processes":["admin_api","mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny","processes":["mcp_server"]},{"route":"unmatched","processes":["admin_api","mcp_server"]}],"metrics":[{"name":"crank_http_requests_total","kind":"counter","unit":"count","help":"Total HTTP requests.","processes":["admin_api","mcp_server"],"labels":[{"name":"route","class":"product_closed","domain_ref":"http_routes"},{"name":"method","class":"product_closed","domain":["GET","POST","PUT","PATCH","DELETE","OPTIONS","HEAD","CONNECT","TRACE","OTHER"]},{"name":"status_class","class":"product_closed","domain":["1xx","2xx","3xx","4xx","5xx","other"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":5160,"rendered_series":5160,"exemplar_slots":0},{"process":"mcp_server","logical_series":480,"rendered_series":480,"exemplar_slots":0}]},{"name":"crank_http_request_duration_seconds","kind":"histogram","unit":"seconds","help":"HTTP request duration in seconds.","processes":["admin_api","mcp_server"],"labels":[{"name":"route","class":"product_closed","domain_ref":"http_routes"},{"name":"method","class":"product_closed","domain":["GET","POST","PUT","PATCH","DELETE","OPTIONS","HEAD","CONNECT","TRACE","OTHER"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"admin_api","logical_series":860,"rendered_series":13760,"exemplar_slots":12040},{"process":"mcp_server","logical_series":80,"rendered_series":1280,"exemplar_slots":1120}]},{"name":"crank_http_inflight","kind":"gauge","unit":"count","help":"HTTP requests currently being processed.","processes":["admin_api","mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":1,"rendered_series":1,"exemplar_slots":0},{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_mcp_requests_total","kind":"counter","unit":"count","help":"Total MCP JSON-RPC requests.","processes":["mcp_server"],"labels":[{"name":"method","class":"product_closed","domain":["initialize","initialized","ping","tools_list","tools_call","notification","unsupported","response","invalid"]},{"name":"response_mode","class":"product_closed","domain":["json","sse","unknown"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","jsonrpc_error","tool_error","aborted","other"]}],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":189,"rendered_series":189,"exemplar_slots":0}]},{"name":"crank_mcp_request_duration_seconds","kind":"histogram","unit":"seconds","help":"MCP JSON-RPC request duration in seconds.","processes":["mcp_server"],"labels":[{"name":"method","class":"product_closed","domain":["initialize","initialized","ping","tools_list","tools_call","notification","unsupported","response","invalid"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","jsonrpc_error","tool_error","aborted","other"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"mcp_server","logical_series":63,"rendered_series":1008,"exemplar_slots":882}]},{"name":"crank_mcp_active_sessions","kind":"gauge","unit":"count","help":"Active persisted MCP transport sessions.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_mcp_session_metrics_fresh","kind":"gauge","unit":"count","help":"Whether the active-session gauge was refreshed successfully.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_mcp_active_streams","kind":"gauge","unit":"count","help":"MCP event streams currently being served.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_tool_invocations_total","kind":"counter","unit":"count","help":"Total tool invocations.","processes":["admin_api","mcp_server"],"labels":[{"name":"source","class":"product_closed","domain":["internal","admin_test_run","agent_tool_call"]},{"name":"outcome","class":"product_closed","domain":["success","error","aborted"]},{"name":"error_kind","class":"product_closed","domain":["none","schema","mapping","rest_adapter","protocol_adapter","unsupported_protocol","unsupported_execution_mode","concurrency_limit","invalid_prepared_request","confirmation_required","invalid_confirmation_token","confirmation_store","idempotency_store","idempotency_in_progress","idempotency_conflict","idempotency_outcome_unknown","missing_auth_profile","missing_secret","missing_secret_version","invalid_auth_secret","secret_crypto","aborted"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":198,"rendered_series":198,"exemplar_slots":0},{"process":"mcp_server","logical_series":198,"rendered_series":198,"exemplar_slots":0}]},{"name":"crank_tool_invocation_duration_seconds","kind":"histogram","unit":"seconds","help":"Tool invocation duration in seconds.","processes":["admin_api","mcp_server"],"labels":[{"name":"source","class":"product_closed","domain":["internal","admin_test_run","agent_tool_call"]},{"name":"outcome","class":"product_closed","domain":["success","error","aborted"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"admin_api","logical_series":9,"rendered_series":144,"exemplar_slots":126},{"process":"mcp_server","logical_series":9,"rendered_series":144,"exemplar_slots":126}]},{"name":"crank_upstream_requests_total","kind":"counter","unit":"count","help":"Total upstream requests.","processes":["admin_api","mcp_server"],"labels":[{"name":"operation_kind","class":"product_closed","domain":["rest"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","unexpected_status","timeout","transport_error","response_too_large","rejected","window_expired","invalid_response","invalid_request","configuration","aborted"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":13,"rendered_series":13,"exemplar_slots":0},{"process":"mcp_server","logical_series":13,"rendered_series":13,"exemplar_slots":0}]},{"name":"crank_upstream_request_duration_seconds","kind":"histogram","unit":"seconds","help":"Upstream request duration in seconds.","processes":["admin_api","mcp_server"],"labels":[{"name":"operation_kind","class":"product_closed","domain":["rest"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","unexpected_status","timeout","transport_error","response_too_large","rejected","window_expired","invalid_response","invalid_request","configuration","aborted"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"admin_api","logical_series":13,"rendered_series":208,"exemplar_slots":182},{"process":"mcp_server","logical_series":13,"rendered_series":208,"exemplar_slots":182}]},{"name":"crank_runtime_inflight","kind":"gauge","unit":"count","help":"Runtime executions currently in progress.","processes":["admin_api","mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":1,"rendered_series":1,"exemplar_slots":0},{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_runtime_limit_rejections_total","kind":"counter","unit":"count","help":"Runtime executions rejected by a bounded limit.","processes":["admin_api","mcp_server"],"labels":[{"name":"stage","class":"product_closed","domain":["concurrency","rate_limit","mcp_stream"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":3,"rendered_series":3,"exemplar_slots":0},{"process":"mcp_server","logical_series":3,"rendered_series":3,"exemplar_slots":0}]},{"name":"crank_runtime_cache_total","kind":"counter","unit":"count","help":"Runtime response cache outcomes.","processes":["admin_api","mcp_server"],"labels":[{"name":"outcome","class":"product_closed","domain":["hit","miss","read_error","decode_error","evict_error","stored","write_error"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":7,"rendered_series":7,"exemplar_slots":0},{"process":"mcp_server","logical_series":7,"rendered_series":7,"exemplar_slots":0}]},{"name":"crank_idempotency_total","kind":"counter","unit":"count","help":"Runtime idempotency outcomes.","processes":["admin_api","mcp_server"],"labels":[{"name":"outcome","class":"product_closed","domain":["execute","replay","completed","conflict","in_progress","outcome_unknown","store_unavailable","error"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":8,"rendered_series":8,"exemplar_slots":0},{"process":"mcp_server","logical_series":8,"rendered_series":8,"exemplar_slots":0}]},{"name":"crank_confirmation_total","kind":"counter","unit":"count","help":"Runtime confirmation outcomes.","processes":["admin_api","mcp_server"],"labels":[{"name":"outcome","class":"product_closed","domain":["approved","required","invalid_token","store_unavailable","error"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":5,"rendered_series":5,"exemplar_slots":0},{"process":"mcp_server","logical_series":5,"rendered_series":5,"exemplar_slots":0}]},{"name":"crank_db_pool_connections","kind":"gauge","unit":"count","help":"PostgreSQL pool connections by state.","processes":["admin_api","mcp_server"],"labels":[{"name":"state","class":"product_closed","domain":["idle","used"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":2,"rendered_series":2,"exemplar_slots":0},{"process":"mcp_server","logical_series":2,"rendered_series":2,"exemplar_slots":0}]},{"name":"crank_catalog_tools","kind":"gauge","unit":"count","help":"Tools in the current published catalog.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_catalog_estimated_context_tokens","kind":"gauge","unit":"count","help":"Estimated context tokens in the current published catalog.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_catalog_warnings","kind":"gauge","unit":"count","help":"Warnings in the current published catalog.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_invocation_history_lost_total","kind":"counter","unit":"count","help":"Invocation history records lost after an action completed.","processes":["admin_api","mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":1,"rendered_series":1,"exemplar_slots":0},{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_telemetry_export_failures_total","kind":"counter","unit":"count","help":"Telemetry export failures.","processes":["admin_api","mcp_server"],"labels":[{"name":"signal_type","class":"product_closed","domain":["trace","invocation_history"]},{"name":"exporter","class":"product_closed","domain":["otlp","postgres"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":4,"rendered_series":4,"exemplar_slots":0},{"process":"mcp_server","logical_series":4,"rendered_series":4,"exemplar_slots":0}]}],"budgets":[{"process":"admin_api","families":15,"logical_series":6285,"rendered_series":19515,"exemplar_slots":12348},{"process":"mcp_server","families":23,"logical_series":1083,"rendered_series":3558,"exemplar_slots":2310}]} +{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6381,"max_rendered_series":19611,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/prompts/{prompt_id}/versions/{version}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/content","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/versions/{version}/content","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/resources/{resource_id}/versions/{version}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/secrets","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/upstreams","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/auth/bootstrap/complete","processes":["admin_api"]},{"route":"/api/auth/bootstrap/status","processes":["admin_api"]},{"route":"/api/auth/login","processes":["admin_api"]},{"route":"/api/auth/logout","processes":["admin_api"]},{"route":"/api/auth/password","processes":["admin_api"]},{"route":"/api/auth/profile","processes":["admin_api"]},{"route":"/api/auth/session","processes":["admin_api"]},{"route":"/api/auth/session/csrf","processes":["admin_api"]},{"route":"/health","processes":["admin_api","mcp_server"]},{"route":"/ready","processes":["admin_api","mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve","processes":["mcp_server"]},{"route":"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny","processes":["mcp_server"]},{"route":"unmatched","processes":["admin_api","mcp_server"]}],"metrics":[{"name":"crank_http_requests_total","kind":"counter","unit":"count","help":"Total HTTP requests.","processes":["admin_api","mcp_server"],"labels":[{"name":"route","class":"product_closed","domain_ref":"http_routes"},{"name":"method","class":"product_closed","domain":["GET","POST","PUT","PATCH","DELETE","OPTIONS","HEAD","CONNECT","TRACE","OTHER"]},{"name":"status_class","class":"product_closed","domain":["1xx","2xx","3xx","4xx","5xx","other"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":5160,"rendered_series":5160,"exemplar_slots":0},{"process":"mcp_server","logical_series":480,"rendered_series":480,"exemplar_slots":0}]},{"name":"crank_http_request_duration_seconds","kind":"histogram","unit":"seconds","help":"HTTP request duration in seconds.","processes":["admin_api","mcp_server"],"labels":[{"name":"route","class":"product_closed","domain_ref":"http_routes"},{"name":"method","class":"product_closed","domain":["GET","POST","PUT","PATCH","DELETE","OPTIONS","HEAD","CONNECT","TRACE","OTHER"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"admin_api","logical_series":860,"rendered_series":13760,"exemplar_slots":12040},{"process":"mcp_server","logical_series":80,"rendered_series":1280,"exemplar_slots":1120}]},{"name":"crank_http_inflight","kind":"gauge","unit":"count","help":"HTTP requests currently being processed.","processes":["admin_api","mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":1,"rendered_series":1,"exemplar_slots":0},{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_mcp_requests_total","kind":"counter","unit":"count","help":"Total MCP JSON-RPC requests.","processes":["mcp_server"],"labels":[{"name":"method","class":"product_closed","domain":["initialize","initialized","ping","tools_list","tools_call","notification","unsupported","response","invalid"]},{"name":"response_mode","class":"product_closed","domain":["json","sse","unknown"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","jsonrpc_error","tool_error","aborted","other"]}],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":189,"rendered_series":189,"exemplar_slots":0}]},{"name":"crank_mcp_request_duration_seconds","kind":"histogram","unit":"seconds","help":"MCP JSON-RPC request duration in seconds.","processes":["mcp_server"],"labels":[{"name":"method","class":"product_closed","domain":["initialize","initialized","ping","tools_list","tools_call","notification","unsupported","response","invalid"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","jsonrpc_error","tool_error","aborted","other"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"mcp_server","logical_series":63,"rendered_series":1008,"exemplar_slots":882}]},{"name":"crank_mcp_active_sessions","kind":"gauge","unit":"count","help":"Active persisted MCP transport sessions.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_mcp_session_metrics_fresh","kind":"gauge","unit":"count","help":"Whether the active-session gauge was refreshed successfully.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_mcp_active_streams","kind":"gauge","unit":"count","help":"MCP event streams currently being served.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_tool_invocations_total","kind":"counter","unit":"count","help":"Total tool invocations.","processes":["admin_api","mcp_server"],"labels":[{"name":"source","class":"product_closed","domain":["internal","admin_test_run","agent_tool_call"]},{"name":"outcome","class":"product_closed","domain":["success","error","aborted"]},{"name":"error_kind","class":"product_closed","domain":["none","schema","mapping","rest_adapter","protocol_adapter","unsupported_protocol","unsupported_execution_mode","concurrency_limit","invalid_prepared_request","confirmation_required","invalid_confirmation_token","confirmation_store","idempotency_store","idempotency_in_progress","idempotency_conflict","idempotency_outcome_unknown","missing_auth_profile","missing_secret","missing_secret_version","invalid_auth_secret","secret_crypto","aborted"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":198,"rendered_series":198,"exemplar_slots":0},{"process":"mcp_server","logical_series":198,"rendered_series":198,"exemplar_slots":0}]},{"name":"crank_tool_invocation_duration_seconds","kind":"histogram","unit":"seconds","help":"Tool invocation duration in seconds.","processes":["admin_api","mcp_server"],"labels":[{"name":"source","class":"product_closed","domain":["internal","admin_test_run","agent_tool_call"]},{"name":"outcome","class":"product_closed","domain":["success","error","aborted"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"admin_api","logical_series":9,"rendered_series":144,"exemplar_slots":126},{"process":"mcp_server","logical_series":9,"rendered_series":144,"exemplar_slots":126}]},{"name":"crank_upstream_requests_total","kind":"counter","unit":"count","help":"Total upstream requests.","processes":["admin_api","mcp_server"],"labels":[{"name":"operation_kind","class":"product_closed","domain":["rest"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","unexpected_status","timeout","transport_error","response_too_large","rejected","window_expired","invalid_response","invalid_request","configuration","aborted"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":13,"rendered_series":13,"exemplar_slots":0},{"process":"mcp_server","logical_series":13,"rendered_series":13,"exemplar_slots":0}]},{"name":"crank_upstream_request_duration_seconds","kind":"histogram","unit":"seconds","help":"Upstream request duration in seconds.","processes":["admin_api","mcp_server"],"labels":[{"name":"operation_kind","class":"product_closed","domain":["rest"]},{"name":"outcome","class":"product_closed","domain":["success","client_error","server_error","unexpected_status","timeout","transport_error","response_too_large","rejected","window_expired","invalid_response","invalid_request","configuration","aborted"]}],"buckets_seconds":[0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],"bounds":[{"process":"admin_api","logical_series":13,"rendered_series":208,"exemplar_slots":182},{"process":"mcp_server","logical_series":13,"rendered_series":208,"exemplar_slots":182}]},{"name":"crank_runtime_inflight","kind":"gauge","unit":"count","help":"Runtime executions currently in progress.","processes":["admin_api","mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":1,"rendered_series":1,"exemplar_slots":0},{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_runtime_limit_rejections_total","kind":"counter","unit":"count","help":"Runtime executions rejected by a bounded limit.","processes":["admin_api","mcp_server"],"labels":[{"name":"stage","class":"product_closed","domain":["concurrency","rate_limit","mcp_stream","admission"]},{"name":"reason","class":"product_closed","domain":["limit","capacity","queue_full","deadline","unsupported","dependency_unavailable"]},{"name":"scope_kind","class":"product_closed","domain":["process","global","agent","tool"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":96,"rendered_series":96,"exemplar_slots":0},{"process":"mcp_server","logical_series":96,"rendered_series":96,"exemplar_slots":0}]},{"name":"crank_runtime_admission_queue_wait_seconds","kind":"gauge","unit":"seconds","help":"Wait of the most recently admitted execution before it received a scoped permit.","processes":["admin_api","mcp_server"],"labels":[{"name":"mode","class":"product_closed","domain":["synchronous","background"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":2,"rendered_series":2,"exemplar_slots":0},{"process":"mcp_server","logical_series":2,"rendered_series":2,"exemplar_slots":0}]},{"name":"crank_runtime_admission_oldest_waiting_seconds","kind":"gauge","unit":"seconds","help":"Age of the oldest waiting admission request.","processes":["admin_api","mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":1,"rendered_series":1,"exemplar_slots":0},{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_runtime_cache_total","kind":"counter","unit":"count","help":"Runtime response cache outcomes.","processes":["admin_api","mcp_server"],"labels":[{"name":"outcome","class":"product_closed","domain":["hit","miss","read_error","decode_error","evict_error","stored","write_error"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":7,"rendered_series":7,"exemplar_slots":0},{"process":"mcp_server","logical_series":7,"rendered_series":7,"exemplar_slots":0}]},{"name":"crank_idempotency_total","kind":"counter","unit":"count","help":"Runtime idempotency outcomes.","processes":["admin_api","mcp_server"],"labels":[{"name":"outcome","class":"product_closed","domain":["execute","replay","completed","conflict","in_progress","outcome_unknown","store_unavailable","error"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":8,"rendered_series":8,"exemplar_slots":0},{"process":"mcp_server","logical_series":8,"rendered_series":8,"exemplar_slots":0}]},{"name":"crank_confirmation_total","kind":"counter","unit":"count","help":"Runtime confirmation outcomes.","processes":["admin_api","mcp_server"],"labels":[{"name":"outcome","class":"product_closed","domain":["approved","required","invalid_token","store_unavailable","error"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":5,"rendered_series":5,"exemplar_slots":0},{"process":"mcp_server","logical_series":5,"rendered_series":5,"exemplar_slots":0}]},{"name":"crank_db_pool_connections","kind":"gauge","unit":"count","help":"PostgreSQL pool connections by state.","processes":["admin_api","mcp_server"],"labels":[{"name":"state","class":"product_closed","domain":["idle","used"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":2,"rendered_series":2,"exemplar_slots":0},{"process":"mcp_server","logical_series":2,"rendered_series":2,"exemplar_slots":0}]},{"name":"crank_catalog_tools","kind":"gauge","unit":"count","help":"Tools in the current published catalog.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_catalog_estimated_context_tokens","kind":"gauge","unit":"count","help":"Estimated context tokens in the current published catalog.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_catalog_warnings","kind":"gauge","unit":"count","help":"Warnings in the current published catalog.","processes":["mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_invocation_history_lost_total","kind":"counter","unit":"count","help":"Invocation history records lost after an action completed.","processes":["admin_api","mcp_server"],"labels":[],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":1,"rendered_series":1,"exemplar_slots":0},{"process":"mcp_server","logical_series":1,"rendered_series":1,"exemplar_slots":0}]},{"name":"crank_telemetry_export_failures_total","kind":"counter","unit":"count","help":"Telemetry export failures.","processes":["admin_api","mcp_server"],"labels":[{"name":"signal_type","class":"product_closed","domain":["trace","invocation_history"]},{"name":"exporter","class":"product_closed","domain":["otlp","postgres"]}],"buckets_seconds":[],"bounds":[{"process":"admin_api","logical_series":4,"rendered_series":4,"exemplar_slots":0},{"process":"mcp_server","logical_series":4,"rendered_series":4,"exemplar_slots":0}]}],"budgets":[{"process":"admin_api","families":17,"logical_series":6381,"rendered_series":19611,"exemplar_slots":12348},{"process":"mcp_server","families":25,"logical_series":1179,"rendered_series":3654,"exemplar_slots":2310}]} diff --git a/docs/schemas/migration-sequence.json b/docs/schemas/migration-sequence.json index 58f5d56..ec7743a 100644 --- a/docs/schemas/migration-sequence.json +++ b/docs/schemas/migration-sequence.json @@ -1 +1 @@ -{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23}]} +{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23},{"backfill":{"kind":"none"},"checksum":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-admission-v24","owner":"crank-registry","phase":"expand","readable_schema_max":24,"readable_schema_min":23,"source_digest":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","transactional":true,"version":24}]} diff --git a/scripts/ci-story-checks.sh b/scripts/ci-story-checks.sh index 86c66fd..a8ad8f2 100755 --- a/scripts/ci-story-checks.sh +++ b/scripts/ci-story-checks.sh @@ -55,3 +55,15 @@ run_atdd "capability advertisement registry" -p crank-community-mcp --lib protoc # smoke) runs against the real endpoint, a real published catalog and a real counted upstream. run_atdd "tool call lifecycle bounds" -p crank-community-mcp --lib app::lifecycle -- --test-threads=1 run_atdd "tool call progress and cancellation" -p mcp-server --test integration tools_lifecycle -- --test-threads=1 + +# Story 4.1 (FR-21, FR-24): a published Tool execution policy and one authoritative admission gate. +# The policy/JSON contract and the fair-scheduler arithmetic are unit-level; the authoritative store +# (scoped permits, bounded queue, typed overload, durable decisions, version pinning, fail-closed +# health) and the runtime wiring (one shared global budget across executors, typed refusals with no +# outbound call, adaptive/synchronous fallback) run against a real PostgreSQL schema. +run_atdd "execution policy contract" -p crank-core --lib operation::tests::execution_policy -- --test-threads=1 +run_atdd "admission scheduler bounds" -p crank-core --lib admission:: -- --test-threads=1 +run_atdd "execution policy validation" -p admin-api --lib operation_validation::tests:: -- --test-threads=1 +run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1 +run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1 +run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1 -- 2.54.0