Compare commits

..
Author SHA1 Message Date
bsodfather 6df6ffac4e Merge pull request 'Story 4.2 — подтверждать durable Task после атомарного PostgreSQL commit (FR-22, FR-23)' (#109) from delivery/epic4-story-4.2 into main
CI / UI Checks (push) Successful in 13s
CI / Rust Checks (push) Successful in 21m12s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 36m42s
Merge accepted Story 4.2 durable Task acceptance
2026-10-27 07:33:30 +00:00
bsodfather 2219a7217d feat(epic4): подтверждать durable Task после атомарного PG-commit (Story 4.2)
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 19m34s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Application entry выдаёт Task ID только после атомарного PostgreSQL commit
Task+intent+capacity; replay/conflict, owner scope, bounded envelope,
terminal CAS и fail-closed readiness проверены на реальном PG.

Также включён принятый план перехода на rmcp (только документ, миграция не
запускается).
2026-10-27 00:09:36 +00:00
bsodfather cc612516a2 Merge pull request 'Story 4.1 — публиковать execution mode и применять admission limits (FR-21, FR-24)' (#104) from delivery/epic4-story-4.1 into main
CI / Rust Checks (push) Successful in 20m52s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / UI Checks (push) Successful in 10s
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
2026-10-26 21:38:06 +00:00
bsodfather 599dc1867b Merge pull request 'ci: безопасно интегрировать принятый Crank и отделить production deploy' (#105) from ops/separate-production-deploy into main
CI / UI Checks (push) Successful in 12s
CI / Rust Checks (push) Successful in 16m0s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 2s
CI / Deploy (push) Skipped
2026-10-26 21:36:09 +00:00
bsodfather e80d30ddb0 feat(epic4): публиковать execution mode и применять общий admission (Story 4.1, FR-21, FR-24)
CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 12m45s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Community Release / release (push) Successful in 40m35s
Tool Draft фиксирует synchronous/background/adaptive, deadline и fallback;
runtime применяет единый снимок политики (concurrency, очередь, approval,
deadline) до внешнего действия. Насыщение отклоняется typed overload с
retry guidance, mandatory safety store fail closed, background/adaptive
используют опубликованный synchronous/unsupported fallback.

Refs: #29
2026-10-26 21:08:08 +00:00
93 changed files with 8095 additions and 167 deletions

No files matched your search

Generated
+1
View File
@@ -826,6 +826,7 @@ dependencies = [
name = "crank-registry"
version = "0.3.1"
dependencies = [
"async-trait",
"crank-artifacts",
"crank-core",
"crank-mapping",
+1
View File
@@ -181,6 +181,7 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+18 -1
View File
@@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
use admin_api::{
app::build_app,
auth::{AuthSettings, BootstrapAdminConfig},
pool_metrics::spawn_postgres_pool_metrics,
pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics},
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
service::AdminServiceBuilder,
state::AppState,
@@ -169,6 +169,7 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let base_url = config
.runtime
@@ -214,6 +215,16 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(std::sync::Arc::new(
crank_registry::PostgresExecutionAdmission::new(
registry.pool().clone(),
crank_registry::AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
},
crank_core::AdmissionSchedulerConfig::default(),
)?,
))
.build();
let identity_provider =
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
@@ -321,6 +332,12 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
+14
View File
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
+27 -2
View File
@@ -37,13 +37,22 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
} else {
"not_ready"
};
let execution_admission = if checks.execution_admission {
"ready"
} else {
"not_ready"
};
if checks.is_ready() {
(
StatusCode::OK,
Json(json!({
"service": "admin-api",
"status": "ready",
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
})),
)
} else {
@@ -52,7 +61,11 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
Json(json!({
"service": "admin-api",
"status": "not_ready",
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
})),
)
}
@@ -67,6 +80,7 @@ mod tests {
let checks = ReadinessChecks {
postgres: false,
artifact_storage: true,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -92,6 +106,7 @@ mod tests {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: false,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -111,4 +126,14 @@ mod tests {
"not_ready"
);
}
#[test]
fn readiness_checks_identify_an_admission_state_failure() {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: true,
execution_admission: false,
};
assert!(!checks.is_ready());
}
}
+9 -3
View File
@@ -55,8 +55,8 @@ use crate::{
storage::LocalArtifactStorage,
};
use operation_validation::{
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_protocol_target, validate_response_cache_policy,
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy,
};
#[derive(Clone)]
@@ -104,11 +104,12 @@ pub use crate::dto::*;
pub struct ReadinessChecks {
pub postgres: bool,
pub artifact_storage: bool,
pub execution_admission: bool,
}
impl ReadinessChecks {
pub const fn is_ready(self) -> bool {
self.postgres && self.artifact_storage
self.postgres && self.artifact_storage && self.execution_admission
}
}
@@ -150,6 +151,8 @@ pub(super) struct CredentialAuditRecord<'a> {
impl AdminService {
pub async fn readiness(&self) -> ReadinessChecks {
let postgres = self.registry.ping().await.is_ok();
// Story 4.1: the mandatory admission state is a readiness dependency of every execution.
let execution_admission = self.registry.execution_admission_health().await.is_ok();
let store = Arc::clone(&self.artifact_store);
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
.await
@@ -157,6 +160,7 @@ impl AdminService {
ReadinessChecks {
postgres,
artifact_storage,
execution_admission,
}
}
@@ -725,6 +729,7 @@ impl AdminService {
validate_response_cache_policy(&payload.target, &payload.execution_config)?;
validate_idempotency_policy(&payload.target, &payload.execution_config)?;
validate_approval_policy(&payload.execution_config)?;
validate_execution_policy(&payload.execution_config)?;
payload.input_mapping.validate_paths()?;
payload.output_mapping.validate_paths()?;
Ok(())
@@ -738,6 +743,7 @@ impl AdminService {
validate_response_cache_policy(&operation.target, &operation.execution_config)?;
validate_idempotency_policy(&operation.target, &operation.execution_config)?;
validate_approval_policy(&operation.execution_config)?;
validate_execution_policy(&operation.execution_config)?;
operation.input_mapping.validate_paths()?;
operation.output_mapping.validate_paths()?;
Ok(())
+1
View File
@@ -441,6 +441,7 @@ fn demo_rest_operation_payload() -> OperationPayload {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+1
View File
@@ -635,6 +635,7 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -610,6 +610,7 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref,
headers: BTreeMap::new(),
},
@@ -160,6 +160,23 @@ pub(super) fn validate_approval_policy(
Ok(())
}
pub(super) fn validate_execution_policy(
execution_config: &crank_core::ExecutionConfig,
) -> Result<(), ApiError> {
let Some(policy) = execution_config.execution_policy.as_ref() else {
return Ok(());
};
if let Err(error) = policy.validate() {
return Err(ApiError::validation_with_context(
error.to_string(),
json!({
"field": "execution_config.execution_policy.deadline_ms",
}),
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
@@ -171,8 +188,8 @@ mod tests {
};
use super::{
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_response_cache_policy,
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_response_cache_policy,
};
fn cacheable_execution_config() -> ExecutionConfig {
@@ -183,6 +200,7 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -201,6 +219,7 @@ mod tests {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -365,4 +384,36 @@ mod tests {
"approval ttl must be between 1 and 300 seconds"
);
}
#[test]
fn accepts_published_execution_policy_and_absent_legacy_policy() {
let mut config = cacheable_execution_config();
validate_execution_policy(&config).unwrap();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Adaptive,
deadline_ms: 120_000,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
});
validate_execution_policy(&config).unwrap();
}
#[test]
fn rejects_zero_and_overflowing_execution_policy_deadline() {
let mut config = cacheable_execution_config();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Synchronous,
deadline_ms: 0,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
let error = validate_execution_policy(&config).unwrap_err();
assert!(matches!(error, crate::error::ApiError::Validation { .. }));
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Background,
deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
assert!(validate_execution_policy(&config).is_err());
}
}
@@ -495,6 +495,7 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+50 -4
View File
@@ -1,7 +1,7 @@
use std::{io, process::ExitCode, time::Duration};
use crank_community_mcp::{
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts,
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts_and_tasks,
session::PostgresTransportSessionStore,
};
use crank_config::{
@@ -14,13 +14,14 @@ use crank_observability::{
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
};
use crank_registry::{
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, PostgresTaskStore,
};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use mcp_server::pool_metrics::spawn_postgres_pool_metrics;
use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics};
use sqlx::postgres::PgConnectOptions;
use tokio::net::TcpListener;
use tracing::info;
@@ -175,6 +176,7 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
let secret_crypto =
@@ -190,12 +192,30 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(execution_admission(
registry.pool().clone(),
runtime_limits,
)?)
.build();
// Published static Resources are read from the same immutable content-addressed root the
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
let app = build_app_with_artifacts(
// Story 4.2: the durable Task application entry shares the registry pool, the published catalog
// refresh interval and the exact admission limits of the runtime; the public wire stays
// synchronous/unsupported and no worker is started.
let task_store = PostgresTaskStore::new(
registry.pool().clone(),
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?;
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
registry.clone(),
Duration::from_millis(config.refresh_ms),
cache_stores.coordination.clone(),
crank_runtime::TaskService::new(std::sync::Arc::new(task_store)),
));
let app = build_app_with_artifacts_and_tasks(
registry,
Duration::from_millis(config.refresh_ms),
config.runtime.base_url.clone(),
@@ -211,6 +231,7 @@ async fn run(
std::sync::Arc::new(CommunityMachineCredentialVerifier),
runtime_limits.max_concurrent_sessions,
artifact_store,
tasks,
);
let listener = TcpListener::bind(config.bind_addr).await?;
@@ -287,6 +308,25 @@ fn init_observability(
)?)
}
fn admission_limits(runtime_limits: RuntimeLimits) -> AdmissionLimits {
AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..AdmissionLimits::default()
}
}
fn execution_admission(
pool: sqlx::PgPool,
runtime_limits: RuntimeLimits,
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
let store = PostgresExecutionAdmission::new(
pool,
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?;
Ok(std::sync::Arc::new(store))
}
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
database_options(&config.database).map_err(|_| invalid("database.source"))?;
@@ -307,6 +347,12 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
+14
View File
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
+874
View File
@@ -0,0 +1,874 @@
//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission.
//!
//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a
//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported
//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal.
//!
//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this
//! app already wires both and this is the real deployment shape.
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
use crank_core::{
AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation,
OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget,
RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy,
WorkspaceId,
};
use crank_mapping::MappingSet;
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits};
use crank_schema::{Schema, SchemaKind};
use serde_json::json;
use sqlx::PgPool;
use time::OffsetDateTime;
struct CountingAdapter {
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
#[async_trait]
impl ProtocolAdapter for CountingAdapter {
fn protocol(&self) -> Protocol {
Protocol::Rest
}
fn supports_mode(&self, mode: ExecutionMode) -> bool {
mode == ExecutionMode::Unary
}
async fn invoke_unary(
&self,
_target: &Target,
_prepared: &crank_core::PreparedRequest,
context: &RuntimeRequestContext,
) -> Result<AdapterResponse, crank_core::ProtocolAdapterError> {
context.mark_dispatch_started();
self.calls.fetch_add(1, Ordering::SeqCst);
let current = self.current.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(current, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(150)).await;
self.current.fetch_sub(1, Ordering::SeqCst);
Ok(AdapterResponse {
status_code: 200,
headers: BTreeMap::new(),
body: json!({"ok": true}),
data: json!({"ok": true}),
})
}
}
fn schema(kind: SchemaKind) -> Schema {
Schema {
kind,
format: None,
description: None,
required: false,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
}
}
fn operation(policy: Option<ToolExecutionPolicy>) -> Operation<Schema, MappingSet> {
Operation {
id: OperationId::new("op_admission_budget"),
name: "admission_budget".to_owned(),
display_name: "Admission budget".to_owned(),
category: "general".to_owned(),
protocol: Protocol::Rest,
security_level: OperationSecurityLevel::Standard,
status: OperationStatus::Published,
version: 7,
target: Target::Rest(RestTarget {
base_url: "https://api.example.com".to_owned(),
method: HttpMethod::Get,
path_template: "/v1/items".to_owned(),
static_headers: BTreeMap::new(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: BTreeMap::new(),
}),
input_schema: schema(SchemaKind::Object),
output_schema: schema(SchemaKind::Object),
input_mapping: MappingSet::default(),
output_mapping: MappingSet::default(),
execution_config: ExecutionConfig {
timeout_ms: 5_000,
execution_policy: policy,
..ExecutionConfig::default()
},
tool_description: ToolDescription {
title: "Admission".to_owned(),
description: "Admission budget probe".to_owned(),
tags: Vec::new(),
examples: Vec::new(),
},
samples: None,
generated_draft: None,
config_export: None,
wizard_state: None,
created_at: OffsetDateTime::now_utc(),
updated_at: OffsetDateTime::now_utc(),
published_at: None,
}
}
async fn execute(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = WorkspaceId::new("ws_test");
let context = crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun);
execute_in(executor, operation, input, &context).await
}
async fn execute_in(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
context: &crank_runtime::RuntimeRequestContext,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = context
.metering_context()
.map(|metering| metering.workspace_id.clone())
.unwrap_or_else(|| WorkspaceId::new("ws_test"));
let request = crank_runtime::RuntimeExecutionRequest::try_new(
&workspace,
ExecutionOrigin::AdminDraft,
None,
operation,
input,
crank_runtime::ExecutionAuthorization::Authorized,
None,
context,
Instant::now() + Duration::from_secs(30),
)
.map_err(|_| {
crank_core::ExecutionFailure::new(
ExecutionErrorCode::RuntimeInternal,
crank_core::CorrelationContext::new(
context.request_id.clone(),
context.trace_context.clone(),
),
)
})?;
executor
.execute_outcome(request)
.await
.map(|success| success.output)
}
async fn schema_pool(schema_url: &str) -> PgPool {
let pool = PgPool::connect(schema_url)
.await
.expect("admission schema pool");
crank_registry::MigrationAuthority::apply(&pool)
.await
.expect("migration authority applies the admission schema");
pool
}
fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc<PostgresExecutionAdmission> {
Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: global,
agent_limit: 8,
tool_limit: 8,
queue_limit: queue,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)
}
fn executor(
store: Arc<PostgresExecutionAdmission>,
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
) -> RuntimeExecutor {
RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current,
peak,
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store)
.build()
}
#[tokio::test]
async fn two_executors_share_one_global_budget() {
let schema_url = crank_test_support::postgres_schema_url("admission_budget").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 8);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let first = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let second = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let mut tasks = Vec::new();
for index in 0..4 {
let executor = if index % 2 == 0 {
first.clone()
} else {
second.clone()
};
let operation = operation.clone();
tasks.push(tokio::spawn(async move {
execute(&executor, &operation, &json!({})).await
}));
}
let mut completed = 0;
for task in tasks {
match task.await.expect("execution task") {
Ok(_) => completed += 1,
Err(failure) => panic!("admission must grant every request: {failure:?}"),
}
}
assert_eq!(completed, 4);
assert_eq!(calls.load(Ordering::SeqCst), 4);
assert!(
peak.load(Ordering::SeqCst) <= 2,
"peak concurrency {} exceeded the shared global limit",
peak.load(Ordering::SeqCst)
);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn saturation_returns_typed_overload_without_an_outbound_call() {
let schema_url = crank_test_support::postgres_schema_url("admission_overload").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 1, 1);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// Another process already holds the only global permit.
let held = match store
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_test".to_owned(),
agent_key: "admin".to_owned(),
operation_id: "op_admission_budget".to_owned(),
operation_version: 7,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_holder".to_owned(),
expires_at_unix_ms: crank_core::current_unix_ms() + 10_000,
})
.await
.expect("admission store is reachable")
{
crank_core::AdmissionOutcome::Permitted(permit) => permit,
other => panic!("expected a permit, got {other:?}"),
};
let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 250,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let waiting_runtime = runtime.clone();
let first_waiting_operation = waiting_operation.clone();
let waiting = tokio::spawn(async move {
execute(&waiting_runtime, &first_waiting_operation, &json!({})).await
});
// Let the first request occupy the bounded queue before the next one arrives.
while store.snapshot().await.unwrap().waiting == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
let overloaded = execute(&runtime, &waiting_operation, &json!({})).await;
let failure = overloaded.expect_err("a full queue must refuse the request");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionOverloaded
);
assert_eq!(failure.stage(), ExecutionStage::Admission);
assert_eq!(failure.retry_after_ms(), Some(250));
// The queued request hits its absolute deadline instead of dispatching. That refusal happened
// inside admission (nothing was ever dispatched), so the client must see an admission-stage
// overload with retry guidance — not an `upstream_timeout`.
let expired = waiting
.await
.expect("waiting task")
.expect_err("deadline must refuse");
assert_eq!(
expired.error_code(),
ExecutionErrorCode::ExecutionOverloaded,
"queue waiting counts against the published deadline"
);
assert_eq!(expired.stage(), ExecutionStage::Admission);
assert_eq!(expired.retry_after_ms(), Some(250));
assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain);
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"no outbound call on refusal"
);
assert!(store.release(&held).await.unwrap());
}
#[tokio::test]
async fn background_policy_without_a_task_transport_is_typed_unsupported() {
let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
}))
.into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("no Task transport exists in Story 4.1");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionModeUnsupported
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let snapshot = store.snapshot().await.unwrap();
assert_eq!(snapshot.waiting, 0);
assert_eq!(snapshot.executing, 0);
}
#[tokio::test]
async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() {
let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let output = execute(&runtime, &operation, &json!({}))
.await
.expect("adaptive resolves through the published synchronous fallback");
// The response shape is unchanged by admission: an operation without output mapping still
// returns the same mapped output object it returned before Story 4.1.
assert_eq!(output, json!({}));
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn admission_store_outage_fails_closed_before_dispatch() {
let schema_url = crank_test_support::postgres_schema_url("admission_outage").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// The mandatory state disappears after the store was built: every execution must fail closed.
sqlx::query("drop table execution_admission_entries")
.execute(&pool)
.await
.expect("drop the mandatory admission state");
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("a missing mandatory store must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(store.health().await.is_err());
}
/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2).
struct RecordingAdmission {
inner: Arc<PostgresExecutionAdmission>,
requests: Mutex<Vec<AdmissionRequest>>,
permits: AtomicUsize,
releases: AtomicUsize,
}
impl RecordingAdmission {
fn new(inner: Arc<PostgresExecutionAdmission>) -> Self {
Self {
inner,
requests: Mutex::new(Vec::new()),
permits: AtomicUsize::new(0),
releases: AtomicUsize::new(0),
}
}
fn requests(&self) -> Vec<AdmissionRequest> {
self.requests.lock().expect("admission log").clone()
}
fn permits(&self) -> usize {
self.permits.load(Ordering::SeqCst)
}
fn releases(&self) -> usize {
self.releases.load(Ordering::SeqCst)
}
}
#[async_trait]
impl ExecutionAdmission for RecordingAdmission {
async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> {
self.inner.health().await
}
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<crank_core::AdmissionOutcome, crank_core::AdmissionStoreError> {
self.requests
.lock()
.expect("admission log")
.push(request.clone());
let outcome = self.inner.admit(request).await?;
if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) {
self.permits.fetch_add(1, Ordering::SeqCst);
}
Ok(outcome)
}
async fn release(
&self,
permit: &crank_core::AdmissionPermit,
) -> Result<bool, crank_core::AdmissionStoreError> {
let released = self.inner.release(permit).await?;
if released {
self.releases.fetch_add(1, Ordering::SeqCst);
}
Ok(released)
}
async fn cancel(
&self,
ticket_id: u64,
owner_token: &str,
) -> Result<bool, crank_core::AdmissionStoreError> {
self.inner.cancel(ticket_id, owner_token).await
}
async fn snapshot(
&self,
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
self.inner.snapshot().await
}
}
fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation {
let mut source = operation(None);
source.id = OperationId::new("op_admission_confirmation");
source.name = "admission_confirmation".to_owned();
source.version = version;
let Target::Rest(target) = &mut source.target;
target.method = HttpMethod::Delete;
source.execution_config.safety = Some(OperationSafetyPolicy {
class: OperationSafetyClass::Destructive,
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
});
source.into()
}
fn confirmation_runtime(
admission: Arc<RecordingAdmission>,
coordination: Option<Arc<dyn crank_core::CoordinationStateStore>>,
calls: Arc<AtomicUsize>,
) -> RuntimeExecutor {
let store: Arc<dyn ExecutionAdmission> = admission;
let builder = RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current: Arc::new(AtomicUsize::new(0)),
peak: Arc::new(AtomicUsize::new(0)),
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store);
match coordination {
Some(coordination) => builder.with_coordination_store(coordination).build(),
None => builder.build(),
}
}
fn confirmation_context() -> crank_runtime::RuntimeRequestContext {
crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_response_cache_scope("ws_test", "agent_a")
.with_metering_context(
WorkspaceId::new("ws_test"),
None,
InvocationSource::AdminTestRun,
)
}
#[tokio::test]
async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let coordination: Arc<dyn crank_core::CoordinationStateStore> =
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default());
let calls = Arc::new(AtomicUsize::new(0));
let runtime = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::clone(&coordination)),
Arc::clone(&calls),
);
let operation = confirmation_operation(3);
let context = confirmation_context();
let input = json!({ "reason": "remove the stale item" });
// 1. No approval yet: the existing challenge is returned and no execution capacity is taken.
let challenge = execute_in(&runtime, &operation, &input, &context)
.await
.expect_err("an unapproved destructive call must only stage confirmation");
assert_eq!(
challenge.error_code(),
ExecutionErrorCode::ConfirmationRequired
);
let token = challenge
.confirmation()
.expect("expected the confirmation challenge")
.token()
.to_owned();
assert_eq!(
recording.permits(),
0,
"an unapproved request must not take an execution permit"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let requests = recording.requests();
assert_eq!(
requests.len(),
1,
"the one admission snapshot is consulted once"
);
assert_eq!(
requests[0].approval,
ApprovalEvidence::NotGranted,
"admission must see the real approval state, not a fixed NotRequired"
);
assert_eq!(requests[0].scope.operation_version, 3);
let snapshot = recording.snapshot().await.expect("admission snapshot");
assert_eq!((snapshot.executing, snapshot.waiting), (0, 0));
// 2. An approval issued for another published version never opens this execution.
let other_version = confirmation_operation(4);
let foreign = execute_in(
&runtime,
&other_version,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a foreign approval must not authorise this version");
assert_eq!(
foreign.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(
recording.requests().len(),
1,
"an unusable approval is refused before any capacity request"
);
// 3. An approval is bound to its exact input as well.
let mismatched_input = json!({ "reason": "remove another item" });
let mismatched = execute_in(
&runtime,
&operation,
&mismatched_input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a confirmation token must not authorise different input");
assert_eq!(
mismatched.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(recording.requests().len(), 1);
// 4. The matching approval passes admission with truthful evidence and executes exactly once.
let executed = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect("a verified approval executes");
assert!(executed.is_object());
assert_eq!(calls.load(Ordering::SeqCst), 1);
let requests = recording.requests();
assert_eq!(requests.len(), 2);
assert_eq!(requests[1].approval, ApprovalEvidence::Granted);
assert_eq!(requests[1].scope.operation_version, 3);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.releases(),
1,
"the permit is released after execution"
);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
// 5. Replay of the consumed confirmation is refused before capacity and without a side effect.
let replay = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token),
)
.await
.expect_err("a confirmation token is single-use");
assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.requests().len(),
2,
"a replayed approval never reaches admission"
);
// 6. The trusted approved-recovery path still passes the same gate with real evidence.
let approved = execute_in(
&runtime,
&operation,
&json!({ "reason": "approved by an operator" }),
&context.with_approval_granted(),
)
.await;
assert!(approved.is_ok());
assert_eq!(calls.load(Ordering::SeqCst), 2);
let requests = recording.requests();
assert_eq!(requests.len(), 3);
assert_eq!(requests[2].approval, ApprovalEvidence::Granted);
assert_eq!(recording.permits(), 2);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
}
#[tokio::test]
async fn mandatory_safety_store_outage_fails_closed_before_admission() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let calls = Arc::new(AtomicUsize::new(0));
// Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail
// closed instead of reserving capacity and hoping for an in-memory fallback.
let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls));
let operation = confirmation_operation(3);
let failure = execute_in(
&runtime,
&operation,
&json!({ "reason": "no store" }),
&confirmation_context(),
)
.await
.expect_err("a mandatory safety store outage must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(recording.permits(), 0);
let after_missing_store = recording.requests().len();
// The same Tool with a claimed approval and a *failing* safety store never reaches the capacity
// store at all: the approval claim stays unverified, so no permit and no outbound call exist.
let failing = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::new(UnavailableSafetyStore)),
Arc::clone(&calls),
);
let outage = execute_in(
&failing,
&operation,
&json!({ "reason": "failing store" }),
&confirmation_context().with_confirmation_token("ct_unavailable"),
)
.await
.expect_err("an unreachable safety store must refuse the claimed approval");
assert_eq!(
outage.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(
recording.requests().len(),
after_missing_store,
"an unverified approval never becomes a capacity request"
);
assert_eq!(recording.permits(), 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
/// Coordination store whose mandatory backend is down: every operation fails closed.
struct UnavailableSafetyStore;
impl UnavailableSafetyStore {
fn error() -> crank_core::CacheStoreError {
crank_core::CacheStoreError::Unavailable {
message: "test backend unavailable".to_owned(),
}
}
}
#[async_trait]
impl crank_core::CoordinationStateStore for UnavailableSafetyStore {
async fn get_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn put_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn delete_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn take_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn reserve_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<crank_core::CoordinationStateReservation, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn compare_and_set_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_expected: &crank_core::CoordinationStateValue,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<bool, crank_core::CacheStoreError> {
Err(Self::error())
}
}
+1
View File
@@ -1,6 +1,7 @@
mod integration {
mod catalog_access;
mod common;
mod durable_tasks;
mod era_isolation;
mod execution_certainty;
mod execution_stages;
@@ -621,6 +621,7 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation<Schema, Ma
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -0,0 +1,591 @@
//! ATDD Story 4.2 (FR-22/FR-23): the durable Task application entry over a real published catalog
//! and a real PostgreSQL schema.
//!
//! These oracles drive the production application entry (`TaskApplication`) that the mcp-server
//! composition root constructs: a verified machine credential is checked against the published Tool,
//! the Task is accepted durably, a freshly constructed application instance reads the same identity
//! and pins, replay/version conflicts never rebind, foreign callers receive no metadata, and a missing
//! Task store fails closed and reports readiness as not ready. The public `tools/call` wire stays
//! synchronous/unsupported: this suite never calls it.
use super::common;
use std::time::Duration;
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
use crank_core::{
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState,
ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
};
use crank_runtime::{InMemoryCoordinationStateStore, TaskService};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "report_gen";
const KEY: &str = "retry-key-1";
fn operation(policy: ToolExecutionPolicy) -> crank_registry::RegistryOperation {
let mut operation = common::test_operation("https://api.example.com", TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
operation
}
fn background_policy() -> ToolExecutionPolicy {
ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
}
}
async fn seed_tool(registry: &PostgresRegistry, policy: ToolExecutionPolicy) {
let operation = operation(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(registry, &operation.id, 1).await;
// A second, explicitly synchronous Tool of the same Agent: the durable entry must refuse it.
let synchronous = common::test_operation("https://api.example.com", "report_sync");
registry
.create_operation(&common::test_workspace_id(), &synchronous, Some("alice"))
.await
.unwrap();
publish(registry, &synchronous.id, 1).await;
let binding = AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
};
let synchronous_binding = AgentOperationBinding {
operation_id: synchronous.id.clone(),
operation_version: 1,
tool_name: synchronous.name.clone(),
tool_title: synchronous.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
};
common::publish_agent_with_bindings(registry, AGENT_SLUG, vec![binding, synchronous_binding])
.await;
}
async fn publish(
registry: &PostgresRegistry,
operation_id: &crank_core::OperationId,
version: u32,
) {
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id,
version,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
}
async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
let mut next = operation(background_policy());
next.version = version;
next.status = crank_core::OperationStatus::Draft;
next.published_at = None;
registry
.create_version(CreateVersionRequest {
workspace_id: &common::test_workspace_id(),
snapshot: &next,
change_note: Some("next publication"),
created_by: Some("alice"),
})
.await
.unwrap();
publish(registry, &next.id, version).await;
// The agent catalog must bind the published Tool version for the new publication to be addressed.
let agent_id = common::test_agent_id(AGENT_SLUG);
let draft = crank_core::AgentVersion {
agent_id: agent_id.clone(),
version: 2,
status: crank_core::AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: Default::default(),
created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
};
registry
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
workspace_id: &common::test_workspace_id(),
agent_id: &agent_id,
version: &draft,
bindings: &[AgentOperationBinding {
operation_id: next.id.clone(),
operation_version: version,
tool_name: next.name.clone(),
tool_title: next.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
registry
.publish_agent(crank_registry::PublishAgentRequest {
workspace_id: &common::test_workspace_id(),
agent_id: &agent_id,
version: 2,
published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
}
fn task_service(pool: PgPool) -> TaskService {
let store = PostgresTaskStore::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration");
TaskService::new(std::sync::Arc::new(store))
}
fn application(registry: &PostgresRegistry) -> TaskApplication {
TaskApplication::new(
registry.clone(),
Duration::from_millis(0),
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
task_service(registry.pool().clone()),
)
}
async fn principal(
registry: &PostgresRegistry,
name: &str,
scopes: &[PlatformApiKeyScope],
) -> TaskPrincipal {
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
// The production machine-access path resolves the presented secret against the active platform
// key of the addressed Agent and derives the scope identity from that verified record.
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("an active platform key resolves");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
fn unsampled_correlation() -> CorrelationContext {
CorrelationContext::new(
crank_core::RequestId::generate(),
crank_core::TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
)
}
async fn counts(pool: &PgPool) -> (i64, i64) {
let row = sqlx::query(
"select
(select count(*)::bigint from tasks) as tasks,
(select count(*)::bigint from execution_admission_entries where task_id is not null)
as intents",
)
.fetch_one(pool)
.await
.unwrap();
(row.get("tasks"), row.get("intents"))
}
#[tokio::test]
async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-accept",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let app = application(&registry);
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&correlation,
)
.await
.expect("acceptance over the real publication");
assert_eq!(acceptance.state, TaskState::Queued);
assert!(!acceptance.replayed);
assert_eq!(acceptance.operation_version, 1);
assert_eq!(acceptance.support.request_id, *correlation.request_id());
assert_eq!(acceptance.support.trace_id, *correlation.trace_id());
assert!(!acceptance.support.trace_sampled);
assert_eq!(
uuid::Uuid::parse_str(&acceptance.task_id.to_string())
.unwrap()
.get_version_num(),
7,
"the durable Task id is UUIDv7"
);
// Task, dispatch intent and queue slot committed together.
assert_eq!(counts(registry.pool()).await, (1, 1));
// A freshly constructed application + service reads the same durable identity and pins.
let reader = application(&registry);
let record = reader
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
.unwrap();
assert_eq!(record.state, TaskState::Queued);
assert_eq!(record.pins.operation_version, 1);
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
assert_eq!(record.support.request_id, acceptance.support.request_id);
assert_eq!(record.support.traceparent, acceptance.support.traceparent);
assert!(!record.support.trace_sampled);
let outcome = reader
.result(&caller, TOOL_NAME, acceptance.task_id)
.await
.unwrap();
assert!(outcome.result.is_none() && outcome.error.is_none());
// Replay of the same scoped key and input returns the accepted Task instead of a new one.
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(replay.task_id, acceptance.task_id);
assert!(replay.replayed);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn a_new_publication_never_rebinds_an_accepted_task() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-rebind",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let accepted = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
let before = app
.status(&caller, TOOL_NAME, accepted.task_id)
.await
.unwrap();
publish_tool_version(&registry, 2).await;
// The same retry keeps the accepted Task and its v1 pin; it never becomes a v2 Task.
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(replay.task_id, accepted.task_id);
assert_eq!(replay.operation_version, 1);
let after = app
.status(&caller, TOOL_NAME, accepted.task_id)
.await
.unwrap();
assert_eq!(after.pins.operation_version, 1);
assert_eq!(after.support.request_id, before.support.request_id);
assert_eq!(counts(registry.pool()).await, (1, 1));
// An explicit other exact version is a typed conflict, not a rebind.
let explicit = app
.accept_background(
&caller,
TOOL_NAME,
Some(2),
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("a different explicit pin cannot rebind the accepted Task");
assert_eq!(explicit, TaskApplicationError::VersionUnavailable);
// The original explicit pin still resolves to the accepted Task.
let pinned = app
.accept_background(
&caller,
TOOL_NAME,
Some(1),
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(pinned.task_id, accepted.task_id);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let owner = principal(
&registry,
"task-owner",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let accepted = app
.accept_background(
&owner,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
// Positive control: the owner reads its Task.
assert!(
app.status(&owner, TOOL_NAME, accepted.task_id)
.await
.is_ok()
);
// A second verified key of the same Agent is another authorization scope: no payload, no metadata.
let foreign_scope =
principal(&registry, "task-other-scope", &[PlatformApiKeyScope::Read]).await;
match app
.status(&foreign_scope, TOOL_NAME, accepted.task_id)
.await
{
Err(TaskApplicationError::NotFound) => {}
other => panic!("foreign scope must be not-found without metadata, got {other:?}"),
}
assert!(matches!(
app.result(&foreign_scope, TOOL_NAME, accepted.task_id)
.await,
Err(TaskApplicationError::NotFound)
));
// A synchronous publication is not accepted as background work at all.
let synchronous_app = application(&registry);
let refused = synchronous_app
.accept_background(
&owner,
"report_sync",
None,
"sync-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("a synchronous policy has no durable acceptance");
assert_eq!(refused, TaskApplicationError::NotBackground);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let app = application(&registry);
let admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_1".to_owned(),
};
let accepted = app
.accept_background(
&admin,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("an authenticated administrator accepts durable work");
assert_eq!(accepted.state, TaskState::Queued);
// Positive control: the same administrator reads its own Task.
assert!(
app.status(&admin, TOOL_NAME, accepted.task_id)
.await
.is_ok()
);
// An administrator is never a wildcard owner: another admin user and the owning Agent of the
// same workspace both see nothing.
let other_admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_2".to_owned(),
};
assert_eq!(
app.status(&other_admin, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
assert_eq!(
app.result(&other_admin, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
let agent = principal(&registry, "task-admin-cross", &[PlatformApiKeyScope::Read]).await;
assert_eq!(
app.status(&agent, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
// An admin identity addressed to another workspace cannot resolve the publication at all.
let foreign_workspace = TaskPrincipal::Admin {
workspace_slug: "other-workspace".to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_1".to_owned(),
};
assert_eq!(
app.accept_background(
&foreign_workspace,
TOOL_NAME,
None,
"foreign-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
let database_url = crank_test_support::postgres_schema_url("durable_tasks_outage").await;
let pool = PgPool::connect(&database_url).await.unwrap();
// No migration: the durable Task state does not exist.
let outage_registry = PostgresRegistry::connect(&database_url).await;
assert!(
outage_registry.is_err(),
"an unmigrated registry cannot be used"
);
crank_registry::MigrationAuthority::apply(&pool)
.await
.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-outage",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
assert!(app.health().await.is_ok());
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
sqlx::query("drop table tasks cascade")
.execute(&pool)
.await
.unwrap();
assert!(app.health().await.is_err());
let error = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("no durable Task store, no acceptance");
assert!(matches!(
error,
TaskApplicationError::Task(crank_core::TaskError::Unavailable { .. })
));
// Readiness of the app that wires the same store reports the durable Task dependency as not ready.
let router = common::build_test_app(registry.clone(), Duration::from_millis(0), None);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
});
let response = reqwest::get(format!("http://{address}/ready"))
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE);
let body: serde_json::Value = response.json().await.unwrap();
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
server.abort();
}
+2
View File
@@ -1036,6 +1036,7 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Secret name is required.',
'wizard.error.secret_value_required': 'Secret value is required.',
'wizard.error.save_upstream_first': 'Save the API host after creating a new authorization profile before continuing.',
'wizard.error.execution_policy_invalid': 'Execution policy is invalid: mode, deadline from 1 to 3600000 ms and no-Tasks fallback are required.',
'wizard.toast.auth_profile_created_title': 'Authorization profile created',
'wizard.toast.auth_profile_created_body': 'New authorization profile is ready and selected.',
'wizard.toast.auth_profile_error_title': 'Authorization profile creation failed',
@@ -2353,6 +2354,7 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Нужно указать имя секрета.',
'wizard.error.secret_value_required': 'Нужно указать значение секрета.',
'wizard.error.save_upstream_first': 'Сначала сохраните API-хост после создания нового профиля авторизации.',
'wizard.error.execution_policy_invalid': 'Политика выполнения некорректна: нужны режим, deadline от 1 до 3600000 мс и поведение без Tasks.',
'wizard.toast.auth_profile_created_title': 'Профиль авторизации создан',
'wizard.toast.auth_profile_created_body': 'Новый профиль авторизации готов и уже выбран.',
'wizard.toast.auth_profile_error_title': 'Не удалось создать профиль авторизации',
+30
View File
@@ -214,10 +214,33 @@ function parseExecutionConfig(text) {
auth_profile_ref: authProfileRef,
headers: headers,
};
var executionPolicy = parseExecutionPolicy(value.execution_policy);
if (executionPolicy) {
config.execution_policy = executionPolicy;
}
return config;
}
// Story 4.1: the published execution policy must survive a wizard save. The editor is a free-form
// document, so unknown/legacy shapes stay untouched: only a fully valid policy is projected, and an
// explicit invalid value is reported instead of being silently replaced by defaults.
function parseExecutionPolicy(raw) {
if (raw === undefined || raw === null) return null;
if (typeof raw !== 'object' || Array.isArray(raw)) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
var mode = raw.mode === undefined ? 'synchronous' : raw.mode;
var fallback = raw.no_tasks_fallback === undefined ? 'synchronous' : raw.no_tasks_fallback;
var deadline = Number(raw.deadline_ms);
if (['synchronous', 'background', 'adaptive'].indexOf(mode) < 0
|| ['synchronous', 'unsupported'].indexOf(fallback) < 0
|| !Number.isInteger(deadline) || deadline < 1 || deadline > 3600000) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback };
}
function currentUpstream() {
var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; });
if (selected) return selected;
@@ -415,6 +438,13 @@ function executionConfigToEditorValue(config) {
var value = {
timeout_ms: config.timeout_ms,
};
if (config.execution_policy) {
value.execution_policy = {
mode: config.execution_policy.mode,
deadline_ms: config.execution_policy.deadline_ms,
no_tasks_fallback: config.execution_policy.no_tasks_fallback,
};
}
if (config.retry_policy) {
value.retry = { max_attempts: config.retry_policy.max_attempts };
}
@@ -0,0 +1,139 @@
const { test, expect } = require('@playwright/test');
const { getCurrentWorkspace, login } = require('./helpers');
// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a
// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit
// invalid value must be reported instead of being silently replaced by defaults.
function draftSnapshot(operationId) {
return {
id: operationId,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
target: {
kind: 'rest',
base_url: 'https://api.example.test',
method: 'GET',
path_template: '/rates',
static_headers: {},
},
input_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
output_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
input_mapping: { rules: [] },
output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] },
execution_config: {
timeout_ms: 10000,
retry_policy: null,
auth_profile_ref: null,
headers: {},
approval_policy: null,
execution_policy: {
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
},
},
tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] },
wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] },
};
}
test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => {
await login(page);
const workspace = await getCurrentWorkspace(page);
const operationId = 'policy_rates_41';
const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`;
const savedPayloads = [];
await page.route(operationPath, async (route) => {
if (route.request().method() === 'PATCH') {
savedPayloads.push(route.request().postDataJSON());
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }),
});
return;
}
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
id: operationId,
workspace_id: workspace.id,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
status: 'draft',
current_draft_version: 1,
draft_version_ref: { version: 1, status: 'draft' },
published_version_ref: null,
agent_refs: [],
}),
});
});
await page.route(`${operationPath}/versions/*`, async (route) => {
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
operation_id: operationId,
workspace_id: workspace.id,
version: 1,
status: 'draft',
snapshot: draftSnapshot(operationId),
}),
});
});
await page.route('**/operations/analyze-quality', async (route) => {
await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) });
});
page.on('dialog', (dialog) => dialog.accept());
await page.goto(`/wizard/?mode=edit&operationId=${operationId}`);
await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates');
const editorValue = await page.locator('#tool-exec-config').inputValue();
expect(editorValue).toContain('execution_policy');
expect(editorValue).toContain('adaptive');
expect(editorValue).toContain('45000');
expect(editorValue).toContain('unsupported');
// A real edit still saves the pinned policy unchanged.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(4));
await page.locator('#tool-display-name').fill('Policy rates edited');
await page.locator('.btn-save-draft').click();
await expect.poll(() => savedPayloads.length).toBe(1);
expect(savedPayloads[0].execution_config.execution_policy).toEqual({
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
});
// An explicit invalid policy is refused instead of being silently rewritten to defaults.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
await expect(page.locator('#tool-exec-config')).toBeVisible();
await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n');
await page.locator('.btn-save-draft').click();
await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/);
expect(savedPayloads.length).toBe(1);
});
+103 -14
View File
@@ -51,12 +51,14 @@ mod auth_resolution;
mod dispatch;
mod endpoint;
mod legacy_dispatch;
mod tasks;
use self::approvals::{
approve_request, deny_request, get_approval_request, list_pending_approvals,
};
pub(super) use self::auth_resolution::resolve_operation_auth;
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
// through `super`, and the plain lib build does not use them (same pattern as
@@ -92,6 +94,9 @@ pub(super) struct AppState {
pub(super) signal_streams: Arc<crate::protocol::StreamRegistry>,
/// Story 3.4: the bounded table of the `tools/call` requests a legacy client can still cancel.
pub(super) in_flight: Arc<self::lifecycle::InFlightCalls>,
/// Story 4.2: durable background Tasks over the same registry pool. The public wire stays
/// synchronous/unsupported; readiness depends on this mandatory store.
tasks: Arc<TaskApplication>,
}
#[derive(Clone)]
@@ -137,6 +142,7 @@ pub fn build_app(
DEFAULT_MAX_CONCURRENT_SESSIONS,
false,
None,
None,
)
}
@@ -192,6 +198,7 @@ pub fn build_app_with_background_workers_and_limits(
max_concurrent_sessions,
true,
None,
None,
)
}
@@ -227,6 +234,41 @@ pub fn build_app_with_artifacts(
max_concurrent_sessions,
true,
Some(artifact_store),
None,
)
}
/// Production launcher entry point with the durable Task application entry constructed by the
/// process composition root from the same registry pool and admission limits as the runtime.
#[allow(clippy::too_many_arguments)]
pub fn build_app_with_artifacts_and_tasks(
registry: PostgresRegistry,
refresh_interval: Duration,
public_base_url: Option<String>,
secret_crypto: SecretCrypto,
runtime: RuntimeExecutor,
api_rate_limiter: RequestRateLimiter,
coordination_store: Arc<dyn CoordinationStateStore>,
sessions: SharedSessionStore,
credential_verifier: SharedMachineCredentialVerifier,
max_concurrent_sessions: usize,
artifact_store: crank_artifacts::ArtifactStore,
tasks: Arc<TaskApplication>,
) -> Router {
build_app_inner(
registry,
refresh_interval,
public_base_url,
secret_crypto,
runtime,
api_rate_limiter,
coordination_store,
sessions,
credential_verifier,
max_concurrent_sessions,
true,
Some(artifact_store),
Some(tasks),
)
}
@@ -244,8 +286,23 @@ fn build_app_inner(
max_concurrent_sessions: usize,
start_background_workers: bool,
artifact_store: Option<crank_artifacts::ArtifactStore>,
tasks: Option<Arc<TaskApplication>>,
) -> Router {
let session_metrics = ActiveSessionMetrics::start(Arc::clone(&sessions));
let tasks = tasks.unwrap_or_else(|| {
let store = crank_registry::PostgresTaskStore::new(
registry.pool().clone(),
crank_registry::AdmissionLimits::default(),
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("default admission limits and scheduler weights are valid");
Arc::new(TaskApplication::new(
registry.clone(),
refresh_interval,
Arc::clone(&coordination_store),
crank_runtime::TaskService::new(Arc::new(store)),
))
});
let state = Arc::new(AppState {
registry: registry.clone(),
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
@@ -263,6 +320,7 @@ fn build_app_inner(
in_flight: Arc::new(lifecycle::InFlightCalls::new(
lifecycle::MAX_IN_FLIGHT_CALLS,
)),
tasks,
});
if start_background_workers {
spawn_approval_recovery(Arc::clone(&state));
@@ -311,24 +369,55 @@ async fn health() -> Json<Value> {
}
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
match state.registry.ping().await {
Ok(()) => Json(json!({
let postgres_ready = state.registry.ping().await.is_ok();
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
// reflect it rather than reporting ready on a bare `select 1`.
let admission_ready = state.registry.execution_admission_health().await.is_ok();
// Story 4.2: the durable Task store is mandatory for a durable acceptance acknowledgement.
let durable_tasks_ready = state.tasks.health().await.is_ok();
if postgres_ready && admission_ready && durable_tasks_ready {
return Json(json!({
"service": "mcp-server",
"status": "ready",
"checks": { "postgres": "ready" }
"checks": {
"postgres": "ready",
"execution_admission": "ready",
"durable_tasks": "ready"
}
}))
.into_response(),
Err(_) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": "database is unavailable"
})),
)
.into_response(),
.into_response();
}
let mut checks = serde_json::Map::new();
checks.insert(
"postgres".to_owned(),
json!(if postgres_ready { "ready" } else { "not_ready" }),
);
checks.insert(
"execution_admission".to_owned(),
json!(if admission_ready {
"ready"
} else {
"not_ready"
}),
);
checks.insert(
"durable_tasks".to_owned(),
json!(if durable_tasks_ready {
"ready"
} else {
"not_ready"
}),
);
(
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": checks,
"error": "a mandatory dependency is unavailable"
})),
)
.into_response()
}
#[cfg(test)]
+295
View File
@@ -0,0 +1,295 @@
//! Application entry of durable background Tasks (Story 4.2, FR-22/FR-23).
//!
//! The entry takes an already authenticated principal (a verified machine credential, or an
//! administrator identity verified by the calling admin surface), resolves the exact published Tool,
//! repeats the caller's authorization, pins the Published Version/policy/deadline and delegates the
//! durable acceptance to the runtime Task service. The repository owns the transaction, so the
//! returned Task ID exists only after Task + dispatch intent + queue slot committed.
//!
//! Deliberately not wired: the public MCP `tools/call`/Tasks wire (Story 4.7) and the worker/leases
//! (Story 4.3). The existing synchronous/unsupported `tools/call` path is unchanged, and this entry
//! never spawns an ephemeral execution future.
use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord,
ToolExecutionMode,
};
use crank_registry::{PostgresRegistry, PublishedAgentTool};
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
use serde_json::Value;
use crate::access::credential_allows_security_level;
use crate::catalog::PublishedToolCatalog;
/// Authenticated principal of one Task operation. The authorization scope identity is derived here
/// from the verified credential/admin identity, never from a caller-supplied request string.
#[derive(Clone, Debug)]
pub enum TaskPrincipal {
Agent {
workspace_slug: String,
agent_slug: String,
credential: crank_core::VerifiedMachineCredential,
},
/// Administrator identity that the admin surface already authenticated. It is never a wildcard:
/// owner, workspace and authorization scope all participate in the stored predicate.
Admin {
workspace_slug: String,
agent_slug: String,
user_id: String,
},
}
impl TaskPrincipal {
fn workspace_slug(&self) -> &str {
match self {
Self::Agent { workspace_slug, .. } | Self::Admin { workspace_slug, .. } => {
workspace_slug
}
}
}
fn agent_slug(&self) -> &str {
match self {
Self::Agent { agent_slug, .. } | Self::Admin { agent_slug, .. } => agent_slug,
}
}
fn owner(&self, tool: &PublishedAgentTool) -> (TaskOwnerKind, String) {
match self {
Self::Agent { .. } => (TaskOwnerKind::Agent, tool.agent_id.as_str().to_owned()),
Self::Admin { user_id, .. } => (TaskOwnerKind::Admin, user_id.clone()),
}
}
/// Bounded server-derived authorization scope identity of this principal.
fn authorization_scope(&self) -> String {
let identity = match self {
Self::Agent { credential, .. } => {
let mut scopes = credential
.scopes
.iter()
.map(|scope| format!("{scope:?}"))
.collect::<Vec<_>>();
scopes.sort();
format!(
"key:{}|mode:{:?}|level:{:?}|scopes:{}",
credential
.platform_api_key_id
.as_ref()
.map(|id| id.as_str().to_owned())
.unwrap_or_else(|| "-".to_owned()),
credential.machine_access_mode,
credential.max_security_level,
scopes.join(",")
)
}
Self::Admin { user_id, .. } => format!("admin:{user_id}"),
};
identity.chars().take(160).collect()
}
}
/// Typed refusal of the application entry. `NotFound`/`Denied` never reveal whether a foreign Task
/// exists, and no variant carries Task payload or fingerprint.
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum TaskApplicationError {
#[error("task request was not found")]
NotFound,
#[error("task request is not authorized")]
Denied,
#[error("the published Tool policy does not accept durable background work")]
NotBackground,
#[error("the requested published Tool version is not available")]
VersionUnavailable,
#[error("this Tool requires a verified approval before acceptance")]
ApprovalRequired,
#[error(transparent)]
Task(#[from] TaskError),
}
#[derive(Clone)]
pub struct TaskApplication {
catalog: PublishedToolCatalog,
service: TaskService,
}
impl TaskApplication {
/// Production construction of the application entry over the process registry, the published
/// catalog and the durable Task service.
pub fn new(
registry: PostgresRegistry,
refresh_interval: Duration,
coordination_store: Arc<dyn crank_core::CoordinationStateStore>,
service: TaskService,
) -> Self {
Self {
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
service,
}
}
/// Readiness of the durable Task store this entry depends on.
pub async fn health(&self) -> Result<(), TaskError> {
self.service.health().await
}
/// Accepts one background request for an already authenticated principal.
///
/// `expected_version` is the caller's explicit Published Version pin. Without it the current
/// publication is pinned for new work and a retry of the same scoped key returns the stored Task
/// even after a newer publication. An explicit pin may only retry the Task accepted with that
/// exact version: a different pin is a typed conflict, never a rebind.
pub async fn accept_background(
&self,
principal: &TaskPrincipal,
tool_name: &str,
expected_version: Option<u32>,
scoped_key: &str,
input: &Value,
correlation: &CorrelationContext,
) -> Result<TaskAcceptance, TaskApplicationError> {
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
let current_version = tool.operation.version;
let Some(policy) = tool.operation.execution_config.execution_policy else {
return Err(TaskApplicationError::NotBackground);
};
if policy.mode == ToolExecutionMode::Synchronous {
return Err(TaskApplicationError::NotBackground);
}
if tool
.operation
.execution_config
.approval_policy
.as_ref()
.is_some_and(|approval| approval.required)
{
// Without a verified, non-consuming approval reference this entry fails closed instead
// of accepting work a worker could never safely execute.
return Err(TaskApplicationError::ApprovalRequired);
}
let draft = |operation_version: u32| {
let (owner_kind, owner_id) = principal.owner(&tool);
TaskAcceptDraft {
workspace_id: tool.workspace_id.as_str().to_owned(),
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
operation_id: tool.operation.id.as_str().to_owned(),
operation_version,
policy_mode: policy.mode,
scoped_key: scoped_key.to_owned(),
input: input.clone(),
correlation: correlation.clone(),
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
retention_ms: TaskService::default_retention_ms(),
approval_reference: None,
}
};
let Some(requested) = expected_version else {
return Ok(self.service.accept(draft(current_version)).await?);
};
let draft = draft(requested);
match self.service.lookup(&draft).await? {
Some(record) => {
if record.input_fingerprint != TaskService::input_fingerprint(input)? {
return Err(TaskApplicationError::Task(TaskError::FingerprintConflict));
}
if record.pins.operation_version != requested {
return Err(TaskApplicationError::VersionUnavailable);
}
Ok(TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support,
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
})
}
None if requested == current_version => Ok(self.service.accept(draft).await?),
// A new Task can only be accepted against the currently published version.
None => Err(TaskApplicationError::VersionUnavailable),
}
}
/// Owner-scoped status read of one Task of the addressed published Tool.
pub async fn status(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<TaskRecord, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.status(&access, task_id).await {
Ok(record) => Ok(record),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped result projection of one Task of the addressed published Tool.
pub async fn result(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<TaskOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.result(&access, task_id).await {
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
async fn task_access(
&self,
principal: &TaskPrincipal,
tool_name: &str,
) -> Result<TaskAccess, TaskApplicationError> {
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
let (owner_kind, owner_id) = principal.owner(&tool);
Ok(TaskAccess {
workspace_id: tool.workspace_id.as_str().to_owned(),
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
})
}
/// Resolves the addressed Tool of the principal's published catalog and repeats its authorization
/// check. A foreign workspace, an unknown Tool and an insufficient security level are typed
/// refusals without metadata.
async fn resolve_authorized_tool(
&self,
principal: &TaskPrincipal,
tool_name: &str,
) -> Result<PublishedAgentTool, TaskApplicationError> {
let catalog = self
.catalog
.get_catalog(principal.workspace_slug(), principal.agent_slug())
.await
.map_err(|_| TaskApplicationError::NotFound)?;
let tool = catalog
.tools
.into_iter()
.find(|tool| tool.tool_name == tool_name)
.ok_or(TaskApplicationError::NotFound)?;
if let TaskPrincipal::Admin { workspace_slug, .. } = principal
&& tool.workspace_slug != *workspace_slug
{
return Err(TaskApplicationError::Denied);
}
if let TaskPrincipal::Agent { credential, .. } = principal
&& !credential_allows_security_level(credential, tool.operation.security_level)
{
return Err(TaskApplicationError::Denied);
}
Ok(tool)
}
}
@@ -550,6 +550,7 @@ pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+2 -1
View File
@@ -16,6 +16,7 @@ mod tool_search;
mod transport;
pub use app::{
build_app, build_app_with_artifacts, build_app_with_background_workers,
TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts,
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
build_app_with_background_workers_and_limits,
};
@@ -136,6 +136,7 @@ fn published_tool(name: &str, title: &str, description: &str, version: u32) -> P
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -284,6 +284,7 @@ fn operation() -> RegistryOperation {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+531
View File
@@ -0,0 +1,531 @@
//! Typed execution-admission seam (Story 4.1, FR-21/FR-24).
//!
//! Admission is the single gate in front of every real Tool execution: it decides whether the
//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide
//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives
//! in PostgreSQL (`crank-registry`), so several processes share one budget.
//!
//! Deliberately absent from this contract: credentials, connection or protocol revision (they must
//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2).
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use crate::{NoTasksFallback, ToolExecutionMode};
pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128;
pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64;
pub const MAX_RETRY_AFTER_MS: u64 = 60_000;
/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for
/// the store and every runtime so the client contract stays stable.
pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250;
/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms).
pub fn current_unix_ms() -> i64 {
i64::try_from(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_millis())
.unwrap_or_default(),
)
.unwrap_or(i64::MAX)
}
/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin
/// bucket that is still bounded by the global limit.
pub const ADMIN_ORIGIN_KEY: &str = "admin";
/// Exact execution identity pinned at admission time.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AdmissionScope {
pub workspace_id: String,
/// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution.
pub agent_key: String,
pub operation_id: String,
/// Quota keys exclude the version; the version is pinned for execution semantics only.
pub operation_version: u32,
}
impl AdmissionScope {
/// Quota key of the Agent bucket: `(workspace, agent)`.
pub fn agent_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.agent_key)
}
/// Quota key of the Tool bucket: `(workspace, operation)` without version.
pub fn tool_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.operation_id)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AdmissionMode {
/// The caller waits for a scoped permit and dispatches inside it.
Synchronous,
/// The store commits a durable enqueue decision and reserves queue capacity.
Background,
}
impl AdmissionMode {
pub const fn as_str(self) -> &'static str {
match self {
Self::Synchronous => "synchronous",
Self::Background => "background",
}
}
}
/// Non-consuming approval evidence gathered by the runtime before admission.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ApprovalEvidence {
/// The operation does not require confirmation.
NotRequired,
/// A verified approval exists for this exact scope and input.
Granted,
/// Confirmation is required and no verified approval exists yet.
NotGranted,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionRequest {
pub scope: AdmissionScope,
pub mode: AdmissionMode,
/// Published policy mode that produced this request.
pub policy_mode: ToolExecutionMode,
pub no_tasks_fallback: NoTasksFallback,
pub approval: ApprovalEvidence,
/// Bounded request identity used only to correlate one admission entry.
pub fingerprint: String,
/// Absolute deadline (unix milliseconds), already including queue waiting time.
pub expires_at_unix_ms: i64,
}
impl AdmissionRequest {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if self.fingerprint.is_empty()
|| self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES
|| self
.fingerprint
.bytes()
.any(|byte| byte.is_ascii_control() || byte == b' ')
{
return Err(AdmissionStoreError::invalid(
"admission fingerprint is outside its allowed bounds",
));
}
if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission workspace is invalid",
));
}
if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 {
return Err(AdmissionStoreError::invalid("admission agent is invalid"));
}
if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission operation is invalid",
));
}
if self.scope.operation_version == 0 {
return Err(AdmissionStoreError::invalid(
"admission operation version must be positive",
));
}
if self.expires_at_unix_ms <= 0 {
return Err(AdmissionStoreError::invalid(
"admission deadline is invalid",
));
}
Ok(())
}
}
/// Scoped permit. Held for the whole local execution future (including retries) and released
/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionPermit {
pub ticket_id: u64,
pub owner_token: String,
pub scope: AdmissionScope,
pub mode: AdmissionMode,
pub policy_mode: ToolExecutionMode,
}
/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics.
/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionDecision {
pub ticket_id: u64,
pub scope: AdmissionScope,
pub expires_at_unix_ms: i64,
pub policy_mode: ToolExecutionMode,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AdmissionOutcome {
Permitted(AdmissionPermit),
Enqueued(AdmissionDecision),
/// Capacity or queue bounds were exhausted before any unbounded allocation.
Overloaded {
retry_after_ms: u64,
},
/// The queue wait reached the absolute deadline; nothing was dispatched.
DeadlineElapsed,
/// Confirmation is mandatory for this operation and no verified approval exists.
ApprovalRequired,
/// The published policy does not advertise durable Tasks and refuses the fallback.
Unsupported,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionQueueSnapshot {
pub waiting: u64,
pub executing: u64,
/// Age of the oldest waiting request in the authoritative queue, in milliseconds.
pub oldest_waiting_ms: Option<u64>,
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum AdmissionStoreError {
/// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed.
#[error("execution admission store is unavailable: {message}")]
Unavailable { message: String },
/// The store rejected a malformed admission request.
#[error("execution admission request is invalid: {message}")]
Invalid { message: String },
}
impl AdmissionStoreError {
pub fn unavailable(message: impl Into<String>) -> Self {
Self::Unavailable {
message: message.into(),
}
}
pub fn invalid(message: impl Into<String>) -> Self {
Self::Invalid {
message: message.into(),
}
}
}
#[async_trait]
pub trait ExecutionAdmission: Send + Sync {
/// Proves the mandatory admission state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), AdmissionStoreError>;
/// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue
/// decision for a background request, or returns a typed refusal.
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<AdmissionOutcome, AdmissionStoreError>;
/// Releases the caller's own permit. Idempotent: a repeated release returns `false`.
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError>;
/// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket.
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError>;
/// Bounded authoritative queue readings for low-cardinality metrics.
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError>;
}
/// One waiting entry as seen by the fairness scheduler.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionQueueEntry {
pub ticket_id: u64,
/// Authoritative insertion order inside `(Agent, Tool)`; never rewritten.
pub sequence: u64,
pub agent_key: String,
pub operation_id: String,
}
/// Weighted round-robin state persisted with the authoritative queue.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionCursor {
pub agent: Option<String>,
pub agent_credit: u32,
pub tool: Option<String>,
pub tool_credit: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionSchedulerConfig {
pub agent_weight: u32,
pub tool_weight: u32,
}
impl Default for AdmissionSchedulerConfig {
fn default() -> Self {
Self {
agent_weight: 1,
tool_weight: 1,
}
}
}
impl AdmissionSchedulerConfig {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) {
return Err(AdmissionStoreError::invalid(
"admission weights must be between 1 and 64",
));
}
Ok(())
}
}
/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the
/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its
/// turn instead of being skipped by newer arrivals.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionSelection {
pub index: usize,
pub cursor: AdmissionCursor,
}
/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside
/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit.
pub fn select_next_grant(
queue: &[AdmissionQueueEntry],
cursor: &AdmissionCursor,
config: AdmissionSchedulerConfig,
) -> Option<AdmissionSelection> {
if queue.is_empty() {
return None;
}
let agent_weight = config.agent_weight.max(1);
let tool_weight = config.tool_weight.max(1);
let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect();
agents.sort_unstable();
agents.dedup();
let mut next = cursor.clone();
let agent = match cursor.agent.as_deref() {
Some(current) if cursor.agent_credit > 0 && agents.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.agent.as_deref() {
None => agents[0].to_owned(),
Some(current) => agents
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(agents[0])
.to_owned(),
};
if cursor.agent.as_deref() != Some(chosen.as_str()) {
next.tool = None;
next.tool_credit = 0;
}
next.agent = Some(chosen.clone());
next.agent_credit = agent_weight;
chosen
}
};
let mut tools: Vec<&str> = queue
.iter()
.filter(|entry| entry.agent_key == agent)
.map(|entry| entry.operation_id.as_str())
.collect();
tools.sort_unstable();
tools.dedup();
let tool = match cursor.tool.as_deref() {
Some(current) if cursor.tool_credit > 0 && tools.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.tool.as_deref() {
None => tools[0].to_owned(),
Some(current) => tools
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(tools[0])
.to_owned(),
};
next.tool = Some(chosen.clone());
next.tool_credit = tool_weight;
chosen
}
};
let index = queue
.iter()
.position(|entry| entry.agent_key == agent && entry.operation_id == tool)?;
next.agent_credit = next.agent_credit.saturating_sub(1);
next.tool_credit = next.tool_credit.saturating_sub(1);
Some(AdmissionSelection {
index,
cursor: next,
})
}
#[cfg(test)]
mod tests {
use super::{
ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest,
AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence,
select_next_grant,
};
use crate::{NoTasksFallback, ToolExecutionMode};
fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry {
super::AdmissionQueueEntry {
ticket_id,
sequence: ticket_id,
agent_key: agent.to_owned(),
operation_id: tool.to_owned(),
}
}
fn request() -> AdmissionRequest {
AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_default".to_owned(),
agent_key: ADMIN_ORIGIN_KEY.to_owned(),
operation_id: "op_leads".to_owned(),
operation_version: 3,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_0123456789".to_owned(),
expires_at_unix_ms: 1_790_000_000_000,
}
}
#[test]
fn quota_keys_exclude_version_and_keep_agent_bucket() {
let scope = request().scope;
assert_eq!(scope.agent_bucket(), "ws_default:admin");
assert_eq!(scope.tool_bucket(), "ws_default:op_leads");
}
#[test]
fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() {
let mut invalid = request();
invalid.expires_at_unix_ms = 0;
assert!(matches!(
invalid.validate(),
Err(AdmissionStoreError::Invalid { .. })
));
let mut invalid = request();
invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1);
assert!(invalid.validate().is_err());
let mut invalid = request();
invalid.scope.operation_version = 0;
assert!(invalid.validate().is_err());
assert!(request().validate().is_ok());
}
#[test]
fn weights_are_bounded_positive_configuration() {
assert!(
AdmissionSchedulerConfig {
agent_weight: 0,
tool_weight: 1
}
.validate()
.is_err()
);
assert!(
AdmissionSchedulerConfig {
agent_weight: 1,
tool_weight: 65
}
.validate()
.is_err()
);
assert!(AdmissionSchedulerConfig::default().validate().is_ok());
}
#[test]
fn weighted_round_robin_serves_each_agent_its_quantum() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_1"),
entry(4, "agent_b", "op_1"),
entry(5, "agent_b", "op_1"),
];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
order.push(queue.remove(selection.index).agent_key.clone());
cursor = selection.cursor;
}
assert_eq!(
order,
vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"]
);
}
#[test]
fn fifo_inside_one_tool_and_round_robin_between_tools() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_2"),
];
let config = AdmissionSchedulerConfig::default();
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
// A granted ticket leaves the waiting queue: granted entries are never re-granted.
order.push(queue.remove(selection.index).ticket_id);
cursor = selection.cursor;
}
// One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue.
assert_eq!(order, vec![1, 3, 2]);
}
#[test]
fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() {
let queue = vec![entry(1, "agent_a", "op_1")];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let cursor = AdmissionCursor::default();
let first = select_next_grant(&queue, &cursor, config).unwrap();
assert_eq!(first.cursor.agent.as_deref(), Some("agent_a"));
assert_eq!(first.cursor.agent_credit, 1);
// A later Agent joins; the cursor keeps its remaining quantum for `agent_a`.
let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")];
let second = select_next_grant(&queue, &first.cursor, config).unwrap();
assert_eq!(queue[second.index].agent_key, "agent_a");
assert_eq!(second.cursor.agent_credit, 0);
let third = select_next_grant(&queue, &second.cursor, config).unwrap();
assert_eq!(queue[third.index].agent_key, "agent_b");
assert_eq!(third.cursor.agent_credit, 1);
}
#[test]
fn empty_queue_keeps_cursor_untouched() {
let cursor = AdmissionCursor {
agent: Some("agent_b".to_owned()),
agent_credit: 1,
tool: Some("op_1".to_owned()),
tool_credit: 0,
};
assert_eq!(
select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()),
None
);
assert_eq!(cursor.agent.as_deref(), Some("agent_b"));
}
}
+35 -9
View File
@@ -1,4 +1,5 @@
pub mod access;
pub mod admission;
pub mod agent;
pub mod approval;
pub mod auth;
@@ -17,6 +18,7 @@ pub mod prompt_composition;
pub mod protocol;
pub mod resource;
pub mod secret;
pub mod task;
pub mod tool_catalog;
pub mod tool_quality;
pub mod tool_search;
@@ -60,10 +62,11 @@ pub mod domain {
pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use crate::operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, WizardState,
};
pub use crate::product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
@@ -71,6 +74,13 @@ pub mod domain {
};
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use crate::task::{
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds,
};
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
pub use crate::tool_quality::{
ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
@@ -108,12 +118,20 @@ pub mod ports {
PreparedRequest, ProtocolAdapter, ProtocolAdapterError, ResponseCacheScope,
RuntimeRequestContext, SharedProtocolAdapter,
};
pub use crate::task::TaskStore;
}
pub use access::{
InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
};
pub use admission::{
ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS,
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission,
MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant,
};
pub use agent::{
Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
@@ -168,13 +186,14 @@ pub use observability::{
pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState,
ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor,
RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample,
WizardState,
IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation,
OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy,
OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract,
RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget,
RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, ToolExecutionPolicyError, WizardState,
};
pub use product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
@@ -182,6 +201,13 @@ pub use product_event::{
};
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use task::{
TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key,
validate_task_envelope,
};
pub use tool_catalog::{
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
ToolCatalogBudget, analyze_tool_catalog,
+144 -2
View File
@@ -270,6 +270,72 @@ pub struct OperationApprovalPolicy {
pub elicitation_message: Option<String>,
}
/// Published execution mode of one Tool (Story 4.1, FR-21).
///
/// This is a scheduling policy of the Published Tool version. It is deliberately separate from
/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolExecutionMode {
#[default]
Synchronous,
Background,
Adaptive,
}
/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NoTasksFallback {
/// Execute through the same synchronous admission path.
#[default]
Synchronous,
/// Reject with a typed unsupported-execution-mode failure; never a Task ID.
Unsupported,
}
pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000;
/// Execution policy pinned by a Published Tool version.
///
/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by
/// `ExecutionConfig::timeout_ms`, with no background path.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ToolExecutionPolicy {
#[serde(default)]
pub mode: ToolExecutionMode,
/// Absolute budget of one execution, counted from the start of admission.
pub deadline_ms: u64,
#[serde(default)]
pub no_tasks_fallback: NoTasksFallback,
}
impl Default for ToolExecutionPolicy {
fn default() -> Self {
Self {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
}
}
impl ToolExecutionPolicy {
pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> {
if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) {
return Err(ToolExecutionPolicyError::DeadlineOutOfRange);
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ToolExecutionPolicyError {
#[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")]
DeadlineOutOfRange,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExecutionConfig {
@@ -285,6 +351,8 @@ pub struct ExecutionConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub approval_policy: Option<OperationApprovalPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub execution_policy: Option<ToolExecutionPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auth_profile_ref: Option<AuthProfileId>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub headers: BTreeMap<String, String>,
@@ -444,9 +512,10 @@ mod tests {
edition::OperationSecurityLevel,
ids::{AuthProfileId, OperationId},
operation::{
ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationStatus,
ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability,
OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus,
OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample,
ToolExecutionMode, ToolExecutionPolicy,
},
protocol::{AuthKind, ExportMode, HttpMethod, Protocol},
};
@@ -664,6 +733,7 @@ updated_at: 2026-03-25T08:10:00Z
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: Some(AuthProfileId::new("auth_01")),
headers: BTreeMap::new(),
},
@@ -684,4 +754,76 @@ updated_at: 2026-03-25T08:10:00Z
published_at: None,
}
}
#[test]
fn execution_policy_round_trips_all_modes_and_fallbacks() {
for (mode, fallback) in [
(ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Unsupported),
(ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported),
] {
let config = ExecutionConfig {
execution_policy: Some(ToolExecutionPolicy {
mode,
deadline_ms: 45_000,
no_tasks_fallback: fallback,
}),
..ExecutionConfig::default()
};
let encoded = serde_json::to_value(&config).unwrap();
assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000));
let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap();
assert_eq!(decoded, config);
assert_eq!(
decoded.execution_policy.unwrap().mode,
mode,
"the published mode must survive the JSON round-trip"
);
}
}
#[test]
fn legacy_execution_config_without_policy_stays_synchronous() {
let legacy: ExecutionConfig =
serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap();
assert_eq!(legacy.execution_policy, None);
assert_eq!(legacy.timeout_ms, 12_000);
}
#[test]
fn execution_policy_rejects_explicit_invalid_deadline() {
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 0,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
.validate()
.is_err()
);
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: NoTasksFallback::Unsupported,
}
.validate()
.is_err()
);
assert!(ToolExecutionPolicy::default().validate().is_ok());
}
#[test]
fn execution_policy_rejects_unknown_fields() {
assert!(
serde_json::from_value::<ToolExecutionPolicy>(json!({
"mode": "background",
"deadline_ms": 1000,
"no_tasks_fallback": "synchronous",
"task_queue": "default"
}))
.is_err()
);
}
}
+730
View File
@@ -0,0 +1,730 @@
//! Durable background Task seam (Story 4.2, FR-22/FR-23).
//!
//! A Task is the internal, protocol-independent record of one accepted background request. This
//! module owns the domain vocabulary only: the lifecycle states, the accepted identity/pins, the
//! bounded persisted envelopes and the `TaskStore` port. The protocol adapter (Story 4.7) and the
//! worker (Story 4.3) do not define these semantics; the authoritative store lives in PostgreSQL
//! (`crank-registry`), so an accepted Task survives a crash or restart.
//!
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
//! Task was executed. A terminal state is written by a conditional transition only.
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
use crate::correlation::{RequestId, TraceContext, TraceId};
use crate::operation::ToolExecutionMode;
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
/// Maximum nesting depth of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
/// Maximum number of JSON members/nodes of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_NODES: usize = 256;
/// Bound of every scope/owner/operation identity column.
pub const TASK_IDENTITY_MAX_BYTES: usize = 160;
/// Bound of the caller-supplied idempotency key before it becomes a scoped digest.
pub const TASK_SCOPED_KEY_MAX_BYTES: usize = 128;
/// Bound of a saved approval reference (an identifier, never a bearer token).
pub const TASK_APPROVAL_REFERENCE_MAX_BYTES: usize = 160;
/// Default retention of a terminal Task after its execution deadline (internal bounded default).
pub const TASK_DEFAULT_RETENTION_MS: i64 = 86_400_000;
/// Upper bound on the retention an accepted Task may pin.
pub const TASK_MAX_RETENTION_MS: i64 = 30 * 86_400_000;
/// Upper bound of one retention/housekeeping batch.
pub const TASK_PURGE_BATCH_LIMIT: u32 = 128;
/// Stable identity of one accepted Task. Generated as UUIDv7 and returned only after the accepting
/// transaction committed, so the ID never exists without its durable record.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct TaskId(Uuid);
impl TaskId {
pub fn generate() -> Self {
Self(Uuid::now_v7())
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
Uuid::parse_str(value)
.map(Self)
.map_err(|_| TaskError::invalid("task identity is not a UUID"))
}
pub fn as_uuid(self) -> Uuid {
self.0
}
}
impl std::fmt::Display for TaskId {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(formatter)
}
}
/// Closed Task lifecycle vocabulary. `outcome_unknown` is a non-terminal state: a value commit that
/// failed to confirm its dispatch is never collapsed into a plain failure.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskState {
Queued,
Working,
InputRequired,
OutcomeUnknown,
Completed,
Failed,
Cancelled,
}
impl TaskState {
pub const fn as_str(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Working => "working",
Self::InputRequired => "input_required",
Self::OutcomeUnknown => "outcome_unknown",
Self::Completed => "completed",
Self::Failed => "failed",
Self::Cancelled => "cancelled",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"queued" => Ok(Self::Queued),
"working" => Ok(Self::Working),
"input_required" => Ok(Self::InputRequired),
"outcome_unknown" => Ok(Self::OutcomeUnknown),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"cancelled" => Ok(Self::Cancelled),
_ => Err(TaskError::invalid("unknown task state")),
}
}
pub const fn is_terminal(self) -> bool {
matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
}
/// One conditional transition may only move along this table. A terminal state has no successor,
/// so a repeated terminal write can never rewrite an accepted result.
pub const fn allows_transition_to(self, next: Self) -> bool {
match self {
Self::Queued => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
Self::Working => matches!(
next,
Self::InputRequired
| Self::OutcomeUnknown
| Self::Completed
| Self::Failed
| Self::Cancelled
),
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
Self::OutcomeUnknown => {
matches!(next, Self::Completed | Self::Failed | Self::Cancelled)
}
Self::Completed | Self::Failed | Self::Cancelled => false,
}
}
}
/// Who owns the accepted Task. An administrator is never a wildcard owner of every Task: the owner
/// identity and the authorization scope are separate, both-pinned columns.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskOwnerKind {
Agent,
Admin,
}
impl TaskOwnerKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Agent => "agent",
Self::Admin => "admin",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"agent" => Ok(Self::Agent),
"admin" => Ok(Self::Admin),
_ => Err(TaskError::invalid("unknown task owner kind")),
}
}
}
/// Support identities captured at acceptance. They are stored and returned independently of
/// sampling, and a replay never rewrites them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskSupportIds {
pub request_id: RequestId,
pub trace_id: TraceId,
pub traceparent: String,
pub trace_sampled: bool,
}
impl TaskSupportIds {
pub fn new(request_id: RequestId, trace_context: &TraceContext) -> Self {
Self {
request_id,
trace_id: trace_context.trace_id().clone(),
traceparent: trace_context.traceparent().to_owned(),
trace_sampled: trace_context.is_sampled(),
}
}
/// Re-validates both the trace identity and the parent context: a Task must never persist an
/// inconsistent or malformed support pair, and an unsampled context is still a valid parent.
pub fn validate(&self) -> Result<(), TaskError> {
if self.trace_id.as_str().is_empty() {
return Err(TaskError::invalid("task trace identity is missing"));
}
let context = TraceContext::parse(&self.traceparent)
.map_err(|_| TaskError::invalid("task trace parent is invalid"))?;
if context.trace_id() != &self.trace_id || context.is_sampled() != self.trace_sampled {
return Err(TaskError::invalid("task trace parent is inconsistent"));
}
Ok(())
}
}
/// Execution semantics pinned at acceptance: a later publication never rewrites them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskPins {
pub operation_version: u32,
pub policy_mode: ToolExecutionMode,
pub execution_deadline_unix_ms: i64,
/// Retention deadline of the record; never the same thing as the execution deadline.
pub retain_until_unix_ms: i64,
/// Verified approval evidence reference (identifier only, never a bearer token).
pub approval_reference: Option<String>,
}
impl TaskPins {
pub fn validate(&self) -> Result<(), TaskError> {
if self.operation_version == 0 {
return Err(TaskError::invalid(
"task operation version must be positive",
));
}
if self.execution_deadline_unix_ms <= 0 {
return Err(TaskError::invalid("task execution deadline is invalid"));
}
if self.retain_until_unix_ms < self.execution_deadline_unix_ms {
return Err(TaskError::invalid(
"task retention deadline precedes execution",
));
}
if self.retain_until_unix_ms - self.execution_deadline_unix_ms > TASK_MAX_RETENTION_MS {
return Err(TaskError::invalid("task retention exceeds its bound"));
}
if let Some(reference) = self.approval_reference.as_deref()
&& (reference.is_empty()
|| reference.len() > TASK_APPROVAL_REFERENCE_MAX_BYTES
|| reference.bytes().any(|byte| byte.is_ascii_control()))
{
return Err(TaskError::invalid("task approval reference is invalid"));
}
Ok(())
}
}
/// Bounded, classified payload of a Task. Construction validates the byte/depth/node budget and
/// redacts secret-bearing fields before anything is hashed or persisted.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskEnvelope {
value: Value,
}
impl TaskEnvelope {
/// Validates the caller-provided value, then stores a redacted copy of it. The caller hashes the
/// original value separately, so redaction never collapses two different requests into one.
pub fn try_redacted(input: &Value) -> Result<Self, TaskError> {
validate_task_envelope(input)?;
Ok(Self {
value: redact_task_envelope(input),
})
}
/// Accepts a value that was already validated and redacted at a trust boundary.
pub fn from_persisted(value: Value) -> Result<Self, TaskError> {
validate_task_envelope(&value)?;
Ok(Self { value })
}
pub fn value(&self) -> &Value {
&self.value
}
pub fn into_value(self) -> Value {
self.value
}
}
/// Validates the size, nesting depth and node count of one Task envelope. Every byte, collection and
/// node is counted before the value is cloned, hashed or persisted.
pub fn validate_task_envelope(value: &Value) -> Result<(), TaskError> {
let encoded = serde_json::to_vec(value)
.map_err(|_| TaskError::invalid("task envelope is not serializable"))?;
if encoded.len() > TASK_ENVELOPE_MAX_BYTES {
return Err(TaskError::invalid("task envelope exceeds its byte budget"));
}
let mut nodes = 0usize;
check_budget(value, 1, &mut nodes)?;
Ok(())
}
fn check_budget(value: &Value, depth: usize, nodes: &mut usize) -> Result<(), TaskError> {
if depth > TASK_ENVELOPE_MAX_DEPTH {
return Err(TaskError::invalid("task envelope exceeds its depth budget"));
}
*nodes += 1;
if *nodes > TASK_ENVELOPE_MAX_NODES {
return Err(TaskError::invalid("task envelope exceeds its node budget"));
}
match value {
Value::Object(object) => {
for (key, child) in object {
check_budget_name(key)?;
check_budget(child, depth + 1, nodes)?;
}
}
Value::Array(values) => {
for child in values {
check_budget(child, depth + 1, nodes)?;
}
}
_ => {}
}
Ok(())
}
fn check_budget_name(key: &str) -> Result<(), TaskError> {
if key.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task envelope key exceeds its budget"));
}
Ok(())
}
/// Field-name classification of secret-bearing values. A matching field keeps its position but never
/// its plaintext in the persisted envelope.
pub fn sensitive_task_key(name: &str) -> bool {
let lowered = name.to_ascii_lowercase();
[
"password",
"passwd",
"secret",
"token",
"api_key",
"apikey",
"authorization",
"credential",
"private_key",
"cookie",
"signature",
]
.iter()
.any(|marker| lowered.contains(marker))
}
/// Redacted projection of an already validated envelope.
pub fn redact_task_envelope(value: &Value) -> Value {
match value {
Value::Object(object) => Value::Object(
object
.iter()
.map(|(key, child)| {
let value = if sensitive_task_key(key) {
Value::String("[redacted]".to_owned())
} else {
redact_task_envelope(child)
};
(key.clone(), value)
})
.collect(),
),
Value::Array(values) => Value::Array(values.iter().map(redact_task_envelope).collect()),
other => other.clone(),
}
}
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
/// queue capacity commit together, and no acknowledgement exists before that commit.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAcceptRequest {
pub task_id: TaskId,
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
pub operation_id: String,
pub pins: TaskPins,
/// Digest of the caller key over the full owner/authorization/Tool scope.
pub scoped_key: String,
/// Digest of the canonical validated executable input.
pub input_fingerprint: String,
pub input: TaskEnvelope,
pub support: TaskSupportIds,
}
impl TaskAcceptRequest {
pub fn validate(&self) -> Result<(), TaskError> {
for (field, value) in [
("workspace", self.workspace_id.as_str()),
("owner", self.owner_id.as_str()),
("authorization scope", self.authorization_scope.as_str()),
("operation", self.operation_id.as_str()),
] {
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
return Err(TaskError::invalid("task scoped key digest is invalid"));
}
if self.input_fingerprint.len() != 64
|| !self
.input_fingerprint
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
{
return Err(TaskError::invalid("task input fingerprint is invalid"));
}
self.pins.validate()?;
self.support.validate()
}
}
/// Successful acceptance, returned only after the accepting transaction committed. A replay returns
/// the stored identity and support IDs of the original acceptance.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAcceptance {
pub task_id: TaskId,
pub state: TaskState,
/// Exact Published Version pinned by this acceptance; a replay returns the stored pin.
pub operation_version: u32,
pub support: TaskSupportIds,
pub created_at_unix_ms: i64,
pub retain_until_unix_ms: i64,
pub replayed: bool,
}
/// Owner-side access proof of a read or a terminal write. It is built by the application layer from
/// the verified principal, never from a caller-supplied marker, and every store query carries the
/// whole predicate so a foreign row is never fetched.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAccess {
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
}
impl TaskAccess {
pub fn validate(&self) -> Result<(), TaskError> {
for (field, value) in [
("workspace", self.workspace_id.as_str()),
("owner", self.owner_id.as_str()),
("authorization scope", self.authorization_scope.as_str()),
] {
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
Ok(())
}
}
/// Read model of one Task. Lifecycle fields, pins and support IDs stay readable together; the
/// protocol adapter only maps them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskRecord {
pub task_id: TaskId,
pub state: TaskState,
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
pub operation_id: String,
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
pub input_fingerprint: String,
pub pins: TaskPins,
pub support: TaskSupportIds,
pub input: TaskEnvelope,
pub progress: Option<TaskEnvelope>,
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
pub attempts: u32,
/// Monotonic lifecycle version used by the terminal CAS.
pub version: u32,
pub created_at_unix_ms: i64,
pub updated_at_unix_ms: i64,
}
/// One conditional terminal transition. Exactly one concurrent writer wins; the losers observe the
/// accepted state and never rewrite the result.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskFinishRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub expected_state: TaskState,
pub expected_version: u32,
pub target_state: TaskState,
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
}
impl TaskFinishRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if !self.target_state.is_terminal() {
return Err(TaskError::invalid("task finish target is not terminal"));
}
if self.expected_state.is_terminal() {
return Err(TaskError::invalid("task finish source is already terminal"));
}
if !self.expected_state.allows_transition_to(self.target_state) {
return Err(TaskError::invalid("task state transition is not allowed"));
}
if self.target_state == TaskState::Completed && self.result.is_none() {
return Err(TaskError::invalid("a completed task requires a result"));
}
Ok(())
}
}
/// Bounded retention outcome.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct TaskPurgeReport {
/// Queued Tasks that passed their execution deadline and were failed with their intent removed.
pub expired_queued: u64,
/// Terminal Tasks that passed their retention deadline.
pub expired_terminal: u64,
}
/// Typed Task failure. A denial and a missing Task share one outward shape, so a foreign caller can
/// never distinguish "not yours" from "does not exist".
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum TaskError {
#[error("task request is invalid: {message}")]
Invalid { message: String },
#[error("task store is unavailable: {message}")]
Unavailable { message: String },
#[error("task was not found")]
NotFound,
#[error("task request conflicts with the accepted request")]
FingerprintConflict,
#[error("task lifecycle transition was rejected")]
StateConflict,
#[error("background task capacity is exhausted")]
Overloaded { retry_after_ms: u64 },
}
impl TaskError {
pub fn invalid(message: impl Into<String>) -> Self {
Self::Invalid {
message: message.into(),
}
}
pub fn unavailable(message: impl Into<String>) -> Self {
Self::Unavailable {
message: message.into(),
}
}
/// Whether the caller may tell this failure apart from an absent Task.
pub fn is_not_found(&self) -> bool {
matches!(self, Self::NotFound)
}
}
/// Authoritative durable Task port.
#[async_trait]
pub trait TaskStore: Send + Sync {
/// Proves the durable Task state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), TaskError>;
/// Atomically accepts a Task: Task record, dispatch intent and one queue slot commit together.
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError>;
/// Reads one Task inside the caller's owner scope.
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError>;
/// Scoped lookup of an already accepted Task by its full-scope idempotency digest. Used to keep a
/// retry of an explicitly pinned version from becoming new work; `None` means "not accepted yet".
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError>;
/// Applies exactly one conditional terminal transition and releases its intent atomically.
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
/// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes
/// expired terminal records. Live and non-terminal work is never deleted.
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn request() -> TaskAcceptRequest {
TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_790_000_000_000,
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
approval_reference: Some("appr_1".to_owned()),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
),
}
}
#[test]
fn uuid_v7_task_ids_are_unique_and_round_trip() {
let first = TaskId::generate();
let second = TaskId::generate();
assert_ne!(first, second);
assert_eq!(TaskId::parse(&first.to_string()).unwrap(), first);
}
#[test]
fn terminal_states_have_no_successor() {
for terminal in [
TaskState::Completed,
TaskState::Failed,
TaskState::Cancelled,
] {
assert!(terminal.is_terminal());
for next in [
TaskState::Queued,
TaskState::Working,
TaskState::Completed,
TaskState::Failed,
] {
assert!(!terminal.allows_transition_to(next));
}
}
assert!(TaskState::Queued.allows_transition_to(TaskState::Working));
assert!(TaskState::Working.allows_transition_to(TaskState::OutcomeUnknown));
assert!(!TaskState::Queued.allows_transition_to(TaskState::Completed));
assert!(!TaskState::InputRequired.allows_transition_to(TaskState::Completed));
}
#[test]
fn envelope_bounds_reject_bytes_depth_and_nodes() {
assert!(TaskEnvelope::try_redacted(&json!({"a": 1})).is_ok());
assert!(
TaskEnvelope::try_redacted(&json!({"a": "x".repeat(TASK_ENVELOPE_MAX_BYTES)})).is_err()
);
let mut deep = json!(1);
for _ in 0..=TASK_ENVELOPE_MAX_DEPTH {
deep = json!({"n": deep});
}
assert!(TaskEnvelope::try_redacted(&deep).is_err());
let wide = Value::Array((0..=TASK_ENVELOPE_MAX_NODES).map(|_| json!(1)).collect());
assert!(TaskEnvelope::try_redacted(&wide).is_err());
}
#[test]
fn secret_bearing_fields_are_redacted_and_keep_their_shape() {
let envelope = TaskEnvelope::try_redacted(&json!({
"query": "leads",
"auth": {"api_key": "sk_live_1", "nested": [{"password": "p"}]}
}))
.unwrap();
let value = envelope.value();
assert_eq!(value["query"], json!("leads"));
assert_eq!(value["auth"]["api_key"], json!("[redacted]"));
assert_eq!(value["auth"]["nested"][0]["password"], json!("[redacted]"));
}
#[test]
fn unsampled_support_ids_are_valid_and_consistent() {
let support = TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
);
assert!(!support.trace_sampled);
assert!(support.traceparent.ends_with("-00"));
support.validate().unwrap();
let mut inconsistent = support.clone();
inconsistent.traceparent = inconsistent.traceparent.replace("-00", "-01");
assert!(inconsistent.validate().is_err());
}
#[test]
fn accept_request_validation_holds_the_identity_and_pin_bounds() {
assert!(request().validate().is_ok());
let mut oversized = request();
oversized.workspace_id = "w".repeat(TASK_IDENTITY_MAX_BYTES + 1);
assert!(oversized.validate().is_err());
let mut bad_fingerprint = request();
bad_fingerprint.input_fingerprint = "XYZ".to_owned();
assert!(bad_fingerprint.validate().is_err());
let mut bad_retention = request();
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
assert!(bad_retention.validate().is_err());
}
#[test]
fn finish_requires_a_terminal_reachable_transition() {
let access = TaskAccess {
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
};
let base = TaskFinishRequest {
access,
task_id: TaskId::generate(),
expected_state: TaskState::Working,
expected_version: 1,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
};
assert!(base.validate().is_ok());
let mut non_terminal = base.clone();
non_terminal.target_state = TaskState::Working;
assert!(non_terminal.validate().is_err());
let mut terminal_source = base.clone();
terminal_source.expected_state = TaskState::Completed;
assert!(terminal_source.validate().is_err());
let mut completed_without_result = base;
completed_without_result.result = None;
assert!(completed_without_result.validate().is_err());
}
}
@@ -231,6 +231,7 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
+1
View File
@@ -564,6 +564,7 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
+28
View File
@@ -217,6 +217,34 @@ string_enum! {
Concurrency => "concurrency",
RateLimit => "rate_limit",
McpStream => "mcp_stream",
Admission => "admission",
}
}
string_enum! {
pub enum RejectionReason {
Limit => "limit",
Capacity => "capacity",
QueueFull => "queue_full",
Deadline => "deadline",
Unsupported => "unsupported",
DependencyUnavailable => "dependency_unavailable",
}
}
string_enum! {
pub enum ScopeKind {
Process => "process",
Global => "global",
Agent => "agent",
Tool => "tool",
}
}
string_enum! {
pub enum AdmissionModeLabel {
Synchronous => "synchronous",
Background => "background",
}
}
+6 -5
View File
@@ -16,14 +16,15 @@ mod schema;
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
pub use labels::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
UpstreamOutcome,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
pub use record::{
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service,
bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome,
bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting,
record_admission_queue_wait, record_admission_rejection, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
+41 -5
View File
@@ -5,10 +5,10 @@ use metrics::Gauge;
use crate::schema::{MetricDefinition, metric_schema};
use crate::{
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter,
HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason,
ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
static BINDING: OnceLock<MetricService> = OnceLock::new();
@@ -352,11 +352,47 @@ pub fn record_limit_rejection(stage: LimitStage) {
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => stage.as_str()
"stage" => stage.as_str(),
"reason" => RejectionReason::Limit.as_str(),
"scope_kind" => ScopeKind::Process.as_str()
)
.increment(1);
}
/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool,
/// request id, fingerprint or arbitrary error text ever becomes a label.
pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) {
if bound_family("crank_runtime_limit_rejections_total").is_none() {
return;
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => LimitStage::Admission.as_str(),
"reason" => reason.as_str(),
"scope_kind" => scope_kind.as_str()
)
.increment(1);
}
pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) {
if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() {
return;
}
metrics::gauge!(
"crank_runtime_admission_queue_wait_seconds",
"mode" => mode.as_str()
)
.set(waited_ms as f64 / 1000.0);
}
pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option<u64>) {
if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() {
return;
}
let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0;
metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds);
}
pub fn record_cache_outcome(outcome: CacheOutcome) {
if bound_family("crank_runtime_cache_total").is_none() {
return;
+38 -8
View File
@@ -3,16 +3,17 @@
use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP};
use crate::labels::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome,
UpstreamOperationKind, UpstreamOutcome,
};
pub const METRIC_SCHEMA_VERSION: u32 = 2;
pub const MAX_METRIC_FAMILIES: usize = 128;
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611;
pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348;
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
@@ -179,6 +180,9 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
const REJECTION_REASONS: &[&str] = RejectionReason::VALUES;
const SCOPE_KINDS: &[&str] = ScopeKind::VALUES;
const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES;
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
@@ -403,13 +407,39 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[
"crank_runtime_limit_rejections_total",
MetricKind::Counter,
MetricUnit::Count,
&["stage"],
&[domain("stage", LIMIT_STAGES)],
&["stage", "reason", "scope_kind"],
&[
domain("stage", LIMIT_STAGES),
domain("reason", REJECTION_REASONS),
domain("scope_kind", SCOPE_KINDS),
],
&[],
BOTH,
3,
96,
"Runtime executions rejected by a bounded limit.",
),
metric(
"crank_runtime_admission_queue_wait_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&["mode"],
&[domain("mode", ADMISSION_MODES)],
&[],
BOTH,
2,
"Wait of the most recently admitted execution before it received a scoped permit.",
),
metric(
"crank_runtime_admission_oldest_waiting_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&[],
&[],
&[],
BOTH,
1,
"Age of the oldest waiting admission request.",
),
metric(
"crank_runtime_cache_total",
MetricKind::Counter,
+13 -6
View File
@@ -1,12 +1,13 @@
use std::{collections::BTreeSet, time::Duration};
use crank_metrics::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics,
bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType,
ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema,
record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection,
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
@@ -71,6 +72,9 @@ fn schema_names_are_unique_and_labels_are_closed() {
"state",
"signal_type",
"exporter",
"reason",
"scope_kind",
"mode",
];
for metric in schema {
assert!(
@@ -265,6 +269,9 @@ fn every_recording_api_matches_the_declared_schema() {
Duration::from_millis(1),
);
record_limit_rejection(LimitStage::Concurrency);
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5);
record_admission_oldest_waiting(Some(5));
record_cache_outcome(CacheOutcome::Hit);
record_idempotency_outcome(IdempotencyOutcome::Replay);
record_confirmation_outcome(ConfirmationOutcome::Required);
+8 -8
View File
@@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
#[test]
fn frozen_budget_edges_are_exact() {
assert!(validate_budget(6_284, 19_514).is_ok());
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 16_000).is_err());
assert!(validate_budget(6_000, 19_516).is_err());
assert!(validate_budget(6_380, 19_610).is_ok());
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_000).is_err());
assert!(validate_budget(6_000, 19_612).is_err());
assert_eq!(
crank_metrics::max_exemplar_slots(MetricService::AdminApi),
12_348
@@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() {
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_285,
rendered_series: 19_515
logical_series: 6_381,
rendered_series: 19_611
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_083,
rendered_series: 3_558
logical_series: 1_179,
rendered_series: 3_654
})
);
}
+11 -11
View File
@@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() {
"root обязан объявлять ровно один ключ budgets"
);
assert!(rendered.ends_with(
",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\
\"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n"
",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\
\"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n"
));
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_285,
rendered_series: 19_515
logical_series: 6_381,
rendered_series: 19_611
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_083,
rendered_series: 3_558
logical_series: 1_179,
rendered_series: 3_654
})
);
}
@@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() {
/// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны.
#[test]
fn frozen_caps_and_edges_are_exact() {
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515);
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611);
assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348);
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 19_515).is_err());
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_611).is_err());
assert!(validate_exemplar_budget(12_347).is_ok());
assert!(validate_exemplar_budget(12_348).is_ok());
assert!(validate_exemplar_budget(12_349).is_err());
+1 -1
View File
@@ -13,7 +13,7 @@ mod schema;
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
pub use crank_metrics::{
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit,
bind_metric_service, bound_metric_service, metric_schema,
bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting,
};
pub use error_reporting::{
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
+1
View File
@@ -7,6 +7,7 @@ publish.workspace = true
version.workspace = true
[dependencies]
async-trait = "0.1"
crank-artifacts = { path = "../crank-artifacts" }
crank-core = { path = "../crank-core" }
crank-mapping = { path = "../crank-mapping" }
+3 -2
View File
@@ -116,6 +116,7 @@ pub use model::{
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
};
pub use postgres::{
AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError,
PostgresRegistry,
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, PostgresTaskStore,
admission_health, admission_snapshot, tasks_health,
};
+4
View File
@@ -6,6 +6,8 @@ mod artifact_cleanup_indexes_v13;
mod artifact_metadata_v12;
mod authority;
mod baseline_v1;
mod durable_tasks_v25;
mod execution_admission_v24;
mod execution_outcome_v5;
mod import_plan_v14;
mod master_key_identity_v7;
@@ -35,6 +37,8 @@ mod schema_guard_v19;
mod schema_guard_v20;
mod schema_guard_v22;
mod schema_guard_v23;
mod schema_guard_v24;
mod schema_guard_v25;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -4,6 +4,8 @@ use super::agent_resource_bindings_v18;
use super::approval_side_effects_v10;
use super::artifact_cleanup_indexes_v13;
use super::artifact_metadata_v12;
use super::durable_tasks_v25;
use super::execution_admission_v24;
use super::execution_outcome_v5;
use super::import_plan_v14;
use super::master_key_identity_v7;
@@ -39,9 +41,9 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 23;
const CURRENT_VERSION: i64 = 25;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -330,6 +332,18 @@ impl MigrationAuthority {
prompt_dependencies_v23::SOURCE_SHA256,
22,
),
expand_descriptor(
24,
execution_admission_v24::NAME,
execution_admission_v24::SOURCE_SHA256,
23,
),
expand_descriptor(
25,
durable_tasks_v25::NAME,
durable_tasks_v25::SOURCE_SHA256,
24,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -489,6 +503,12 @@ impl MigrationAuthority {
if from < 23 {
prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?;
}
if from < 24 {
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
}
if from < 25 {
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -140,6 +140,8 @@ pub(super) fn validate_descriptors(
!= agent_resource_bindings_v18::SOURCE_SHA256
|| sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes())
!= resource_security_revocations_v19::SOURCE_SHA256
|| sha256_hex(execution_admission_v24::SOURCE.as_bytes())
!= execution_admission_v24::SOURCE_SHA256
{
return Err(MigrationError::new(
"invalid_contract",
@@ -148,6 +148,7 @@ pub(super) async fn inspect(
owned_relations::PROMPTS,
owned_relations::PROMPT_PUBLICATIONS,
owned_relations::PROMPT_DEPENDENCIES,
owned_relations::EXECUTION_ADMISSION,
] {
validate_required_relations(connection, relations, CURRENT_VERSION).await?;
}
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v25";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v25.sql");
pub(super) const SOURCE_SHA256: &str =
"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v25",
Some(25),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.canonical_ledger",
Some(25),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,90 @@
-- V25: durable background Tasks (Story 4.2, FR-22, FR-23).
--
-- Одна строка `tasks` — принятая durable работа. Идентификатор Task (UUIDv7), owner scope,
-- authorization scope, запрошенный Tool и точная published-версия, canonical fingerprint входа,
-- support IDs (Request/Trace/parent TraceContext независимо от sampling) и bounded envelopes
-- закрепляются в той же PostgreSQL transaction, что и dispatch intent
-- (`execution_admission_entries.task_id`) и его queue slot. До commit наружу не выдаётся ни ID, ни ack.
--
-- `scoped_key` — digest caller-ключа по полному (workspace, owner kind/id, authorization scope,
-- operation) scope; UNIQUE по этому набору и есть защита идемпотентности. Повтор того же ключа с тем
-- же canonical fingerprint возвращает сохранённую Task, другой fingerprint — typed conflict.
--
-- `retain_until` — срок хранения записи, а не execution deadline: пока запись в retention, replay
-- находит тот же Task. Удаление terminal-записей возможно только после retain_until; queued/working/
-- input_required/outcome_unknown не удаляются никогда ради лимита.
--
-- Строка не хранит credentials, bearer/prepared headers и confirmation tokens: секретные поля входа
-- редактируются до записи, а approval хранится безопасной ссылкой.
create table tasks (
task_id uuid not null,
workspace_id text not null,
owner_kind text not null,
owner_id text not null,
authorization_scope text not null,
operation_id text not null,
operation_version integer not null,
scoped_key text not null,
input_fingerprint text not null,
request_id text not null,
trace_id text not null,
traceparent text not null,
trace_sampled boolean not null,
policy_mode text not null,
approval_reference text,
state text not null,
version integer not null default 1,
attempts integer not null default 0,
input_envelope jsonb not null,
progress_envelope jsonb,
result_envelope jsonb,
error_envelope jsonb,
execution_deadline timestamptz not null,
retain_until timestamptz not null,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint tasks_pkey primary key (task_id),
constraint tasks_workspace_check check (octet_length(workspace_id) between 1 and 160),
constraint tasks_owner_kind_check check (owner_kind in ('agent', 'admin')),
constraint tasks_owner_check check (octet_length(owner_id) between 1 and 160),
constraint tasks_authorization_scope_check check (
octet_length(authorization_scope) between 1 and 160
),
constraint tasks_operation_check check (octet_length(operation_id) between 1 and 160),
constraint tasks_operation_version_check check (operation_version >= 1),
constraint tasks_scoped_key_check check (octet_length(scoped_key) between 16 and 128),
constraint tasks_fingerprint_check check (octet_length(input_fingerprint) = 64),
constraint tasks_request_check check (octet_length(request_id) between 1 and 128),
constraint tasks_trace_check check (octet_length(trace_id) = 32),
constraint tasks_traceparent_check check (octet_length(traceparent) = 55),
constraint tasks_policy_mode_check check (
policy_mode in ('synchronous', 'background', 'adaptive')
),
constraint tasks_state_check check (
state in ('queued', 'working', 'input_required', 'outcome_unknown',
'completed', 'failed', 'cancelled')
),
constraint tasks_lifecycle_check check (version >= 1 and attempts >= 0),
constraint tasks_retention_check check (retain_until >= execution_deadline),
constraint tasks_approval_check check (
approval_reference is null or octet_length(approval_reference) between 1 and 160
)
);
create unique index tasks_scoped_key_uidx
on tasks (workspace_id, owner_kind, owner_id, authorization_scope, operation_id, scoped_key);
create index tasks_retention_idx on tasks (state, retain_until, task_id);
create index tasks_deadline_idx on tasks (state, execution_deadline, task_id);
-- Dispatch intent: прежняя reserved-строка admission становится точкой связи Task и capacity.
-- FK без cascade: удаление принятой Task не маскируется housekeeping-удалением intent.
alter table execution_admission_entries
add column task_id uuid,
add constraint execution_admission_entries_task_fkey
foreign key (task_id) references tasks (task_id);
create unique index execution_admission_entries_task_uidx
on execution_admission_entries (task_id) where task_id is not null;
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "execution-admission-v24";
pub(super) const SOURCE: &str = include_str!("execution_admission_v24.sql");
pub(super) const SOURCE_SHA256: &str =
"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.execution_admission_v24",
Some(24),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.canonical_ledger",
Some(24),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,92 @@
-- V24: authoritative execution admission state (Story 4.1, FR-21, FR-24).
--
-- Один deployment-wide admission: PostgreSQL хранит cursor/fairness state, ожидающие тикеты,
-- выданные permits и durable enqueue decisions, поэтому несколько процессов делят общий бюджет.
--
-- `execution_admission_entries` — bounded набор: waiting (ожидает capacity), executing (scoped
-- permit выдан, держится до явного release) и reserved (durable enqueue decision с зарезервированным
-- queue slot; не исполняется и не рекламируется как Task до Story 4.2).
--
-- Строка не хранит ни credentials, ни connection/session id, ни prepared headers, ни тело запроса:
-- quota key — (workspace, agent) и (workspace, operation) без version. `operation_version` пишется
-- только как pin исполняемых semantics и не участвует в квотах.
--
-- `expires_at` — абсолютный deadline. Queue decisions без dispatch можно expire по deadline; активный
-- permit НЕ освобождается по TTL (иначе paused/crashed owner обойдёт global limit): его снимает только
-- явный release/cancel владельца.
create table execution_admission_controllers (
workspace_id text not null,
cursor_agent text,
cursor_agent_credit integer not null default 0,
cursor_tool text,
cursor_tool_credit integer not null default 0,
updated_at timestamptz not null default now(),
constraint execution_admission_controllers_pkey primary key (workspace_id),
constraint execution_admission_controllers_workspace_check check (
octet_length(workspace_id) between 1 and 160
),
constraint execution_admission_controllers_agent_credit_check check (
cursor_agent_credit between 0 and 64
),
constraint execution_admission_controllers_tool_credit_check check (
cursor_tool_credit between 0 and 64
)
);
create table execution_admission_entries (
ticket_id bigint generated always as identity,
workspace_id text not null,
agent_key text not null,
operation_id text not null,
operation_version integer not null,
mode text not null,
policy_mode text not null,
state text not null,
owner_token text not null,
fingerprint text not null,
created_at timestamptz not null default now(),
expires_at timestamptz not null,
granted_at timestamptz,
constraint execution_admission_entries_pkey primary key (ticket_id),
constraint execution_admission_entries_workspace_check check (
octet_length(workspace_id) between 1 and 160
),
constraint execution_admission_entries_agent_check check (
octet_length(agent_key) between 1 and 160
),
constraint execution_admission_entries_operation_check check (
octet_length(operation_id) between 1 and 160
),
constraint execution_admission_entries_version_check check (operation_version >= 1),
constraint execution_admission_entries_mode_check check (
mode in ('synchronous', 'background')
),
constraint execution_admission_entries_policy_mode_check check (
policy_mode in ('synchronous', 'background', 'adaptive')
),
constraint execution_admission_entries_state_check check (
state in ('waiting', 'executing', 'reserved')
),
constraint execution_admission_entries_owner_check check (
octet_length(owner_token) between 1 and 64
),
constraint execution_admission_entries_fingerprint_check check (
octet_length(fingerprint) between 1 and 128
),
constraint execution_admission_entries_reserved_mode_check check (
state <> 'reserved' or mode = 'background'
)
);
create unique index execution_admission_entries_owner_uidx
on execution_admission_entries (owner_token);
create index execution_admission_entries_state_idx
on execution_admission_entries (state, workspace_id, ticket_id);
create index execution_admission_entries_agent_idx
on execution_admission_entries (state, workspace_id, agent_key);
create index execution_admission_entries_operation_idx
on execution_admission_entries (state, workspace_id, operation_id);
@@ -55,6 +55,11 @@ pub(super) const PROMPT_PUBLICATIONS: &[&str] = &["prompt_publications", "agent_
pub(super) const PROMPT_DEPENDENCIES: &[&str] =
&["prompt_dependencies", "prompt_message_resource_refs"];
pub(super) const EXECUTION_ADMISSION: &[&str] = &[
"execution_admission_controllers",
"execution_admission_entries",
];
pub(super) const ARTIFACT_METADATA: &[&str] = &["artifact_blobs", "artifact_sources"];
pub(super) const IMPORT_PLAN: &[&str] = &["import_plans", "import_plan_applications"];
@@ -468,6 +468,21 @@ pub(super) async fn validate_schema_fingerprint(
} else {
super::schema_guard_v23::validate_v23_prompt_dependencies(connection).await?;
}
if current_version < 24 {
if !super::schema_guard_v24::validate_v24_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v24::validate_v24_execution_admission(connection, current_version)
.await?;
}
if current_version < 25 {
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
}
Ok(())
}
@@ -51,6 +51,8 @@ pub(crate) const OWNED_RELATIONS: &[&str] = &[
"prompt_messages",
"prompt_dependencies",
"prompt_message_resource_refs",
"execution_admission_controllers",
"execution_admission_entries",
];
pub(crate) const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[
@@ -0,0 +1,277 @@
//! V24 schema guard: authoritative execution admission state (Story 4.1, FR-21/FR-24).
//!
//! Схема V24 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
//! ожидаемой форме: controller-row с weighted round-robin cursor и bounded entries с закрытым
//! словарём mode/policy_mode/state, уникальным owner token и индексами квот.
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const CONTROLLERS: &str = "execution_admission_controllers";
const ENTRIES: &str = "execution_admission_entries";
const OWNED_RELATIONS: &[&str] = &[CONTROLLERS, ENTRIES];
const CONTROLLER_COLUMNS: &[(&str, &str, bool)] = &[
("workspace_id", "text", false),
("cursor_agent", "text", true),
("cursor_agent_credit", "integer", false),
("cursor_tool", "text", true),
("cursor_tool_credit", "integer", false),
("updated_at", "timestamp with time zone", false),
];
const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[
("ticket_id", "bigint", false),
("workspace_id", "text", false),
("agent_key", "text", false),
("operation_id", "text", false),
("operation_version", "integer", false),
("mode", "text", false),
("policy_mode", "text", false),
("state", "text", false),
("owner_token", "text", false),
("fingerprint", "text", false),
("created_at", "timestamp with time zone", false),
("expires_at", "timestamp with time zone", false),
("granted_at", "timestamp with time zone", true),
];
/// V25 adds the nullable Task dispatch-intent link to the same reserved row. At version 24 the
/// column must not exist yet, so the guard validates the shape of the version it describes.
fn entry_columns(current_version: i64) -> Vec<(&'static str, &'static str, bool)> {
let mut columns = ENTRY_COLUMNS.to_vec();
if current_version >= 25 {
columns.push(("task_id", "uuid", true));
}
columns
}
fn constraint_count(table: &str, current_version: i64) -> Option<usize> {
match table {
CONTROLLERS => Some(4),
ENTRIES if current_version >= 25 => Some(12),
ENTRIES => Some(11),
_ => None,
}
}
const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"];
fn v24_error(stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(24),
"restore_known_good_backup",
)
}
/// Схема V24 отсутствует целиком (для current_version < 24).
pub(super) async fn validate_v24_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
for relation in OWNED_RELATIONS {
if relation_exists(connection, relation).await? {
return Ok(false);
}
}
for index in UNIQUE_INDEXES {
if index_exists(connection, index).await? {
return Ok(false);
}
}
Ok(true)
}
/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24).
pub(super) async fn validate_v24_execution_admission(
connection: &mut PgConnection,
current_version: i64,
) -> Result<(), MigrationError> {
for relation in OWNED_RELATIONS {
if !relation_exists(connection, relation).await? {
return Err(v24_error("v24.relations"));
}
}
let tables = [
(CONTROLLERS, CONTROLLER_COLUMNS.to_vec()),
(ENTRIES, entry_columns(current_version)),
];
for (table, columns) in tables {
validate_columns(connection, table, &columns).await?;
}
for table in [CONTROLLERS, ENTRIES] {
let expected = constraint_count(table, current_version).unwrap_or_default();
let count: i64 = query(
"select count(*)::bigint as count
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1",
)
.bind(table)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(expected).unwrap_or_default() {
return Err(v24_error("v24.constraints"));
}
}
for (table, constraint) in [
(CONTROLLERS, "execution_admission_controllers_pkey"),
(ENTRIES, "execution_admission_entries_pkey"),
(ENTRIES, "execution_admission_entries_state_check"),
] {
if !constraint_exists(connection, table, constraint).await? {
return Err(v24_error("v24.constraint_missing"));
}
}
if !normalized_expression(
connection,
ENTRIES,
"execution_admission_entries_state_check",
)
.await?
.is_some_and(|expression| {
super::schema_guard::enum_constraint_matches(
&expression,
"state",
&["waiting", "executing", "reserved"],
)
}) {
return Err(v24_error("v24.state_vocabulary"));
}
for index in UNIQUE_INDEXES {
let unique = index_unique(connection, index).await?;
if unique != Some(true) {
return Err(v24_error("v24.indexes"));
}
}
Ok(())
}
async fn validate_columns(
connection: &mut PgConnection,
table: &str,
columns: &[(&str, &str, bool)],
) -> Result<(), MigrationError> {
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(table)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(columns.len()).unwrap_or_default() {
return Err(v24_error("v24.columns"));
}
for (column, data_type, nullable) in columns {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(table)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if *nullable { "YES" } else { "NO" })
});
if !valid {
return Err(v24_error("v24.columns"));
}
}
Ok(())
}
async fn constraint_exists(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
) as present",
)
.bind(table)
.bind(constraint)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn normalized_expression(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<Option<String>, MigrationError> {
let row = query(
"select pg_get_expr(c.conbin, c.conrelid) as expression
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
and c.contype = 'c'",
)
.bind(table)
.bind(constraint)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
}
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_class idx
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1
) as present",
)
.bind(index)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn index_unique(
connection: &mut PgConnection,
index: &str,
) -> Result<Option<bool>, MigrationError> {
let row = query(
"select i.indisunique, i.indisvalid
from pg_catalog.pg_index i
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1",
)
.bind(index)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| {
let unique = row.try_get::<bool, _>("indisunique").ok()?;
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
Some(unique && valid)
}))
}
@@ -0,0 +1,247 @@
//! V25 schema guard: durable background Tasks (Story 4.2, FR-22/FR-23).
//!
//! Схема V25 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
//! ожидаемой форме: таблица `tasks` с закрытым словарём state/policy_mode и scoped uniqueness, плюс
//! nullable `task_id` на существующей reserved-строке admission с FK и UNIQUE-индексом.
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const TASKS: &str = "tasks";
const ENTRIES: &str = "execution_admission_entries";
const SCOPED_INDEX: &str = "tasks_scoped_key_uidx";
const TASK_INDEX: &str = "execution_admission_entries_task_uidx";
const TASK_FK: &str = "execution_admission_entries_task_fkey";
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
("task_id", "uuid", false),
("workspace_id", "text", false),
("owner_kind", "text", false),
("owner_id", "text", false),
("authorization_scope", "text", false),
("operation_id", "text", false),
("operation_version", "integer", false),
("scoped_key", "text", false),
("input_fingerprint", "text", false),
("request_id", "text", false),
("trace_id", "text", false),
("traceparent", "text", false),
("trace_sampled", "boolean", false),
("policy_mode", "text", false),
("approval_reference", "text", true),
("state", "text", false),
("version", "integer", false),
("attempts", "integer", false),
("input_envelope", "jsonb", false),
("progress_envelope", "jsonb", true),
("result_envelope", "jsonb", true),
("error_envelope", "jsonb", true),
("execution_deadline", "timestamp with time zone", false),
("retain_until", "timestamp with time zone", false),
("created_at", "timestamp with time zone", false),
("updated_at", "timestamp with time zone", false),
];
fn v25_error(stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(25),
"restore_known_good_backup",
)
}
/// Схема V25 отсутствует целиком (для current_version < 25).
pub(super) async fn validate_v25_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
return Ok(false);
}
if super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
return Ok(false);
}
for index in [SCOPED_INDEX, TASK_INDEX] {
if index_exists(connection, index).await? {
return Ok(false);
}
}
Ok(true)
}
/// Схема V25 присутствует в ожидаемой форме (для current_version >= 25).
pub(super) async fn validate_v25_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
if !relation_exists(connection, TASKS).await? {
return Err(v25_error("v25.relations"));
}
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
return Err(v25_error("v25.columns"));
}
for (column, data_type, nullable) in TASK_COLUMNS {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if *nullable { "YES" } else { "NO" })
});
if !valid {
return Err(v25_error("v25.columns"));
}
}
if !super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
return Err(v25_error("v25.intent_column"));
}
for (constraint, vocabulary) in [
(
"tasks_state_check",
[
"queued",
"working",
"input_required",
"outcome_unknown",
"completed",
"failed",
"cancelled",
]
.as_slice(),
),
(
"tasks_policy_mode_check",
["synchronous", "background", "adaptive"].as_slice(),
),
] {
let expression = normalized_expression(connection, TASKS, constraint).await?;
if !expression.is_some_and(|expression| {
super::schema_guard::enum_constraint_matches(
&expression,
column_of(constraint),
vocabulary,
)
}) {
return Err(v25_error("v25.vocabulary"));
}
}
for (index, table) in [(SCOPED_INDEX, TASKS), (TASK_INDEX, ENTRIES)] {
if index_unique(connection, table, index).await? != Some(true) {
return Err(v25_error("v25.indexes"));
}
}
if !constraint_exists(connection, ENTRIES, TASK_FK).await? {
return Err(v25_error("v25.intent_fk"));
}
Ok(())
}
fn column_of(constraint: &str) -> &'static str {
if constraint.contains("policy_mode") {
"policy_mode"
} else {
"state"
}
}
async fn constraint_exists(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
) as present",
)
.bind(table)
.bind(constraint)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn normalized_expression(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<Option<String>, MigrationError> {
let row = query(
"select pg_get_expr(c.conbin, c.conrelid) as expression
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
and c.contype = 'c'",
)
.bind(table)
.bind(constraint)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
}
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_class idx
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1
) as present",
)
.bind(index)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn index_unique(
connection: &mut PgConnection,
table: &str,
index: &str,
) -> Result<Option<bool>, MigrationError> {
let row = query(
"select i.indisunique, i.indisvalid
from pg_catalog.pg_index i
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
join pg_catalog.pg_class t on t.oid = i.indrelid
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1 and t.relname = $2",
)
.bind(index)
.bind(table)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| {
let unique = row.try_get::<bool, _>("indisunique").ok()?;
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
Some(unique && valid)
}))
}
@@ -0,0 +1,680 @@
//! PostgreSQL implementation of the mandatory execution-admission store (Story 4.1, FR-21/FR-24).
//!
//! Source of truth for capacity and durable enqueue decisions of every process: a short transaction
//! locks one deployment controller row and one workspace controller row in a fixed order, then reads
//! bounded counters, inserts/transitions at most one entry and persists the weighted round-robin
//! cursor. No network call happens under the transaction and no connection is held while a waiter is
//! idle between attempts.
use std::collections::BTreeMap;
use std::time::Duration;
use async_trait::async_trait;
use crank_core::{
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, TaskError,
select_next_grant,
};
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
const DEPLOYMENT_CONTROLLER: &str = "*";
const OVERLOAD_RETRY_AFTER_MS: u64 = crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS;
const MAX_WAIT_BACKOFF_MS: u64 = 250;
const INITIAL_WAIT_BACKOFF_MS: u64 = 5;
const EXPIRED_CLEANUP_BATCH: i64 = 128;
const QUEUE_READ_LIMIT: i64 = 1024;
/// Deployment-wide bounds of one admission store. They come from validated deployment
/// configuration, never from a request.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionLimits {
pub global_limit: usize,
pub agent_limit: usize,
pub tool_limit: usize,
pub queue_limit: usize,
}
impl Default for AdmissionLimits {
fn default() -> Self {
Self {
global_limit: 64,
agent_limit: 16,
tool_limit: 8,
queue_limit: 32,
}
}
}
impl AdmissionLimits {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
for (field, value) in [
("global_limit", self.global_limit),
("agent_limit", self.agent_limit),
("tool_limit", self.tool_limit),
] {
if !(1..=65_535).contains(&value) {
return Err(AdmissionStoreError::invalid(format!(
"admission {field} is outside its allowed bounds"
)));
}
}
if !(1..=1_024).contains(&self.queue_limit) {
return Err(AdmissionStoreError::invalid(
"admission queue_limit is outside its allowed bounds",
));
}
Ok(())
}
}
#[derive(Clone)]
pub struct PostgresExecutionAdmission {
pool: PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
}
impl PostgresExecutionAdmission {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> Result<Self, AdmissionStoreError> {
limits.validate()?;
scheduler.validate()?;
Ok(Self {
pool,
limits,
scheduler,
})
}
fn unavailable(error: impl std::fmt::Display) -> AdmissionStoreError {
AdmissionStoreError::unavailable(format!(
"execution admission store is unavailable: {error}"
))
}
}
struct AttemptContext<'a> {
request: &'a AdmissionRequest,
owner_token: &'a str,
}
enum Attempt {
Granted,
Enqueued(u64),
Waiting,
Overloaded,
Expired,
}
#[async_trait]
impl ExecutionAdmission for PostgresExecutionAdmission {
async fn health(&self) -> Result<(), AdmissionStoreError> {
admission_health(&self.pool).await
}
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<AdmissionOutcome, AdmissionStoreError> {
request.validate()?;
if request.approval == ApprovalEvidence::NotGranted {
return Ok(AdmissionOutcome::ApprovalRequired);
}
if request.mode == AdmissionMode::Background
&& request.no_tasks_fallback == NoTasksFallback::Unsupported
{
return Ok(AdmissionOutcome::Unsupported);
}
let owner_token = format!("adm_{}", Uuid::now_v7().simple());
let mut ticket: Option<u64> = None;
let mut backoff = INITIAL_WAIT_BACKOFF_MS;
loop {
if crank_core::current_unix_ms() >= request.expires_at_unix_ms {
if let Some(ticket_id) = ticket {
self.cancel(ticket_id, &owner_token).await?;
}
return Ok(AdmissionOutcome::DeadlineElapsed);
}
let attempt = self
.attempt(
AttemptContext {
request: &request,
owner_token: &owner_token,
},
&mut ticket,
)
.await?;
match attempt {
Attempt::Granted => {
return Ok(AdmissionOutcome::Permitted(AdmissionPermit {
ticket_id: ticket.unwrap_or_default(),
owner_token,
scope: request.scope.clone(),
mode: request.mode,
policy_mode: request.policy_mode,
}));
}
Attempt::Enqueued(ticket_id) => {
return Ok(AdmissionOutcome::Enqueued(AdmissionDecision {
ticket_id,
scope: request.scope.clone(),
expires_at_unix_ms: request.expires_at_unix_ms,
policy_mode: request.policy_mode,
}));
}
Attempt::Expired => {
return Ok(AdmissionOutcome::DeadlineElapsed);
}
Attempt::Overloaded => {
return Ok(AdmissionOutcome::Overloaded {
retry_after_ms: OVERLOAD_RETRY_AFTER_MS,
});
}
Attempt::Waiting => {
let remaining = request
.expires_at_unix_ms
.saturating_sub(crank_core::current_unix_ms())
.max(1);
let sleep_ms = i64::try_from(backoff).unwrap_or(remaining).min(remaining);
tokio::time::sleep(Duration::from_millis(
u64::try_from(sleep_ms).unwrap_or(backoff),
))
.await;
backoff = (backoff * 2).min(MAX_WAIT_BACKOFF_MS);
}
}
}
}
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError> {
let result = sqlx::query(
"delete from execution_admission_entries
where ticket_id = $1 and owner_token = $2 and state = 'executing'
and task_id is null",
)
.bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX))
.bind(&permit.owner_token)
.execute(&self.pool)
.await
.map_err(Self::unavailable)?;
Ok(result.rows_affected() > 0)
}
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError> {
let result = sqlx::query(
"delete from execution_admission_entries
where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')
and task_id is null",
)
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
.bind(owner_token)
.execute(&self.pool)
.await
.map_err(Self::unavailable)?;
Ok(result.rows_affected() > 0)
}
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
admission_snapshot(&self.pool).await
}
}
/// Readiness probe of the mandatory admission state: both tables must be reachable and readable.
pub async fn admission_health(pool: &PgPool) -> Result<(), AdmissionStoreError> {
sqlx::query(
"select count(*)::bigint from execution_admission_controllers where workspace_id = $1",
)
.bind(DEPLOYMENT_CONTROLLER)
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
sqlx::query("select count(*)::bigint from execution_admission_entries where false")
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
Ok(())
}
/// Authoritative queue readings of the deployment-wide admission state.
pub async fn admission_snapshot(
pool: &PgPool,
) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
let row = sqlx::query(
"select
count(*) filter (where state in ('waiting', 'reserved')) as queued,
count(*) filter (where state = 'executing') as executing,
coalesce(
(extract(epoch from (now() - min(created_at) filter (where state in ('waiting', 'reserved')))) * 1000)::bigint,
-1
) as oldest_waiting_ms
from execution_admission_entries",
)
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
let queued: i64 = row
.try_get("queued")
.map_err(PostgresExecutionAdmission::unavailable)?;
let executing: i64 = row
.try_get("executing")
.map_err(PostgresExecutionAdmission::unavailable)?;
let oldest: i64 = row
.try_get("oldest_waiting_ms")
.map_err(PostgresExecutionAdmission::unavailable)?;
Ok(AdmissionQueueSnapshot {
waiting: u64::try_from(queued).unwrap_or_default(),
executing: u64::try_from(executing).unwrap_or_default(),
oldest_waiting_ms: (oldest >= 0).then(|| u64::try_from(oldest).unwrap_or_default()),
})
}
impl PostgresExecutionAdmission {
async fn attempt(
&self,
context: AttemptContext<'_>,
ticket: &mut Option<u64>,
) -> Result<Attempt, AdmissionStoreError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
match self.attempt_locked(&mut transaction, context, ticket).await {
Ok(attempt) => {
transaction.commit().await.map_err(Self::unavailable)?;
Ok(attempt)
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Ok(Attempt::Waiting)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Ok(Attempt::Overloaded)
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(AdmissionStoreError::unavailable(error.to_string()))
}
Err(AttemptError::AdmissionFailure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
Err(AttemptError::UniqueViolation) => {
transaction.rollback().await.ok();
Err(AdmissionStoreError::unavailable(
"execution admission hit an unexpected unique conflict",
))
}
}
}
async fn attempt_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
context: AttemptContext<'_>,
ticket: &mut Option<u64>,
) -> Result<Attempt, AttemptError> {
let workspace_id = context.request.scope.workspace_id.as_str();
let cursor = lock_admission_in_tx(transaction, workspace_id).await?;
// Expired queue decisions and waiting tickets leave the bounded queue; held permits and
// Task-linked dispatch intents never do (Story 4.2 owns the durable Task lifetime).
cleanup_expired_unlinked(transaction).await?;
if ticket.is_none() {
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let state = match context.request.mode {
AdmissionMode::Synchronous => "waiting",
AdmissionMode::Background => "reserved",
};
let ticket_id = insert_admission_entry(
transaction,
context.request,
context.owner_token,
state,
None,
)
.await?;
*ticket = Some(ticket_id);
if context.request.mode == AdmissionMode::Background {
return Ok(Attempt::Enqueued(ticket_id));
}
}
self.grant_next(transaction, workspace_id, &cursor).await?;
let Some(ticket_id) = *ticket else {
return Ok(Attempt::Waiting);
};
let state: Option<String> =
sqlx::query("select state from execution_admission_entries where ticket_id = $1")
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.map(|row| row.try_get("state"))
.transpose()
.map_err(AttemptError::from_sql)?;
match state.as_deref() {
Some("executing") => Ok(Attempt::Granted),
Some(_) => Ok(Attempt::Waiting),
// The ticket expired inside the store (or was removed by another cleanup).
None => Ok(Attempt::Expired),
}
}
/// Grants at most one waiting ticket per transaction, following the persisted weighted
/// round-robin cursor. A blocked head keeps its turn: nothing later is granted in its place.
async fn grant_next(
&self,
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
cursor: &AdmissionCursor,
) -> Result<(), AttemptError> {
let rows = sqlx::query(
"select ticket_id, agent_key, operation_id from execution_admission_entries
where workspace_id = $1 and state = 'waiting'
order by ticket_id
limit $2",
)
.bind(workspace_id)
.bind(QUEUE_READ_LIMIT)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
if rows.is_empty() {
return Ok(());
}
let queue = rows
.iter()
.map(|row| {
Ok(AdmissionQueueEntry {
ticket_id: u64::try_from(
row.try_get::<i64, _>("ticket_id")
.map_err(AttemptError::from_sql)?,
)
.unwrap_or_default(),
sequence: 0,
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
operation_id: row
.try_get("operation_id")
.map_err(AttemptError::from_sql)?,
})
})
.collect::<Result<Vec<_>, AttemptError>>()?;
let Some(selection) = select_next_grant(&queue, cursor, self.scheduler) else {
return Ok(());
};
let head = &queue[selection.index];
let global_executing: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries
where state = 'executing'",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
let agent_counts = scoped_counts(transaction, workspace_id, "agent_key").await?;
let tool_counts = scoped_counts(transaction, workspace_id, "operation_id").await?;
if u64::try_from(global_executing).unwrap_or(u64::MAX)
>= u64::try_from(self.limits.global_limit).unwrap_or(u64::MAX)
{
return Ok(());
}
let agent_executing = agent_counts
.get(head.agent_key.as_str())
.copied()
.unwrap_or_default();
if agent_executing >= u64::try_from(self.limits.agent_limit).unwrap_or(u64::MAX) {
return Ok(());
}
let tool_executing = tool_counts
.get(head.operation_id.as_str())
.copied()
.unwrap_or_default();
if tool_executing >= u64::try_from(self.limits.tool_limit).unwrap_or(u64::MAX) {
return Ok(());
}
sqlx::query(
"update execution_admission_entries
set state = 'executing', granted_at = now()
where ticket_id = $1 and state = 'waiting'",
)
.bind(i64::try_from(head.ticket_id).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
sqlx::query(
"update execution_admission_controllers
set cursor_agent = $2,
cursor_agent_credit = $3,
cursor_tool = $4,
cursor_tool_credit = $5,
updated_at = now()
where workspace_id = $1",
)
.bind(workspace_id)
.bind(selection.cursor.agent.as_deref())
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
.bind(selection.cursor.tool.as_deref())
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(())
}
}
/// Takes both controller locks in the fixed `'*'` -> workspace order. Shared by the synchronous
/// admission attempt and the durable Task repository, so both serialize on the same controller rows
/// and one task/queue invariant holds. Callers own the transaction; nothing here commits or sleeps.
pub(super) async fn lock_admission_in_tx(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
) -> Result<AdmissionCursor, AttemptError> {
sqlx::query(
"insert into execution_admission_controllers (workspace_id)
values ($1), ($2)
on conflict (workspace_id) do nothing",
)
.bind(DEPLOYMENT_CONTROLLER)
.bind(workspace_id)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?;
lock_controller(transaction, workspace_id).await
}
/// Bounded cleanup of expired waiting/reserved rows that are not a durable Task dispatch intent.
pub(super) async fn cleanup_expired_unlinked(
transaction: &mut Transaction<'_, Postgres>,
) -> Result<(), AttemptError> {
sqlx::query(
"delete from execution_admission_entries
where ticket_id in (
select ticket_id from execution_admission_entries
where state in ('waiting', 'reserved') and expires_at <= now()
and task_id is null
order by ticket_id
limit $1
)",
)
.bind(EXPIRED_CLEANUP_BATCH)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(())
}
/// Authoritative bounded queue length: waiting and reserved rows share one budget.
pub(super) async fn bounded_queue_len(
transaction: &mut Transaction<'_, Postgres>,
) -> Result<u64, AttemptError> {
let queued: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries
where state in ('waiting', 'reserved')",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
Ok(u64::try_from(queued).unwrap_or(u64::MAX))
}
/// Inserts one bounded admission row. A Task-linked row is the durable dispatch intent of the Task
/// accepted in the same transaction, so its capacity slot can never be silently reclaimed.
pub(super) async fn insert_admission_entry(
transaction: &mut Transaction<'_, Postgres>,
request: &AdmissionRequest,
owner_token: &str,
state: &str,
task_id: Option<Uuid>,
) -> Result<u64, AttemptError> {
let row = sqlx::query(
"insert into execution_admission_entries (
workspace_id, agent_key, operation_id, operation_version, mode, policy_mode,
state, owner_token, fingerprint, expires_at, task_id
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9,
to_timestamp($10::double precision / 1000.0), $11)
returning ticket_id",
)
.bind(&request.scope.workspace_id)
.bind(&request.scope.agent_key)
.bind(&request.scope.operation_id)
.bind(i32::try_from(request.scope.operation_version).unwrap_or(i32::MAX))
.bind(request.mode.as_str())
.bind(policy_mode_str(request))
.bind(state)
.bind(owner_token)
.bind(&request.fingerprint)
.bind(request.expires_at_unix_ms)
.bind(task_id)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
Ok(u64::try_from(ticket_id).unwrap_or_default())
}
pub(super) enum AttemptError {
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
Contended,
/// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait.
Overloaded,
/// A foreign unique constraint aborted the transaction; the caller rolls back and rereads.
UniqueViolation,
/// Durable-Task side failure (the Task repository maps it to its own typed error).
Failure(TaskError),
/// Admission side failure.
AdmissionFailure(AdmissionStoreError),
}
impl AttemptError {
pub(super) fn from_sql(error: sqlx::Error) -> Self {
if let Some(database) = error.as_database_error()
&& matches!(database.code().as_deref(), Some("55P03" | "57014"))
{
return Self::Contended;
}
Self::AdmissionFailure(PostgresExecutionAdmission::unavailable(error))
}
}
async fn lock_controller(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
) -> Result<AdmissionCursor, AttemptError> {
sqlx::query(
"update execution_admission_controllers set updated_at = updated_at
where workspace_id = $1",
)
.bind(workspace_id)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let row = sqlx::query(
"select cursor_agent, cursor_agent_credit, cursor_tool, cursor_tool_credit
from execution_admission_controllers where workspace_id = $1",
)
.bind(workspace_id)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(AdmissionCursor {
agent: row
.try_get("cursor_agent")
.map_err(AttemptError::from_sql)?,
agent_credit: u32::try_from(
row.try_get::<i32, _>("cursor_agent_credit")
.map_err(AttemptError::from_sql)?
.max(0),
)
.unwrap_or_default(),
tool: row.try_get("cursor_tool").map_err(AttemptError::from_sql)?,
tool_credit: u32::try_from(
row.try_get::<i32, _>("cursor_tool_credit")
.map_err(AttemptError::from_sql)?
.max(0),
)
.unwrap_or_default(),
})
}
async fn scoped_counts(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
column: &str,
) -> Result<BTreeMap<String, u64>, AttemptError> {
let statement = if column == "agent_key" {
"select agent_key as key, count(*)::bigint as count
from execution_admission_entries
where workspace_id = $1 and state = 'executing'
group by agent_key"
} else {
"select operation_id as key, count(*)::bigint as count
from execution_admission_entries
where workspace_id = $1 and state = 'executing'
group by operation_id"
};
let rows = sqlx::query(statement)
.bind(workspace_id)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let mut counts = BTreeMap::new();
for row in rows {
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
counts.insert(key, u64::try_from(count).unwrap_or_default());
}
Ok(counts)
}
fn policy_mode_str(request: &AdmissionRequest) -> &'static str {
match request.policy_mode {
crank_core::ToolExecutionMode::Synchronous => "synchronous",
crank_core::ToolExecutionMode::Background => "background",
crank_core::ToolExecutionMode::Adaptive => "adaptive",
}
}
@@ -46,6 +46,24 @@ impl PostgresRegistry {
Ok(())
}
/// Readiness probe of the mandatory execution-admission state (Story 4.1). A missing or
/// unreadable admission schema is a dependency failure, never a degraded-but-ready state.
pub async fn execution_admission_health(&self) -> Result<(), crank_core::AdmissionStoreError> {
super::admission::admission_health(&self.pool).await
}
/// Readiness probe of the durable Task state (Story 4.2).
pub async fn durable_tasks_health(&self) -> Result<(), crank_core::TaskError> {
super::tasks::tasks_health(&self.pool).await
}
/// Authoritative admission queue readings for low-cardinality metrics.
pub async fn execution_admission_snapshot(
&self,
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
super::admission::admission_snapshot(&self.pool).await
}
async fn connect_in_schema(
database_url: &str,
schema: Option<&str>,
@@ -1,3 +1,4 @@
mod admission;
mod agent;
mod agent_catalog;
mod agent_prompt_bindings;
@@ -20,6 +21,7 @@ mod product_event;
mod prompt;
mod resource;
mod secret;
mod tasks;
mod upstream;
mod workspace;
mod yaml_import;
@@ -36,8 +38,13 @@ use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow, types::Json};
use time::OffsetDateTime;
pub use admission::{
AdmissionLimits, PostgresExecutionAdmission, admission_health, admission_snapshot,
};
pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError};
pub use resource::{AuthorizedResourceRead, BoundResourceAdmission};
pub use tasks::PostgresTaskStore;
pub use tasks::tasks_health;
use crate::{
error::RegistryError,
+586
View File
@@ -0,0 +1,586 @@
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
//!
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
//!
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
use async_trait::async_trait;
use crank_core::{
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode,
};
use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
use super::admission::{
AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked,
insert_admission_entry, lock_admission_in_tx,
};
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode,
approval_reference,
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
result_envelope, error_envelope, attempts, version,
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
(extract(epoch from created_at) * 1000)::bigint as created_ms,
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
#[derive(Clone)]
pub struct PostgresTaskStore {
pool: PgPool,
limits: AdmissionLimits,
}
impl PostgresTaskStore {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
) -> Result<Self, TaskError> {
limits
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
scheduler
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
Ok(Self { pool, limits })
}
fn unavailable(error: impl std::fmt::Display) -> TaskError {
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
}
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
Ok(transaction)
}
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
/// query, so a foreign row is never fetched.
async fn stored_acceptance(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let Some(row) = row else {
return Ok(None);
};
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
Ok(Some((
acceptance_from_record(&record),
record.input_fingerprint,
)))
}
}
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support.clone(),
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
}
}
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(pool)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}
/// One transaction's decision of the acceptance path.
enum Accepted {
Created(TaskAcceptance),
Replayed(TaskAcceptance),
Conflict,
}
#[async_trait]
impl TaskStore for PostgresTaskStore {
async fn health(&self) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(&self.pool)
.await
.map_err(Self::unavailable)?;
super::admission::admission_health(&self.pool)
.await
.map_err(|error| TaskError::unavailable(error.to_string()))?;
Ok(())
}
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let outcome = self.accept_locked(&mut transaction, &request).await;
match outcome {
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
// The acknowledgement of a created Task is only possible after a real COMMIT.
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskAcceptance {
replayed: acceptance.replayed,
..acceptance
})
}
Ok(Accepted::Conflict) => {
transaction.rollback().await.ok();
Err(TaskError::FingerprintConflict)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Err(TaskError::Overloaded {
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
})
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(
"durable task acceptance could not take the admission locks",
))
}
Err(AttemptError::UniqueViolation) => {
// Roll back the aborted transaction, then reread the scoped key once outside it.
transaction.rollback().await.ok();
self.reread_after_unique(&request).await
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
Err(AttemptError::AdmissionFailure(error)) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(error.to_string()))
}
}
}
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
record_from_row(&row).map_err(Self::unavailable)
}
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(operation_id)
.bind(scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
row.map(|row| record_from_row(&row)).transpose()
}
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now()
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = update else {
transaction.rollback().await.ok();
return Err(self.classify_missed_finish(&request).await);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
let expired = sqlx::query(
"with expired as (
update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
version = version + 1,
updated_at = now()
where task_id in (
select task_id from tasks
where state = 'queued' and execution_deadline <= now()
order by task_id
limit $1
)
returning task_id
), released as (
delete from execution_admission_entries
where task_id in (select task_id from expired)
returning task_id
)
select (select count(*) from expired)::bigint as expired_queued",
)
.bind(batch)
.fetch_one(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let expired_queued: i64 = expired
.try_get("expired_queued")
.map_err(Self::unavailable)?;
let purged = sqlx::query(
"delete from tasks
where task_id in (
select task_id from tasks
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
order by task_id
limit $1
)",
)
.bind(batch)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskPurgeReport {
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
expired_terminal: purged.rows_affected(),
})
}
}
impl PostgresTaskStore {
async fn accept_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Accepted, AttemptError> {
// Replay first: an already accepted Task is returned without capacity mutation, so it works
// with a full queue and after the original execution deadline while the record is retained.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
lock_admission_in_tx(transaction, &request.workspace_id).await?;
// The competitor may have committed while this transaction waited for the controller lock.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
cleanup_expired_unlinked(transaction).await?;
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let created_ms: i64 = sqlx::query(
"insert into tasks (
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued',
$16, to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0)
)
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
)
.bind(request.task_id.as_uuid())
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
.bind(&request.scoped_key)
.bind(&request.input_fingerprint)
.bind(request.support.request_id.as_str())
.bind(request.support.trace_id.as_str())
.bind(&request.support.traceparent)
.bind(request.support.trace_sampled)
.bind(policy_mode_str(request.pins.policy_mode))
.bind(request.pins.approval_reference.as_deref())
.bind(request.input.value().clone())
.bind(request.pins.execution_deadline_unix_ms)
.bind(request.pins.retain_until_unix_ms)
.fetch_one(&mut **transaction)
.await
.map_err(task_sql_error)?
.try_get("created_ms")
.map_err(AttemptError::from_sql)?;
let admission_request = AdmissionRequest {
scope: AdmissionScope {
workspace_id: request.workspace_id.clone(),
agent_key: request.owner_id.clone(),
operation_id: request.operation_id.clone(),
operation_version: request.pins.operation_version,
},
mode: AdmissionMode::Background,
policy_mode: request.pins.policy_mode,
no_tasks_fallback: NoTasksFallback::Unsupported,
// Never reached: the intent row is inserted by this transaction, not by `admit`.
approval: ApprovalEvidence::NotRequired,
fingerprint: request.input_fingerprint.clone(),
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
};
let owner_token = format!("task_{}", request.task_id);
insert_admission_entry(
transaction,
&admission_request,
&owner_token,
"reserved",
Some(request.task_id.as_uuid()),
)
.await?;
Ok(Accepted::Created(TaskAcceptance {
task_id: request.task_id,
state: TaskState::Queued,
operation_version: request.pins.operation_version,
support: request.support.clone(),
created_at_unix_ms: created_ms,
retain_until_unix_ms: request.pins.retain_until_unix_ms,
replayed: false,
}))
}
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
async fn reread_after_unique(
&self,
request: &TaskAcceptRequest,
) -> Result<TaskAcceptance, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::unavailable(
"durable task acceptance ended with an unresolved unique conflict",
));
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
return Err(TaskError::FingerprintConflict);
}
Ok(acceptance_from_record(&record))
}
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
let access = &request.access;
match sqlx::query(
"select state, version from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
{
Ok(Some(_)) => TaskError::StateConflict,
Ok(None) => TaskError::NotFound,
Err(error) => Self::unavailable(error),
}
}
}
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
match mode {
ToolExecutionMode::Synchronous => "synchronous",
ToolExecutionMode::Background => "background",
ToolExecutionMode::Adaptive => "adaptive",
}
}
fn task_sql_error(error: sqlx::Error) -> AttemptError {
if error
.as_database_error()
.and_then(|database| database.code())
.as_deref()
== Some("23505")
{
return AttemptError::UniqueViolation;
}
AttemptError::from_sql(error)
}
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let state: String = row.try_get("state").map_err(unavailable_row)?;
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
value.map(TaskEnvelope::from_persisted).transpose()
};
Ok(TaskRecord {
task_id: TaskId::parse(&task_id.to_string())?,
state: TaskState::parse(&state)?,
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
authorization_scope: row
.try_get("authorization_scope")
.map_err(unavailable_row)?,
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
pins: TaskPins {
operation_version: u32::try_from(
row.try_get::<i32, _>("operation_version")
.map_err(unavailable_row)?,
)
.unwrap_or_default(),
policy_mode: match policy_mode.as_str() {
"background" => ToolExecutionMode::Background,
"adaptive" => ToolExecutionMode::Adaptive,
_ => ToolExecutionMode::Synchronous,
},
execution_deadline_unix_ms: row
.try_get("execution_deadline_ms")
.map_err(unavailable_row)?,
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
},
support: TaskSupportIds {
request_id: crank_core::RequestId::parse(&request_id)
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
trace_id: crank_core::TraceId::parse(&trace_id)
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
},
input: envelope("input_envelope")?
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
progress: envelope("progress_envelope")?,
result: envelope("result_envelope")?,
error: envelope("error_envelope")?,
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
.unwrap_or_default(),
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
.unwrap_or_default(),
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
})
}
fn unavailable_row(error: sqlx::Error) -> TaskError {
PostgresTaskStore::unavailable(error)
}
@@ -1,4 +1,5 @@
mod integration {
mod admission;
mod agent_resource_bindings;
mod agents_usage;
mod approval;
@@ -16,5 +17,6 @@ mod integration {
mod prompt_drafts;
mod prompt_publication;
mod resource_drafts;
mod tasks;
mod workspace_access;
}
@@ -0,0 +1,496 @@
//! ATDD Story 4.1 (FR-21/FR-24): authoritative PostgreSQL execution admission.
//!
//! These oracles drive the real store against a real PostgreSQL schema: scoped permits, bounded
//! queue, typed overload, durable enqueue decisions, deadline propagation, quota keys that ignore
//! the operation version, fairness order and fail-closed health.
use super::common;
use std::time::{Duration, Instant};
use crank_core::{
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
AdmissionSchedulerConfig, AdmissionScope, ApprovalEvidence, ExecutionAdmission,
NoTasksFallback, ToolExecutionMode, current_unix_ms,
};
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
use tokio::sync::mpsc;
fn limits(global: usize, agent: usize, tool: usize, queue: usize) -> AdmissionLimits {
AdmissionLimits {
global_limit: global,
agent_limit: agent,
tool_limit: tool,
queue_limit: queue,
}
}
fn store(
pool: sqlx::PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> PostgresExecutionAdmission {
PostgresExecutionAdmission::new(pool, limits, scheduler).expect("valid admission store")
}
fn scope(agent: &str, operation: &str, version: u32) -> AdmissionScope {
AdmissionScope {
workspace_id: "ws_default".to_owned(),
agent_key: agent.to_owned(),
operation_id: operation.to_owned(),
operation_version: version,
}
}
fn request(scope: AdmissionScope, deadline_in_ms: i64) -> AdmissionRequest {
AdmissionRequest {
scope,
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: format!("fp_{}", current_unix_ms()),
expires_at_unix_ms: current_unix_ms() + deadline_in_ms,
}
}
fn background(scope: AdmissionScope, fallback: NoTasksFallback) -> AdmissionRequest {
AdmissionRequest {
mode: AdmissionMode::Background,
no_tasks_fallback: fallback,
..request(scope, 30_000)
}
}
fn permit(outcome: AdmissionOutcome) -> AdmissionPermit {
match outcome {
AdmissionOutcome::Permitted(permit) => permit,
other => panic!("expected a scoped permit, got {other:?}"),
}
}
async fn wait_for_metric_poll() {
tokio::time::sleep(Duration::from_millis(20)).await;
}
#[tokio::test]
async fn permits_are_scoped_released_once_and_respect_the_agent_budget() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 1, 4, 8),
Default::default(),
);
let first = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
assert_eq!(first.scope.agent_key, "agent_a");
// The same Agent is already at its scoped budget: the second request waits and expires instead of
// over-admitting, and it never dispatches.
let waiting = admission
.admit(request(scope("agent_a", "op_leads", 2), 120))
.await
.expect("admission store is reachable");
assert_eq!(waiting, AdmissionOutcome::DeadlineElapsed);
// Another Agent still has the global capacity it is entitled to.
let other = permit(
admission
.admit(request(scope("agent_b", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
assert_eq!(other.scope.agent_key, "agent_b");
assert!(admission.release(&other).await.unwrap());
// Repeated release is idempotent and never frees a foreign or already released permit.
assert!(!admission.release(&other).await.unwrap());
assert!(admission.release(&first).await.unwrap());
let snapshot = admission.snapshot().await.unwrap();
assert_eq!(snapshot.executing, 0);
assert_eq!(snapshot.waiting, 0);
database.cleanup().await;
}
#[tokio::test]
async fn full_queue_returns_a_typed_overload_with_retry_guidance() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 1, 1, 2),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable"),
);
// Two bounded waiting entries fill the queue.
let mut waiters = Vec::new();
for version in 1..=2 {
let admission = admission.clone();
waiters.push(tokio::spawn(async move {
admission
.admit(request(scope("agent_b", "op_leads", version), 5_000))
.await
}));
}
while admission.snapshot().await.unwrap().waiting < 2 {
wait_for_metric_poll().await;
}
let rejected = admission
.admit(request(scope("agent_c", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable");
match rejected {
AdmissionOutcome::Overloaded { retry_after_ms } => {
assert!((1..=crank_core::MAX_RETRY_AFTER_MS).contains(&retry_after_ms));
}
other => panic!("expected typed overload, got {other:?}"),
}
// Cancelling the held permit lets the queued waiters make progress instead of leaking capacity.
assert!(admission.release(&held).await.unwrap());
for waiter in waiters.drain(..) {
let outcome = waiter.await.expect("waiter task");
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
let _ = admission.release(&permit).await;
}
}
database.cleanup().await;
}
#[tokio::test]
async fn background_decisions_are_durable_pinned_and_released_once() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let decision = match admission
.admit(background(
scope("agent_a", "op_report", 3),
NoTasksFallback::Synchronous,
))
.await
.expect("admission store is reachable")
{
AdmissionOutcome::Enqueued(decision) => decision,
other => panic!("expected a durable enqueue decision, got {other:?}"),
};
assert_eq!(decision.scope.operation_version, 3);
assert_eq!(decision.policy_mode, ToolExecutionMode::Synchronous);
// A second process (fresh pool and store over the same schema) must read the committed decision:
// it occupies bounded queue capacity and is not a runnable execution.
let reconnected = store(
database.concurrent_registry(2).await.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let snapshot = reconnected.snapshot().await.unwrap();
assert_eq!(snapshot.waiting, 1);
assert_eq!(snapshot.executing, 0);
assert!(
reconnected
.cancel(decision.ticket_id, "not_the_owner")
.await
.unwrap()
.eq(&false),
"a foreign cancel must never release the decision"
);
assert!(
reconnected.snapshot().await.unwrap().waiting.eq(&1),
"the foreign cancel must leave the decision intact"
);
database.cleanup().await;
}
#[tokio::test]
async fn background_without_a_task_transport_is_typed_unsupported() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let outcome = admission
.admit(background(
scope("agent_a", "op_report", 1),
NoTasksFallback::Unsupported,
))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::Unsupported);
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
database.cleanup().await;
}
#[tokio::test]
async fn unverified_approval_never_produces_a_decision() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let mut denied = background(
scope("agent_a", "op_report", 1),
NoTasksFallback::Synchronous,
);
denied.approval = ApprovalEvidence::NotGranted;
assert_eq!(
admission.admit(denied).await.unwrap(),
AdmissionOutcome::ApprovalRequired
);
let mut denied_sync = request(scope("agent_a", "op_report", 1), 1_000);
denied_sync.approval = ApprovalEvidence::NotGranted;
assert_eq!(
admission.admit(denied_sync).await.unwrap(),
AdmissionOutcome::ApprovalRequired
);
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
assert_eq!(admission.snapshot().await.unwrap().executing, 0);
database.cleanup().await;
}
#[tokio::test]
async fn quota_keys_ignore_the_operation_version() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 1, 8),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
// A different Published Version of the same Tool shares the same per-Tool budget.
let outcome = admission
.admit(request(scope("agent_a", "op_leads", 2), 120))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
assert!(admission.release(&held).await.unwrap());
database.cleanup().await;
}
#[tokio::test]
async fn weighted_agents_are_served_their_quantum_in_order() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 4, 4, 8),
AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
},
);
let held = permit(
admission
.admit(request(scope("agent_z", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable"),
);
let (sender, mut receiver) = mpsc::channel(3);
let mut tasks = Vec::new();
for (agent, version) in [("agent_a", 1), ("agent_a", 2), ("agent_b", 1)] {
let admission = admission.clone();
let sender = sender.clone();
tasks.push(tokio::spawn(async move {
let outcome = admission
.admit(request(scope(agent, "op_leads", version), 5_000))
.await;
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
let _ = sender.send(permit.scope.agent_key.clone()).await;
// Hold briefly so the grant order, not task scheduling, decides the sequence.
tokio::time::sleep(Duration::from_millis(30)).await;
let _ = admission.release(&permit).await;
}
}));
}
drop(sender);
assert!(admission.release(&held).await.unwrap());
let mut order = Vec::new();
while let Some(agent) = receiver.recv().await {
order.push(agent);
if order.len() == 3 {
break;
}
}
for task in tasks {
let _ = tokio::time::timeout(Duration::from_secs(5), task).await;
}
// agent_a carries weight 2 and therefore receives two consecutive grants before agent_b.
assert_eq!(order, vec!["agent_a", "agent_a", "agent_b"]);
database.cleanup().await;
}
#[tokio::test]
async fn mandatory_state_failure_fails_closed_and_reports_not_ready() {
let database = common::TestDatabase::new().await;
// No migration on this schema: the mandatory admission state does not exist yet.
let unmigrated = database.raw_pool().await;
let admission = store(unmigrated, limits(4, 4, 4, 8), Default::default());
assert!(admission.health().await.is_err());
assert!(
admission
.admit(request(scope("agent_a", "op_leads", 1), 1_000))
.await
.is_err()
);
database.cleanup().await;
}
#[tokio::test]
async fn held_permits_survive_a_new_process_view() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 4, 4, 8),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
let reconnected = store(
database.concurrent_registry(2).await.pool().clone(),
limits(1, 4, 4, 8),
Default::default(),
);
assert_eq!(reconnected.snapshot().await.unwrap().executing, 1);
let outcome = reconnected
.admit(request(scope("agent_b", "op_leads", 1), 120))
.await
.expect("admission store is reachable");
assert_eq!(
outcome,
AdmissionOutcome::DeadlineElapsed,
"a held permit is not silently reused by another process"
);
assert!(reconnected.release(&held).await.unwrap());
database.cleanup().await;
}
#[tokio::test]
async fn published_execution_policy_is_pinned_across_a_new_draft_version() {
use crank_core::{NoTasksFallback, OperationStatus, ToolExecutionMode, ToolExecutionPolicy};
use crank_registry::{CreateVersionRequest, PublishRequest};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let workspace = common::test_workspace_id();
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 45_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
let mut version_one = common::test_operation("op_policy", 1, OperationStatus::Draft);
version_one.execution_config.execution_policy = Some(policy);
registry
.create_operation(&workspace, &version_one, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &workspace,
operation_id: &version_one.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-25T12:10:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
let published = registry
.get_published_operation(&version_one.id)
.await
.unwrap()
.expect("published operation");
assert_eq!(published.execution_config.execution_policy, Some(policy));
// A later Draft with another policy never rewrites the already accepted publication.
let mut version_two = common::test_operation("op_policy", 2, OperationStatus::Draft);
version_two.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 1_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
});
registry
.create_version(CreateVersionRequest {
workspace_id: &workspace,
snapshot: &version_two,
change_note: Some("next draft"),
created_by: Some("alice"),
})
.await
.unwrap();
let unchanged = registry
.get_published_operation(&version_one.id)
.await
.unwrap()
.expect("published operation");
assert_eq!(unchanged.version, 1);
assert_eq!(unchanged.execution_config.execution_policy, Some(policy));
database.cleanup().await;
}
#[tokio::test]
async fn admin_origin_bucket_is_bounded_together_with_agents() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 1, 1, 4),
Default::default(),
);
let held = permit(
admission
.admit(request(scope(ADMIN_ORIGIN_KEY, "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
let started = Instant::now();
let outcome = admission
.admit(request(scope("agent_a", "op_leads", 1), 120))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
assert!(started.elapsed() >= Duration::from_millis(100));
assert!(admission.release(&held).await.unwrap());
database.cleanup().await;
}
@@ -70,6 +70,8 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
(21, "prompts-v21"),
(22, "prompt-publications-v22"),
(23, "prompt-dependencies-v23"),
(24, "execution-admission-v24"),
(25, "durable-tasks-v25"),
];
assert_eq!(rows.len(), expected.len());
for (row, (version, name)) in rows.iter().zip(expected) {
@@ -180,7 +182,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
);
assert_eq!(
MigrationAuthority::preflight(first.pool()).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
#[tokio::test]
@@ -228,7 +230,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
.expect("the V18-local guard must be restorable");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restoring the exact V18 contract must return the current verdict"
);
}
@@ -354,7 +356,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -467,13 +469,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 2,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
let trace_column: bool = sqlx::query_scalar(
"select exists (
@@ -516,7 +518,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 3,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -576,7 +578,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 4,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -830,6 +832,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 23,
target: 25,
}
);
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
.unwrap();
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
for (drift, restore) in [
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"a rejected primary-key nullability change must leave the schema current"
);
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"the baseline before the drift matrix must be the current V16 schema"
);
let mut observations: Vec<String> = Vec::new();
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
.await
.expect("restoring the exact definition must apply");
let restored = MigrationAuthority::preflight(&pool).await;
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 23 }));
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 }));
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
// мнением того же guard-а, который принимает неправильные определения (F4).
let restored_exact = catalog_definition_snapshot(&pool).await == before;
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"baseline before inserting the shadow schema"
);
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
// Тень не меняет вердикт guard-а по текущей схеме.
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"a same-named object in another schema must not change the verdict for the current schema"
);
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"the exact restoration must return the current V16 schema"
);
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
assert!(
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
let tables = sqlx::query(
"select table_name
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 13,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
assert_eq!(operation_count, 1);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
assert!(supports_elicitation);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
.fetch_one(&pool)
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 23,
target: 25,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 18, to: 23 }
MigrationApplyResult::Applied { from: 18, to: 25 }
);
let after: Vec<(i64, String, String)> =
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -84,18 +84,18 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() {
let database_url = crank_test_support::postgres_schema_url("test_v19_future_version").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
// The same `version > CURRENT_VERSION` branch an old V23 binary takes when it
// meets a V24 database; a synthetic V24 row exercises it without a second binary.
// The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer
// database; a synthetic version above the current one exercises it without a second binary.
sqlx::query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values (24, 'future-v24', repeat('a', 64), 'expand', 'n-minus-one-readable')",
values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')",
)
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "future_version");
assert_eq!(error.version(), Some(24));
assert_eq!(error.version(), Some(26));
assert_eq!(error.recovery(), "install_matching_application");
assert!(
MigrationAuthority::require_current(&pool).await.is_err(),
@@ -155,7 +155,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 23,
target: 25,
}
);
}
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 23,
target: 25,
}
);
}
@@ -38,6 +38,7 @@ pub(crate) async fn remove_v4_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v5_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v6_schema(pool).await;
sqlx::raw_sql(
"drop index if exists invocation_logs_workspace_operation_version_idx;
@@ -154,6 +155,7 @@ pub(crate) async fn remove_v11_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v14_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v15_schema(pool).await;
sqlx::raw_sql(
"drop table if exists import_plan_applications;
@@ -224,6 +226,7 @@ pub(crate) async fn remove_v18_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v19_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v20_schema(pool).await;
sqlx::raw_sql(
"drop table if exists resource_security_revocations;
@@ -307,6 +310,7 @@ pub(crate) async fn remove_v20_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v14_schema(pool).await;
sqlx::raw_sql(
"drop index if exists import_jobs_openapi_source_idx;
@@ -323,3 +327,27 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
.await
.unwrap();
}
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
sqlx::raw_sql(
"drop table if exists tasks cascade;
delete from __crank_migrations where version = 25;",
)
.execute(pool)
.await
.unwrap();
}
/// Story 4.1: снять V24 (authoritative execution admission) перед возвратом к < 24.
pub(crate) async fn remove_v24_schema(pool: &sqlx::PgPool) {
remove_v25_schema(pool).await;
sqlx::raw_sql(
"drop table if exists execution_admission_entries;
drop table if exists execution_admission_controllers;
delete from __crank_migrations where version = 24;",
)
.execute(pool)
.await
.unwrap();
}
@@ -0,0 +1,682 @@
//! ATDD Story 4.2 (FR-22/FR-23): durable Task acceptance on a real PostgreSQL schema.
//!
//! These oracles drive the real store: Task + dispatch intent + queue slot in one transaction, replay
//! and typed conflict before any capacity mutation, bounded shared capacity, the Task-linked intent
//! guard of the legacy admission paths, one-winner terminal CAS with an immutable result, bounded
//! retention, owner/authorization-scope isolation and fail-closed health.
use super::common;
use crank_core::{
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms,
};
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore};
use serde_json::json;
use sqlx::{PgPool, Row};
const WORKSPACE: &str = "ws_default";
const AGENT: &str = "agent_a";
fn limits(queue: usize) -> AdmissionLimits {
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: queue,
}
}
fn store_with(pool: PgPool, queue: usize) -> PostgresTaskStore {
PostgresTaskStore::new(pool, limits(queue), Default::default()).expect("valid store config")
}
fn unsampled_support() -> TaskSupportIds {
TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00").unwrap(),
)
}
fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> TaskAcceptRequest {
let deadline = current_unix_ms() + deadline_in_ms;
TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: WORKSPACE.to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: agent.to_owned(),
authorization_scope: format!("credential:pk_{agent}"),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: deadline,
retain_until_unix_ms: deadline + 3_600_000,
approval_reference: Some("appr_1".to_owned()),
},
scoped_key: key.to_owned(),
input_fingerprint: fingerprint.to_owned(),
input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(),
support: unsampled_support(),
}
}
fn access(agent: &str) -> TaskAccess {
TaskAccess {
workspace_id: WORKSPACE.to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: agent.to_owned(),
authorization_scope: format!("credential:pk_{agent}"),
}
}
fn key(seed: &str) -> String {
format!("{seed:0<64}").chars().take(64).collect()
}
fn digest(seed: &str) -> String {
format!("{seed:0<64}").chars().take(64).collect()
}
async fn row_counts(pool: &PgPool) -> (i64, i64) {
let row = sqlx::query(
"select
(select count(*)::bigint from tasks) as tasks,
(select count(*)::bigint from execution_admission_entries where task_id is not null)
as intents",
)
.fetch_one(pool)
.await
.unwrap();
(row.get("tasks"), row.get("intents"))
}
#[tokio::test]
async fn acceptance_commits_task_intent_and_slot_together() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let request = request(&key("k1"), &digest("f1"), AGENT, 30_000);
let task_id = request.task_id;
let support = request.support.clone();
// Another connection holds both controller rows: the acceptance cannot commit yet.
let mut blocker = registry.pool().begin().await.unwrap();
sqlx::query(
"insert into execution_admission_controllers (workspace_id) values ('*'), ($1)
on conflict (workspace_id) do nothing",
)
.bind(WORKSPACE)
.execute(&mut *blocker)
.await
.unwrap();
sqlx::query("update execution_admission_controllers set updated_at = updated_at")
.execute(&mut *blocker)
.await
.unwrap();
let pool = registry.pool().clone();
let acceptance = tokio::spawn(async move { store.accept(request).await });
tokio::time::sleep(std::time::Duration::from_millis(150)).await;
// No partial Task, no intent and no accepted response exist before COMMIT.
assert!(!acceptance.is_finished());
assert_eq!(row_counts(&pool).await, (0, 0));
blocker.commit().await.unwrap();
let acceptance = acceptance.await.unwrap().expect("acceptance commits");
assert_eq!(acceptance.state, TaskState::Queued);
assert!(!acceptance.replayed);
assert_eq!(acceptance.task_id, task_id);
assert_eq!(acceptance.support, support);
assert_eq!(row_counts(registry.pool()).await, (1, 1));
// A fresh connection and store read the committed pins/identities back.
let reader = store_with(database.concurrent_registry(2).await.pool().clone(), 8);
let record = reader.read(&access(AGENT), task_id).await.unwrap();
assert_eq!(record.state, TaskState::Queued);
assert_eq!(record.pins.operation_version, 3);
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
assert_eq!(record.pins.approval_reference.as_deref(), Some("appr_1"));
assert_eq!(record.support.request_id, support.request_id);
assert_eq!(record.support.trace_id, support.trace_id);
assert_eq!(record.support.traceparent, support.traceparent);
assert!(!record.support.trace_sampled);
// Secret-bearing input is classified before persistence.
assert_eq!(record.input.value()["api_key"], json!("[redacted]"));
assert_eq!(record.input.value()["report"], json!("q3"));
assert!(record.result.is_none());
database.cleanup().await;
}
#[tokio::test]
async fn replay_returns_the_same_task_and_one_slot() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let first = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect("first acceptance");
let before = store.read(&access(AGENT), first.task_id).await.unwrap();
let replay = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect("replay returns the accepted Task");
assert_eq!(replay.task_id, first.task_id);
assert!(replay.replayed);
assert_eq!(replay.support.request_id, first.support.request_id);
assert_eq!(row_counts(registry.pool()).await, (1, 1));
// Replay never rewrites the accepted row or its audit evidence.
let after = store.read(&access(AGENT), first.task_id).await.unwrap();
assert_eq!(after.version, before.version);
assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms);
assert_eq!(after.created_at_unix_ms, before.created_at_unix_ms);
database.cleanup().await;
}
#[tokio::test]
async fn replay_stays_available_with_a_full_queue_and_past_execution_deadline() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 1);
// A short deadline: the Task is still retained after its execution deadline elapsed.
let first = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
.await
.expect("first acceptance");
sqlx::query(
"update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1",
)
.bind(first.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
let replayed = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
.await
.expect("replay works with a full queue and an elapsed execution deadline");
assert_eq!(replayed.task_id, first.task_id);
assert!(replayed.replayed);
database.cleanup().await;
}
#[tokio::test]
async fn a_different_fingerprint_on_the_same_key_is_a_typed_conflict() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let first = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect("first acceptance");
let before = store.read(&access(AGENT), first.task_id).await.unwrap();
let conflict = store
.accept(request(&key("k1"), &digest("f2"), AGENT, 30_000))
.await
.expect_err("different input is a conflict");
assert_eq!(conflict, TaskError::FingerprintConflict);
assert_eq!(row_counts(registry.pool()).await, (1, 1));
let after = store.read(&access(AGENT), first.task_id).await.unwrap();
assert_eq!(after.version, before.version);
assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms);
database.cleanup().await;
}
#[tokio::test]
async fn bounded_capacity_is_shared_and_overload_creates_nothing() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 1);
let admission =
PostgresExecutionAdmission::new(registry.pool().clone(), limits(1), Default::default())
.unwrap();
// A durable enqueue decision and a durable Task share the same bounded queue budget.
let waiting = admission
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: WORKSPACE.to_owned(),
agent_key: AGENT.to_owned(),
operation_id: "op_sync".to_owned(),
operation_version: 1,
},
mode: AdmissionMode::Background,
policy_mode: ToolExecutionMode::Background,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_sync".to_owned(),
expires_at_unix_ms: current_unix_ms() + 30_000,
})
.await
.unwrap();
assert!(matches!(waiting, AdmissionOutcome::Enqueued(_)));
let overloaded = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect_err("queue is full");
assert!(matches!(overloaded, TaskError::Overloaded { .. }));
assert_eq!(row_counts(registry.pool()).await, (0, 0));
assert_eq!(
admission.snapshot().await.unwrap().waiting,
1,
"the durable refusal changed no capacity"
);
database.cleanup().await;
}
#[tokio::test]
async fn legacy_cleanup_and_cancel_never_remove_a_task_linked_intent() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let admission =
PostgresExecutionAdmission::new(registry.pool().clone(), limits(8), Default::default())
.unwrap();
let accepted = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
.await
.expect("acceptance");
// The linked reserved row looks expired; the legacy TTL cleanup must ignore it.
sqlx::query("update execution_admission_entries set expires_at = now() - interval '1 hour'")
.execute(registry.pool())
.await
.unwrap();
let ticket: i64 =
sqlx::query("select ticket_id from execution_admission_entries where task_id = $1")
.bind(accepted.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap()
.get("ticket_id");
let owner_token: String =
sqlx::query("select owner_token from execution_admission_entries where ticket_id = $1")
.bind(ticket)
.fetch_one(registry.pool())
.await
.unwrap()
.get("owner_token");
// A legacy attempt runs the bounded expiry cleanup; a foreign/own legacy cancel is refused.
let _ = admission
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: WORKSPACE.to_owned(),
agent_key: "agent_b".to_owned(),
operation_id: "op_sync".to_owned(),
operation_version: 1,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_sync".to_owned(),
expires_at_unix_ms: current_unix_ms() + 50,
})
.await;
assert!(!admission.cancel(ticket as u64, &owner_token).await.unwrap());
assert_eq!(row_counts(registry.pool()).await, (1, 1));
assert_eq!(
store
.read(&access(AGENT), accepted.task_id)
.await
.unwrap()
.state,
TaskState::Queued
);
database.cleanup().await;
}
#[tokio::test]
async fn terminal_transition_has_one_winner_and_an_immutable_result() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let accepted = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
let queued = store.read(&access(AGENT), accepted.task_id).await.unwrap();
assert_eq!(queued.version, 1);
// Moving a queued Task straight to completed is not an allowed transition.
let skipped = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Queued,
expected_version: queued.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
};
assert_eq!(
store
.finish(skipped)
.await
.expect_err("transition rejected"),
TaskError::Invalid {
message: "task state transition is not allowed".to_owned()
}
);
let started = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Queued,
expected_version: queued.version,
target_state: TaskState::Working,
result: None,
error: None,
};
assert_eq!(
store
.finish(started)
.await
.expect_err("working is not terminal"),
TaskError::Invalid {
message: "task finish target is not terminal".to_owned()
}
);
sqlx::query("update tasks set state = 'working' where task_id = $1")
.bind(accepted.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let working = store.read(&access(AGENT), accepted.task_id).await.unwrap();
let mut finishers = Vec::new();
for marker in ["first", "second"] {
let store = store.clone();
let mut finish = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Working,
expected_version: working.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"marker": marker})).unwrap()),
error: None,
};
finish.target_state = TaskState::Completed;
finishers.push(tokio::spawn(async move { store.finish(finish).await }));
}
let first = finishers.remove(0).await.unwrap();
let second = finishers.remove(0).await.unwrap();
let winners = [&first, &second]
.iter()
.filter(|result| result.is_ok())
.count();
assert_eq!(winners, 1, "exactly one concurrent CAS wins");
let loser = if first.is_err() { &first } else { &second };
assert_eq!(
loser.as_ref().err().unwrap(),
&TaskError::StateConflict,
"the loser observes a rejected transition, not a rewrite"
);
let terminal = store.read(&access(AGENT), accepted.task_id).await.unwrap();
assert_eq!(terminal.state, TaskState::Completed);
assert_eq!(terminal.version, working.version + 1);
let marker = terminal.result.as_ref().unwrap().value()["marker"]
.as_str()
.unwrap()
.to_owned();
assert!(marker == "first" || marker == "second");
// A repeated terminal write and a terminal->terminal move are rejected.
let repeated = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Completed,
expected_version: terminal.version,
target_state: TaskState::Failed,
result: None,
error: Some(TaskEnvelope::try_redacted(&json!({"reason": "x"})).unwrap()),
};
assert!(repeated.validate().is_err());
// Terminal Task and its dispatch intent left together.
assert_eq!(row_counts(registry.pool()).await, (1, 0));
database.cleanup().await;
}
#[tokio::test]
async fn retention_is_bounded_and_never_deletes_live_work() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let queued = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
let expired_queued = store
.accept(request(&key("k2"), &digest("f2"), AGENT, 1))
.await
.unwrap();
sqlx::query(
"update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1",
)
.bind(expired_queued.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
// A retained terminal Task whose retention has not expired yet and one that has.
let terminal = store
.accept(request(&key("k3"), &digest("f3"), AGENT, 30_000))
.await
.unwrap();
sqlx::query(
"update tasks set state = 'completed', result_envelope = '{\"ok\":true}'::jsonb,
retain_until = now() + interval '1 hour' where task_id = $1",
)
.bind(terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let expired_terminal = store
.accept(request(&key("k4"), &digest("f4"), AGENT, 30_000))
.await
.unwrap();
sqlx::query(
"update tasks set state = 'cancelled', execution_deadline = now() - interval '2 hours',
retain_until = now() - interval '1 hour'
where task_id = $1",
)
.bind(expired_terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(expired_terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let report = store.purge_expired(64).await.unwrap();
assert_eq!(report.expired_queued, 1);
assert_eq!(report.expired_terminal, 1);
// Live queued work is never deleted or failed; the expired queued Task released its slot.
assert_eq!(
store
.read(&access(AGENT), queued.task_id)
.await
.unwrap()
.state,
TaskState::Queued
);
assert_eq!(
store
.read(&access(AGENT), terminal.task_id)
.await
.unwrap()
.state,
TaskState::Completed
);
assert!(matches!(
store
.read(&access(AGENT), expired_queued.task_id)
.await
.unwrap()
.state,
TaskState::Failed
));
assert!(
store
.read(&access(AGENT), expired_terminal.task_id)
.await
.is_err()
);
let remaining: i64 = sqlx::query("select count(*)::bigint as count from execution_admission_entries where task_id is not null")
.fetch_one(registry.pool())
.await
.unwrap()
.get("count");
assert_eq!(remaining, 1, "only the live queued Task keeps its intent");
database.cleanup().await;
}
#[tokio::test]
async fn foreign_scope_reads_are_not_found_and_never_leak_payload() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let accepted = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
// Positive control: the owner reads its Task.
assert!(store.read(&access(AGENT), accepted.task_id).await.is_ok());
// Foreign owner, foreign authorization scope and foreign workspace are all not-found.
let mut foreign_owner = access(AGENT);
foreign_owner.owner_id = "agent_b".to_owned();
assert_eq!(
store
.read(&foreign_owner, accepted.task_id)
.await
.unwrap_err(),
TaskError::NotFound
);
let mut foreign_scope = access(AGENT);
foreign_scope.authorization_scope = "credential:pk_other".to_owned();
assert_eq!(
store
.read(&foreign_scope, accepted.task_id)
.await
.unwrap_err(),
TaskError::NotFound
);
let mut foreign_workspace = access(AGENT);
foreign_workspace.workspace_id = "ws_other".to_owned();
assert_eq!(
store
.read(&foreign_workspace, accepted.task_id)
.await
.unwrap_err(),
TaskError::NotFound
);
// The same caller key under a foreign owner creates its own Task, never a visible collision.
let other = store
.accept(request(&key("k1"), &digest("f1"), "agent_b", 30_000))
.await
.unwrap();
assert_ne!(other.task_id, accepted.task_id);
assert_eq!(row_counts(registry.pool()).await, (2, 2));
database.cleanup().await;
}
#[tokio::test]
async fn invalid_bounded_envelope_is_rejected_before_persistence() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
assert!(
TaskEnvelope::try_redacted(&json!({"blob": "x".repeat(64 * 1024)})).is_err(),
"an oversized envelope is rejected before any accept"
);
let mut invalid = request(&key("k1"), &digest("f1"), AGENT, 30_000);
invalid.scoped_key = "short".to_owned();
assert!(matches!(invalid.validate(), Err(TaskError::Invalid { .. })));
assert_eq!(
store.accept(invalid).await.unwrap_err(),
TaskError::Invalid {
message: "task scoped key digest is invalid".to_owned()
}
);
assert_eq!(row_counts(registry.pool()).await, (0, 0));
database.cleanup().await;
}
#[tokio::test]
async fn store_failure_fails_closed_without_records() {
let database = common::TestDatabase::new().await;
// No migration on this schema: the durable Task state does not exist yet.
let unmigrated = database.raw_pool().await;
let store = store_with(unmigrated, 8);
assert!(store.health().await.is_err());
let error = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect_err("acceptance fails closed");
assert!(matches!(error, TaskError::Unavailable { .. }));
database.cleanup().await;
}
+255
View File
@@ -0,0 +1,255 @@
//! Common execution admission seam of the runtime (Story 4.1, FR-21/FR-24).
//!
//! Every real execution — synchronous, or a published `background`/`adaptive` policy that falls back
//! to synchronous because no durable Task transport exists yet — passes this one gate before any
//! adapter work. The gate asks the authoritative store for a scoped permit bounded by the published
//! Tool policy, the verified approval state of the exact Published Version/scope/input and the caller
//! deadline, and it releases the permit explicitly when the local execution future ends.
use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
AdmissionScope, CoordinationStateStore, ExecutionAdmission, NoTasksFallback, ToolExecutionMode,
current_unix_ms,
};
use crank_metrics::{
AdmissionModeLabel, RejectionReason, ScopeKind, record_admission_queue_wait,
record_admission_rejection,
};
use tracing::warn;
use crate::{RuntimeError, RuntimeOperation, RuntimeRequestContext};
/// Effective absolute deadline of one execution: the published policy deadline never extends the
/// caller deadline and always includes queue waiting time.
pub(crate) fn effective_deadline(
operation: &RuntimeOperation,
caller_deadline: Instant,
) -> Instant {
match operation.execution_config.execution_policy {
Some(policy) => {
caller_deadline.min(Instant::now() + Duration::from_millis(policy.deadline_ms))
}
None => caller_deadline,
}
}
/// Effective execution path for this deployment.
///
/// Story 4.1 has no Task transport, so a `background` or `adaptive` policy resolves through its
/// published `no_tasks_fallback`: either the same synchronous admission path, or a typed
/// unsupported refusal with no admission entry, no Task id and no outbound call.
pub(crate) fn effective_mode(
operation: &RuntimeOperation,
) -> Result<ToolExecutionMode, RuntimeError> {
let Some(policy) = operation.execution_config.execution_policy else {
return Ok(ToolExecutionMode::Synchronous);
};
match policy.mode {
ToolExecutionMode::Synchronous => Ok(ToolExecutionMode::Synchronous),
mode => match policy.no_tasks_fallback {
NoTasksFallback::Synchronous => Ok(ToolExecutionMode::Synchronous),
NoTasksFallback::Unsupported => Err(RuntimeError::ExecutionPolicyUnsupported {
operation_id: operation.operation_id.as_str().to_owned(),
mode,
}),
},
}
}
pub(crate) struct AdmissionLease {
store: Arc<dyn ExecutionAdmission>,
permit: Option<AdmissionPermit>,
runtime: tokio::runtime::Handle,
}
impl AdmissionLease {
fn new(store: Arc<dyn ExecutionAdmission>, permit: AdmissionPermit) -> Self {
Self {
store,
permit: Some(permit),
runtime: tokio::runtime::Handle::current(),
}
}
/// Explicit release on the normal/error path of the local execution. The `Drop` cleanup stays as
/// the cancellation path; both CAS on the same owner token, so only one of them can free it.
pub(crate) async fn release(&mut self) {
let Some(permit) = self.permit.take() else {
return;
};
if let Err(error) = self.store.release(&permit).await {
warn!(
name: "runtime.admission.release_failed",
error = %error,
"failed to release an admission permit; the reservation stays fail-closed"
);
}
}
}
impl Drop for AdmissionLease {
fn drop(&mut self) {
let Some(permit) = self.permit.take() else {
return;
};
let store = Arc::clone(&self.store);
// Bounded cleanup: the explicit release path already ran on success/error; cancellation
// reaches here and releases by owner token without ever freeing a foreign permit. A failed
// release keeps the reservation fail-closed (no TTL reuse of an active permit).
self.runtime.spawn(async move {
match store.release(&permit).await {
Ok(true) => {}
Ok(false) => warn!(
name: "runtime.admission.release_skipped",
"admission permit was already released"
),
Err(error) => warn!(
name: "runtime.admission.release_failed",
error = %error,
"failed to release an admission permit; the reservation stays fail-closed"
),
}
});
}
}
/// Acquires a scoped permit from the authoritative store. Returns `None` when this deployment has no
/// admission store wired (embedded/unit-test executors keep the pre-4.1 local limiter).
pub(crate) async fn acquire(
store: Option<&Arc<dyn ExecutionAdmission>>,
coordination_store: Option<&dyn CoordinationStateStore>,
request: &crate::RuntimeExecutionRequest<'_>,
deadline: Instant,
) -> Result<Option<AdmissionLease>, RuntimeError> {
let Some(store) = store else {
return Ok(None);
};
let operation = request.operation();
let input = request.input();
let request_context = request.request_context();
let operation_id = operation.operation_id.as_str().to_owned();
let policy_mode = effective_mode(operation)?;
// The one admission snapshot carries the authoritative approval state of this exact Published
// Version, scope, input and deadline. A confirmation-requiring request without a verified
// approval never takes capacity and never dispatches: the pending human decision is answered
// here with the existing challenge, and the single-use token is consumed only at dispatch.
let approval = crate::confirmation::approval_evidence(
coordination_store,
operation,
input,
Some(request_context),
)
.await?;
let agent_key = request.agent_id().map(|agent| agent.as_str());
let workspace_id = request.workspace_id().as_str().to_owned();
let request = AdmissionRequest {
scope: AdmissionScope {
workspace_id,
agent_key: agent_key.unwrap_or(ADMIN_ORIGIN_KEY).to_owned(),
operation_id: operation_id.clone(),
operation_version: operation.operation_version,
},
mode: AdmissionMode::Synchronous,
policy_mode,
no_tasks_fallback: operation
.execution_config
.execution_policy
.map(|policy| policy.no_tasks_fallback)
.unwrap_or(NoTasksFallback::Synchronous),
approval,
fingerprint: admission_fingerprint(request_context, operation),
expires_at_unix_ms: current_unix_ms().saturating_add(
deadline_remaining_ms(deadline)
.try_into()
.unwrap_or(i64::MAX),
),
};
let started = Instant::now();
let outcome = store.admit(request).await.map_err(|error| {
record_admission_rejection(RejectionReason::DependencyUnavailable, ScopeKind::Global);
RuntimeError::AdmissionStoreUnavailable {
operation_id: operation_id.clone(),
details: error.to_string(),
}
})?;
let waited_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or_default();
match outcome {
AdmissionOutcome::Permitted(permit) => {
record_admission_queue_wait(AdmissionModeLabel::Synchronous, waited_ms);
Ok(Some(AdmissionLease::new(Arc::clone(store), permit)))
}
AdmissionOutcome::Overloaded { retry_after_ms } => {
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
Err(RuntimeError::AdmissionOverloaded {
operation_id,
retry_after_ms,
})
}
AdmissionOutcome::DeadlineElapsed => {
// Queue wait exhausted the absolute deadline before a permit was granted: nothing was
// dispatched, so this is an admission refusal, not an upstream timeout.
Err(deadline_refusal(operation))
}
AdmissionOutcome::ApprovalRequired => {
// The store refused before reserving any capacity or queue entry. Answer with the
// existing single-use challenge so the caller can confirm out of band.
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
Err(crate::confirmation::challenge(
coordination_store,
operation,
input,
Some(request_context),
)
.await)
}
AdmissionOutcome::Unsupported | AdmissionOutcome::Enqueued(_) => {
// No Task transport exists in this deployment: an enqueue decision can never become a
// runnable execution here, so it must not be reported as accepted work.
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
Err(RuntimeError::ExecutionPolicyUnsupported {
operation_id,
mode: policy_mode,
})
}
}
}
/// Typed refusal for an admission that never granted a permit within the absolute deadline. Reuses the
/// existing `execution_overloaded` vocabulary (stage `admission`, retry guidance, certain outcome) so
/// no new client error code or API shape appears, while reporting zero dispatch truthfully.
pub(crate) fn deadline_refusal(operation: &RuntimeOperation) -> RuntimeError {
record_admission_rejection(RejectionReason::Deadline, ScopeKind::Global);
RuntimeError::AdmissionOverloaded {
operation_id: operation.operation_id.as_str().to_owned(),
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
}
}
fn deadline_remaining_ms(deadline: Instant) -> u64 {
u64::try_from(
deadline
.saturating_duration_since(Instant::now())
.as_millis(),
)
.unwrap_or_default()
.max(1)
}
/// Bounded request identity for correlating one admission entry. Never contains payload, headers or
/// credentials.
fn admission_fingerprint(
request_context: &RuntimeRequestContext,
operation: &RuntimeOperation,
) -> String {
let mut fingerprint = format!(
"{}:{}:{}",
operation.operation_id.as_str(),
operation.operation_version,
request_context.request_id
);
fingerprint.truncate(crank_core::ADMISSION_FINGERPRINT_MAX_BYTES);
fingerprint
}
+120 -23
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod, OperationSafetyClass,
OperationSafetyPolicy, Target,
ApprovalEvidence, CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod,
OperationSafetyClass, OperationSafetyPolicy, Target,
};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
@@ -26,32 +26,14 @@ pub async fn confirm_operation(
}
let Some(store) = store else {
return Err(RuntimeError::ConfirmationStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
});
return Err(confirmation_store_unavailable(operation));
};
let scope = confirmation_scope(operation, request_context)?;
let input_hash = hash_json(input)?;
let provided_token = request_context
.and_then(RuntimeRequestContext::confirmation_token)
.or_else(|| confirmation_token_from_input(input));
let Some(provided_token) = provided_token else {
let token = issue_confirmation_token(
store,
operation.operation_id.as_str(),
&scope,
&input_hash,
&safety,
)
.await?;
return Err(RuntimeError::ConfirmationRequired {
operation_id: operation.operation_id.as_str().to_owned(),
safety_class: safety.class,
confirmation_token: token,
expires_in_ms: confirmation_ttl_ms(&safety),
});
let Some(provided_token) = provided_confirmation_token(request_context, input) else {
return Err(confirmation_required(store, operation, &scope, &input_hash, &safety).await);
};
consume_confirmation_token(
@@ -64,6 +46,121 @@ pub async fn confirm_operation(
.await
}
/// Read-only approval proof for the shared admission gate (Story 4.1, AC2).
///
/// It reports the authoritative approval state of this exact Published Version, scope and input and
/// never consumes the single-use confirmation token: the authoritative consumption stays in
/// [`confirm_operation`] immediately before the adapter call. A missing approval is `NotGranted`, so
/// admission refuses the request before taking execution capacity while the human decision is
/// pending; an unusable token or an unreachable safety store is a typed refusal too, never a permit.
pub(crate) async fn approval_evidence(
store: Option<&dyn CoordinationStateStore>,
operation: &RuntimeOperation,
input: &Value,
request_context: Option<&RuntimeRequestContext>,
) -> Result<ApprovalEvidence, RuntimeError> {
let safety = effective_safety_policy(operation);
if !safety.class.requires_confirmation() {
return Ok(ApprovalEvidence::NotRequired);
}
if request_context.is_some_and(RuntimeRequestContext::approval_granted) {
return Ok(ApprovalEvidence::Granted);
}
let Some(store) = store else {
return Err(confirmation_store_unavailable(operation));
};
let scope = confirmation_scope(operation, request_context)?;
let input_hash = hash_json(input)?;
let Some(token) = provided_confirmation_token(request_context, input) else {
return Ok(ApprovalEvidence::NotGranted);
};
let stored = store
.get_value(
CacheScope::Coordination,
&confirmation_cache_key(&scope, token),
)
.await
.map_err(|_| confirmation_store_unavailable(operation))?;
let verified = stored.is_some_and(|value| {
value.payload.get("scope").and_then(Value::as_str) == Some(scope.as_str())
&& value.payload.get("input_hash").and_then(Value::as_str) == Some(input_hash.as_str())
});
if verified {
Ok(ApprovalEvidence::Granted)
} else {
Err(RuntimeError::InvalidConfirmationToken {
operation_id: operation.operation_id.as_str().to_owned(),
})
}
}
/// Issues the existing single-use confirmation challenge and returns the typed refusal that carries
/// it. Admission calls this when no verified approval exists yet, so waiting for the human decision
/// never holds execution capacity.
pub(crate) async fn challenge(
store: Option<&dyn CoordinationStateStore>,
operation: &RuntimeOperation,
input: &Value,
request_context: Option<&RuntimeRequestContext>,
) -> RuntimeError {
let safety = effective_safety_policy(operation);
let Some(store) = store else {
return confirmation_store_unavailable(operation);
};
let scope = match confirmation_scope(operation, request_context) {
Ok(scope) => scope,
Err(error) => return error,
};
let input_hash = match hash_json(input) {
Ok(input_hash) => input_hash,
Err(error) => return error,
};
confirmation_required(store, operation, &scope, &input_hash, &safety).await
}
fn provided_confirmation_token<'a>(
request_context: Option<&'a RuntimeRequestContext>,
input: &'a Value,
) -> Option<&'a str> {
request_context
.and_then(RuntimeRequestContext::confirmation_token)
.or_else(|| confirmation_token_from_input(input))
}
fn confirmation_store_unavailable(operation: &RuntimeOperation) -> RuntimeError {
RuntimeError::ConfirmationStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
}
}
async fn confirmation_required(
store: &dyn CoordinationStateStore,
operation: &RuntimeOperation,
scope: &str,
input_hash: &str,
safety: &OperationSafetyPolicy,
) -> RuntimeError {
match issue_confirmation_token(
store,
operation.operation_id.as_str(),
scope,
input_hash,
safety,
)
.await
{
Ok(token) => RuntimeError::ConfirmationRequired {
operation_id: operation.operation_id.as_str().to_owned(),
safety_class: safety.class,
confirmation_token: token,
expires_in_ms: confirmation_ttl_ms(safety),
},
Err(error) => error,
}
}
pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool {
effective_safety_policy(operation)
.class
+21
View File
@@ -43,6 +43,27 @@ pub enum RuntimeError {
confirmation_token: String,
expires_in_ms: u64,
},
#[error(
"execution mode {mode:?} of operation {operation_id} is unavailable without a durable Task transport"
)]
ExecutionPolicyUnsupported {
operation_id: String,
mode: crank_core::ToolExecutionMode,
},
#[error(
"execution admission is overloaded for operation {operation_id}; retry after {retry_after_ms} ms"
)]
AdmissionOverloaded {
operation_id: String,
retry_after_ms: u64,
},
#[error("execution admission requires a verified approval for operation {operation_id}")]
AdmissionApprovalRequired { operation_id: String },
#[error("execution admission store is unavailable for operation {operation_id}: {details}")]
AdmissionStoreUnavailable {
operation_id: String,
details: String,
},
#[error("confirmation token for operation {operation_id} is invalid, expired, or already used")]
InvalidConfirmationToken { operation_id: String },
#[error("confirmation store is unavailable for operation {operation_id}")]
@@ -57,6 +57,11 @@ fn normalize_runtime_error_with_effect(
| RuntimeError::IdempotencyCompletionUnknown { .. } => {
failure = failure.with_dispatch_uncertainty();
}
RuntimeError::AdmissionOverloaded { retry_after_ms, .. } => {
if let Ok(with_retry) = failure.clone().try_with_retry_after_ms(*retry_after_ms) {
failure = with_retry;
}
}
RuntimeError::ConfirmationRequired {
confirmation_token,
expires_in_ms,
@@ -113,6 +118,14 @@ fn error_code(error: &RuntimeError) -> ExecutionErrorCode {
RuntimeError::UnsupportedExecutionMode { .. } => {
ExecutionErrorCode::ExecutionModeUnsupported
}
RuntimeError::ExecutionPolicyUnsupported { .. } => {
ExecutionErrorCode::ExecutionModeUnsupported
}
RuntimeError::AdmissionOverloaded { .. } => ExecutionErrorCode::ExecutionOverloaded,
RuntimeError::AdmissionApprovalRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
RuntimeError::AdmissionStoreUnavailable { .. } => {
ExecutionErrorCode::SafetyStoreUnavailable
}
RuntimeError::ConcurrencyLimitExceeded { .. } => ExecutionErrorCode::ExecutionOverloaded,
RuntimeError::InvalidPreparedRequest { .. } => ExecutionErrorCode::PreparedRequestInvalid,
RuntimeError::ConfirmationRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
@@ -220,6 +220,7 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+53 -10
View File
@@ -7,9 +7,9 @@ use std::time::{Duration, Instant};
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
use crank_core::{
AdapterRegistry, CoordinationStateStore, ExecutionMode, HttpMethod, InvocationSource,
InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
Target,
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
SharedProtocolAdapter, Target,
};
use crank_metrics::{
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
@@ -41,6 +41,7 @@ pub struct RuntimeExecutor {
unary_limiter: Arc<Semaphore>,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
@@ -139,6 +140,7 @@ impl RuntimeExecutor {
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
) -> Self {
Self {
@@ -147,6 +149,7 @@ impl RuntimeExecutor {
limits,
response_cache,
coordination_store,
admission,
metering_sink,
}
}
@@ -167,6 +170,11 @@ impl RuntimeExecutor {
self
}
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub(crate) async fn execute_request(
&self,
request: RuntimeExecutionRequest<'_>,
@@ -188,11 +196,16 @@ impl RuntimeExecutor {
request.agent_id()
);
let operation_may_mutate = operation_may_mutate(operation);
let execution_deadline =
crate::admission::effective_deadline(operation, request.deadline());
let runtime_span = Stage::RuntimeExecute.span();
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
log_runtime_event("unary.execute", operation, Some(request_context));
let started_at = Instant::now();
let dispatch_started = Arc::new(AtomicBool::new(false));
// True until the admission gate answered. A deadline that fires while the request is still
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
metric_invocation_source(Some(request_context)),
request_context
@@ -206,6 +219,17 @@ impl RuntimeExecutor {
.flatten(),
);
let execution = async {
// One admission gate for every executable path (Story 4.1): the published policy decides
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
// is held for the whole local execution future, including retries.
let mut lease = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
let _permit = self.acquire_unary_permit(operation)?;
let _inflight = InFlightGuard::runtime();
let mapping_span = Stage::RuntimeArgumentsMap.span();
@@ -215,7 +239,7 @@ impl RuntimeExecutor {
let prepared_request = prepared_request?;
let request_preview = sanitized_request_preview(&prepared_request);
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
let output = self
let result = self
.execute_prepared(
operation,
input,
@@ -223,19 +247,27 @@ impl RuntimeExecutor {
Some(request_context),
Arc::clone(&dispatch_started),
)
.await?;
.await;
if let Some(lease) = lease.as_mut() {
lease.release().await;
}
let output = result?;
Ok((output, request_preview))
};
let result = tokio::time::timeout_at(
request.deadline().into(),
execution_deadline.into(),
execution.instrument(runtime_span.clone()),
)
.await
.unwrap_or_else(|_| {
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
if admission_pending.load(Ordering::Acquire) {
Err(crate::admission::deadline_refusal(operation))
} else {
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
}
});
record_runtime_result(&runtime_span, &result);
drop(runtime_span);
@@ -952,6 +984,10 @@ fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
| RuntimeError::UnsupportedProtocol { .. }
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
RuntimeError::AdmissionOverloaded { .. }
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
RuntimeError::ConfirmationRequired { .. }
| RuntimeError::InvalidConfirmationToken { .. }
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
@@ -1008,6 +1044,13 @@ fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
// label domain stays fixed; the precise code lives in the execution failure/invocation log
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
+12 -2
View File
@@ -2,8 +2,8 @@ use std::sync::Arc;
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter};
use crank_core::{
AdapterRegistry, CoordinationStateStore, MeteringSink, NoopMeteringSink, ResponseCacheStore,
SharedMeteringSink, SharedProtocolAdapter,
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, MeteringSink, NoopMeteringSink,
ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
};
use crate::{RuntimeExecutor, RuntimeLimits};
@@ -13,6 +13,7 @@ pub struct RuntimeExecutorBuilder {
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
@@ -23,6 +24,7 @@ impl RuntimeExecutorBuilder {
adapters: AdapterRegistry::empty(),
response_cache: None,
coordination_store: None,
admission: None,
metering_sink: Arc::new(NoopMeteringSink) as Arc<dyn MeteringSink>,
}
}
@@ -47,6 +49,13 @@ impl RuntimeExecutorBuilder {
self
}
/// Wires the mandatory deployment-wide admission store (Story 4.1). Production processes wire
/// the PostgreSQL implementation; an executor without it keeps the pre-4.1 local limit.
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub fn with_metering_sink(mut self, sink: SharedMeteringSink) -> Self {
self.metering_sink = sink;
self
@@ -58,6 +67,7 @@ impl RuntimeExecutorBuilder {
self.adapters,
self.response_cache,
self.coordination_store,
self.admission,
self.metering_sink,
)
}
+2 -1
View File
@@ -314,7 +314,7 @@ fn request_fingerprint(input: &Value) -> Result<String, RuntimeError> {
Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded)))
}
fn canonical_json(value: &Value) -> Value {
pub(crate) fn canonical_json(value: &Value) -> Value {
match value {
Value::Object(object) => {
let mut entries = object.iter().collect::<Vec<_>>();
@@ -446,6 +446,7 @@ mod tests {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+3
View File
@@ -1,3 +1,4 @@
mod admission;
mod auth;
mod cache;
mod cache_factory;
@@ -14,6 +15,7 @@ mod rate_limit;
mod request_context;
mod request_preparation;
mod response_cache;
mod tasks;
pub fn validate_resource_template_parameter(
schema: &crank_schema::Schema,
value: &serde_json::Value,
@@ -113,3 +115,4 @@ pub use rate_limit::{
};
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
pub use secret_crypto::SecretCrypto;
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
+266
View File
@@ -0,0 +1,266 @@
//! Runtime application service of durable Tasks (Story 4.2, FR-22/FR-23).
//!
//! The service is the only place that turns an approved background request into the durable domain
//! request: bounded validated input, canonical input fingerprint, full-scope idempotency digest,
//! pinned Published Version/deadline/retention and the support identities of the request. It holds an
//! owned request, never `ResolvedAuth` or raw secrets, and it never dispatches: the durable store
//! commits Task + dispatch intent + queue slot in one transaction and only then produces an
//! acknowledgement.
//!
//! Reads go through the same owner-scoped access proof, so the durable row is only fetched for the
//! caller that owns it. There is no in-memory fallback: a store failure is a typed refusal.
use std::sync::Arc;
use std::time::Instant;
use crank_core::{
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds,
ToolExecutionMode, current_unix_ms, validate_task_envelope,
};
use sha2::{Digest, Sha256};
/// One owned background request, already authorized by the application layer.
#[derive(Clone, Debug)]
pub struct TaskAcceptDraft {
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
/// Server-derived authorization scope identity of the verified principal.
pub authorization_scope: String,
pub operation_id: String,
pub operation_version: u32,
pub policy_mode: ToolExecutionMode,
/// Caller key, bounded before it becomes a full-scope digest.
pub scoped_key: String,
/// Canonical executable input; it is validated, fingerprinted, then persisted redacted.
pub input: serde_json::Value,
pub correlation: CorrelationContext,
/// Absolute execution deadline of the accepted work.
pub deadline: Instant,
/// Retention after the execution deadline.
pub retention_ms: i64,
pub approval_reference: Option<String>,
}
/// Projection of the durable status/result read. `result`/`error` are only present once a terminal
/// transition committed; a non-terminal Task reports its state and inputs without inventing output.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskOutcome {
pub task_id: TaskId,
pub state: crank_core::TaskState,
pub result: Option<serde_json::Value>,
pub error: Option<serde_json::Value>,
}
impl TaskOutcome {
fn from_record(record: TaskRecord) -> Self {
Self {
task_id: record.task_id,
state: record.state,
result: record.result.map(|envelope| envelope.value().clone()),
error: record.error.map(|envelope| envelope.value().clone()),
}
}
}
#[derive(Clone)]
pub struct TaskService {
store: Arc<dyn TaskStore>,
}
impl TaskService {
pub fn new(store: Arc<dyn TaskStore>) -> Self {
Self { store }
}
/// Readiness of the mandatory durable Task state.
pub async fn health(&self) -> Result<(), TaskError> {
self.store.health().await
}
/// Durable acceptance: the returned identity exists only after the store committed it.
pub async fn accept(&self, draft: TaskAcceptDraft) -> Result<TaskAcceptance, TaskError> {
validate_task_envelope(&draft.input)?;
let (scoped_key, input_fingerprint) = (
scoped_key_digest(&draft)?,
canonical_input_fingerprint(&draft.input)?,
);
let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline);
let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: draft.workspace_id,
owner_kind: draft.owner_kind,
owner_id: draft.owner_id,
authorization_scope: draft.authorization_scope,
operation_id: draft.operation_id,
pins: TaskPins {
operation_version: draft.operation_version,
policy_mode: draft.policy_mode,
execution_deadline_unix_ms,
retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms),
approval_reference: draft.approval_reference,
},
scoped_key,
input_fingerprint,
input: TaskEnvelope::try_redacted(&draft.input)?,
support: TaskSupportIds::new(
draft.correlation.request_id().clone(),
draft.correlation.trace_context(),
),
};
self.store.accept(request).await
}
/// Owner-scoped status read. A foreign or absent Task is one typed not-found.
pub async fn status(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<TaskRecord, TaskError> {
self.store.read(access, task_id).await
}
/// Owner-scoped result read, projected from the same authorized row.
pub async fn result(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<TaskOutcome, TaskError> {
Ok(TaskOutcome::from_record(
self.store.read(access, task_id).await?,
))
}
/// Default retention pinned by the application layer when it does not choose one explicitly.
pub fn default_retention_ms() -> i64 {
TASK_DEFAULT_RETENTION_MS
}
/// Scoped lookup of an already accepted Task by the same full-scope identity, used to keep a
/// retry of an explicitly pinned version from being accepted as new work.
pub async fn lookup(&self, draft: &TaskAcceptDraft) -> Result<Option<TaskRecord>, TaskError> {
let scoped_key = scoped_key_digest(draft)?;
let access = TaskAccess {
workspace_id: draft.workspace_id.clone(),
owner_kind: draft.owner_kind,
owner_id: draft.owner_id.clone(),
authorization_scope: draft.authorization_scope.clone(),
};
self.store
.find(&access, &draft.operation_id, &scoped_key)
.await
}
/// Canonical digest of a validated input, exposed so the application layer can compare a stored
/// Task against the request it is retrying without duplicating the canonicalization.
pub fn input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
canonical_input_fingerprint(input)
}
}
fn deadline_unix_ms(deadline: Instant) -> i64 {
let remaining = u64::try_from(
deadline
.saturating_duration_since(Instant::now())
.as_millis(),
)
.unwrap_or_default();
current_unix_ms().saturating_add(i64::try_from(remaining.max(1)).unwrap_or(i64::MAX))
}
/// Digest of the caller key over the full Task identity. The scope is never abbreviated to a
/// connection, protocol revision or raw credential, so two callers cannot collide on one key.
fn scoped_key_digest(draft: &TaskAcceptDraft) -> Result<String, TaskError> {
let key = draft.scoped_key.as_str();
if key.is_empty()
|| key.len() > TASK_SCOPED_KEY_MAX_BYTES
|| key.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task idempotency key is invalid"));
}
let material = format!(
"{}\n{}\n{}\n{}\n{}\n{}",
draft.workspace_id,
draft.owner_kind.as_str(),
draft.owner_id,
draft.authorization_scope,
draft.operation_id,
key
);
Ok(hex_digest(material.as_bytes()))
}
/// Canonical digest of the validated executable input. Object key order never changes the digest,
/// while any material business or secret value does.
fn canonical_input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
let canonical = crate::idempotency::canonical_json(input);
let encoded = serde_json::to_vec(&canonical)
.map_err(|_| TaskError::invalid("task input is not canonicalizable"))?;
Ok(hex_digest(&encoded))
}
fn hex_digest(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn draft(key: &str, input: serde_json::Value) -> TaskAcceptDraft {
TaskAcceptDraft {
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
operation_id: "op_report".to_owned(),
operation_version: 2,
policy_mode: ToolExecutionMode::Background,
scoped_key: key.to_owned(),
input,
correlation: CorrelationContext::generate(),
deadline: Instant::now() + std::time::Duration::from_secs(30),
retention_ms: TASK_DEFAULT_RETENTION_MS,
approval_reference: None,
}
}
#[test]
fn canonical_fingerprint_ignores_key_order_and_tracks_values() {
let first = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 2}})).unwrap();
let reordered = canonical_input_fingerprint(&json!({"b": {"c": 2}, "a": 1})).unwrap();
assert_eq!(first, reordered);
assert_eq!(first.len(), 64);
let changed = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 3}})).unwrap();
assert_ne!(first, changed);
}
#[test]
fn scoped_key_digest_covers_the_whole_identity() {
let base = scoped_key_digest(&draft("key_1", json!({}))).unwrap();
let mut foreign = draft("key_1", json!({}));
foreign.owner_id = "agent_b".to_owned();
assert_ne!(base, scoped_key_digest(&foreign).unwrap());
let mut other_scope = draft("key_1", json!({}));
other_scope.authorization_scope = "credential:pk_2".to_owned();
assert_ne!(base, scoped_key_digest(&other_scope).unwrap());
assert!(scoped_key_digest(&draft("", json!({}))).is_err());
assert!(
scoped_key_digest(&draft(
&"k".repeat(TASK_SCOPED_KEY_MAX_BYTES + 1),
json!({})
))
.is_err()
);
}
}
@@ -333,6 +333,7 @@ fn runtime_operation(draft: RestImportCandidate) -> RuntimeOperation {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -96,6 +96,7 @@ paths:
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -334,6 +334,7 @@ fn destructive_delete_operation() -> Operation<Schema, MappingSet> {
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
}),
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -340,6 +340,7 @@ fn idempotent_post_operation() -> Operation<Schema, MappingSet> {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -109,6 +109,7 @@ fn no_input_get_operation() -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -573,6 +573,7 @@ fn operation(
idempotency,
safety,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+8 -1
View File
@@ -10,7 +10,7 @@ use std::{
use async_trait::async_trait;
use crank_core::{
AdapterResponse, ConfirmationPolicy, CorrelationContext, ExecutionConfig, ExecutionErrorCode,
ExecutionMode, ExecutionOrigin, HttpMethod, IdempotencyMode, IdempotencyPolicy,
ExecutionMode, ExecutionOrigin, ExecutionStage, HttpMethod, IdempotencyMode, IdempotencyPolicy,
InvocationSource, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy,
OperationSecurityLevel, OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter,
ProtocolAdapterError, RestTarget, RetryPolicy, Retryability, Target, ToolDescription,
@@ -64,6 +64,10 @@ async fn deadline_after_dispatch_is_normalized_as_unknown_outcome() {
failure.outcome_certainty(),
OutcomeCertainty::OutcomeUnknown
);
// A deadline that fires after the adapter was called stays an upstream timeout: the admission
// refusal vocabulary (Story 4.1) only covers deadlines reached before any dispatch.
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
@@ -98,6 +102,8 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
let failure = executor.execute_outcome(request).await.unwrap_err();
assert_eq!(failure.retryability(), Retryability::AfterDelay);
assert_eq!(failure.outcome_certainty(), OutcomeCertainty::Certain);
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
@@ -632,6 +638,7 @@ fn operation() -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+78
View File
@@ -0,0 +1,78 @@
# Переход Crank на официальный MCP SDK rmcp
Статус: планирование одобрено владельцем 27.10.2026. Полная миграция и production-переключение этим планом не запускаются. Владелец планирования и очереди — hermes_tl; реализация — hermes_dev; независимая проверка — hermes_qa; Git/CI/релизы — hermes_devops. ARC подключается к конкретному техническому затруднению, не как обязательный новый этап.
## Результат
Стандартный протокольный слой MCP обслуживает официальный SDK. Crank сохраняет авторизацию, права Agent/workspace, опубликованные версии, REST mapping, admission и исполнение команд. Обе действующие версии протокола и функции Tools/Resources/Prompts остаются доступны. Снижается объём собственного протокольного кода; ускорение и производительность не объявляются измеренными заранее.
Исходная точка: принятый main после 4.1, известный merge cc612516a29d48f2831ba6bdf58c1dc777aa6abf. Перед исполнением получить актуальный main и убедиться, что в него вошла проверенная поставка. Плановый кандидат SDK — опубликованный rmcp 3.7.0; окончательный pin и lock проверяются на первом этапе, автоматического обновления на другую версию нет.
## 1. Проверить тонкий адаптер на реальном MCP-пути
Исполнитель: DEV, независимая QA. Риск: высокий — ошибка может нарушить доступ к инструментам.
Сделать изолированный исполняемый прототип SDK на существующей границе типизированных команд, сохранив авторизацию и динамический каталог. Публичный production endpoint пока обслуживает прежний путь.
Проверяемый результат:
- Динамические tools/list и tools/call работают через SDK с действующими сервисами Crank, включая запрет чужого Agent/неверных credentials и сохранение ошибок инструмента.
- Через настоящий HTTP-путь проверены обе revisions: legacy initialize/session и modern discover/stateless, JSON/SSE, headers, progress и cancellation.
- Отдельно проверены пригодность SessionManager для существующего хранения сессий, переподключение и владение отменой; наличие SDK-типа само по себе не считается доказательством.
- Установлены существенные gaps для Resources/Prompts/notifications/subscriptions и текущих execution modes. Наличие интерфейса не выдаётся за полную совместимость.
- На небольшом smoke-наборе проверено, что adapter не обходит действующие ограничения; замеченный риск памяти/очередей записан. Нагрузочная квалификация относится к завершённой функции, не превращается в условие этого прототипа. Существующие ceilings не меняются.
- Результат — работающий код и краткое решение TL: готов перенос либо конкретная несовместимость с минимальным вариантом решения. Если потребуется ослабить контракт, заменить хранилище или создать большой дополнительный framework — вопрос владельцу до продолжения.
Это один достаточный опыт, а не серия исследований SDK. Использовать существующие wire fixtures и точечные проверки, сохранять непроверенные пункты явно.
## 2. Перенести действующие MCP-функции на SDK
Зависит от положительного результата этапа 1 и решения TL о достаточности совместимости. Исполнитель: DEV, независимая QA. Риск: высокий — затрагивается общий клиентский интерфейс.
Подключить SDK к действующему пути и перенести стандартные envelopes, dispatch, lifecycle и transport. Порядок переноса устанавливается по результату первого опыта, без параллельных writers одних MCP-файлов.
Проверяемый результат:
- Через SDK проходят действующие Tools, Resources, Prompts, completion, уведомления и подписки в применимых revisions, а также progress/cancel и существующая logging policy.
- Сохранены авторизация/отзыв доступа, agent scope, advertised capabilities, admission, deadlines и семантика ошибок. SDK defaults не расширяют объявленные capabilities или список разрешённых revisions.
- Сессии, reconnect и ограничения памяти/очередей сохраняют действующий контракт; in-memory хранилище не заменяет существующее долговременное хранение без решения.
- Перенесённые участки собственного протокольного кода удаляются по мере принятия. Временный способ безопасного сравнения/отката не превращается во второй постоянный MCP runtime.
- Существующие affected ATDD подтверждают старое поведение; тесты меняются только там, где проверяли приватное устройство старого адаптера, а не нужный клиенту контракт.
Новые бизнес-возможности, переработка REST runtime, базы, модели или UI в эту работу не входят.
## 3. Завершить замену и поставить результат
Зависит от принятого этапа 2. Исполнители: DEV/QA, затем DevOps. Риск: высокий — финальный переход общего MCP-пути. Внешняя проверка относительно реализации: независимая QA на существующем клиентском/wire-наборе обеих revisions.
Проверяемый результат:
- Удалены ставшие ненужными собственные parser/dispatch/transport участки, временные переключатели и неиспользуемые зависимости. Поддержка legacy protocol при этом сохраняется средствами SDK.
- Один необходимый финальный feature-прогон подтверждает реальный клиентский путь обеих revisions, изоляцию доступа, streaming/cancel/session и применимые ограничения ресурсов. Полный набор не повторяется отдельно каждой ролью; обязательные CI-гейты сохраняются.
- DevOps выполнил commit/push/PR, protected merge и честный release с проверенными артефактами. Документация описывает активный SDK, поддерживаемые версии и реальные ограничения.
- План отката указывает предыдущую принятую версию и проверяет совместимость состояния. Production-переключение выполняется только отдельным разрешённым manual deployment по существующему runbook; эта карточка не даёт разрешения на выкатку.
## Порядок относительно текущей разработки
1. Закончить уже выполняемую проверку поставки 4.1, не открывать её разработку заново.
2. Планировать SDK-адаптер перед дальнейшим расширением заменяемого transport/protocol слоя. Этап 1 не ждёт завершения будущих 4.2–4.7 или всей нагрузочной кампании.
3. Доменные части будущих задач могут идти параллельно при отсутствии общего writer и явных интерфейсных зависимостях. Новую протокольную реализацию и миграцию одних файлов параллельно не вести.
4. TL сопоставляет существующие 4.2–4.7 с SDK, чтобы не реализовать вручную уже предоставленный механизм. Действующие продуктовые AC не сокращаются. Неизвестная совместимость — проверка этапа 1, а не повод считать будущие функции готовыми.
## Постановка и контроль
Достаточно трёх последовательных исполнений по этапам выше, с same-card QA и действующей приёмкой TL; не создавать отдельный новый бизнес-эпик, PRD, новый тестовый framework или цепочку архитектурных согласований.
На текущем шаге TL фиксирует эти этапы в существующем backlog/Gitea, зависимости и приоритет относительно текущей работы. Будущие исполнительские карточки оставить в triage/backlog; не запускать полную миграцию автоматически из задачи планирования. Сроки не выдумывать до первого опыта. Подтверждённая непригодность SDK эскалируется владельцу с фактами и вариантами, а не молчаливым возвращением к самописному MCP.
## Основания
Канонические пути ниже находятся на Hermes `192.168.1.127` (SSH-пользователь `codex`); относительные пути считаются от `/home/codex/code/crank`. Это доступный TL источник R, а не текущая локальная папка оператора. Локальная копия аудита у оператора: `/home/bsodfather/code/hermes/deploy/mcp-sdk-audit-20261027/REPORT.md`.
- Решение владельца 27.10.2026: запланировать переход на rmcp.
- Аудит: `/home/codex/.hermes/codex-operator/handoffs/crank-rmcp-audit-20261027.md`.
- Действующие AC: `_bmad-output/initiative-crank/epic-mcp-capabilities/story-mcp-revision-capabilities.md`, в том числе оценка SDK AT-3.1-14; остальные MCP AC остаются действующими.
- Официальный SDK: https://github.com/modelcontextprotocol/rust-sdk ; документация ServerHandler: https://docs.rs/rmcp/latest/rmcp/handler/server/trait.ServerHandler.html .
- Общий процесс: `/home/codex/.hermes/team/development-delivery.md`, `validation-policy.md`, `delivery-efficiency.md`.
Предыдущее default-правило «сохранять собственный transport» не является техническим доказательством отказа от SDK. Цель нового плана — проверить и принять SDK-путь; продуктовые и security-контракты сохраняются.
+16
View File
@@ -35,3 +35,19 @@ version, stage, stable error code, retryability и certainty. Для legacy v4
Пути Admin API, MCP methods и успешные response fields не изменены. Поля ошибки
добавлены обратно совместимо. Outbound DNS/redirect/retry/SSRF policy принадлежит
Story 1.9 и этой границей не считается завершённой.
## Durable background Task (Story 4.2)
Принятие фоновой работы идёт через `TaskApplication → crank-runtime::TaskService →
TaskStore`. PostgreSQL-реализация выполняет одну транзакцию: Task, dispatch intent
(`execution_admission_entries.task_id`) и queue slot коммитятся вместе, и только
после commit выдаётся ack с Task ID/поддержкой Request/Trace/parent TraceContext.
Повтор того же полного scoped key возвращает сохранённую Task, другой canonical
input fingerprint — типизированный конфликт; явная другая published-версия при том
же ключе не перезакрепляет уже принятую работу. Owner-скоуп (workspace, Agent/admin,
authorization scope) входит в каждый read; чужой Task неотличим от отсутствующего.
Границы этой story: публичный MCP wire Tasks/capabilities/activation остаётся
Story 4.7, worker/leases/recovery — Story 4.3. Существующий `tools/call` сохраняет
synchronous/unsupported поведение; durable acceptance не запускает ephemeral
execution и не рекламирует клиенту background capability.
+2
View File
@@ -178,6 +178,8 @@ scrape_configs:
- `crank_upstream_requests_total`,
`crank_upstream_request_duration_seconds`;
- `crank_runtime_inflight`, `crank_runtime_limit_rejections_total`,
`crank_runtime_admission_queue_wait_seconds`,
`crank_runtime_admission_oldest_waiting_seconds`,
`crank_runtime_cache_total`, `crank_idempotency_total`,
`crank_confirmation_total`;
- `crank_db_pool_connections`;
+1 -1
View File
@@ -1 +1 @@
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6285,"max_rendered_series":19515,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6381,"max_rendered_series":19611,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
+1 -1
View File
@@ -1 +1 @@
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
+23
View File
@@ -55,3 +55,26 @@ run_atdd "capability advertisement registry" -p crank-community-mcp --lib protoc
# smoke) runs against the real endpoint, a real published catalog and a real counted upstream.
run_atdd "tool call lifecycle bounds" -p crank-community-mcp --lib app::lifecycle -- --test-threads=1
run_atdd "tool call progress and cancellation" -p mcp-server --test integration tools_lifecycle -- --test-threads=1
# Story 4.1 (FR-21, FR-24): a published Tool execution policy and one authoritative admission gate.
# The policy/JSON contract and the fair-scheduler arithmetic are unit-level; the authoritative store
# (scoped permits, bounded queue, typed overload, durable decisions, version pinning, fail-closed
# health) and the runtime wiring (one shared global budget across executors, typed refusals with no
# outbound call, adaptive/synchronous fallback) run against a real PostgreSQL schema.
run_atdd "execution policy contract" -p crank-core --lib operation::tests::execution_policy -- --test-threads=1
run_atdd "admission scheduler bounds" -p crank-core --lib admission:: -- --test-threads=1
run_atdd "execution policy validation" -p admin-api --lib operation_validation::tests:: -- --test-threads=1
run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1
run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1
run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1
# Story 4.2 (FR-22, FR-23): durable background Tasks. The lifecycle/envelope bounds are unit-level; the
# authoritative store (one-transaction acceptance, replay/typed conflict, shared bounded capacity,
# Task-linked intent guard, terminal CAS, retention, scope isolation, fail-closed health) and the
# application entry (verified principal, exact publication pin, foreign-scope denial, readiness) run
# against a real PostgreSQL schema. The migration suite proves a fresh and an already-migrated v25.
run_atdd "durable task domain bounds" -p crank-core --lib task:: -- --test-threads=1
run_atdd "runtime task identity" -p crank-runtime --lib tasks:: -- --test-threads=1
run_atdd "durable task store" -p crank-registry --test integration tasks:: -- --test-threads=1
run_atdd "durable task application entry" -p mcp-server --test integration durable_tasks:: -- --test-threads=1
run_atdd "durable task migration" -p crank-registry --test integration migrations:: -- --test-threads=1