Compare commits

..
Author SHA1 Message Date
bsodfather 6df6ffac4e Merge pull request 'Story 4.2 — подтверждать durable Task после атомарного PostgreSQL commit (FR-22, FR-23)' (#109) from delivery/epic4-story-4.2 into main
CI / UI Checks (push) Successful in 13s
CI / Rust Checks (push) Successful in 21m12s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
Community Release / release (push) Successful in 36m42s
Merge accepted Story 4.2 durable Task acceptance
2026-10-27 07:33:30 +00:00
bsodfather 2219a7217d feat(epic4): подтверждать durable Task после атомарного PG-commit (Story 4.2)
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 19m34s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Application entry выдаёт Task ID только после атомарного PostgreSQL commit
Task+intent+capacity; replay/conflict, owner scope, bounded envelope,
terminal CAS и fail-closed readiness проверены на реальном PG.

Также включён принятый план перехода на rmcp (только документ, миграция не
запускается).
2026-10-27 00:09:36 +00:00
bsodfather cc612516a2 Merge pull request 'Story 4.1 — публиковать execution mode и применять admission limits (FR-21, FR-24)' (#104) from delivery/epic4-story-4.1 into main
CI / Rust Checks (push) Successful in 20m52s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / UI Checks (push) Successful in 10s
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 1s
CI / Deploy (push) Skipped
2026-10-26 21:38:06 +00:00
bsodfather 599dc1867b Merge pull request 'ci: безопасно интегрировать принятый Crank и отделить production deploy' (#105) from ops/separate-production-deploy into main
CI / UI Checks (push) Successful in 12s
CI / Rust Checks (push) Successful in 16m0s
CI / Rust Feature Validation (push) Skipped
CI / Frontend E2E (push) Skipped
CI / Community Image Smoke (push) Skipped
CI / Required Checks (push) Successful in 2s
CI / Deploy (push) Skipped
2026-10-26 21:36:09 +00:00
Codex 4e7440f25e ci: separate protected source integration from explicit production deployment
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 12m54s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 3s
CI / Deploy (pull_request) Skipped
2026-10-27 00:21:37 +03:00
bsodfather e80d30ddb0 feat(epic4): публиковать execution mode и применять общий admission (Story 4.1, FR-21, FR-24)
CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 12m45s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Community Release / release (push) Successful in 40m35s
Tool Draft фиксирует synchronous/background/adaptive, deadline и fallback;
runtime применяет единый снимок политики (concurrency, очередь, approval,
deadline) до внешнего действия. Насыщение отклоняется typed overload с
retry guidance, mandatory safety store fail closed, background/adaptive
используют опубликованный synchronous/unsupported fallback.

Refs: #29
2026-10-26 21:08:08 +00:00
97 changed files with 8366 additions and 170 deletions

No files matched your search

+7 -1
View File
@@ -11,6 +11,12 @@ on:
options: [story, feature]
default: story
required: true
deploy_target:
description: Deployment target for this run
type: choice
options: [none, production]
default: none
required: true
push:
branches:
- main
@@ -499,7 +505,7 @@ jobs:
runs-on: ubuntu-latest
needs:
- required-checks
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
if: ${{ gitea.event_name == 'workflow_dispatch' && gitea.ref == 'refs/heads/main' && inputs.validation_level == 'feature' && inputs.deploy_target == 'production' }}
env:
REGISTRY: git.itexp.me
IMAGE_TAG: ${{ gitea.sha }}
Generated
+1
View File
@@ -826,6 +826,7 @@ dependencies = [
name = "crank-registry"
version = "0.3.1"
dependencies = [
"async-trait",
"crank-artifacts",
"crank-core",
"crank-mapping",
+1
View File
@@ -181,6 +181,7 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+18 -1
View File
@@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
use admin_api::{
app::build_app,
auth::{AuthSettings, BootstrapAdminConfig},
pool_metrics::spawn_postgres_pool_metrics,
pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics},
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
service::AdminServiceBuilder,
state::AppState,
@@ -169,6 +169,7 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let base_url = config
.runtime
@@ -214,6 +215,16 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(std::sync::Arc::new(
crank_registry::PostgresExecutionAdmission::new(
registry.pool().clone(),
crank_registry::AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
},
crank_core::AdmissionSchedulerConfig::default(),
)?,
))
.build();
let identity_provider =
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
@@ -321,6 +332,12 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
+14
View File
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
+27 -2
View File
@@ -37,13 +37,22 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
} else {
"not_ready"
};
let execution_admission = if checks.execution_admission {
"ready"
} else {
"not_ready"
};
if checks.is_ready() {
(
StatusCode::OK,
Json(json!({
"service": "admin-api",
"status": "ready",
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
})),
)
} else {
@@ -52,7 +61,11 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
Json(json!({
"service": "admin-api",
"status": "not_ready",
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
})),
)
}
@@ -67,6 +80,7 @@ mod tests {
let checks = ReadinessChecks {
postgres: false,
artifact_storage: true,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -92,6 +106,7 @@ mod tests {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: false,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -111,4 +126,14 @@ mod tests {
"not_ready"
);
}
#[test]
fn readiness_checks_identify_an_admission_state_failure() {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: true,
execution_admission: false,
};
assert!(!checks.is_ready());
}
}
+9 -3
View File
@@ -55,8 +55,8 @@ use crate::{
storage::LocalArtifactStorage,
};
use operation_validation::{
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_protocol_target, validate_response_cache_policy,
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy,
};
#[derive(Clone)]
@@ -104,11 +104,12 @@ pub use crate::dto::*;
pub struct ReadinessChecks {
pub postgres: bool,
pub artifact_storage: bool,
pub execution_admission: bool,
}
impl ReadinessChecks {
pub const fn is_ready(self) -> bool {
self.postgres && self.artifact_storage
self.postgres && self.artifact_storage && self.execution_admission
}
}
@@ -150,6 +151,8 @@ pub(super) struct CredentialAuditRecord<'a> {
impl AdminService {
pub async fn readiness(&self) -> ReadinessChecks {
let postgres = self.registry.ping().await.is_ok();
// Story 4.1: the mandatory admission state is a readiness dependency of every execution.
let execution_admission = self.registry.execution_admission_health().await.is_ok();
let store = Arc::clone(&self.artifact_store);
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
.await
@@ -157,6 +160,7 @@ impl AdminService {
ReadinessChecks {
postgres,
artifact_storage,
execution_admission,
}
}
@@ -725,6 +729,7 @@ impl AdminService {
validate_response_cache_policy(&payload.target, &payload.execution_config)?;
validate_idempotency_policy(&payload.target, &payload.execution_config)?;
validate_approval_policy(&payload.execution_config)?;
validate_execution_policy(&payload.execution_config)?;
payload.input_mapping.validate_paths()?;
payload.output_mapping.validate_paths()?;
Ok(())
@@ -738,6 +743,7 @@ impl AdminService {
validate_response_cache_policy(&operation.target, &operation.execution_config)?;
validate_idempotency_policy(&operation.target, &operation.execution_config)?;
validate_approval_policy(&operation.execution_config)?;
validate_execution_policy(&operation.execution_config)?;
operation.input_mapping.validate_paths()?;
operation.output_mapping.validate_paths()?;
Ok(())
+1
View File
@@ -441,6 +441,7 @@ fn demo_rest_operation_payload() -> OperationPayload {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+1
View File
@@ -635,6 +635,7 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -610,6 +610,7 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref,
headers: BTreeMap::new(),
},
@@ -160,6 +160,23 @@ pub(super) fn validate_approval_policy(
Ok(())
}
pub(super) fn validate_execution_policy(
execution_config: &crank_core::ExecutionConfig,
) -> Result<(), ApiError> {
let Some(policy) = execution_config.execution_policy.as_ref() else {
return Ok(());
};
if let Err(error) = policy.validate() {
return Err(ApiError::validation_with_context(
error.to_string(),
json!({
"field": "execution_config.execution_policy.deadline_ms",
}),
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
@@ -171,8 +188,8 @@ mod tests {
};
use super::{
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_response_cache_policy,
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_response_cache_policy,
};
fn cacheable_execution_config() -> ExecutionConfig {
@@ -183,6 +200,7 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -201,6 +219,7 @@ mod tests {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -365,4 +384,36 @@ mod tests {
"approval ttl must be between 1 and 300 seconds"
);
}
#[test]
fn accepts_published_execution_policy_and_absent_legacy_policy() {
let mut config = cacheable_execution_config();
validate_execution_policy(&config).unwrap();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Adaptive,
deadline_ms: 120_000,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
});
validate_execution_policy(&config).unwrap();
}
#[test]
fn rejects_zero_and_overflowing_execution_policy_deadline() {
let mut config = cacheable_execution_config();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Synchronous,
deadline_ms: 0,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
let error = validate_execution_policy(&config).unwrap_err();
assert!(matches!(error, crate::error::ApiError::Validation { .. }));
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Background,
deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
assert!(validate_execution_policy(&config).is_err());
}
}
@@ -495,6 +495,7 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+50 -4
View File
@@ -1,7 +1,7 @@
use std::{io, process::ExitCode, time::Duration};
use crank_community_mcp::{
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts,
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts_and_tasks,
session::PostgresTransportSessionStore,
};
use crank_config::{
@@ -14,13 +14,14 @@ use crank_observability::{
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
};
use crank_registry::{
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, PostgresTaskStore,
};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use mcp_server::pool_metrics::spawn_postgres_pool_metrics;
use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics};
use sqlx::postgres::PgConnectOptions;
use tokio::net::TcpListener;
use tracing::info;
@@ -175,6 +176,7 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
let secret_crypto =
@@ -190,12 +192,30 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(execution_admission(
registry.pool().clone(),
runtime_limits,
)?)
.build();
// Published static Resources are read from the same immutable content-addressed root the
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
let app = build_app_with_artifacts(
// Story 4.2: the durable Task application entry shares the registry pool, the published catalog
// refresh interval and the exact admission limits of the runtime; the public wire stays
// synchronous/unsupported and no worker is started.
let task_store = PostgresTaskStore::new(
registry.pool().clone(),
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?;
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
registry.clone(),
Duration::from_millis(config.refresh_ms),
cache_stores.coordination.clone(),
crank_runtime::TaskService::new(std::sync::Arc::new(task_store)),
));
let app = build_app_with_artifacts_and_tasks(
registry,
Duration::from_millis(config.refresh_ms),
config.runtime.base_url.clone(),
@@ -211,6 +231,7 @@ async fn run(
std::sync::Arc::new(CommunityMachineCredentialVerifier),
runtime_limits.max_concurrent_sessions,
artifact_store,
tasks,
);
let listener = TcpListener::bind(config.bind_addr).await?;
@@ -287,6 +308,25 @@ fn init_observability(
)?)
}
fn admission_limits(runtime_limits: RuntimeLimits) -> AdmissionLimits {
AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..AdmissionLimits::default()
}
}
fn execution_admission(
pool: sqlx::PgPool,
runtime_limits: RuntimeLimits,
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
let store = PostgresExecutionAdmission::new(
pool,
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?;
Ok(std::sync::Arc::new(store))
}
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
database_options(&config.database).map_err(|_| invalid("database.source"))?;
@@ -307,6 +347,12 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
+14
View File
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
+874
View File
@@ -0,0 +1,874 @@
//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission.
//!
//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a
//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported
//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal.
//!
//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this
//! app already wires both and this is the real deployment shape.
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
use crank_core::{
AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation,
OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget,
RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy,
WorkspaceId,
};
use crank_mapping::MappingSet;
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits};
use crank_schema::{Schema, SchemaKind};
use serde_json::json;
use sqlx::PgPool;
use time::OffsetDateTime;
struct CountingAdapter {
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
#[async_trait]
impl ProtocolAdapter for CountingAdapter {
fn protocol(&self) -> Protocol {
Protocol::Rest
}
fn supports_mode(&self, mode: ExecutionMode) -> bool {
mode == ExecutionMode::Unary
}
async fn invoke_unary(
&self,
_target: &Target,
_prepared: &crank_core::PreparedRequest,
context: &RuntimeRequestContext,
) -> Result<AdapterResponse, crank_core::ProtocolAdapterError> {
context.mark_dispatch_started();
self.calls.fetch_add(1, Ordering::SeqCst);
let current = self.current.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(current, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(150)).await;
self.current.fetch_sub(1, Ordering::SeqCst);
Ok(AdapterResponse {
status_code: 200,
headers: BTreeMap::new(),
body: json!({"ok": true}),
data: json!({"ok": true}),
})
}
}
fn schema(kind: SchemaKind) -> Schema {
Schema {
kind,
format: None,
description: None,
required: false,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
}
}
fn operation(policy: Option<ToolExecutionPolicy>) -> Operation<Schema, MappingSet> {
Operation {
id: OperationId::new("op_admission_budget"),
name: "admission_budget".to_owned(),
display_name: "Admission budget".to_owned(),
category: "general".to_owned(),
protocol: Protocol::Rest,
security_level: OperationSecurityLevel::Standard,
status: OperationStatus::Published,
version: 7,
target: Target::Rest(RestTarget {
base_url: "https://api.example.com".to_owned(),
method: HttpMethod::Get,
path_template: "/v1/items".to_owned(),
static_headers: BTreeMap::new(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: BTreeMap::new(),
}),
input_schema: schema(SchemaKind::Object),
output_schema: schema(SchemaKind::Object),
input_mapping: MappingSet::default(),
output_mapping: MappingSet::default(),
execution_config: ExecutionConfig {
timeout_ms: 5_000,
execution_policy: policy,
..ExecutionConfig::default()
},
tool_description: ToolDescription {
title: "Admission".to_owned(),
description: "Admission budget probe".to_owned(),
tags: Vec::new(),
examples: Vec::new(),
},
samples: None,
generated_draft: None,
config_export: None,
wizard_state: None,
created_at: OffsetDateTime::now_utc(),
updated_at: OffsetDateTime::now_utc(),
published_at: None,
}
}
async fn execute(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = WorkspaceId::new("ws_test");
let context = crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun);
execute_in(executor, operation, input, &context).await
}
async fn execute_in(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
context: &crank_runtime::RuntimeRequestContext,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = context
.metering_context()
.map(|metering| metering.workspace_id.clone())
.unwrap_or_else(|| WorkspaceId::new("ws_test"));
let request = crank_runtime::RuntimeExecutionRequest::try_new(
&workspace,
ExecutionOrigin::AdminDraft,
None,
operation,
input,
crank_runtime::ExecutionAuthorization::Authorized,
None,
context,
Instant::now() + Duration::from_secs(30),
)
.map_err(|_| {
crank_core::ExecutionFailure::new(
ExecutionErrorCode::RuntimeInternal,
crank_core::CorrelationContext::new(
context.request_id.clone(),
context.trace_context.clone(),
),
)
})?;
executor
.execute_outcome(request)
.await
.map(|success| success.output)
}
async fn schema_pool(schema_url: &str) -> PgPool {
let pool = PgPool::connect(schema_url)
.await
.expect("admission schema pool");
crank_registry::MigrationAuthority::apply(&pool)
.await
.expect("migration authority applies the admission schema");
pool
}
fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc<PostgresExecutionAdmission> {
Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: global,
agent_limit: 8,
tool_limit: 8,
queue_limit: queue,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)
}
fn executor(
store: Arc<PostgresExecutionAdmission>,
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
) -> RuntimeExecutor {
RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current,
peak,
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store)
.build()
}
#[tokio::test]
async fn two_executors_share_one_global_budget() {
let schema_url = crank_test_support::postgres_schema_url("admission_budget").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 8);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let first = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let second = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let mut tasks = Vec::new();
for index in 0..4 {
let executor = if index % 2 == 0 {
first.clone()
} else {
second.clone()
};
let operation = operation.clone();
tasks.push(tokio::spawn(async move {
execute(&executor, &operation, &json!({})).await
}));
}
let mut completed = 0;
for task in tasks {
match task.await.expect("execution task") {
Ok(_) => completed += 1,
Err(failure) => panic!("admission must grant every request: {failure:?}"),
}
}
assert_eq!(completed, 4);
assert_eq!(calls.load(Ordering::SeqCst), 4);
assert!(
peak.load(Ordering::SeqCst) <= 2,
"peak concurrency {} exceeded the shared global limit",
peak.load(Ordering::SeqCst)
);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn saturation_returns_typed_overload_without_an_outbound_call() {
let schema_url = crank_test_support::postgres_schema_url("admission_overload").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 1, 1);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// Another process already holds the only global permit.
let held = match store
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_test".to_owned(),
agent_key: "admin".to_owned(),
operation_id: "op_admission_budget".to_owned(),
operation_version: 7,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_holder".to_owned(),
expires_at_unix_ms: crank_core::current_unix_ms() + 10_000,
})
.await
.expect("admission store is reachable")
{
crank_core::AdmissionOutcome::Permitted(permit) => permit,
other => panic!("expected a permit, got {other:?}"),
};
let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 250,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let waiting_runtime = runtime.clone();
let first_waiting_operation = waiting_operation.clone();
let waiting = tokio::spawn(async move {
execute(&waiting_runtime, &first_waiting_operation, &json!({})).await
});
// Let the first request occupy the bounded queue before the next one arrives.
while store.snapshot().await.unwrap().waiting == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
let overloaded = execute(&runtime, &waiting_operation, &json!({})).await;
let failure = overloaded.expect_err("a full queue must refuse the request");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionOverloaded
);
assert_eq!(failure.stage(), ExecutionStage::Admission);
assert_eq!(failure.retry_after_ms(), Some(250));
// The queued request hits its absolute deadline instead of dispatching. That refusal happened
// inside admission (nothing was ever dispatched), so the client must see an admission-stage
// overload with retry guidance — not an `upstream_timeout`.
let expired = waiting
.await
.expect("waiting task")
.expect_err("deadline must refuse");
assert_eq!(
expired.error_code(),
ExecutionErrorCode::ExecutionOverloaded,
"queue waiting counts against the published deadline"
);
assert_eq!(expired.stage(), ExecutionStage::Admission);
assert_eq!(expired.retry_after_ms(), Some(250));
assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain);
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"no outbound call on refusal"
);
assert!(store.release(&held).await.unwrap());
}
#[tokio::test]
async fn background_policy_without_a_task_transport_is_typed_unsupported() {
let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
}))
.into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("no Task transport exists in Story 4.1");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionModeUnsupported
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let snapshot = store.snapshot().await.unwrap();
assert_eq!(snapshot.waiting, 0);
assert_eq!(snapshot.executing, 0);
}
#[tokio::test]
async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() {
let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let output = execute(&runtime, &operation, &json!({}))
.await
.expect("adaptive resolves through the published synchronous fallback");
// The response shape is unchanged by admission: an operation without output mapping still
// returns the same mapped output object it returned before Story 4.1.
assert_eq!(output, json!({}));
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn admission_store_outage_fails_closed_before_dispatch() {
let schema_url = crank_test_support::postgres_schema_url("admission_outage").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// The mandatory state disappears after the store was built: every execution must fail closed.
sqlx::query("drop table execution_admission_entries")
.execute(&pool)
.await
.expect("drop the mandatory admission state");
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("a missing mandatory store must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(store.health().await.is_err());
}
/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2).
struct RecordingAdmission {
inner: Arc<PostgresExecutionAdmission>,
requests: Mutex<Vec<AdmissionRequest>>,
permits: AtomicUsize,
releases: AtomicUsize,
}
impl RecordingAdmission {
fn new(inner: Arc<PostgresExecutionAdmission>) -> Self {
Self {
inner,
requests: Mutex::new(Vec::new()),
permits: AtomicUsize::new(0),
releases: AtomicUsize::new(0),
}
}
fn requests(&self) -> Vec<AdmissionRequest> {
self.requests.lock().expect("admission log").clone()
}
fn permits(&self) -> usize {
self.permits.load(Ordering::SeqCst)
}
fn releases(&self) -> usize {
self.releases.load(Ordering::SeqCst)
}
}
#[async_trait]
impl ExecutionAdmission for RecordingAdmission {
async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> {
self.inner.health().await
}
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<crank_core::AdmissionOutcome, crank_core::AdmissionStoreError> {
self.requests
.lock()
.expect("admission log")
.push(request.clone());
let outcome = self.inner.admit(request).await?;
if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) {
self.permits.fetch_add(1, Ordering::SeqCst);
}
Ok(outcome)
}
async fn release(
&self,
permit: &crank_core::AdmissionPermit,
) -> Result<bool, crank_core::AdmissionStoreError> {
let released = self.inner.release(permit).await?;
if released {
self.releases.fetch_add(1, Ordering::SeqCst);
}
Ok(released)
}
async fn cancel(
&self,
ticket_id: u64,
owner_token: &str,
) -> Result<bool, crank_core::AdmissionStoreError> {
self.inner.cancel(ticket_id, owner_token).await
}
async fn snapshot(
&self,
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
self.inner.snapshot().await
}
}
fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation {
let mut source = operation(None);
source.id = OperationId::new("op_admission_confirmation");
source.name = "admission_confirmation".to_owned();
source.version = version;
let Target::Rest(target) = &mut source.target;
target.method = HttpMethod::Delete;
source.execution_config.safety = Some(OperationSafetyPolicy {
class: OperationSafetyClass::Destructive,
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
});
source.into()
}
fn confirmation_runtime(
admission: Arc<RecordingAdmission>,
coordination: Option<Arc<dyn crank_core::CoordinationStateStore>>,
calls: Arc<AtomicUsize>,
) -> RuntimeExecutor {
let store: Arc<dyn ExecutionAdmission> = admission;
let builder = RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current: Arc::new(AtomicUsize::new(0)),
peak: Arc::new(AtomicUsize::new(0)),
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store);
match coordination {
Some(coordination) => builder.with_coordination_store(coordination).build(),
None => builder.build(),
}
}
fn confirmation_context() -> crank_runtime::RuntimeRequestContext {
crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_response_cache_scope("ws_test", "agent_a")
.with_metering_context(
WorkspaceId::new("ws_test"),
None,
InvocationSource::AdminTestRun,
)
}
#[tokio::test]
async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let coordination: Arc<dyn crank_core::CoordinationStateStore> =
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default());
let calls = Arc::new(AtomicUsize::new(0));
let runtime = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::clone(&coordination)),
Arc::clone(&calls),
);
let operation = confirmation_operation(3);
let context = confirmation_context();
let input = json!({ "reason": "remove the stale item" });
// 1. No approval yet: the existing challenge is returned and no execution capacity is taken.
let challenge = execute_in(&runtime, &operation, &input, &context)
.await
.expect_err("an unapproved destructive call must only stage confirmation");
assert_eq!(
challenge.error_code(),
ExecutionErrorCode::ConfirmationRequired
);
let token = challenge
.confirmation()
.expect("expected the confirmation challenge")
.token()
.to_owned();
assert_eq!(
recording.permits(),
0,
"an unapproved request must not take an execution permit"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let requests = recording.requests();
assert_eq!(
requests.len(),
1,
"the one admission snapshot is consulted once"
);
assert_eq!(
requests[0].approval,
ApprovalEvidence::NotGranted,
"admission must see the real approval state, not a fixed NotRequired"
);
assert_eq!(requests[0].scope.operation_version, 3);
let snapshot = recording.snapshot().await.expect("admission snapshot");
assert_eq!((snapshot.executing, snapshot.waiting), (0, 0));
// 2. An approval issued for another published version never opens this execution.
let other_version = confirmation_operation(4);
let foreign = execute_in(
&runtime,
&other_version,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a foreign approval must not authorise this version");
assert_eq!(
foreign.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(
recording.requests().len(),
1,
"an unusable approval is refused before any capacity request"
);
// 3. An approval is bound to its exact input as well.
let mismatched_input = json!({ "reason": "remove another item" });
let mismatched = execute_in(
&runtime,
&operation,
&mismatched_input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a confirmation token must not authorise different input");
assert_eq!(
mismatched.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(recording.requests().len(), 1);
// 4. The matching approval passes admission with truthful evidence and executes exactly once.
let executed = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect("a verified approval executes");
assert!(executed.is_object());
assert_eq!(calls.load(Ordering::SeqCst), 1);
let requests = recording.requests();
assert_eq!(requests.len(), 2);
assert_eq!(requests[1].approval, ApprovalEvidence::Granted);
assert_eq!(requests[1].scope.operation_version, 3);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.releases(),
1,
"the permit is released after execution"
);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
// 5. Replay of the consumed confirmation is refused before capacity and without a side effect.
let replay = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token),
)
.await
.expect_err("a confirmation token is single-use");
assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.requests().len(),
2,
"a replayed approval never reaches admission"
);
// 6. The trusted approved-recovery path still passes the same gate with real evidence.
let approved = execute_in(
&runtime,
&operation,
&json!({ "reason": "approved by an operator" }),
&context.with_approval_granted(),
)
.await;
assert!(approved.is_ok());
assert_eq!(calls.load(Ordering::SeqCst), 2);
let requests = recording.requests();
assert_eq!(requests.len(), 3);
assert_eq!(requests[2].approval, ApprovalEvidence::Granted);
assert_eq!(recording.permits(), 2);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
}
#[tokio::test]
async fn mandatory_safety_store_outage_fails_closed_before_admission() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let calls = Arc::new(AtomicUsize::new(0));
// Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail
// closed instead of reserving capacity and hoping for an in-memory fallback.
let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls));
let operation = confirmation_operation(3);
let failure = execute_in(
&runtime,
&operation,
&json!({ "reason": "no store" }),
&confirmation_context(),
)
.await
.expect_err("a mandatory safety store outage must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(recording.permits(), 0);
let after_missing_store = recording.requests().len();
// The same Tool with a claimed approval and a *failing* safety store never reaches the capacity
// store at all: the approval claim stays unverified, so no permit and no outbound call exist.
let failing = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::new(UnavailableSafetyStore)),
Arc::clone(&calls),
);
let outage = execute_in(
&failing,
&operation,
&json!({ "reason": "failing store" }),
&confirmation_context().with_confirmation_token("ct_unavailable"),
)
.await
.expect_err("an unreachable safety store must refuse the claimed approval");
assert_eq!(
outage.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(
recording.requests().len(),
after_missing_store,
"an unverified approval never becomes a capacity request"
);
assert_eq!(recording.permits(), 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
/// Coordination store whose mandatory backend is down: every operation fails closed.
struct UnavailableSafetyStore;
impl UnavailableSafetyStore {
fn error() -> crank_core::CacheStoreError {
crank_core::CacheStoreError::Unavailable {
message: "test backend unavailable".to_owned(),
}
}
}
#[async_trait]
impl crank_core::CoordinationStateStore for UnavailableSafetyStore {
async fn get_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn put_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn delete_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn take_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn reserve_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<crank_core::CoordinationStateReservation, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn compare_and_set_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_expected: &crank_core::CoordinationStateValue,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<bool, crank_core::CacheStoreError> {
Err(Self::error())
}
}
+1
View File
@@ -1,6 +1,7 @@
mod integration {
mod catalog_access;
mod common;
mod durable_tasks;
mod era_isolation;
mod execution_certainty;
mod execution_stages;
@@ -621,6 +621,7 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation<Schema, Ma
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -0,0 +1,591 @@
//! ATDD Story 4.2 (FR-22/FR-23): the durable Task application entry over a real published catalog
//! and a real PostgreSQL schema.
//!
//! These oracles drive the production application entry (`TaskApplication`) that the mcp-server
//! composition root constructs: a verified machine credential is checked against the published Tool,
//! the Task is accepted durably, a freshly constructed application instance reads the same identity
//! and pins, replay/version conflicts never rebind, foreign callers receive no metadata, and a missing
//! Task store fails closed and reports readiness as not ready. The public `tools/call` wire stays
//! synchronous/unsupported: this suite never calls it.
use super::common;
use std::time::Duration;
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
use crank_core::{
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState,
ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
};
use crank_runtime::{InMemoryCoordinationStateStore, TaskService};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "report_gen";
const KEY: &str = "retry-key-1";
fn operation(policy: ToolExecutionPolicy) -> crank_registry::RegistryOperation {
let mut operation = common::test_operation("https://api.example.com", TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
operation
}
fn background_policy() -> ToolExecutionPolicy {
ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
}
}
async fn seed_tool(registry: &PostgresRegistry, policy: ToolExecutionPolicy) {
let operation = operation(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(registry, &operation.id, 1).await;
// A second, explicitly synchronous Tool of the same Agent: the durable entry must refuse it.
let synchronous = common::test_operation("https://api.example.com", "report_sync");
registry
.create_operation(&common::test_workspace_id(), &synchronous, Some("alice"))
.await
.unwrap();
publish(registry, &synchronous.id, 1).await;
let binding = AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
};
let synchronous_binding = AgentOperationBinding {
operation_id: synchronous.id.clone(),
operation_version: 1,
tool_name: synchronous.name.clone(),
tool_title: synchronous.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
};
common::publish_agent_with_bindings(registry, AGENT_SLUG, vec![binding, synchronous_binding])
.await;
}
async fn publish(
registry: &PostgresRegistry,
operation_id: &crank_core::OperationId,
version: u32,
) {
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id,
version,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
}
async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
let mut next = operation(background_policy());
next.version = version;
next.status = crank_core::OperationStatus::Draft;
next.published_at = None;
registry
.create_version(CreateVersionRequest {
workspace_id: &common::test_workspace_id(),
snapshot: &next,
change_note: Some("next publication"),
created_by: Some("alice"),
})
.await
.unwrap();
publish(registry, &next.id, version).await;
// The agent catalog must bind the published Tool version for the new publication to be addressed.
let agent_id = common::test_agent_id(AGENT_SLUG);
let draft = crank_core::AgentVersion {
agent_id: agent_id.clone(),
version: 2,
status: crank_core::AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: Default::default(),
created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
};
registry
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
workspace_id: &common::test_workspace_id(),
agent_id: &agent_id,
version: &draft,
bindings: &[AgentOperationBinding {
operation_id: next.id.clone(),
operation_version: version,
tool_name: next.name.clone(),
tool_title: next.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
registry
.publish_agent(crank_registry::PublishAgentRequest {
workspace_id: &common::test_workspace_id(),
agent_id: &agent_id,
version: 2,
published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
}
fn task_service(pool: PgPool) -> TaskService {
let store = PostgresTaskStore::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration");
TaskService::new(std::sync::Arc::new(store))
}
fn application(registry: &PostgresRegistry) -> TaskApplication {
TaskApplication::new(
registry.clone(),
Duration::from_millis(0),
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
task_service(registry.pool().clone()),
)
}
async fn principal(
registry: &PostgresRegistry,
name: &str,
scopes: &[PlatformApiKeyScope],
) -> TaskPrincipal {
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
// The production machine-access path resolves the presented secret against the active platform
// key of the addressed Agent and derives the scope identity from that verified record.
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("an active platform key resolves");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
fn unsampled_correlation() -> CorrelationContext {
CorrelationContext::new(
crank_core::RequestId::generate(),
crank_core::TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
)
}
async fn counts(pool: &PgPool) -> (i64, i64) {
let row = sqlx::query(
"select
(select count(*)::bigint from tasks) as tasks,
(select count(*)::bigint from execution_admission_entries where task_id is not null)
as intents",
)
.fetch_one(pool)
.await
.unwrap();
(row.get("tasks"), row.get("intents"))
}
#[tokio::test]
async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-accept",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let app = application(&registry);
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&correlation,
)
.await
.expect("acceptance over the real publication");
assert_eq!(acceptance.state, TaskState::Queued);
assert!(!acceptance.replayed);
assert_eq!(acceptance.operation_version, 1);
assert_eq!(acceptance.support.request_id, *correlation.request_id());
assert_eq!(acceptance.support.trace_id, *correlation.trace_id());
assert!(!acceptance.support.trace_sampled);
assert_eq!(
uuid::Uuid::parse_str(&acceptance.task_id.to_string())
.unwrap()
.get_version_num(),
7,
"the durable Task id is UUIDv7"
);
// Task, dispatch intent and queue slot committed together.
assert_eq!(counts(registry.pool()).await, (1, 1));
// A freshly constructed application + service reads the same durable identity and pins.
let reader = application(&registry);
let record = reader
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
.unwrap();
assert_eq!(record.state, TaskState::Queued);
assert_eq!(record.pins.operation_version, 1);
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
assert_eq!(record.support.request_id, acceptance.support.request_id);
assert_eq!(record.support.traceparent, acceptance.support.traceparent);
assert!(!record.support.trace_sampled);
let outcome = reader
.result(&caller, TOOL_NAME, acceptance.task_id)
.await
.unwrap();
assert!(outcome.result.is_none() && outcome.error.is_none());
// Replay of the same scoped key and input returns the accepted Task instead of a new one.
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(replay.task_id, acceptance.task_id);
assert!(replay.replayed);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn a_new_publication_never_rebinds_an_accepted_task() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-rebind",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let accepted = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
let before = app
.status(&caller, TOOL_NAME, accepted.task_id)
.await
.unwrap();
publish_tool_version(&registry, 2).await;
// The same retry keeps the accepted Task and its v1 pin; it never becomes a v2 Task.
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(replay.task_id, accepted.task_id);
assert_eq!(replay.operation_version, 1);
let after = app
.status(&caller, TOOL_NAME, accepted.task_id)
.await
.unwrap();
assert_eq!(after.pins.operation_version, 1);
assert_eq!(after.support.request_id, before.support.request_id);
assert_eq!(counts(registry.pool()).await, (1, 1));
// An explicit other exact version is a typed conflict, not a rebind.
let explicit = app
.accept_background(
&caller,
TOOL_NAME,
Some(2),
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("a different explicit pin cannot rebind the accepted Task");
assert_eq!(explicit, TaskApplicationError::VersionUnavailable);
// The original explicit pin still resolves to the accepted Task.
let pinned = app
.accept_background(
&caller,
TOOL_NAME,
Some(1),
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(pinned.task_id, accepted.task_id);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let owner = principal(
&registry,
"task-owner",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
let accepted = app
.accept_background(
&owner,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
// Positive control: the owner reads its Task.
assert!(
app.status(&owner, TOOL_NAME, accepted.task_id)
.await
.is_ok()
);
// A second verified key of the same Agent is another authorization scope: no payload, no metadata.
let foreign_scope =
principal(&registry, "task-other-scope", &[PlatformApiKeyScope::Read]).await;
match app
.status(&foreign_scope, TOOL_NAME, accepted.task_id)
.await
{
Err(TaskApplicationError::NotFound) => {}
other => panic!("foreign scope must be not-found without metadata, got {other:?}"),
}
assert!(matches!(
app.result(&foreign_scope, TOOL_NAME, accepted.task_id)
.await,
Err(TaskApplicationError::NotFound)
));
// A synchronous publication is not accepted as background work at all.
let synchronous_app = application(&registry);
let refused = synchronous_app
.accept_background(
&owner,
"report_sync",
None,
"sync-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("a synchronous policy has no durable acceptance");
assert_eq!(refused, TaskApplicationError::NotBackground);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let app = application(&registry);
let admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_1".to_owned(),
};
let accepted = app
.accept_background(
&admin,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("an authenticated administrator accepts durable work");
assert_eq!(accepted.state, TaskState::Queued);
// Positive control: the same administrator reads its own Task.
assert!(
app.status(&admin, TOOL_NAME, accepted.task_id)
.await
.is_ok()
);
// An administrator is never a wildcard owner: another admin user and the owning Agent of the
// same workspace both see nothing.
let other_admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_2".to_owned(),
};
assert_eq!(
app.status(&other_admin, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
assert_eq!(
app.result(&other_admin, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
let agent = principal(&registry, "task-admin-cross", &[PlatformApiKeyScope::Read]).await;
assert_eq!(
app.status(&agent, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
// An admin identity addressed to another workspace cannot resolve the publication at all.
let foreign_workspace = TaskPrincipal::Admin {
workspace_slug: "other-workspace".to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_1".to_owned(),
};
assert_eq!(
app.accept_background(
&foreign_workspace,
TOOL_NAME,
None,
"foreign-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
let database_url = crank_test_support::postgres_schema_url("durable_tasks_outage").await;
let pool = PgPool::connect(&database_url).await.unwrap();
// No migration: the durable Task state does not exist.
let outage_registry = PostgresRegistry::connect(&database_url).await;
assert!(
outage_registry.is_err(),
"an unmigrated registry cannot be used"
);
crank_registry::MigrationAuthority::apply(&pool)
.await
.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-outage",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry);
assert!(app.health().await.is_ok());
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
sqlx::query("drop table tasks cascade")
.execute(&pool)
.await
.unwrap();
assert!(app.health().await.is_err());
let error = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("no durable Task store, no acceptance");
assert!(matches!(
error,
TaskApplicationError::Task(crank_core::TaskError::Unavailable { .. })
));
// Readiness of the app that wires the same store reports the durable Task dependency as not ready.
let router = common::build_test_app(registry.clone(), Duration::from_millis(0), None);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
});
let response = reqwest::get(format!("http://{address}/ready"))
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE);
let body: serde_json::Value = response.json().await.unwrap();
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
server.abort();
}
+2
View File
@@ -1036,6 +1036,7 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Secret name is required.',
'wizard.error.secret_value_required': 'Secret value is required.',
'wizard.error.save_upstream_first': 'Save the API host after creating a new authorization profile before continuing.',
'wizard.error.execution_policy_invalid': 'Execution policy is invalid: mode, deadline from 1 to 3600000 ms and no-Tasks fallback are required.',
'wizard.toast.auth_profile_created_title': 'Authorization profile created',
'wizard.toast.auth_profile_created_body': 'New authorization profile is ready and selected.',
'wizard.toast.auth_profile_error_title': 'Authorization profile creation failed',
@@ -2353,6 +2354,7 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Нужно указать имя секрета.',
'wizard.error.secret_value_required': 'Нужно указать значение секрета.',
'wizard.error.save_upstream_first': 'Сначала сохраните API-хост после создания нового профиля авторизации.',
'wizard.error.execution_policy_invalid': 'Политика выполнения некорректна: нужны режим, deadline от 1 до 3600000 мс и поведение без Tasks.',
'wizard.toast.auth_profile_created_title': 'Профиль авторизации создан',
'wizard.toast.auth_profile_created_body': 'Новый профиль авторизации готов и уже выбран.',
'wizard.toast.auth_profile_error_title': 'Не удалось создать профиль авторизации',
+30
View File
@@ -214,10 +214,33 @@ function parseExecutionConfig(text) {
auth_profile_ref: authProfileRef,
headers: headers,
};
var executionPolicy = parseExecutionPolicy(value.execution_policy);
if (executionPolicy) {
config.execution_policy = executionPolicy;
}
return config;
}
// Story 4.1: the published execution policy must survive a wizard save. The editor is a free-form
// document, so unknown/legacy shapes stay untouched: only a fully valid policy is projected, and an
// explicit invalid value is reported instead of being silently replaced by defaults.
function parseExecutionPolicy(raw) {
if (raw === undefined || raw === null) return null;
if (typeof raw !== 'object' || Array.isArray(raw)) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
var mode = raw.mode === undefined ? 'synchronous' : raw.mode;
var fallback = raw.no_tasks_fallback === undefined ? 'synchronous' : raw.no_tasks_fallback;
var deadline = Number(raw.deadline_ms);
if (['synchronous', 'background', 'adaptive'].indexOf(mode) < 0
|| ['synchronous', 'unsupported'].indexOf(fallback) < 0
|| !Number.isInteger(deadline) || deadline < 1 || deadline > 3600000) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback };
}
function currentUpstream() {
var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; });
if (selected) return selected;
@@ -415,6 +438,13 @@ function executionConfigToEditorValue(config) {
var value = {
timeout_ms: config.timeout_ms,
};
if (config.execution_policy) {
value.execution_policy = {
mode: config.execution_policy.mode,
deadline_ms: config.execution_policy.deadline_ms,
no_tasks_fallback: config.execution_policy.no_tasks_fallback,
};
}
if (config.retry_policy) {
value.retry = { max_attempts: config.retry_policy.max_attempts };
}
@@ -0,0 +1,139 @@
const { test, expect } = require('@playwright/test');
const { getCurrentWorkspace, login } = require('./helpers');
// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a
// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit
// invalid value must be reported instead of being silently replaced by defaults.
function draftSnapshot(operationId) {
return {
id: operationId,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
target: {
kind: 'rest',
base_url: 'https://api.example.test',
method: 'GET',
path_template: '/rates',
static_headers: {},
},
input_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
output_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
input_mapping: { rules: [] },
output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] },
execution_config: {
timeout_ms: 10000,
retry_policy: null,
auth_profile_ref: null,
headers: {},
approval_policy: null,
execution_policy: {
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
},
},
tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] },
wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] },
};
}
test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => {
await login(page);
const workspace = await getCurrentWorkspace(page);
const operationId = 'policy_rates_41';
const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`;
const savedPayloads = [];
await page.route(operationPath, async (route) => {
if (route.request().method() === 'PATCH') {
savedPayloads.push(route.request().postDataJSON());
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }),
});
return;
}
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
id: operationId,
workspace_id: workspace.id,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
status: 'draft',
current_draft_version: 1,
draft_version_ref: { version: 1, status: 'draft' },
published_version_ref: null,
agent_refs: [],
}),
});
});
await page.route(`${operationPath}/versions/*`, async (route) => {
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
operation_id: operationId,
workspace_id: workspace.id,
version: 1,
status: 'draft',
snapshot: draftSnapshot(operationId),
}),
});
});
await page.route('**/operations/analyze-quality', async (route) => {
await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) });
});
page.on('dialog', (dialog) => dialog.accept());
await page.goto(`/wizard/?mode=edit&operationId=${operationId}`);
await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates');
const editorValue = await page.locator('#tool-exec-config').inputValue();
expect(editorValue).toContain('execution_policy');
expect(editorValue).toContain('adaptive');
expect(editorValue).toContain('45000');
expect(editorValue).toContain('unsupported');
// A real edit still saves the pinned policy unchanged.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(4));
await page.locator('#tool-display-name').fill('Policy rates edited');
await page.locator('.btn-save-draft').click();
await expect.poll(() => savedPayloads.length).toBe(1);
expect(savedPayloads[0].execution_config.execution_policy).toEqual({
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
});
// An explicit invalid policy is refused instead of being silently rewritten to defaults.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
await expect(page.locator('#tool-exec-config')).toBeVisible();
await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n');
await page.locator('.btn-save-draft').click();
await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/);
expect(savedPayloads.length).toBe(1);
});
+103 -14
View File
@@ -51,12 +51,14 @@ mod auth_resolution;
mod dispatch;
mod endpoint;
mod legacy_dispatch;
mod tasks;
use self::approvals::{
approve_request, deny_request, get_approval_request, list_pending_approvals,
};
pub(super) use self::auth_resolution::resolve_operation_auth;
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
// through `super`, and the plain lib build does not use them (same pattern as
@@ -92,6 +94,9 @@ pub(super) struct AppState {
pub(super) signal_streams: Arc<crate::protocol::StreamRegistry>,
/// Story 3.4: the bounded table of the `tools/call` requests a legacy client can still cancel.
pub(super) in_flight: Arc<self::lifecycle::InFlightCalls>,
/// Story 4.2: durable background Tasks over the same registry pool. The public wire stays
/// synchronous/unsupported; readiness depends on this mandatory store.
tasks: Arc<TaskApplication>,
}
#[derive(Clone)]
@@ -137,6 +142,7 @@ pub fn build_app(
DEFAULT_MAX_CONCURRENT_SESSIONS,
false,
None,
None,
)
}
@@ -192,6 +198,7 @@ pub fn build_app_with_background_workers_and_limits(
max_concurrent_sessions,
true,
None,
None,
)
}
@@ -227,6 +234,41 @@ pub fn build_app_with_artifacts(
max_concurrent_sessions,
true,
Some(artifact_store),
None,
)
}
/// Production launcher entry point with the durable Task application entry constructed by the
/// process composition root from the same registry pool and admission limits as the runtime.
#[allow(clippy::too_many_arguments)]
pub fn build_app_with_artifacts_and_tasks(
registry: PostgresRegistry,
refresh_interval: Duration,
public_base_url: Option<String>,
secret_crypto: SecretCrypto,
runtime: RuntimeExecutor,
api_rate_limiter: RequestRateLimiter,
coordination_store: Arc<dyn CoordinationStateStore>,
sessions: SharedSessionStore,
credential_verifier: SharedMachineCredentialVerifier,
max_concurrent_sessions: usize,
artifact_store: crank_artifacts::ArtifactStore,
tasks: Arc<TaskApplication>,
) -> Router {
build_app_inner(
registry,
refresh_interval,
public_base_url,
secret_crypto,
runtime,
api_rate_limiter,
coordination_store,
sessions,
credential_verifier,
max_concurrent_sessions,
true,
Some(artifact_store),
Some(tasks),
)
}
@@ -244,8 +286,23 @@ fn build_app_inner(
max_concurrent_sessions: usize,
start_background_workers: bool,
artifact_store: Option<crank_artifacts::ArtifactStore>,
tasks: Option<Arc<TaskApplication>>,
) -> Router {
let session_metrics = ActiveSessionMetrics::start(Arc::clone(&sessions));
let tasks = tasks.unwrap_or_else(|| {
let store = crank_registry::PostgresTaskStore::new(
registry.pool().clone(),
crank_registry::AdmissionLimits::default(),
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("default admission limits and scheduler weights are valid");
Arc::new(TaskApplication::new(
registry.clone(),
refresh_interval,
Arc::clone(&coordination_store),
crank_runtime::TaskService::new(Arc::new(store)),
))
});
let state = Arc::new(AppState {
registry: registry.clone(),
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
@@ -263,6 +320,7 @@ fn build_app_inner(
in_flight: Arc::new(lifecycle::InFlightCalls::new(
lifecycle::MAX_IN_FLIGHT_CALLS,
)),
tasks,
});
if start_background_workers {
spawn_approval_recovery(Arc::clone(&state));
@@ -311,24 +369,55 @@ async fn health() -> Json<Value> {
}
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
match state.registry.ping().await {
Ok(()) => Json(json!({
let postgres_ready = state.registry.ping().await.is_ok();
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
// reflect it rather than reporting ready on a bare `select 1`.
let admission_ready = state.registry.execution_admission_health().await.is_ok();
// Story 4.2: the durable Task store is mandatory for a durable acceptance acknowledgement.
let durable_tasks_ready = state.tasks.health().await.is_ok();
if postgres_ready && admission_ready && durable_tasks_ready {
return Json(json!({
"service": "mcp-server",
"status": "ready",
"checks": { "postgres": "ready" }
"checks": {
"postgres": "ready",
"execution_admission": "ready",
"durable_tasks": "ready"
}
}))
.into_response(),
Err(_) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": "database is unavailable"
})),
)
.into_response(),
.into_response();
}
let mut checks = serde_json::Map::new();
checks.insert(
"postgres".to_owned(),
json!(if postgres_ready { "ready" } else { "not_ready" }),
);
checks.insert(
"execution_admission".to_owned(),
json!(if admission_ready {
"ready"
} else {
"not_ready"
}),
);
checks.insert(
"durable_tasks".to_owned(),
json!(if durable_tasks_ready {
"ready"
} else {
"not_ready"
}),
);
(
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": checks,
"error": "a mandatory dependency is unavailable"
})),
)
.into_response()
}
#[cfg(test)]
+295
View File
@@ -0,0 +1,295 @@
//! Application entry of durable background Tasks (Story 4.2, FR-22/FR-23).
//!
//! The entry takes an already authenticated principal (a verified machine credential, or an
//! administrator identity verified by the calling admin surface), resolves the exact published Tool,
//! repeats the caller's authorization, pins the Published Version/policy/deadline and delegates the
//! durable acceptance to the runtime Task service. The repository owns the transaction, so the
//! returned Task ID exists only after Task + dispatch intent + queue slot committed.
//!
//! Deliberately not wired: the public MCP `tools/call`/Tasks wire (Story 4.7) and the worker/leases
//! (Story 4.3). The existing synchronous/unsupported `tools/call` path is unchanged, and this entry
//! never spawns an ephemeral execution future.
use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord,
ToolExecutionMode,
};
use crank_registry::{PostgresRegistry, PublishedAgentTool};
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
use serde_json::Value;
use crate::access::credential_allows_security_level;
use crate::catalog::PublishedToolCatalog;
/// Authenticated principal of one Task operation. The authorization scope identity is derived here
/// from the verified credential/admin identity, never from a caller-supplied request string.
#[derive(Clone, Debug)]
pub enum TaskPrincipal {
Agent {
workspace_slug: String,
agent_slug: String,
credential: crank_core::VerifiedMachineCredential,
},
/// Administrator identity that the admin surface already authenticated. It is never a wildcard:
/// owner, workspace and authorization scope all participate in the stored predicate.
Admin {
workspace_slug: String,
agent_slug: String,
user_id: String,
},
}
impl TaskPrincipal {
fn workspace_slug(&self) -> &str {
match self {
Self::Agent { workspace_slug, .. } | Self::Admin { workspace_slug, .. } => {
workspace_slug
}
}
}
fn agent_slug(&self) -> &str {
match self {
Self::Agent { agent_slug, .. } | Self::Admin { agent_slug, .. } => agent_slug,
}
}
fn owner(&self, tool: &PublishedAgentTool) -> (TaskOwnerKind, String) {
match self {
Self::Agent { .. } => (TaskOwnerKind::Agent, tool.agent_id.as_str().to_owned()),
Self::Admin { user_id, .. } => (TaskOwnerKind::Admin, user_id.clone()),
}
}
/// Bounded server-derived authorization scope identity of this principal.
fn authorization_scope(&self) -> String {
let identity = match self {
Self::Agent { credential, .. } => {
let mut scopes = credential
.scopes
.iter()
.map(|scope| format!("{scope:?}"))
.collect::<Vec<_>>();
scopes.sort();
format!(
"key:{}|mode:{:?}|level:{:?}|scopes:{}",
credential
.platform_api_key_id
.as_ref()
.map(|id| id.as_str().to_owned())
.unwrap_or_else(|| "-".to_owned()),
credential.machine_access_mode,
credential.max_security_level,
scopes.join(",")
)
}
Self::Admin { user_id, .. } => format!("admin:{user_id}"),
};
identity.chars().take(160).collect()
}
}
/// Typed refusal of the application entry. `NotFound`/`Denied` never reveal whether a foreign Task
/// exists, and no variant carries Task payload or fingerprint.
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum TaskApplicationError {
#[error("task request was not found")]
NotFound,
#[error("task request is not authorized")]
Denied,
#[error("the published Tool policy does not accept durable background work")]
NotBackground,
#[error("the requested published Tool version is not available")]
VersionUnavailable,
#[error("this Tool requires a verified approval before acceptance")]
ApprovalRequired,
#[error(transparent)]
Task(#[from] TaskError),
}
#[derive(Clone)]
pub struct TaskApplication {
catalog: PublishedToolCatalog,
service: TaskService,
}
impl TaskApplication {
/// Production construction of the application entry over the process registry, the published
/// catalog and the durable Task service.
pub fn new(
registry: PostgresRegistry,
refresh_interval: Duration,
coordination_store: Arc<dyn crank_core::CoordinationStateStore>,
service: TaskService,
) -> Self {
Self {
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
service,
}
}
/// Readiness of the durable Task store this entry depends on.
pub async fn health(&self) -> Result<(), TaskError> {
self.service.health().await
}
/// Accepts one background request for an already authenticated principal.
///
/// `expected_version` is the caller's explicit Published Version pin. Without it the current
/// publication is pinned for new work and a retry of the same scoped key returns the stored Task
/// even after a newer publication. An explicit pin may only retry the Task accepted with that
/// exact version: a different pin is a typed conflict, never a rebind.
pub async fn accept_background(
&self,
principal: &TaskPrincipal,
tool_name: &str,
expected_version: Option<u32>,
scoped_key: &str,
input: &Value,
correlation: &CorrelationContext,
) -> Result<TaskAcceptance, TaskApplicationError> {
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
let current_version = tool.operation.version;
let Some(policy) = tool.operation.execution_config.execution_policy else {
return Err(TaskApplicationError::NotBackground);
};
if policy.mode == ToolExecutionMode::Synchronous {
return Err(TaskApplicationError::NotBackground);
}
if tool
.operation
.execution_config
.approval_policy
.as_ref()
.is_some_and(|approval| approval.required)
{
// Without a verified, non-consuming approval reference this entry fails closed instead
// of accepting work a worker could never safely execute.
return Err(TaskApplicationError::ApprovalRequired);
}
let draft = |operation_version: u32| {
let (owner_kind, owner_id) = principal.owner(&tool);
TaskAcceptDraft {
workspace_id: tool.workspace_id.as_str().to_owned(),
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
operation_id: tool.operation.id.as_str().to_owned(),
operation_version,
policy_mode: policy.mode,
scoped_key: scoped_key.to_owned(),
input: input.clone(),
correlation: correlation.clone(),
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
retention_ms: TaskService::default_retention_ms(),
approval_reference: None,
}
};
let Some(requested) = expected_version else {
return Ok(self.service.accept(draft(current_version)).await?);
};
let draft = draft(requested);
match self.service.lookup(&draft).await? {
Some(record) => {
if record.input_fingerprint != TaskService::input_fingerprint(input)? {
return Err(TaskApplicationError::Task(TaskError::FingerprintConflict));
}
if record.pins.operation_version != requested {
return Err(TaskApplicationError::VersionUnavailable);
}
Ok(TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support,
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
})
}
None if requested == current_version => Ok(self.service.accept(draft).await?),
// A new Task can only be accepted against the currently published version.
None => Err(TaskApplicationError::VersionUnavailable),
}
}
/// Owner-scoped status read of one Task of the addressed published Tool.
pub async fn status(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<TaskRecord, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.status(&access, task_id).await {
Ok(record) => Ok(record),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped result projection of one Task of the addressed published Tool.
pub async fn result(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<TaskOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.result(&access, task_id).await {
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
async fn task_access(
&self,
principal: &TaskPrincipal,
tool_name: &str,
) -> Result<TaskAccess, TaskApplicationError> {
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
let (owner_kind, owner_id) = principal.owner(&tool);
Ok(TaskAccess {
workspace_id: tool.workspace_id.as_str().to_owned(),
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
})
}
/// Resolves the addressed Tool of the principal's published catalog and repeats its authorization
/// check. A foreign workspace, an unknown Tool and an insufficient security level are typed
/// refusals without metadata.
async fn resolve_authorized_tool(
&self,
principal: &TaskPrincipal,
tool_name: &str,
) -> Result<PublishedAgentTool, TaskApplicationError> {
let catalog = self
.catalog
.get_catalog(principal.workspace_slug(), principal.agent_slug())
.await
.map_err(|_| TaskApplicationError::NotFound)?;
let tool = catalog
.tools
.into_iter()
.find(|tool| tool.tool_name == tool_name)
.ok_or(TaskApplicationError::NotFound)?;
if let TaskPrincipal::Admin { workspace_slug, .. } = principal
&& tool.workspace_slug != *workspace_slug
{
return Err(TaskApplicationError::Denied);
}
if let TaskPrincipal::Agent { credential, .. } = principal
&& !credential_allows_security_level(credential, tool.operation.security_level)
{
return Err(TaskApplicationError::Denied);
}
Ok(tool)
}
}
@@ -550,6 +550,7 @@ pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+2 -1
View File
@@ -16,6 +16,7 @@ mod tool_search;
mod transport;
pub use app::{
build_app, build_app_with_artifacts, build_app_with_background_workers,
TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts,
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
build_app_with_background_workers_and_limits,
};
@@ -136,6 +136,7 @@ fn published_tool(name: &str, title: &str, description: &str, version: u32) -> P
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -284,6 +284,7 @@ fn operation() -> RegistryOperation {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+531
View File
@@ -0,0 +1,531 @@
//! Typed execution-admission seam (Story 4.1, FR-21/FR-24).
//!
//! Admission is the single gate in front of every real Tool execution: it decides whether the
//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide
//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives
//! in PostgreSQL (`crank-registry`), so several processes share one budget.
//!
//! Deliberately absent from this contract: credentials, connection or protocol revision (they must
//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2).
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use crate::{NoTasksFallback, ToolExecutionMode};
pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128;
pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64;
pub const MAX_RETRY_AFTER_MS: u64 = 60_000;
/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for
/// the store and every runtime so the client contract stays stable.
pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250;
/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms).
pub fn current_unix_ms() -> i64 {
i64::try_from(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_millis())
.unwrap_or_default(),
)
.unwrap_or(i64::MAX)
}
/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin
/// bucket that is still bounded by the global limit.
pub const ADMIN_ORIGIN_KEY: &str = "admin";
/// Exact execution identity pinned at admission time.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AdmissionScope {
pub workspace_id: String,
/// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution.
pub agent_key: String,
pub operation_id: String,
/// Quota keys exclude the version; the version is pinned for execution semantics only.
pub operation_version: u32,
}
impl AdmissionScope {
/// Quota key of the Agent bucket: `(workspace, agent)`.
pub fn agent_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.agent_key)
}
/// Quota key of the Tool bucket: `(workspace, operation)` without version.
pub fn tool_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.operation_id)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AdmissionMode {
/// The caller waits for a scoped permit and dispatches inside it.
Synchronous,
/// The store commits a durable enqueue decision and reserves queue capacity.
Background,
}
impl AdmissionMode {
pub const fn as_str(self) -> &'static str {
match self {
Self::Synchronous => "synchronous",
Self::Background => "background",
}
}
}
/// Non-consuming approval evidence gathered by the runtime before admission.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ApprovalEvidence {
/// The operation does not require confirmation.
NotRequired,
/// A verified approval exists for this exact scope and input.
Granted,
/// Confirmation is required and no verified approval exists yet.
NotGranted,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionRequest {
pub scope: AdmissionScope,
pub mode: AdmissionMode,
/// Published policy mode that produced this request.
pub policy_mode: ToolExecutionMode,
pub no_tasks_fallback: NoTasksFallback,
pub approval: ApprovalEvidence,
/// Bounded request identity used only to correlate one admission entry.
pub fingerprint: String,
/// Absolute deadline (unix milliseconds), already including queue waiting time.
pub expires_at_unix_ms: i64,
}
impl AdmissionRequest {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if self.fingerprint.is_empty()
|| self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES
|| self
.fingerprint
.bytes()
.any(|byte| byte.is_ascii_control() || byte == b' ')
{
return Err(AdmissionStoreError::invalid(
"admission fingerprint is outside its allowed bounds",
));
}
if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission workspace is invalid",
));
}
if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 {
return Err(AdmissionStoreError::invalid("admission agent is invalid"));
}
if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission operation is invalid",
));
}
if self.scope.operation_version == 0 {
return Err(AdmissionStoreError::invalid(
"admission operation version must be positive",
));
}
if self.expires_at_unix_ms <= 0 {
return Err(AdmissionStoreError::invalid(
"admission deadline is invalid",
));
}
Ok(())
}
}
/// Scoped permit. Held for the whole local execution future (including retries) and released
/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionPermit {
pub ticket_id: u64,
pub owner_token: String,
pub scope: AdmissionScope,
pub mode: AdmissionMode,
pub policy_mode: ToolExecutionMode,
}
/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics.
/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionDecision {
pub ticket_id: u64,
pub scope: AdmissionScope,
pub expires_at_unix_ms: i64,
pub policy_mode: ToolExecutionMode,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AdmissionOutcome {
Permitted(AdmissionPermit),
Enqueued(AdmissionDecision),
/// Capacity or queue bounds were exhausted before any unbounded allocation.
Overloaded {
retry_after_ms: u64,
},
/// The queue wait reached the absolute deadline; nothing was dispatched.
DeadlineElapsed,
/// Confirmation is mandatory for this operation and no verified approval exists.
ApprovalRequired,
/// The published policy does not advertise durable Tasks and refuses the fallback.
Unsupported,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionQueueSnapshot {
pub waiting: u64,
pub executing: u64,
/// Age of the oldest waiting request in the authoritative queue, in milliseconds.
pub oldest_waiting_ms: Option<u64>,
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum AdmissionStoreError {
/// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed.
#[error("execution admission store is unavailable: {message}")]
Unavailable { message: String },
/// The store rejected a malformed admission request.
#[error("execution admission request is invalid: {message}")]
Invalid { message: String },
}
impl AdmissionStoreError {
pub fn unavailable(message: impl Into<String>) -> Self {
Self::Unavailable {
message: message.into(),
}
}
pub fn invalid(message: impl Into<String>) -> Self {
Self::Invalid {
message: message.into(),
}
}
}
#[async_trait]
pub trait ExecutionAdmission: Send + Sync {
/// Proves the mandatory admission state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), AdmissionStoreError>;
/// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue
/// decision for a background request, or returns a typed refusal.
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<AdmissionOutcome, AdmissionStoreError>;
/// Releases the caller's own permit. Idempotent: a repeated release returns `false`.
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError>;
/// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket.
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError>;
/// Bounded authoritative queue readings for low-cardinality metrics.
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError>;
}
/// One waiting entry as seen by the fairness scheduler.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionQueueEntry {
pub ticket_id: u64,
/// Authoritative insertion order inside `(Agent, Tool)`; never rewritten.
pub sequence: u64,
pub agent_key: String,
pub operation_id: String,
}
/// Weighted round-robin state persisted with the authoritative queue.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionCursor {
pub agent: Option<String>,
pub agent_credit: u32,
pub tool: Option<String>,
pub tool_credit: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionSchedulerConfig {
pub agent_weight: u32,
pub tool_weight: u32,
}
impl Default for AdmissionSchedulerConfig {
fn default() -> Self {
Self {
agent_weight: 1,
tool_weight: 1,
}
}
}
impl AdmissionSchedulerConfig {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) {
return Err(AdmissionStoreError::invalid(
"admission weights must be between 1 and 64",
));
}
Ok(())
}
}
/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the
/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its
/// turn instead of being skipped by newer arrivals.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionSelection {
pub index: usize,
pub cursor: AdmissionCursor,
}
/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside
/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit.
pub fn select_next_grant(
queue: &[AdmissionQueueEntry],
cursor: &AdmissionCursor,
config: AdmissionSchedulerConfig,
) -> Option<AdmissionSelection> {
if queue.is_empty() {
return None;
}
let agent_weight = config.agent_weight.max(1);
let tool_weight = config.tool_weight.max(1);
let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect();
agents.sort_unstable();
agents.dedup();
let mut next = cursor.clone();
let agent = match cursor.agent.as_deref() {
Some(current) if cursor.agent_credit > 0 && agents.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.agent.as_deref() {
None => agents[0].to_owned(),
Some(current) => agents
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(agents[0])
.to_owned(),
};
if cursor.agent.as_deref() != Some(chosen.as_str()) {
next.tool = None;
next.tool_credit = 0;
}
next.agent = Some(chosen.clone());
next.agent_credit = agent_weight;
chosen
}
};
let mut tools: Vec<&str> = queue
.iter()
.filter(|entry| entry.agent_key == agent)
.map(|entry| entry.operation_id.as_str())
.collect();
tools.sort_unstable();
tools.dedup();
let tool = match cursor.tool.as_deref() {
Some(current) if cursor.tool_credit > 0 && tools.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.tool.as_deref() {
None => tools[0].to_owned(),
Some(current) => tools
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(tools[0])
.to_owned(),
};
next.tool = Some(chosen.clone());
next.tool_credit = tool_weight;
chosen
}
};
let index = queue
.iter()
.position(|entry| entry.agent_key == agent && entry.operation_id == tool)?;
next.agent_credit = next.agent_credit.saturating_sub(1);
next.tool_credit = next.tool_credit.saturating_sub(1);
Some(AdmissionSelection {
index,
cursor: next,
})
}
#[cfg(test)]
mod tests {
use super::{
ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest,
AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence,
select_next_grant,
};
use crate::{NoTasksFallback, ToolExecutionMode};
fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry {
super::AdmissionQueueEntry {
ticket_id,
sequence: ticket_id,
agent_key: agent.to_owned(),
operation_id: tool.to_owned(),
}
}
fn request() -> AdmissionRequest {
AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_default".to_owned(),
agent_key: ADMIN_ORIGIN_KEY.to_owned(),
operation_id: "op_leads".to_owned(),
operation_version: 3,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_0123456789".to_owned(),
expires_at_unix_ms: 1_790_000_000_000,
}
}
#[test]
fn quota_keys_exclude_version_and_keep_agent_bucket() {
let scope = request().scope;
assert_eq!(scope.agent_bucket(), "ws_default:admin");
assert_eq!(scope.tool_bucket(), "ws_default:op_leads");
}
#[test]
fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() {
let mut invalid = request();
invalid.expires_at_unix_ms = 0;
assert!(matches!(
invalid.validate(),
Err(AdmissionStoreError::Invalid { .. })
));
let mut invalid = request();
invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1);
assert!(invalid.validate().is_err());
let mut invalid = request();
invalid.scope.operation_version = 0;
assert!(invalid.validate().is_err());
assert!(request().validate().is_ok());
}
#[test]
fn weights_are_bounded_positive_configuration() {
assert!(
AdmissionSchedulerConfig {
agent_weight: 0,
tool_weight: 1
}
.validate()
.is_err()
);
assert!(
AdmissionSchedulerConfig {
agent_weight: 1,
tool_weight: 65
}
.validate()
.is_err()
);
assert!(AdmissionSchedulerConfig::default().validate().is_ok());
}
#[test]
fn weighted_round_robin_serves_each_agent_its_quantum() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_1"),
entry(4, "agent_b", "op_1"),
entry(5, "agent_b", "op_1"),
];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
order.push(queue.remove(selection.index).agent_key.clone());
cursor = selection.cursor;
}
assert_eq!(
order,
vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"]
);
}
#[test]
fn fifo_inside_one_tool_and_round_robin_between_tools() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_2"),
];
let config = AdmissionSchedulerConfig::default();
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
// A granted ticket leaves the waiting queue: granted entries are never re-granted.
order.push(queue.remove(selection.index).ticket_id);
cursor = selection.cursor;
}
// One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue.
assert_eq!(order, vec![1, 3, 2]);
}
#[test]
fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() {
let queue = vec![entry(1, "agent_a", "op_1")];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let cursor = AdmissionCursor::default();
let first = select_next_grant(&queue, &cursor, config).unwrap();
assert_eq!(first.cursor.agent.as_deref(), Some("agent_a"));
assert_eq!(first.cursor.agent_credit, 1);
// A later Agent joins; the cursor keeps its remaining quantum for `agent_a`.
let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")];
let second = select_next_grant(&queue, &first.cursor, config).unwrap();
assert_eq!(queue[second.index].agent_key, "agent_a");
assert_eq!(second.cursor.agent_credit, 0);
let third = select_next_grant(&queue, &second.cursor, config).unwrap();
assert_eq!(queue[third.index].agent_key, "agent_b");
assert_eq!(third.cursor.agent_credit, 1);
}
#[test]
fn empty_queue_keeps_cursor_untouched() {
let cursor = AdmissionCursor {
agent: Some("agent_b".to_owned()),
agent_credit: 1,
tool: Some("op_1".to_owned()),
tool_credit: 0,
};
assert_eq!(
select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()),
None
);
assert_eq!(cursor.agent.as_deref(), Some("agent_b"));
}
}
+35 -9
View File
@@ -1,4 +1,5 @@
pub mod access;
pub mod admission;
pub mod agent;
pub mod approval;
pub mod auth;
@@ -17,6 +18,7 @@ pub mod prompt_composition;
pub mod protocol;
pub mod resource;
pub mod secret;
pub mod task;
pub mod tool_catalog;
pub mod tool_quality;
pub mod tool_search;
@@ -60,10 +62,11 @@ pub mod domain {
pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use crate::operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, WizardState,
};
pub use crate::product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
@@ -71,6 +74,13 @@ pub mod domain {
};
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use crate::task::{
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds,
};
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
pub use crate::tool_quality::{
ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
@@ -108,12 +118,20 @@ pub mod ports {
PreparedRequest, ProtocolAdapter, ProtocolAdapterError, ResponseCacheScope,
RuntimeRequestContext, SharedProtocolAdapter,
};
pub use crate::task::TaskStore;
}
pub use access::{
InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
};
pub use admission::{
ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS,
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission,
MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant,
};
pub use agent::{
Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
@@ -168,13 +186,14 @@ pub use observability::{
pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState,
ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor,
RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample,
WizardState,
IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation,
OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy,
OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract,
RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget,
RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, ToolExecutionPolicyError, WizardState,
};
pub use product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
@@ -182,6 +201,13 @@ pub use product_event::{
};
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use task::{
TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key,
validate_task_envelope,
};
pub use tool_catalog::{
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
ToolCatalogBudget, analyze_tool_catalog,
+144 -2
View File
@@ -270,6 +270,72 @@ pub struct OperationApprovalPolicy {
pub elicitation_message: Option<String>,
}
/// Published execution mode of one Tool (Story 4.1, FR-21).
///
/// This is a scheduling policy of the Published Tool version. It is deliberately separate from
/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolExecutionMode {
#[default]
Synchronous,
Background,
Adaptive,
}
/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NoTasksFallback {
/// Execute through the same synchronous admission path.
#[default]
Synchronous,
/// Reject with a typed unsupported-execution-mode failure; never a Task ID.
Unsupported,
}
pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000;
/// Execution policy pinned by a Published Tool version.
///
/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by
/// `ExecutionConfig::timeout_ms`, with no background path.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ToolExecutionPolicy {
#[serde(default)]
pub mode: ToolExecutionMode,
/// Absolute budget of one execution, counted from the start of admission.
pub deadline_ms: u64,
#[serde(default)]
pub no_tasks_fallback: NoTasksFallback,
}
impl Default for ToolExecutionPolicy {
fn default() -> Self {
Self {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
}
}
impl ToolExecutionPolicy {
pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> {
if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) {
return Err(ToolExecutionPolicyError::DeadlineOutOfRange);
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ToolExecutionPolicyError {
#[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")]
DeadlineOutOfRange,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExecutionConfig {
@@ -285,6 +351,8 @@ pub struct ExecutionConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub approval_policy: Option<OperationApprovalPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub execution_policy: Option<ToolExecutionPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auth_profile_ref: Option<AuthProfileId>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub headers: BTreeMap<String, String>,
@@ -444,9 +512,10 @@ mod tests {
edition::OperationSecurityLevel,
ids::{AuthProfileId, OperationId},
operation::{
ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationStatus,
ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability,
OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus,
OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample,
ToolExecutionMode, ToolExecutionPolicy,
},
protocol::{AuthKind, ExportMode, HttpMethod, Protocol},
};
@@ -664,6 +733,7 @@ updated_at: 2026-03-25T08:10:00Z
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: Some(AuthProfileId::new("auth_01")),
headers: BTreeMap::new(),
},
@@ -684,4 +754,76 @@ updated_at: 2026-03-25T08:10:00Z
published_at: None,
}
}
#[test]
fn execution_policy_round_trips_all_modes_and_fallbacks() {
for (mode, fallback) in [
(ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Unsupported),
(ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported),
] {
let config = ExecutionConfig {
execution_policy: Some(ToolExecutionPolicy {
mode,
deadline_ms: 45_000,
no_tasks_fallback: fallback,
}),
..ExecutionConfig::default()
};
let encoded = serde_json::to_value(&config).unwrap();
assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000));
let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap();
assert_eq!(decoded, config);
assert_eq!(
decoded.execution_policy.unwrap().mode,
mode,
"the published mode must survive the JSON round-trip"
);
}
}
#[test]
fn legacy_execution_config_without_policy_stays_synchronous() {
let legacy: ExecutionConfig =
serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap();
assert_eq!(legacy.execution_policy, None);
assert_eq!(legacy.timeout_ms, 12_000);
}
#[test]
fn execution_policy_rejects_explicit_invalid_deadline() {
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 0,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
.validate()
.is_err()
);
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: NoTasksFallback::Unsupported,
}
.validate()
.is_err()
);
assert!(ToolExecutionPolicy::default().validate().is_ok());
}
#[test]
fn execution_policy_rejects_unknown_fields() {
assert!(
serde_json::from_value::<ToolExecutionPolicy>(json!({
"mode": "background",
"deadline_ms": 1000,
"no_tasks_fallback": "synchronous",
"task_queue": "default"
}))
.is_err()
);
}
}
+730
View File
@@ -0,0 +1,730 @@
//! Durable background Task seam (Story 4.2, FR-22/FR-23).
//!
//! A Task is the internal, protocol-independent record of one accepted background request. This
//! module owns the domain vocabulary only: the lifecycle states, the accepted identity/pins, the
//! bounded persisted envelopes and the `TaskStore` port. The protocol adapter (Story 4.7) and the
//! worker (Story 4.3) do not define these semantics; the authoritative store lives in PostgreSQL
//! (`crank-registry`), so an accepted Task survives a crash or restart.
//!
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
//! Task was executed. A terminal state is written by a conditional transition only.
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
use crate::correlation::{RequestId, TraceContext, TraceId};
use crate::operation::ToolExecutionMode;
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
/// Maximum nesting depth of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
/// Maximum number of JSON members/nodes of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_NODES: usize = 256;
/// Bound of every scope/owner/operation identity column.
pub const TASK_IDENTITY_MAX_BYTES: usize = 160;
/// Bound of the caller-supplied idempotency key before it becomes a scoped digest.
pub const TASK_SCOPED_KEY_MAX_BYTES: usize = 128;
/// Bound of a saved approval reference (an identifier, never a bearer token).
pub const TASK_APPROVAL_REFERENCE_MAX_BYTES: usize = 160;
/// Default retention of a terminal Task after its execution deadline (internal bounded default).
pub const TASK_DEFAULT_RETENTION_MS: i64 = 86_400_000;
/// Upper bound on the retention an accepted Task may pin.
pub const TASK_MAX_RETENTION_MS: i64 = 30 * 86_400_000;
/// Upper bound of one retention/housekeeping batch.
pub const TASK_PURGE_BATCH_LIMIT: u32 = 128;
/// Stable identity of one accepted Task. Generated as UUIDv7 and returned only after the accepting
/// transaction committed, so the ID never exists without its durable record.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct TaskId(Uuid);
impl TaskId {
pub fn generate() -> Self {
Self(Uuid::now_v7())
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
Uuid::parse_str(value)
.map(Self)
.map_err(|_| TaskError::invalid("task identity is not a UUID"))
}
pub fn as_uuid(self) -> Uuid {
self.0
}
}
impl std::fmt::Display for TaskId {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(formatter)
}
}
/// Closed Task lifecycle vocabulary. `outcome_unknown` is a non-terminal state: a value commit that
/// failed to confirm its dispatch is never collapsed into a plain failure.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskState {
Queued,
Working,
InputRequired,
OutcomeUnknown,
Completed,
Failed,
Cancelled,
}
impl TaskState {
pub const fn as_str(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Working => "working",
Self::InputRequired => "input_required",
Self::OutcomeUnknown => "outcome_unknown",
Self::Completed => "completed",
Self::Failed => "failed",
Self::Cancelled => "cancelled",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"queued" => Ok(Self::Queued),
"working" => Ok(Self::Working),
"input_required" => Ok(Self::InputRequired),
"outcome_unknown" => Ok(Self::OutcomeUnknown),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"cancelled" => Ok(Self::Cancelled),
_ => Err(TaskError::invalid("unknown task state")),
}
}
pub const fn is_terminal(self) -> bool {
matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
}
/// One conditional transition may only move along this table. A terminal state has no successor,
/// so a repeated terminal write can never rewrite an accepted result.
pub const fn allows_transition_to(self, next: Self) -> bool {
match self {
Self::Queued => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
Self::Working => matches!(
next,
Self::InputRequired
| Self::OutcomeUnknown
| Self::Completed
| Self::Failed
| Self::Cancelled
),
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
Self::OutcomeUnknown => {
matches!(next, Self::Completed | Self::Failed | Self::Cancelled)
}
Self::Completed | Self::Failed | Self::Cancelled => false,
}
}
}
/// Who owns the accepted Task. An administrator is never a wildcard owner of every Task: the owner
/// identity and the authorization scope are separate, both-pinned columns.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskOwnerKind {
Agent,
Admin,
}
impl TaskOwnerKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Agent => "agent",
Self::Admin => "admin",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"agent" => Ok(Self::Agent),
"admin" => Ok(Self::Admin),
_ => Err(TaskError::invalid("unknown task owner kind")),
}
}
}
/// Support identities captured at acceptance. They are stored and returned independently of
/// sampling, and a replay never rewrites them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskSupportIds {
pub request_id: RequestId,
pub trace_id: TraceId,
pub traceparent: String,
pub trace_sampled: bool,
}
impl TaskSupportIds {
pub fn new(request_id: RequestId, trace_context: &TraceContext) -> Self {
Self {
request_id,
trace_id: trace_context.trace_id().clone(),
traceparent: trace_context.traceparent().to_owned(),
trace_sampled: trace_context.is_sampled(),
}
}
/// Re-validates both the trace identity and the parent context: a Task must never persist an
/// inconsistent or malformed support pair, and an unsampled context is still a valid parent.
pub fn validate(&self) -> Result<(), TaskError> {
if self.trace_id.as_str().is_empty() {
return Err(TaskError::invalid("task trace identity is missing"));
}
let context = TraceContext::parse(&self.traceparent)
.map_err(|_| TaskError::invalid("task trace parent is invalid"))?;
if context.trace_id() != &self.trace_id || context.is_sampled() != self.trace_sampled {
return Err(TaskError::invalid("task trace parent is inconsistent"));
}
Ok(())
}
}
/// Execution semantics pinned at acceptance: a later publication never rewrites them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskPins {
pub operation_version: u32,
pub policy_mode: ToolExecutionMode,
pub execution_deadline_unix_ms: i64,
/// Retention deadline of the record; never the same thing as the execution deadline.
pub retain_until_unix_ms: i64,
/// Verified approval evidence reference (identifier only, never a bearer token).
pub approval_reference: Option<String>,
}
impl TaskPins {
pub fn validate(&self) -> Result<(), TaskError> {
if self.operation_version == 0 {
return Err(TaskError::invalid(
"task operation version must be positive",
));
}
if self.execution_deadline_unix_ms <= 0 {
return Err(TaskError::invalid("task execution deadline is invalid"));
}
if self.retain_until_unix_ms < self.execution_deadline_unix_ms {
return Err(TaskError::invalid(
"task retention deadline precedes execution",
));
}
if self.retain_until_unix_ms - self.execution_deadline_unix_ms > TASK_MAX_RETENTION_MS {
return Err(TaskError::invalid("task retention exceeds its bound"));
}
if let Some(reference) = self.approval_reference.as_deref()
&& (reference.is_empty()
|| reference.len() > TASK_APPROVAL_REFERENCE_MAX_BYTES
|| reference.bytes().any(|byte| byte.is_ascii_control()))
{
return Err(TaskError::invalid("task approval reference is invalid"));
}
Ok(())
}
}
/// Bounded, classified payload of a Task. Construction validates the byte/depth/node budget and
/// redacts secret-bearing fields before anything is hashed or persisted.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskEnvelope {
value: Value,
}
impl TaskEnvelope {
/// Validates the caller-provided value, then stores a redacted copy of it. The caller hashes the
/// original value separately, so redaction never collapses two different requests into one.
pub fn try_redacted(input: &Value) -> Result<Self, TaskError> {
validate_task_envelope(input)?;
Ok(Self {
value: redact_task_envelope(input),
})
}
/// Accepts a value that was already validated and redacted at a trust boundary.
pub fn from_persisted(value: Value) -> Result<Self, TaskError> {
validate_task_envelope(&value)?;
Ok(Self { value })
}
pub fn value(&self) -> &Value {
&self.value
}
pub fn into_value(self) -> Value {
self.value
}
}
/// Validates the size, nesting depth and node count of one Task envelope. Every byte, collection and
/// node is counted before the value is cloned, hashed or persisted.
pub fn validate_task_envelope(value: &Value) -> Result<(), TaskError> {
let encoded = serde_json::to_vec(value)
.map_err(|_| TaskError::invalid("task envelope is not serializable"))?;
if encoded.len() > TASK_ENVELOPE_MAX_BYTES {
return Err(TaskError::invalid("task envelope exceeds its byte budget"));
}
let mut nodes = 0usize;
check_budget(value, 1, &mut nodes)?;
Ok(())
}
fn check_budget(value: &Value, depth: usize, nodes: &mut usize) -> Result<(), TaskError> {
if depth > TASK_ENVELOPE_MAX_DEPTH {
return Err(TaskError::invalid("task envelope exceeds its depth budget"));
}
*nodes += 1;
if *nodes > TASK_ENVELOPE_MAX_NODES {
return Err(TaskError::invalid("task envelope exceeds its node budget"));
}
match value {
Value::Object(object) => {
for (key, child) in object {
check_budget_name(key)?;
check_budget(child, depth + 1, nodes)?;
}
}
Value::Array(values) => {
for child in values {
check_budget(child, depth + 1, nodes)?;
}
}
_ => {}
}
Ok(())
}
fn check_budget_name(key: &str) -> Result<(), TaskError> {
if key.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task envelope key exceeds its budget"));
}
Ok(())
}
/// Field-name classification of secret-bearing values. A matching field keeps its position but never
/// its plaintext in the persisted envelope.
pub fn sensitive_task_key(name: &str) -> bool {
let lowered = name.to_ascii_lowercase();
[
"password",
"passwd",
"secret",
"token",
"api_key",
"apikey",
"authorization",
"credential",
"private_key",
"cookie",
"signature",
]
.iter()
.any(|marker| lowered.contains(marker))
}
/// Redacted projection of an already validated envelope.
pub fn redact_task_envelope(value: &Value) -> Value {
match value {
Value::Object(object) => Value::Object(
object
.iter()
.map(|(key, child)| {
let value = if sensitive_task_key(key) {
Value::String("[redacted]".to_owned())
} else {
redact_task_envelope(child)
};
(key.clone(), value)
})
.collect(),
),
Value::Array(values) => Value::Array(values.iter().map(redact_task_envelope).collect()),
other => other.clone(),
}
}
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
/// queue capacity commit together, and no acknowledgement exists before that commit.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAcceptRequest {
pub task_id: TaskId,
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
pub operation_id: String,
pub pins: TaskPins,
/// Digest of the caller key over the full owner/authorization/Tool scope.
pub scoped_key: String,
/// Digest of the canonical validated executable input.
pub input_fingerprint: String,
pub input: TaskEnvelope,
pub support: TaskSupportIds,
}
impl TaskAcceptRequest {
pub fn validate(&self) -> Result<(), TaskError> {
for (field, value) in [
("workspace", self.workspace_id.as_str()),
("owner", self.owner_id.as_str()),
("authorization scope", self.authorization_scope.as_str()),
("operation", self.operation_id.as_str()),
] {
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
return Err(TaskError::invalid("task scoped key digest is invalid"));
}
if self.input_fingerprint.len() != 64
|| !self
.input_fingerprint
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
{
return Err(TaskError::invalid("task input fingerprint is invalid"));
}
self.pins.validate()?;
self.support.validate()
}
}
/// Successful acceptance, returned only after the accepting transaction committed. A replay returns
/// the stored identity and support IDs of the original acceptance.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAcceptance {
pub task_id: TaskId,
pub state: TaskState,
/// Exact Published Version pinned by this acceptance; a replay returns the stored pin.
pub operation_version: u32,
pub support: TaskSupportIds,
pub created_at_unix_ms: i64,
pub retain_until_unix_ms: i64,
pub replayed: bool,
}
/// Owner-side access proof of a read or a terminal write. It is built by the application layer from
/// the verified principal, never from a caller-supplied marker, and every store query carries the
/// whole predicate so a foreign row is never fetched.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAccess {
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
}
impl TaskAccess {
pub fn validate(&self) -> Result<(), TaskError> {
for (field, value) in [
("workspace", self.workspace_id.as_str()),
("owner", self.owner_id.as_str()),
("authorization scope", self.authorization_scope.as_str()),
] {
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
Ok(())
}
}
/// Read model of one Task. Lifecycle fields, pins and support IDs stay readable together; the
/// protocol adapter only maps them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskRecord {
pub task_id: TaskId,
pub state: TaskState,
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
pub operation_id: String,
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
pub input_fingerprint: String,
pub pins: TaskPins,
pub support: TaskSupportIds,
pub input: TaskEnvelope,
pub progress: Option<TaskEnvelope>,
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
pub attempts: u32,
/// Monotonic lifecycle version used by the terminal CAS.
pub version: u32,
pub created_at_unix_ms: i64,
pub updated_at_unix_ms: i64,
}
/// One conditional terminal transition. Exactly one concurrent writer wins; the losers observe the
/// accepted state and never rewrite the result.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskFinishRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub expected_state: TaskState,
pub expected_version: u32,
pub target_state: TaskState,
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
}
impl TaskFinishRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if !self.target_state.is_terminal() {
return Err(TaskError::invalid("task finish target is not terminal"));
}
if self.expected_state.is_terminal() {
return Err(TaskError::invalid("task finish source is already terminal"));
}
if !self.expected_state.allows_transition_to(self.target_state) {
return Err(TaskError::invalid("task state transition is not allowed"));
}
if self.target_state == TaskState::Completed && self.result.is_none() {
return Err(TaskError::invalid("a completed task requires a result"));
}
Ok(())
}
}
/// Bounded retention outcome.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct TaskPurgeReport {
/// Queued Tasks that passed their execution deadline and were failed with their intent removed.
pub expired_queued: u64,
/// Terminal Tasks that passed their retention deadline.
pub expired_terminal: u64,
}
/// Typed Task failure. A denial and a missing Task share one outward shape, so a foreign caller can
/// never distinguish "not yours" from "does not exist".
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum TaskError {
#[error("task request is invalid: {message}")]
Invalid { message: String },
#[error("task store is unavailable: {message}")]
Unavailable { message: String },
#[error("task was not found")]
NotFound,
#[error("task request conflicts with the accepted request")]
FingerprintConflict,
#[error("task lifecycle transition was rejected")]
StateConflict,
#[error("background task capacity is exhausted")]
Overloaded { retry_after_ms: u64 },
}
impl TaskError {
pub fn invalid(message: impl Into<String>) -> Self {
Self::Invalid {
message: message.into(),
}
}
pub fn unavailable(message: impl Into<String>) -> Self {
Self::Unavailable {
message: message.into(),
}
}
/// Whether the caller may tell this failure apart from an absent Task.
pub fn is_not_found(&self) -> bool {
matches!(self, Self::NotFound)
}
}
/// Authoritative durable Task port.
#[async_trait]
pub trait TaskStore: Send + Sync {
/// Proves the durable Task state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), TaskError>;
/// Atomically accepts a Task: Task record, dispatch intent and one queue slot commit together.
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError>;
/// Reads one Task inside the caller's owner scope.
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError>;
/// Scoped lookup of an already accepted Task by its full-scope idempotency digest. Used to keep a
/// retry of an explicitly pinned version from becoming new work; `None` means "not accepted yet".
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError>;
/// Applies exactly one conditional terminal transition and releases its intent atomically.
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
/// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes
/// expired terminal records. Live and non-terminal work is never deleted.
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn request() -> TaskAcceptRequest {
TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_790_000_000_000,
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
approval_reference: Some("appr_1".to_owned()),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
),
}
}
#[test]
fn uuid_v7_task_ids_are_unique_and_round_trip() {
let first = TaskId::generate();
let second = TaskId::generate();
assert_ne!(first, second);
assert_eq!(TaskId::parse(&first.to_string()).unwrap(), first);
}
#[test]
fn terminal_states_have_no_successor() {
for terminal in [
TaskState::Completed,
TaskState::Failed,
TaskState::Cancelled,
] {
assert!(terminal.is_terminal());
for next in [
TaskState::Queued,
TaskState::Working,
TaskState::Completed,
TaskState::Failed,
] {
assert!(!terminal.allows_transition_to(next));
}
}
assert!(TaskState::Queued.allows_transition_to(TaskState::Working));
assert!(TaskState::Working.allows_transition_to(TaskState::OutcomeUnknown));
assert!(!TaskState::Queued.allows_transition_to(TaskState::Completed));
assert!(!TaskState::InputRequired.allows_transition_to(TaskState::Completed));
}
#[test]
fn envelope_bounds_reject_bytes_depth_and_nodes() {
assert!(TaskEnvelope::try_redacted(&json!({"a": 1})).is_ok());
assert!(
TaskEnvelope::try_redacted(&json!({"a": "x".repeat(TASK_ENVELOPE_MAX_BYTES)})).is_err()
);
let mut deep = json!(1);
for _ in 0..=TASK_ENVELOPE_MAX_DEPTH {
deep = json!({"n": deep});
}
assert!(TaskEnvelope::try_redacted(&deep).is_err());
let wide = Value::Array((0..=TASK_ENVELOPE_MAX_NODES).map(|_| json!(1)).collect());
assert!(TaskEnvelope::try_redacted(&wide).is_err());
}
#[test]
fn secret_bearing_fields_are_redacted_and_keep_their_shape() {
let envelope = TaskEnvelope::try_redacted(&json!({
"query": "leads",
"auth": {"api_key": "sk_live_1", "nested": [{"password": "p"}]}
}))
.unwrap();
let value = envelope.value();
assert_eq!(value["query"], json!("leads"));
assert_eq!(value["auth"]["api_key"], json!("[redacted]"));
assert_eq!(value["auth"]["nested"][0]["password"], json!("[redacted]"));
}
#[test]
fn unsampled_support_ids_are_valid_and_consistent() {
let support = TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
);
assert!(!support.trace_sampled);
assert!(support.traceparent.ends_with("-00"));
support.validate().unwrap();
let mut inconsistent = support.clone();
inconsistent.traceparent = inconsistent.traceparent.replace("-00", "-01");
assert!(inconsistent.validate().is_err());
}
#[test]
fn accept_request_validation_holds_the_identity_and_pin_bounds() {
assert!(request().validate().is_ok());
let mut oversized = request();
oversized.workspace_id = "w".repeat(TASK_IDENTITY_MAX_BYTES + 1);
assert!(oversized.validate().is_err());
let mut bad_fingerprint = request();
bad_fingerprint.input_fingerprint = "XYZ".to_owned();
assert!(bad_fingerprint.validate().is_err());
let mut bad_retention = request();
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
assert!(bad_retention.validate().is_err());
}
#[test]
fn finish_requires_a_terminal_reachable_transition() {
let access = TaskAccess {
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
};
let base = TaskFinishRequest {
access,
task_id: TaskId::generate(),
expected_state: TaskState::Working,
expected_version: 1,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
};
assert!(base.validate().is_ok());
let mut non_terminal = base.clone();
non_terminal.target_state = TaskState::Working;
assert!(non_terminal.validate().is_err());
let mut terminal_source = base.clone();
terminal_source.expected_state = TaskState::Completed;
assert!(terminal_source.validate().is_err());
let mut completed_without_result = base;
completed_without_result.result = None;
assert!(completed_without_result.validate().is_err());
}
}
@@ -231,6 +231,7 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
+1
View File
@@ -564,6 +564,7 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
+28
View File
@@ -217,6 +217,34 @@ string_enum! {
Concurrency => "concurrency",
RateLimit => "rate_limit",
McpStream => "mcp_stream",
Admission => "admission",
}
}
string_enum! {
pub enum RejectionReason {
Limit => "limit",
Capacity => "capacity",
QueueFull => "queue_full",
Deadline => "deadline",
Unsupported => "unsupported",
DependencyUnavailable => "dependency_unavailable",
}
}
string_enum! {
pub enum ScopeKind {
Process => "process",
Global => "global",
Agent => "agent",
Tool => "tool",
}
}
string_enum! {
pub enum AdmissionModeLabel {
Synchronous => "synchronous",
Background => "background",
}
}
+6 -5
View File
@@ -16,14 +16,15 @@ mod schema;
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
pub use labels::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
UpstreamOutcome,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
pub use record::{
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service,
bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome,
bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting,
record_admission_queue_wait, record_admission_rejection, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
+41 -5
View File
@@ -5,10 +5,10 @@ use metrics::Gauge;
use crate::schema::{MetricDefinition, metric_schema};
use crate::{
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter,
HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason,
ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
static BINDING: OnceLock<MetricService> = OnceLock::new();
@@ -352,11 +352,47 @@ pub fn record_limit_rejection(stage: LimitStage) {
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => stage.as_str()
"stage" => stage.as_str(),
"reason" => RejectionReason::Limit.as_str(),
"scope_kind" => ScopeKind::Process.as_str()
)
.increment(1);
}
/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool,
/// request id, fingerprint or arbitrary error text ever becomes a label.
pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) {
if bound_family("crank_runtime_limit_rejections_total").is_none() {
return;
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => LimitStage::Admission.as_str(),
"reason" => reason.as_str(),
"scope_kind" => scope_kind.as_str()
)
.increment(1);
}
pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) {
if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() {
return;
}
metrics::gauge!(
"crank_runtime_admission_queue_wait_seconds",
"mode" => mode.as_str()
)
.set(waited_ms as f64 / 1000.0);
}
pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option<u64>) {
if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() {
return;
}
let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0;
metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds);
}
pub fn record_cache_outcome(outcome: CacheOutcome) {
if bound_family("crank_runtime_cache_total").is_none() {
return;
+38 -8
View File
@@ -3,16 +3,17 @@
use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP};
use crate::labels::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome,
UpstreamOperationKind, UpstreamOutcome,
};
pub const METRIC_SCHEMA_VERSION: u32 = 2;
pub const MAX_METRIC_FAMILIES: usize = 128;
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611;
pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348;
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
@@ -179,6 +180,9 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
const REJECTION_REASONS: &[&str] = RejectionReason::VALUES;
const SCOPE_KINDS: &[&str] = ScopeKind::VALUES;
const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES;
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
@@ -403,13 +407,39 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[
"crank_runtime_limit_rejections_total",
MetricKind::Counter,
MetricUnit::Count,
&["stage"],
&[domain("stage", LIMIT_STAGES)],
&["stage", "reason", "scope_kind"],
&[
domain("stage", LIMIT_STAGES),
domain("reason", REJECTION_REASONS),
domain("scope_kind", SCOPE_KINDS),
],
&[],
BOTH,
3,
96,
"Runtime executions rejected by a bounded limit.",
),
metric(
"crank_runtime_admission_queue_wait_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&["mode"],
&[domain("mode", ADMISSION_MODES)],
&[],
BOTH,
2,
"Wait of the most recently admitted execution before it received a scoped permit.",
),
metric(
"crank_runtime_admission_oldest_waiting_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&[],
&[],
&[],
BOTH,
1,
"Age of the oldest waiting admission request.",
),
metric(
"crank_runtime_cache_total",
MetricKind::Counter,
+13 -6
View File
@@ -1,12 +1,13 @@
use std::{collections::BTreeSet, time::Duration};
use crank_metrics::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics,
bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType,
ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema,
record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection,
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
@@ -71,6 +72,9 @@ fn schema_names_are_unique_and_labels_are_closed() {
"state",
"signal_type",
"exporter",
"reason",
"scope_kind",
"mode",
];
for metric in schema {
assert!(
@@ -265,6 +269,9 @@ fn every_recording_api_matches_the_declared_schema() {
Duration::from_millis(1),
);
record_limit_rejection(LimitStage::Concurrency);
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5);
record_admission_oldest_waiting(Some(5));
record_cache_outcome(CacheOutcome::Hit);
record_idempotency_outcome(IdempotencyOutcome::Replay);
record_confirmation_outcome(ConfirmationOutcome::Required);
+8 -8
View File
@@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
#[test]
fn frozen_budget_edges_are_exact() {
assert!(validate_budget(6_284, 19_514).is_ok());
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 16_000).is_err());
assert!(validate_budget(6_000, 19_516).is_err());
assert!(validate_budget(6_380, 19_610).is_ok());
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_000).is_err());
assert!(validate_budget(6_000, 19_612).is_err());
assert_eq!(
crank_metrics::max_exemplar_slots(MetricService::AdminApi),
12_348
@@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() {
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_285,
rendered_series: 19_515
logical_series: 6_381,
rendered_series: 19_611
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_083,
rendered_series: 3_558
logical_series: 1_179,
rendered_series: 3_654
})
);
}
+11 -11
View File
@@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() {
"root обязан объявлять ровно один ключ budgets"
);
assert!(rendered.ends_with(
",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\
\"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n"
",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\
\"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n"
));
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_285,
rendered_series: 19_515
logical_series: 6_381,
rendered_series: 19_611
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_083,
rendered_series: 3_558
logical_series: 1_179,
rendered_series: 3_654
})
);
}
@@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() {
/// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны.
#[test]
fn frozen_caps_and_edges_are_exact() {
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515);
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611);
assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348);
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 19_515).is_err());
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_611).is_err());
assert!(validate_exemplar_budget(12_347).is_ok());
assert!(validate_exemplar_budget(12_348).is_ok());
assert!(validate_exemplar_budget(12_349).is_err());
+1 -1
View File
@@ -13,7 +13,7 @@ mod schema;
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
pub use crank_metrics::{
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit,
bind_metric_service, bound_metric_service, metric_schema,
bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting,
};
pub use error_reporting::{
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
+1
View File
@@ -7,6 +7,7 @@ publish.workspace = true
version.workspace = true
[dependencies]
async-trait = "0.1"
crank-artifacts = { path = "../crank-artifacts" }
crank-core = { path = "../crank-core" }
crank-mapping = { path = "../crank-mapping" }
+3 -2
View File
@@ -116,6 +116,7 @@ pub use model::{
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
};
pub use postgres::{
AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError,
PostgresRegistry,
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, PostgresTaskStore,
admission_health, admission_snapshot, tasks_health,
};
+4
View File
@@ -6,6 +6,8 @@ mod artifact_cleanup_indexes_v13;
mod artifact_metadata_v12;
mod authority;
mod baseline_v1;
mod durable_tasks_v25;
mod execution_admission_v24;
mod execution_outcome_v5;
mod import_plan_v14;
mod master_key_identity_v7;
@@ -35,6 +37,8 @@ mod schema_guard_v19;
mod schema_guard_v20;
mod schema_guard_v22;
mod schema_guard_v23;
mod schema_guard_v24;
mod schema_guard_v25;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -4,6 +4,8 @@ use super::agent_resource_bindings_v18;
use super::approval_side_effects_v10;
use super::artifact_cleanup_indexes_v13;
use super::artifact_metadata_v12;
use super::durable_tasks_v25;
use super::execution_admission_v24;
use super::execution_outcome_v5;
use super::import_plan_v14;
use super::master_key_identity_v7;
@@ -39,9 +41,9 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 23;
const CURRENT_VERSION: i64 = 25;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -330,6 +332,18 @@ impl MigrationAuthority {
prompt_dependencies_v23::SOURCE_SHA256,
22,
),
expand_descriptor(
24,
execution_admission_v24::NAME,
execution_admission_v24::SOURCE_SHA256,
23,
),
expand_descriptor(
25,
durable_tasks_v25::NAME,
durable_tasks_v25::SOURCE_SHA256,
24,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -489,6 +503,12 @@ impl MigrationAuthority {
if from < 23 {
prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?;
}
if from < 24 {
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
}
if from < 25 {
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -140,6 +140,8 @@ pub(super) fn validate_descriptors(
!= agent_resource_bindings_v18::SOURCE_SHA256
|| sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes())
!= resource_security_revocations_v19::SOURCE_SHA256
|| sha256_hex(execution_admission_v24::SOURCE.as_bytes())
!= execution_admission_v24::SOURCE_SHA256
{
return Err(MigrationError::new(
"invalid_contract",
@@ -148,6 +148,7 @@ pub(super) async fn inspect(
owned_relations::PROMPTS,
owned_relations::PROMPT_PUBLICATIONS,
owned_relations::PROMPT_DEPENDENCIES,
owned_relations::EXECUTION_ADMISSION,
] {
validate_required_relations(connection, relations, CURRENT_VERSION).await?;
}
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v25";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v25.sql");
pub(super) const SOURCE_SHA256: &str =
"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v25",
Some(25),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.canonical_ledger",
Some(25),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,90 @@
-- V25: durable background Tasks (Story 4.2, FR-22, FR-23).
--
-- Одна строка `tasks` — принятая durable работа. Идентификатор Task (UUIDv7), owner scope,
-- authorization scope, запрошенный Tool и точная published-версия, canonical fingerprint входа,
-- support IDs (Request/Trace/parent TraceContext независимо от sampling) и bounded envelopes
-- закрепляются в той же PostgreSQL transaction, что и dispatch intent
-- (`execution_admission_entries.task_id`) и его queue slot. До commit наружу не выдаётся ни ID, ни ack.
--
-- `scoped_key` — digest caller-ключа по полному (workspace, owner kind/id, authorization scope,
-- operation) scope; UNIQUE по этому набору и есть защита идемпотентности. Повтор того же ключа с тем
-- же canonical fingerprint возвращает сохранённую Task, другой fingerprint — typed conflict.
--
-- `retain_until` — срок хранения записи, а не execution deadline: пока запись в retention, replay
-- находит тот же Task. Удаление terminal-записей возможно только после retain_until; queued/working/
-- input_required/outcome_unknown не удаляются никогда ради лимита.
--
-- Строка не хранит credentials, bearer/prepared headers и confirmation tokens: секретные поля входа
-- редактируются до записи, а approval хранится безопасной ссылкой.
create table tasks (
task_id uuid not null,
workspace_id text not null,
owner_kind text not null,
owner_id text not null,
authorization_scope text not null,
operation_id text not null,
operation_version integer not null,
scoped_key text not null,
input_fingerprint text not null,
request_id text not null,
trace_id text not null,
traceparent text not null,
trace_sampled boolean not null,
policy_mode text not null,
approval_reference text,
state text not null,
version integer not null default 1,
attempts integer not null default 0,
input_envelope jsonb not null,
progress_envelope jsonb,
result_envelope jsonb,
error_envelope jsonb,
execution_deadline timestamptz not null,
retain_until timestamptz not null,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint tasks_pkey primary key (task_id),
constraint tasks_workspace_check check (octet_length(workspace_id) between 1 and 160),
constraint tasks_owner_kind_check check (owner_kind in ('agent', 'admin')),
constraint tasks_owner_check check (octet_length(owner_id) between 1 and 160),
constraint tasks_authorization_scope_check check (
octet_length(authorization_scope) between 1 and 160
),
constraint tasks_operation_check check (octet_length(operation_id) between 1 and 160),
constraint tasks_operation_version_check check (operation_version >= 1),
constraint tasks_scoped_key_check check (octet_length(scoped_key) between 16 and 128),
constraint tasks_fingerprint_check check (octet_length(input_fingerprint) = 64),
constraint tasks_request_check check (octet_length(request_id) between 1 and 128),
constraint tasks_trace_check check (octet_length(trace_id) = 32),
constraint tasks_traceparent_check check (octet_length(traceparent) = 55),
constraint tasks_policy_mode_check check (
policy_mode in ('synchronous', 'background', 'adaptive')
),
constraint tasks_state_check check (
state in ('queued', 'working', 'input_required', 'outcome_unknown',
'completed', 'failed', 'cancelled')
),
constraint tasks_lifecycle_check check (version >= 1 and attempts >= 0),
constraint tasks_retention_check check (retain_until >= execution_deadline),
constraint tasks_approval_check check (
approval_reference is null or octet_length(approval_reference) between 1 and 160
)
);
create unique index tasks_scoped_key_uidx
on tasks (workspace_id, owner_kind, owner_id, authorization_scope, operation_id, scoped_key);
create index tasks_retention_idx on tasks (state, retain_until, task_id);
create index tasks_deadline_idx on tasks (state, execution_deadline, task_id);
-- Dispatch intent: прежняя reserved-строка admission становится точкой связи Task и capacity.
-- FK без cascade: удаление принятой Task не маскируется housekeeping-удалением intent.
alter table execution_admission_entries
add column task_id uuid,
add constraint execution_admission_entries_task_fkey
foreign key (task_id) references tasks (task_id);
create unique index execution_admission_entries_task_uidx
on execution_admission_entries (task_id) where task_id is not null;
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "execution-admission-v24";
pub(super) const SOURCE: &str = include_str!("execution_admission_v24.sql");
pub(super) const SOURCE_SHA256: &str =
"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.execution_admission_v24",
Some(24),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.canonical_ledger",
Some(24),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,92 @@
-- V24: authoritative execution admission state (Story 4.1, FR-21, FR-24).
--
-- Один deployment-wide admission: PostgreSQL хранит cursor/fairness state, ожидающие тикеты,
-- выданные permits и durable enqueue decisions, поэтому несколько процессов делят общий бюджет.
--
-- `execution_admission_entries` — bounded набор: waiting (ожидает capacity), executing (scoped
-- permit выдан, держится до явного release) и reserved (durable enqueue decision с зарезервированным
-- queue slot; не исполняется и не рекламируется как Task до Story 4.2).
--
-- Строка не хранит ни credentials, ни connection/session id, ни prepared headers, ни тело запроса:
-- quota key — (workspace, agent) и (workspace, operation) без version. `operation_version` пишется
-- только как pin исполняемых semantics и не участвует в квотах.
--
-- `expires_at` — абсолютный deadline. Queue decisions без dispatch можно expire по deadline; активный
-- permit НЕ освобождается по TTL (иначе paused/crashed owner обойдёт global limit): его снимает только
-- явный release/cancel владельца.
create table execution_admission_controllers (
workspace_id text not null,
cursor_agent text,
cursor_agent_credit integer not null default 0,
cursor_tool text,
cursor_tool_credit integer not null default 0,
updated_at timestamptz not null default now(),
constraint execution_admission_controllers_pkey primary key (workspace_id),
constraint execution_admission_controllers_workspace_check check (
octet_length(workspace_id) between 1 and 160
),
constraint execution_admission_controllers_agent_credit_check check (
cursor_agent_credit between 0 and 64
),
constraint execution_admission_controllers_tool_credit_check check (
cursor_tool_credit between 0 and 64
)
);
create table execution_admission_entries (
ticket_id bigint generated always as identity,
workspace_id text not null,
agent_key text not null,
operation_id text not null,
operation_version integer not null,
mode text not null,
policy_mode text not null,
state text not null,
owner_token text not null,
fingerprint text not null,
created_at timestamptz not null default now(),
expires_at timestamptz not null,
granted_at timestamptz,
constraint execution_admission_entries_pkey primary key (ticket_id),
constraint execution_admission_entries_workspace_check check (
octet_length(workspace_id) between 1 and 160
),
constraint execution_admission_entries_agent_check check (
octet_length(agent_key) between 1 and 160
),
constraint execution_admission_entries_operation_check check (
octet_length(operation_id) between 1 and 160
),
constraint execution_admission_entries_version_check check (operation_version >= 1),
constraint execution_admission_entries_mode_check check (
mode in ('synchronous', 'background')
),
constraint execution_admission_entries_policy_mode_check check (
policy_mode in ('synchronous', 'background', 'adaptive')
),
constraint execution_admission_entries_state_check check (
state in ('waiting', 'executing', 'reserved')
),
constraint execution_admission_entries_owner_check check (
octet_length(owner_token) between 1 and 64
),
constraint execution_admission_entries_fingerprint_check check (
octet_length(fingerprint) between 1 and 128
),
constraint execution_admission_entries_reserved_mode_check check (
state <> 'reserved' or mode = 'background'
)
);
create unique index execution_admission_entries_owner_uidx
on execution_admission_entries (owner_token);
create index execution_admission_entries_state_idx
on execution_admission_entries (state, workspace_id, ticket_id);
create index execution_admission_entries_agent_idx
on execution_admission_entries (state, workspace_id, agent_key);
create index execution_admission_entries_operation_idx
on execution_admission_entries (state, workspace_id, operation_id);
@@ -55,6 +55,11 @@ pub(super) const PROMPT_PUBLICATIONS: &[&str] = &["prompt_publications", "agent_
pub(super) const PROMPT_DEPENDENCIES: &[&str] =
&["prompt_dependencies", "prompt_message_resource_refs"];
pub(super) const EXECUTION_ADMISSION: &[&str] = &[
"execution_admission_controllers",
"execution_admission_entries",
];
pub(super) const ARTIFACT_METADATA: &[&str] = &["artifact_blobs", "artifact_sources"];
pub(super) const IMPORT_PLAN: &[&str] = &["import_plans", "import_plan_applications"];
@@ -468,6 +468,21 @@ pub(super) async fn validate_schema_fingerprint(
} else {
super::schema_guard_v23::validate_v23_prompt_dependencies(connection).await?;
}
if current_version < 24 {
if !super::schema_guard_v24::validate_v24_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v24::validate_v24_execution_admission(connection, current_version)
.await?;
}
if current_version < 25 {
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
}
Ok(())
}
@@ -51,6 +51,8 @@ pub(crate) const OWNED_RELATIONS: &[&str] = &[
"prompt_messages",
"prompt_dependencies",
"prompt_message_resource_refs",
"execution_admission_controllers",
"execution_admission_entries",
];
pub(crate) const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[
@@ -0,0 +1,277 @@
//! V24 schema guard: authoritative execution admission state (Story 4.1, FR-21/FR-24).
//!
//! Схема V24 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
//! ожидаемой форме: controller-row с weighted round-robin cursor и bounded entries с закрытым
//! словарём mode/policy_mode/state, уникальным owner token и индексами квот.
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const CONTROLLERS: &str = "execution_admission_controllers";
const ENTRIES: &str = "execution_admission_entries";
const OWNED_RELATIONS: &[&str] = &[CONTROLLERS, ENTRIES];
const CONTROLLER_COLUMNS: &[(&str, &str, bool)] = &[
("workspace_id", "text", false),
("cursor_agent", "text", true),
("cursor_agent_credit", "integer", false),
("cursor_tool", "text", true),
("cursor_tool_credit", "integer", false),
("updated_at", "timestamp with time zone", false),
];
const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[
("ticket_id", "bigint", false),
("workspace_id", "text", false),
("agent_key", "text", false),
("operation_id", "text", false),
("operation_version", "integer", false),
("mode", "text", false),
("policy_mode", "text", false),
("state", "text", false),
("owner_token", "text", false),
("fingerprint", "text", false),
("created_at", "timestamp with time zone", false),
("expires_at", "timestamp with time zone", false),
("granted_at", "timestamp with time zone", true),
];
/// V25 adds the nullable Task dispatch-intent link to the same reserved row. At version 24 the
/// column must not exist yet, so the guard validates the shape of the version it describes.
fn entry_columns(current_version: i64) -> Vec<(&'static str, &'static str, bool)> {
let mut columns = ENTRY_COLUMNS.to_vec();
if current_version >= 25 {
columns.push(("task_id", "uuid", true));
}
columns
}
fn constraint_count(table: &str, current_version: i64) -> Option<usize> {
match table {
CONTROLLERS => Some(4),
ENTRIES if current_version >= 25 => Some(12),
ENTRIES => Some(11),
_ => None,
}
}
const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"];
fn v24_error(stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(24),
"restore_known_good_backup",
)
}
/// Схема V24 отсутствует целиком (для current_version < 24).
pub(super) async fn validate_v24_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
for relation in OWNED_RELATIONS {
if relation_exists(connection, relation).await? {
return Ok(false);
}
}
for index in UNIQUE_INDEXES {
if index_exists(connection, index).await? {
return Ok(false);
}
}
Ok(true)
}
/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24).
pub(super) async fn validate_v24_execution_admission(
connection: &mut PgConnection,
current_version: i64,
) -> Result<(), MigrationError> {
for relation in OWNED_RELATIONS {
if !relation_exists(connection, relation).await? {
return Err(v24_error("v24.relations"));
}
}
let tables = [
(CONTROLLERS, CONTROLLER_COLUMNS.to_vec()),
(ENTRIES, entry_columns(current_version)),
];
for (table, columns) in tables {
validate_columns(connection, table, &columns).await?;
}
for table in [CONTROLLERS, ENTRIES] {
let expected = constraint_count(table, current_version).unwrap_or_default();
let count: i64 = query(
"select count(*)::bigint as count
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1",
)
.bind(table)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(expected).unwrap_or_default() {
return Err(v24_error("v24.constraints"));
}
}
for (table, constraint) in [
(CONTROLLERS, "execution_admission_controllers_pkey"),
(ENTRIES, "execution_admission_entries_pkey"),
(ENTRIES, "execution_admission_entries_state_check"),
] {
if !constraint_exists(connection, table, constraint).await? {
return Err(v24_error("v24.constraint_missing"));
}
}
if !normalized_expression(
connection,
ENTRIES,
"execution_admission_entries_state_check",
)
.await?
.is_some_and(|expression| {
super::schema_guard::enum_constraint_matches(
&expression,
"state",
&["waiting", "executing", "reserved"],
)
}) {
return Err(v24_error("v24.state_vocabulary"));
}
for index in UNIQUE_INDEXES {
let unique = index_unique(connection, index).await?;
if unique != Some(true) {
return Err(v24_error("v24.indexes"));
}
}
Ok(())
}
async fn validate_columns(
connection: &mut PgConnection,
table: &str,
columns: &[(&str, &str, bool)],
) -> Result<(), MigrationError> {
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(table)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(columns.len()).unwrap_or_default() {
return Err(v24_error("v24.columns"));
}
for (column, data_type, nullable) in columns {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(table)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if *nullable { "YES" } else { "NO" })
});
if !valid {
return Err(v24_error("v24.columns"));
}
}
Ok(())
}
async fn constraint_exists(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
) as present",
)
.bind(table)
.bind(constraint)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn normalized_expression(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<Option<String>, MigrationError> {
let row = query(
"select pg_get_expr(c.conbin, c.conrelid) as expression
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
and c.contype = 'c'",
)
.bind(table)
.bind(constraint)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
}
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_class idx
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1
) as present",
)
.bind(index)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn index_unique(
connection: &mut PgConnection,
index: &str,
) -> Result<Option<bool>, MigrationError> {
let row = query(
"select i.indisunique, i.indisvalid
from pg_catalog.pg_index i
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1",
)
.bind(index)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| {
let unique = row.try_get::<bool, _>("indisunique").ok()?;
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
Some(unique && valid)
}))
}
@@ -0,0 +1,247 @@
//! V25 schema guard: durable background Tasks (Story 4.2, FR-22/FR-23).
//!
//! Схема V25 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
//! ожидаемой форме: таблица `tasks` с закрытым словарём state/policy_mode и scoped uniqueness, плюс
//! nullable `task_id` на существующей reserved-строке admission с FK и UNIQUE-индексом.
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::relation_exists;
const TASKS: &str = "tasks";
const ENTRIES: &str = "execution_admission_entries";
const SCOPED_INDEX: &str = "tasks_scoped_key_uidx";
const TASK_INDEX: &str = "execution_admission_entries_task_uidx";
const TASK_FK: &str = "execution_admission_entries_task_fkey";
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
("task_id", "uuid", false),
("workspace_id", "text", false),
("owner_kind", "text", false),
("owner_id", "text", false),
("authorization_scope", "text", false),
("operation_id", "text", false),
("operation_version", "integer", false),
("scoped_key", "text", false),
("input_fingerprint", "text", false),
("request_id", "text", false),
("trace_id", "text", false),
("traceparent", "text", false),
("trace_sampled", "boolean", false),
("policy_mode", "text", false),
("approval_reference", "text", true),
("state", "text", false),
("version", "integer", false),
("attempts", "integer", false),
("input_envelope", "jsonb", false),
("progress_envelope", "jsonb", true),
("result_envelope", "jsonb", true),
("error_envelope", "jsonb", true),
("execution_deadline", "timestamp with time zone", false),
("retain_until", "timestamp with time zone", false),
("created_at", "timestamp with time zone", false),
("updated_at", "timestamp with time zone", false),
];
fn v25_error(stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(25),
"restore_known_good_backup",
)
}
/// Схема V25 отсутствует целиком (для current_version < 25).
pub(super) async fn validate_v25_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
return Ok(false);
}
if super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
return Ok(false);
}
for index in [SCOPED_INDEX, TASK_INDEX] {
if index_exists(connection, index).await? {
return Ok(false);
}
}
Ok(true)
}
/// Схема V25 присутствует в ожидаемой форме (для current_version >= 25).
pub(super) async fn validate_v25_durable_tasks(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
if !relation_exists(connection, TASKS).await? {
return Err(v25_error("v25.relations"));
}
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
return Err(v25_error("v25.columns"));
}
for (column, data_type, nullable) in TASK_COLUMNS {
let row = query(
"select data_type, is_nullable from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2",
)
.bind(TASKS)
.bind(column)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
let valid = row.is_some_and(|row| {
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
== Some(if *nullable { "YES" } else { "NO" })
});
if !valid {
return Err(v25_error("v25.columns"));
}
}
if !super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
return Err(v25_error("v25.intent_column"));
}
for (constraint, vocabulary) in [
(
"tasks_state_check",
[
"queued",
"working",
"input_required",
"outcome_unknown",
"completed",
"failed",
"cancelled",
]
.as_slice(),
),
(
"tasks_policy_mode_check",
["synchronous", "background", "adaptive"].as_slice(),
),
] {
let expression = normalized_expression(connection, TASKS, constraint).await?;
if !expression.is_some_and(|expression| {
super::schema_guard::enum_constraint_matches(
&expression,
column_of(constraint),
vocabulary,
)
}) {
return Err(v25_error("v25.vocabulary"));
}
}
for (index, table) in [(SCOPED_INDEX, TASKS), (TASK_INDEX, ENTRIES)] {
if index_unique(connection, table, index).await? != Some(true) {
return Err(v25_error("v25.indexes"));
}
}
if !constraint_exists(connection, ENTRIES, TASK_FK).await? {
return Err(v25_error("v25.intent_fk"));
}
Ok(())
}
fn column_of(constraint: &str) -> &'static str {
if constraint.contains("policy_mode") {
"policy_mode"
} else {
"state"
}
}
async fn constraint_exists(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
) as present",
)
.bind(table)
.bind(constraint)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn normalized_expression(
connection: &mut PgConnection,
table: &str,
constraint: &str,
) -> Result<Option<String>, MigrationError> {
let row = query(
"select pg_get_expr(c.conbin, c.conrelid) as expression
from pg_catalog.pg_constraint c
join pg_catalog.pg_class t on t.oid = c.conrelid
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
and c.contype = 'c'",
)
.bind(table)
.bind(constraint)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
}
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
query(
"select exists (
select 1 from pg_catalog.pg_class idx
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1
) as present",
)
.bind(index)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get::<bool, _>("present")
.map_err(|_| MigrationError::storage("preflight.schema"))
}
async fn index_unique(
connection: &mut PgConnection,
table: &str,
index: &str,
) -> Result<Option<bool>, MigrationError> {
let row = query(
"select i.indisunique, i.indisvalid
from pg_catalog.pg_index i
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
join pg_catalog.pg_class t on t.oid = i.indrelid
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
where n.nspname = current_schema() and idx.relname = $1 and t.relname = $2",
)
.bind(index)
.bind(table)
.fetch_optional(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?;
Ok(row.and_then(|row| {
let unique = row.try_get::<bool, _>("indisunique").ok()?;
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
Some(unique && valid)
}))
}
@@ -0,0 +1,680 @@
//! PostgreSQL implementation of the mandatory execution-admission store (Story 4.1, FR-21/FR-24).
//!
//! Source of truth for capacity and durable enqueue decisions of every process: a short transaction
//! locks one deployment controller row and one workspace controller row in a fixed order, then reads
//! bounded counters, inserts/transitions at most one entry and persists the weighted round-robin
//! cursor. No network call happens under the transaction and no connection is held while a waiter is
//! idle between attempts.
use std::collections::BTreeMap;
use std::time::Duration;
use async_trait::async_trait;
use crank_core::{
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, TaskError,
select_next_grant,
};
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
const DEPLOYMENT_CONTROLLER: &str = "*";
const OVERLOAD_RETRY_AFTER_MS: u64 = crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS;
const MAX_WAIT_BACKOFF_MS: u64 = 250;
const INITIAL_WAIT_BACKOFF_MS: u64 = 5;
const EXPIRED_CLEANUP_BATCH: i64 = 128;
const QUEUE_READ_LIMIT: i64 = 1024;
/// Deployment-wide bounds of one admission store. They come from validated deployment
/// configuration, never from a request.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionLimits {
pub global_limit: usize,
pub agent_limit: usize,
pub tool_limit: usize,
pub queue_limit: usize,
}
impl Default for AdmissionLimits {
fn default() -> Self {
Self {
global_limit: 64,
agent_limit: 16,
tool_limit: 8,
queue_limit: 32,
}
}
}
impl AdmissionLimits {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
for (field, value) in [
("global_limit", self.global_limit),
("agent_limit", self.agent_limit),
("tool_limit", self.tool_limit),
] {
if !(1..=65_535).contains(&value) {
return Err(AdmissionStoreError::invalid(format!(
"admission {field} is outside its allowed bounds"
)));
}
}
if !(1..=1_024).contains(&self.queue_limit) {
return Err(AdmissionStoreError::invalid(
"admission queue_limit is outside its allowed bounds",
));
}
Ok(())
}
}
#[derive(Clone)]
pub struct PostgresExecutionAdmission {
pool: PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
}
impl PostgresExecutionAdmission {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> Result<Self, AdmissionStoreError> {
limits.validate()?;
scheduler.validate()?;
Ok(Self {
pool,
limits,
scheduler,
})
}
fn unavailable(error: impl std::fmt::Display) -> AdmissionStoreError {
AdmissionStoreError::unavailable(format!(
"execution admission store is unavailable: {error}"
))
}
}
struct AttemptContext<'a> {
request: &'a AdmissionRequest,
owner_token: &'a str,
}
enum Attempt {
Granted,
Enqueued(u64),
Waiting,
Overloaded,
Expired,
}
#[async_trait]
impl ExecutionAdmission for PostgresExecutionAdmission {
async fn health(&self) -> Result<(), AdmissionStoreError> {
admission_health(&self.pool).await
}
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<AdmissionOutcome, AdmissionStoreError> {
request.validate()?;
if request.approval == ApprovalEvidence::NotGranted {
return Ok(AdmissionOutcome::ApprovalRequired);
}
if request.mode == AdmissionMode::Background
&& request.no_tasks_fallback == NoTasksFallback::Unsupported
{
return Ok(AdmissionOutcome::Unsupported);
}
let owner_token = format!("adm_{}", Uuid::now_v7().simple());
let mut ticket: Option<u64> = None;
let mut backoff = INITIAL_WAIT_BACKOFF_MS;
loop {
if crank_core::current_unix_ms() >= request.expires_at_unix_ms {
if let Some(ticket_id) = ticket {
self.cancel(ticket_id, &owner_token).await?;
}
return Ok(AdmissionOutcome::DeadlineElapsed);
}
let attempt = self
.attempt(
AttemptContext {
request: &request,
owner_token: &owner_token,
},
&mut ticket,
)
.await?;
match attempt {
Attempt::Granted => {
return Ok(AdmissionOutcome::Permitted(AdmissionPermit {
ticket_id: ticket.unwrap_or_default(),
owner_token,
scope: request.scope.clone(),
mode: request.mode,
policy_mode: request.policy_mode,
}));
}
Attempt::Enqueued(ticket_id) => {
return Ok(AdmissionOutcome::Enqueued(AdmissionDecision {
ticket_id,
scope: request.scope.clone(),
expires_at_unix_ms: request.expires_at_unix_ms,
policy_mode: request.policy_mode,
}));
}
Attempt::Expired => {
return Ok(AdmissionOutcome::DeadlineElapsed);
}
Attempt::Overloaded => {
return Ok(AdmissionOutcome::Overloaded {
retry_after_ms: OVERLOAD_RETRY_AFTER_MS,
});
}
Attempt::Waiting => {
let remaining = request
.expires_at_unix_ms
.saturating_sub(crank_core::current_unix_ms())
.max(1);
let sleep_ms = i64::try_from(backoff).unwrap_or(remaining).min(remaining);
tokio::time::sleep(Duration::from_millis(
u64::try_from(sleep_ms).unwrap_or(backoff),
))
.await;
backoff = (backoff * 2).min(MAX_WAIT_BACKOFF_MS);
}
}
}
}
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError> {
let result = sqlx::query(
"delete from execution_admission_entries
where ticket_id = $1 and owner_token = $2 and state = 'executing'
and task_id is null",
)
.bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX))
.bind(&permit.owner_token)
.execute(&self.pool)
.await
.map_err(Self::unavailable)?;
Ok(result.rows_affected() > 0)
}
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError> {
let result = sqlx::query(
"delete from execution_admission_entries
where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')
and task_id is null",
)
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
.bind(owner_token)
.execute(&self.pool)
.await
.map_err(Self::unavailable)?;
Ok(result.rows_affected() > 0)
}
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
admission_snapshot(&self.pool).await
}
}
/// Readiness probe of the mandatory admission state: both tables must be reachable and readable.
pub async fn admission_health(pool: &PgPool) -> Result<(), AdmissionStoreError> {
sqlx::query(
"select count(*)::bigint from execution_admission_controllers where workspace_id = $1",
)
.bind(DEPLOYMENT_CONTROLLER)
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
sqlx::query("select count(*)::bigint from execution_admission_entries where false")
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
Ok(())
}
/// Authoritative queue readings of the deployment-wide admission state.
pub async fn admission_snapshot(
pool: &PgPool,
) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
let row = sqlx::query(
"select
count(*) filter (where state in ('waiting', 'reserved')) as queued,
count(*) filter (where state = 'executing') as executing,
coalesce(
(extract(epoch from (now() - min(created_at) filter (where state in ('waiting', 'reserved')))) * 1000)::bigint,
-1
) as oldest_waiting_ms
from execution_admission_entries",
)
.fetch_one(pool)
.await
.map_err(PostgresExecutionAdmission::unavailable)?;
let queued: i64 = row
.try_get("queued")
.map_err(PostgresExecutionAdmission::unavailable)?;
let executing: i64 = row
.try_get("executing")
.map_err(PostgresExecutionAdmission::unavailable)?;
let oldest: i64 = row
.try_get("oldest_waiting_ms")
.map_err(PostgresExecutionAdmission::unavailable)?;
Ok(AdmissionQueueSnapshot {
waiting: u64::try_from(queued).unwrap_or_default(),
executing: u64::try_from(executing).unwrap_or_default(),
oldest_waiting_ms: (oldest >= 0).then(|| u64::try_from(oldest).unwrap_or_default()),
})
}
impl PostgresExecutionAdmission {
async fn attempt(
&self,
context: AttemptContext<'_>,
ticket: &mut Option<u64>,
) -> Result<Attempt, AdmissionStoreError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
match self.attempt_locked(&mut transaction, context, ticket).await {
Ok(attempt) => {
transaction.commit().await.map_err(Self::unavailable)?;
Ok(attempt)
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Ok(Attempt::Waiting)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Ok(Attempt::Overloaded)
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(AdmissionStoreError::unavailable(error.to_string()))
}
Err(AttemptError::AdmissionFailure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
Err(AttemptError::UniqueViolation) => {
transaction.rollback().await.ok();
Err(AdmissionStoreError::unavailable(
"execution admission hit an unexpected unique conflict",
))
}
}
}
async fn attempt_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
context: AttemptContext<'_>,
ticket: &mut Option<u64>,
) -> Result<Attempt, AttemptError> {
let workspace_id = context.request.scope.workspace_id.as_str();
let cursor = lock_admission_in_tx(transaction, workspace_id).await?;
// Expired queue decisions and waiting tickets leave the bounded queue; held permits and
// Task-linked dispatch intents never do (Story 4.2 owns the durable Task lifetime).
cleanup_expired_unlinked(transaction).await?;
if ticket.is_none() {
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let state = match context.request.mode {
AdmissionMode::Synchronous => "waiting",
AdmissionMode::Background => "reserved",
};
let ticket_id = insert_admission_entry(
transaction,
context.request,
context.owner_token,
state,
None,
)
.await?;
*ticket = Some(ticket_id);
if context.request.mode == AdmissionMode::Background {
return Ok(Attempt::Enqueued(ticket_id));
}
}
self.grant_next(transaction, workspace_id, &cursor).await?;
let Some(ticket_id) = *ticket else {
return Ok(Attempt::Waiting);
};
let state: Option<String> =
sqlx::query("select state from execution_admission_entries where ticket_id = $1")
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.map(|row| row.try_get("state"))
.transpose()
.map_err(AttemptError::from_sql)?;
match state.as_deref() {
Some("executing") => Ok(Attempt::Granted),
Some(_) => Ok(Attempt::Waiting),
// The ticket expired inside the store (or was removed by another cleanup).
None => Ok(Attempt::Expired),
}
}
/// Grants at most one waiting ticket per transaction, following the persisted weighted
/// round-robin cursor. A blocked head keeps its turn: nothing later is granted in its place.
async fn grant_next(
&self,
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
cursor: &AdmissionCursor,
) -> Result<(), AttemptError> {
let rows = sqlx::query(
"select ticket_id, agent_key, operation_id from execution_admission_entries
where workspace_id = $1 and state = 'waiting'
order by ticket_id
limit $2",
)
.bind(workspace_id)
.bind(QUEUE_READ_LIMIT)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
if rows.is_empty() {
return Ok(());
}
let queue = rows
.iter()
.map(|row| {
Ok(AdmissionQueueEntry {
ticket_id: u64::try_from(
row.try_get::<i64, _>("ticket_id")
.map_err(AttemptError::from_sql)?,
)
.unwrap_or_default(),
sequence: 0,
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
operation_id: row
.try_get("operation_id")
.map_err(AttemptError::from_sql)?,
})
})
.collect::<Result<Vec<_>, AttemptError>>()?;
let Some(selection) = select_next_grant(&queue, cursor, self.scheduler) else {
return Ok(());
};
let head = &queue[selection.index];
let global_executing: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries
where state = 'executing'",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
let agent_counts = scoped_counts(transaction, workspace_id, "agent_key").await?;
let tool_counts = scoped_counts(transaction, workspace_id, "operation_id").await?;
if u64::try_from(global_executing).unwrap_or(u64::MAX)
>= u64::try_from(self.limits.global_limit).unwrap_or(u64::MAX)
{
return Ok(());
}
let agent_executing = agent_counts
.get(head.agent_key.as_str())
.copied()
.unwrap_or_default();
if agent_executing >= u64::try_from(self.limits.agent_limit).unwrap_or(u64::MAX) {
return Ok(());
}
let tool_executing = tool_counts
.get(head.operation_id.as_str())
.copied()
.unwrap_or_default();
if tool_executing >= u64::try_from(self.limits.tool_limit).unwrap_or(u64::MAX) {
return Ok(());
}
sqlx::query(
"update execution_admission_entries
set state = 'executing', granted_at = now()
where ticket_id = $1 and state = 'waiting'",
)
.bind(i64::try_from(head.ticket_id).unwrap_or(i64::MAX))
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
sqlx::query(
"update execution_admission_controllers
set cursor_agent = $2,
cursor_agent_credit = $3,
cursor_tool = $4,
cursor_tool_credit = $5,
updated_at = now()
where workspace_id = $1",
)
.bind(workspace_id)
.bind(selection.cursor.agent.as_deref())
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
.bind(selection.cursor.tool.as_deref())
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(())
}
}
/// Takes both controller locks in the fixed `'*'` -> workspace order. Shared by the synchronous
/// admission attempt and the durable Task repository, so both serialize on the same controller rows
/// and one task/queue invariant holds. Callers own the transaction; nothing here commits or sleeps.
pub(super) async fn lock_admission_in_tx(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
) -> Result<AdmissionCursor, AttemptError> {
sqlx::query(
"insert into execution_admission_controllers (workspace_id)
values ($1), ($2)
on conflict (workspace_id) do nothing",
)
.bind(DEPLOYMENT_CONTROLLER)
.bind(workspace_id)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?;
lock_controller(transaction, workspace_id).await
}
/// Bounded cleanup of expired waiting/reserved rows that are not a durable Task dispatch intent.
pub(super) async fn cleanup_expired_unlinked(
transaction: &mut Transaction<'_, Postgres>,
) -> Result<(), AttemptError> {
sqlx::query(
"delete from execution_admission_entries
where ticket_id in (
select ticket_id from execution_admission_entries
where state in ('waiting', 'reserved') and expires_at <= now()
and task_id is null
order by ticket_id
limit $1
)",
)
.bind(EXPIRED_CLEANUP_BATCH)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(())
}
/// Authoritative bounded queue length: waiting and reserved rows share one budget.
pub(super) async fn bounded_queue_len(
transaction: &mut Transaction<'_, Postgres>,
) -> Result<u64, AttemptError> {
let queued: i64 = sqlx::query(
"select count(*)::bigint as count from execution_admission_entries
where state in ('waiting', 'reserved')",
)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?
.try_get("count")
.map_err(AttemptError::from_sql)?;
Ok(u64::try_from(queued).unwrap_or(u64::MAX))
}
/// Inserts one bounded admission row. A Task-linked row is the durable dispatch intent of the Task
/// accepted in the same transaction, so its capacity slot can never be silently reclaimed.
pub(super) async fn insert_admission_entry(
transaction: &mut Transaction<'_, Postgres>,
request: &AdmissionRequest,
owner_token: &str,
state: &str,
task_id: Option<Uuid>,
) -> Result<u64, AttemptError> {
let row = sqlx::query(
"insert into execution_admission_entries (
workspace_id, agent_key, operation_id, operation_version, mode, policy_mode,
state, owner_token, fingerprint, expires_at, task_id
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9,
to_timestamp($10::double precision / 1000.0), $11)
returning ticket_id",
)
.bind(&request.scope.workspace_id)
.bind(&request.scope.agent_key)
.bind(&request.scope.operation_id)
.bind(i32::try_from(request.scope.operation_version).unwrap_or(i32::MAX))
.bind(request.mode.as_str())
.bind(policy_mode_str(request))
.bind(state)
.bind(owner_token)
.bind(&request.fingerprint)
.bind(request.expires_at_unix_ms)
.bind(task_id)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
Ok(u64::try_from(ticket_id).unwrap_or_default())
}
pub(super) enum AttemptError {
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
Contended,
/// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait.
Overloaded,
/// A foreign unique constraint aborted the transaction; the caller rolls back and rereads.
UniqueViolation,
/// Durable-Task side failure (the Task repository maps it to its own typed error).
Failure(TaskError),
/// Admission side failure.
AdmissionFailure(AdmissionStoreError),
}
impl AttemptError {
pub(super) fn from_sql(error: sqlx::Error) -> Self {
if let Some(database) = error.as_database_error()
&& matches!(database.code().as_deref(), Some("55P03" | "57014"))
{
return Self::Contended;
}
Self::AdmissionFailure(PostgresExecutionAdmission::unavailable(error))
}
}
async fn lock_controller(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
) -> Result<AdmissionCursor, AttemptError> {
sqlx::query(
"update execution_admission_controllers set updated_at = updated_at
where workspace_id = $1",
)
.bind(workspace_id)
.execute(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let row = sqlx::query(
"select cursor_agent, cursor_agent_credit, cursor_tool, cursor_tool_credit
from execution_admission_controllers where workspace_id = $1",
)
.bind(workspace_id)
.fetch_one(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
Ok(AdmissionCursor {
agent: row
.try_get("cursor_agent")
.map_err(AttemptError::from_sql)?,
agent_credit: u32::try_from(
row.try_get::<i32, _>("cursor_agent_credit")
.map_err(AttemptError::from_sql)?
.max(0),
)
.unwrap_or_default(),
tool: row.try_get("cursor_tool").map_err(AttemptError::from_sql)?,
tool_credit: u32::try_from(
row.try_get::<i32, _>("cursor_tool_credit")
.map_err(AttemptError::from_sql)?
.max(0),
)
.unwrap_or_default(),
})
}
async fn scoped_counts(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &str,
column: &str,
) -> Result<BTreeMap<String, u64>, AttemptError> {
let statement = if column == "agent_key" {
"select agent_key as key, count(*)::bigint as count
from execution_admission_entries
where workspace_id = $1 and state = 'executing'
group by agent_key"
} else {
"select operation_id as key, count(*)::bigint as count
from execution_admission_entries
where workspace_id = $1 and state = 'executing'
group by operation_id"
};
let rows = sqlx::query(statement)
.bind(workspace_id)
.fetch_all(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let mut counts = BTreeMap::new();
for row in rows {
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
counts.insert(key, u64::try_from(count).unwrap_or_default());
}
Ok(counts)
}
fn policy_mode_str(request: &AdmissionRequest) -> &'static str {
match request.policy_mode {
crank_core::ToolExecutionMode::Synchronous => "synchronous",
crank_core::ToolExecutionMode::Background => "background",
crank_core::ToolExecutionMode::Adaptive => "adaptive",
}
}
@@ -46,6 +46,24 @@ impl PostgresRegistry {
Ok(())
}
/// Readiness probe of the mandatory execution-admission state (Story 4.1). A missing or
/// unreadable admission schema is a dependency failure, never a degraded-but-ready state.
pub async fn execution_admission_health(&self) -> Result<(), crank_core::AdmissionStoreError> {
super::admission::admission_health(&self.pool).await
}
/// Readiness probe of the durable Task state (Story 4.2).
pub async fn durable_tasks_health(&self) -> Result<(), crank_core::TaskError> {
super::tasks::tasks_health(&self.pool).await
}
/// Authoritative admission queue readings for low-cardinality metrics.
pub async fn execution_admission_snapshot(
&self,
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
super::admission::admission_snapshot(&self.pool).await
}
async fn connect_in_schema(
database_url: &str,
schema: Option<&str>,
@@ -1,3 +1,4 @@
mod admission;
mod agent;
mod agent_catalog;
mod agent_prompt_bindings;
@@ -20,6 +21,7 @@ mod product_event;
mod prompt;
mod resource;
mod secret;
mod tasks;
mod upstream;
mod workspace;
mod yaml_import;
@@ -36,8 +38,13 @@ use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow, types::Json};
use time::OffsetDateTime;
pub use admission::{
AdmissionLimits, PostgresExecutionAdmission, admission_health, admission_snapshot,
};
pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError};
pub use resource::{AuthorizedResourceRead, BoundResourceAdmission};
pub use tasks::PostgresTaskStore;
pub use tasks::tasks_health;
use crate::{
error::RegistryError,
+586
View File
@@ -0,0 +1,586 @@
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
//!
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
//!
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
use async_trait::async_trait;
use crank_core::{
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode,
};
use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
use super::admission::{
AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked,
insert_admission_entry, lock_admission_in_tx,
};
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode,
approval_reference,
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
result_envelope, error_envelope, attempts, version,
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
(extract(epoch from created_at) * 1000)::bigint as created_ms,
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
#[derive(Clone)]
pub struct PostgresTaskStore {
pool: PgPool,
limits: AdmissionLimits,
}
impl PostgresTaskStore {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
) -> Result<Self, TaskError> {
limits
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
scheduler
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
Ok(Self { pool, limits })
}
fn unavailable(error: impl std::fmt::Display) -> TaskError {
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
}
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
Ok(transaction)
}
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
/// query, so a foreign row is never fetched.
async fn stored_acceptance(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let Some(row) = row else {
return Ok(None);
};
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
Ok(Some((
acceptance_from_record(&record),
record.input_fingerprint,
)))
}
}
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support.clone(),
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
}
}
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(pool)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}
/// One transaction's decision of the acceptance path.
enum Accepted {
Created(TaskAcceptance),
Replayed(TaskAcceptance),
Conflict,
}
#[async_trait]
impl TaskStore for PostgresTaskStore {
async fn health(&self) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(&self.pool)
.await
.map_err(Self::unavailable)?;
super::admission::admission_health(&self.pool)
.await
.map_err(|error| TaskError::unavailable(error.to_string()))?;
Ok(())
}
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let outcome = self.accept_locked(&mut transaction, &request).await;
match outcome {
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
// The acknowledgement of a created Task is only possible after a real COMMIT.
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskAcceptance {
replayed: acceptance.replayed,
..acceptance
})
}
Ok(Accepted::Conflict) => {
transaction.rollback().await.ok();
Err(TaskError::FingerprintConflict)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Err(TaskError::Overloaded {
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
})
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(
"durable task acceptance could not take the admission locks",
))
}
Err(AttemptError::UniqueViolation) => {
// Roll back the aborted transaction, then reread the scoped key once outside it.
transaction.rollback().await.ok();
self.reread_after_unique(&request).await
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
Err(AttemptError::AdmissionFailure(error)) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(error.to_string()))
}
}
}
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
record_from_row(&row).map_err(Self::unavailable)
}
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(operation_id)
.bind(scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
row.map(|row| record_from_row(&row)).transpose()
}
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now()
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = update else {
transaction.rollback().await.ok();
return Err(self.classify_missed_finish(&request).await);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
let expired = sqlx::query(
"with expired as (
update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
version = version + 1,
updated_at = now()
where task_id in (
select task_id from tasks
where state = 'queued' and execution_deadline <= now()
order by task_id
limit $1
)
returning task_id
), released as (
delete from execution_admission_entries
where task_id in (select task_id from expired)
returning task_id
)
select (select count(*) from expired)::bigint as expired_queued",
)
.bind(batch)
.fetch_one(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let expired_queued: i64 = expired
.try_get("expired_queued")
.map_err(Self::unavailable)?;
let purged = sqlx::query(
"delete from tasks
where task_id in (
select task_id from tasks
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
order by task_id
limit $1
)",
)
.bind(batch)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskPurgeReport {
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
expired_terminal: purged.rows_affected(),
})
}
}
impl PostgresTaskStore {
async fn accept_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Accepted, AttemptError> {
// Replay first: an already accepted Task is returned without capacity mutation, so it works
// with a full queue and after the original execution deadline while the record is retained.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
lock_admission_in_tx(transaction, &request.workspace_id).await?;
// The competitor may have committed while this transaction waited for the controller lock.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
cleanup_expired_unlinked(transaction).await?;
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let created_ms: i64 = sqlx::query(
"insert into tasks (
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued',
$16, to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0)
)
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
)
.bind(request.task_id.as_uuid())
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
.bind(&request.scoped_key)
.bind(&request.input_fingerprint)
.bind(request.support.request_id.as_str())
.bind(request.support.trace_id.as_str())
.bind(&request.support.traceparent)
.bind(request.support.trace_sampled)
.bind(policy_mode_str(request.pins.policy_mode))
.bind(request.pins.approval_reference.as_deref())
.bind(request.input.value().clone())
.bind(request.pins.execution_deadline_unix_ms)
.bind(request.pins.retain_until_unix_ms)
.fetch_one(&mut **transaction)
.await
.map_err(task_sql_error)?
.try_get("created_ms")
.map_err(AttemptError::from_sql)?;
let admission_request = AdmissionRequest {
scope: AdmissionScope {
workspace_id: request.workspace_id.clone(),
agent_key: request.owner_id.clone(),
operation_id: request.operation_id.clone(),
operation_version: request.pins.operation_version,
},
mode: AdmissionMode::Background,
policy_mode: request.pins.policy_mode,
no_tasks_fallback: NoTasksFallback::Unsupported,
// Never reached: the intent row is inserted by this transaction, not by `admit`.
approval: ApprovalEvidence::NotRequired,
fingerprint: request.input_fingerprint.clone(),
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
};
let owner_token = format!("task_{}", request.task_id);
insert_admission_entry(
transaction,
&admission_request,
&owner_token,
"reserved",
Some(request.task_id.as_uuid()),
)
.await?;
Ok(Accepted::Created(TaskAcceptance {
task_id: request.task_id,
state: TaskState::Queued,
operation_version: request.pins.operation_version,
support: request.support.clone(),
created_at_unix_ms: created_ms,
retain_until_unix_ms: request.pins.retain_until_unix_ms,
replayed: false,
}))
}
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
async fn reread_after_unique(
&self,
request: &TaskAcceptRequest,
) -> Result<TaskAcceptance, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::unavailable(
"durable task acceptance ended with an unresolved unique conflict",
));
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
return Err(TaskError::FingerprintConflict);
}
Ok(acceptance_from_record(&record))
}
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
let access = &request.access;
match sqlx::query(
"select state, version from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
{
Ok(Some(_)) => TaskError::StateConflict,
Ok(None) => TaskError::NotFound,
Err(error) => Self::unavailable(error),
}
}
}
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
match mode {
ToolExecutionMode::Synchronous => "synchronous",
ToolExecutionMode::Background => "background",
ToolExecutionMode::Adaptive => "adaptive",
}
}
fn task_sql_error(error: sqlx::Error) -> AttemptError {
if error
.as_database_error()
.and_then(|database| database.code())
.as_deref()
== Some("23505")
{
return AttemptError::UniqueViolation;
}
AttemptError::from_sql(error)
}
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let state: String = row.try_get("state").map_err(unavailable_row)?;
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
value.map(TaskEnvelope::from_persisted).transpose()
};
Ok(TaskRecord {
task_id: TaskId::parse(&task_id.to_string())?,
state: TaskState::parse(&state)?,
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
authorization_scope: row
.try_get("authorization_scope")
.map_err(unavailable_row)?,
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
pins: TaskPins {
operation_version: u32::try_from(
row.try_get::<i32, _>("operation_version")
.map_err(unavailable_row)?,
)
.unwrap_or_default(),
policy_mode: match policy_mode.as_str() {
"background" => ToolExecutionMode::Background,
"adaptive" => ToolExecutionMode::Adaptive,
_ => ToolExecutionMode::Synchronous,
},
execution_deadline_unix_ms: row
.try_get("execution_deadline_ms")
.map_err(unavailable_row)?,
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
},
support: TaskSupportIds {
request_id: crank_core::RequestId::parse(&request_id)
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
trace_id: crank_core::TraceId::parse(&trace_id)
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
},
input: envelope("input_envelope")?
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
progress: envelope("progress_envelope")?,
result: envelope("result_envelope")?,
error: envelope("error_envelope")?,
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
.unwrap_or_default(),
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
.unwrap_or_default(),
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
})
}
fn unavailable_row(error: sqlx::Error) -> TaskError {
PostgresTaskStore::unavailable(error)
}
@@ -1,4 +1,5 @@
mod integration {
mod admission;
mod agent_resource_bindings;
mod agents_usage;
mod approval;
@@ -16,5 +17,6 @@ mod integration {
mod prompt_drafts;
mod prompt_publication;
mod resource_drafts;
mod tasks;
mod workspace_access;
}
@@ -0,0 +1,496 @@
//! ATDD Story 4.1 (FR-21/FR-24): authoritative PostgreSQL execution admission.
//!
//! These oracles drive the real store against a real PostgreSQL schema: scoped permits, bounded
//! queue, typed overload, durable enqueue decisions, deadline propagation, quota keys that ignore
//! the operation version, fairness order and fail-closed health.
use super::common;
use std::time::{Duration, Instant};
use crank_core::{
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
AdmissionSchedulerConfig, AdmissionScope, ApprovalEvidence, ExecutionAdmission,
NoTasksFallback, ToolExecutionMode, current_unix_ms,
};
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
use tokio::sync::mpsc;
fn limits(global: usize, agent: usize, tool: usize, queue: usize) -> AdmissionLimits {
AdmissionLimits {
global_limit: global,
agent_limit: agent,
tool_limit: tool,
queue_limit: queue,
}
}
fn store(
pool: sqlx::PgPool,
limits: AdmissionLimits,
scheduler: AdmissionSchedulerConfig,
) -> PostgresExecutionAdmission {
PostgresExecutionAdmission::new(pool, limits, scheduler).expect("valid admission store")
}
fn scope(agent: &str, operation: &str, version: u32) -> AdmissionScope {
AdmissionScope {
workspace_id: "ws_default".to_owned(),
agent_key: agent.to_owned(),
operation_id: operation.to_owned(),
operation_version: version,
}
}
fn request(scope: AdmissionScope, deadline_in_ms: i64) -> AdmissionRequest {
AdmissionRequest {
scope,
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: format!("fp_{}", current_unix_ms()),
expires_at_unix_ms: current_unix_ms() + deadline_in_ms,
}
}
fn background(scope: AdmissionScope, fallback: NoTasksFallback) -> AdmissionRequest {
AdmissionRequest {
mode: AdmissionMode::Background,
no_tasks_fallback: fallback,
..request(scope, 30_000)
}
}
fn permit(outcome: AdmissionOutcome) -> AdmissionPermit {
match outcome {
AdmissionOutcome::Permitted(permit) => permit,
other => panic!("expected a scoped permit, got {other:?}"),
}
}
async fn wait_for_metric_poll() {
tokio::time::sleep(Duration::from_millis(20)).await;
}
#[tokio::test]
async fn permits_are_scoped_released_once_and_respect_the_agent_budget() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 1, 4, 8),
Default::default(),
);
let first = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
assert_eq!(first.scope.agent_key, "agent_a");
// The same Agent is already at its scoped budget: the second request waits and expires instead of
// over-admitting, and it never dispatches.
let waiting = admission
.admit(request(scope("agent_a", "op_leads", 2), 120))
.await
.expect("admission store is reachable");
assert_eq!(waiting, AdmissionOutcome::DeadlineElapsed);
// Another Agent still has the global capacity it is entitled to.
let other = permit(
admission
.admit(request(scope("agent_b", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
assert_eq!(other.scope.agent_key, "agent_b");
assert!(admission.release(&other).await.unwrap());
// Repeated release is idempotent and never frees a foreign or already released permit.
assert!(!admission.release(&other).await.unwrap());
assert!(admission.release(&first).await.unwrap());
let snapshot = admission.snapshot().await.unwrap();
assert_eq!(snapshot.executing, 0);
assert_eq!(snapshot.waiting, 0);
database.cleanup().await;
}
#[tokio::test]
async fn full_queue_returns_a_typed_overload_with_retry_guidance() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 1, 1, 2),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable"),
);
// Two bounded waiting entries fill the queue.
let mut waiters = Vec::new();
for version in 1..=2 {
let admission = admission.clone();
waiters.push(tokio::spawn(async move {
admission
.admit(request(scope("agent_b", "op_leads", version), 5_000))
.await
}));
}
while admission.snapshot().await.unwrap().waiting < 2 {
wait_for_metric_poll().await;
}
let rejected = admission
.admit(request(scope("agent_c", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable");
match rejected {
AdmissionOutcome::Overloaded { retry_after_ms } => {
assert!((1..=crank_core::MAX_RETRY_AFTER_MS).contains(&retry_after_ms));
}
other => panic!("expected typed overload, got {other:?}"),
}
// Cancelling the held permit lets the queued waiters make progress instead of leaking capacity.
assert!(admission.release(&held).await.unwrap());
for waiter in waiters.drain(..) {
let outcome = waiter.await.expect("waiter task");
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
let _ = admission.release(&permit).await;
}
}
database.cleanup().await;
}
#[tokio::test]
async fn background_decisions_are_durable_pinned_and_released_once() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let decision = match admission
.admit(background(
scope("agent_a", "op_report", 3),
NoTasksFallback::Synchronous,
))
.await
.expect("admission store is reachable")
{
AdmissionOutcome::Enqueued(decision) => decision,
other => panic!("expected a durable enqueue decision, got {other:?}"),
};
assert_eq!(decision.scope.operation_version, 3);
assert_eq!(decision.policy_mode, ToolExecutionMode::Synchronous);
// A second process (fresh pool and store over the same schema) must read the committed decision:
// it occupies bounded queue capacity and is not a runnable execution.
let reconnected = store(
database.concurrent_registry(2).await.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let snapshot = reconnected.snapshot().await.unwrap();
assert_eq!(snapshot.waiting, 1);
assert_eq!(snapshot.executing, 0);
assert!(
reconnected
.cancel(decision.ticket_id, "not_the_owner")
.await
.unwrap()
.eq(&false),
"a foreign cancel must never release the decision"
);
assert!(
reconnected.snapshot().await.unwrap().waiting.eq(&1),
"the foreign cancel must leave the decision intact"
);
database.cleanup().await;
}
#[tokio::test]
async fn background_without_a_task_transport_is_typed_unsupported() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let outcome = admission
.admit(background(
scope("agent_a", "op_report", 1),
NoTasksFallback::Unsupported,
))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::Unsupported);
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
database.cleanup().await;
}
#[tokio::test]
async fn unverified_approval_never_produces_a_decision() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 4, 8),
Default::default(),
);
let mut denied = background(
scope("agent_a", "op_report", 1),
NoTasksFallback::Synchronous,
);
denied.approval = ApprovalEvidence::NotGranted;
assert_eq!(
admission.admit(denied).await.unwrap(),
AdmissionOutcome::ApprovalRequired
);
let mut denied_sync = request(scope("agent_a", "op_report", 1), 1_000);
denied_sync.approval = ApprovalEvidence::NotGranted;
assert_eq!(
admission.admit(denied_sync).await.unwrap(),
AdmissionOutcome::ApprovalRequired
);
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
assert_eq!(admission.snapshot().await.unwrap().executing, 0);
database.cleanup().await;
}
#[tokio::test]
async fn quota_keys_ignore_the_operation_version() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(4, 4, 1, 8),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
// A different Published Version of the same Tool shares the same per-Tool budget.
let outcome = admission
.admit(request(scope("agent_a", "op_leads", 2), 120))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
assert!(admission.release(&held).await.unwrap());
database.cleanup().await;
}
#[tokio::test]
async fn weighted_agents_are_served_their_quantum_in_order() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 4, 4, 8),
AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
},
);
let held = permit(
admission
.admit(request(scope("agent_z", "op_leads", 1), 5_000))
.await
.expect("admission store is reachable"),
);
let (sender, mut receiver) = mpsc::channel(3);
let mut tasks = Vec::new();
for (agent, version) in [("agent_a", 1), ("agent_a", 2), ("agent_b", 1)] {
let admission = admission.clone();
let sender = sender.clone();
tasks.push(tokio::spawn(async move {
let outcome = admission
.admit(request(scope(agent, "op_leads", version), 5_000))
.await;
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
let _ = sender.send(permit.scope.agent_key.clone()).await;
// Hold briefly so the grant order, not task scheduling, decides the sequence.
tokio::time::sleep(Duration::from_millis(30)).await;
let _ = admission.release(&permit).await;
}
}));
}
drop(sender);
assert!(admission.release(&held).await.unwrap());
let mut order = Vec::new();
while let Some(agent) = receiver.recv().await {
order.push(agent);
if order.len() == 3 {
break;
}
}
for task in tasks {
let _ = tokio::time::timeout(Duration::from_secs(5), task).await;
}
// agent_a carries weight 2 and therefore receives two consecutive grants before agent_b.
assert_eq!(order, vec!["agent_a", "agent_a", "agent_b"]);
database.cleanup().await;
}
#[tokio::test]
async fn mandatory_state_failure_fails_closed_and_reports_not_ready() {
let database = common::TestDatabase::new().await;
// No migration on this schema: the mandatory admission state does not exist yet.
let unmigrated = database.raw_pool().await;
let admission = store(unmigrated, limits(4, 4, 4, 8), Default::default());
assert!(admission.health().await.is_err());
assert!(
admission
.admit(request(scope("agent_a", "op_leads", 1), 1_000))
.await
.is_err()
);
database.cleanup().await;
}
#[tokio::test]
async fn held_permits_survive_a_new_process_view() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 4, 4, 8),
Default::default(),
);
let held = permit(
admission
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
let reconnected = store(
database.concurrent_registry(2).await.pool().clone(),
limits(1, 4, 4, 8),
Default::default(),
);
assert_eq!(reconnected.snapshot().await.unwrap().executing, 1);
let outcome = reconnected
.admit(request(scope("agent_b", "op_leads", 1), 120))
.await
.expect("admission store is reachable");
assert_eq!(
outcome,
AdmissionOutcome::DeadlineElapsed,
"a held permit is not silently reused by another process"
);
assert!(reconnected.release(&held).await.unwrap());
database.cleanup().await;
}
#[tokio::test]
async fn published_execution_policy_is_pinned_across_a_new_draft_version() {
use crank_core::{NoTasksFallback, OperationStatus, ToolExecutionMode, ToolExecutionPolicy};
use crank_registry::{CreateVersionRequest, PublishRequest};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let workspace = common::test_workspace_id();
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 45_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
let mut version_one = common::test_operation("op_policy", 1, OperationStatus::Draft);
version_one.execution_config.execution_policy = Some(policy);
registry
.create_operation(&workspace, &version_one, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &workspace,
operation_id: &version_one.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-25T12:10:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
let published = registry
.get_published_operation(&version_one.id)
.await
.unwrap()
.expect("published operation");
assert_eq!(published.execution_config.execution_policy, Some(policy));
// A later Draft with another policy never rewrites the already accepted publication.
let mut version_two = common::test_operation("op_policy", 2, OperationStatus::Draft);
version_two.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 1_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
});
registry
.create_version(CreateVersionRequest {
workspace_id: &workspace,
snapshot: &version_two,
change_note: Some("next draft"),
created_by: Some("alice"),
})
.await
.unwrap();
let unchanged = registry
.get_published_operation(&version_one.id)
.await
.unwrap()
.expect("published operation");
assert_eq!(unchanged.version, 1);
assert_eq!(unchanged.execution_config.execution_policy, Some(policy));
database.cleanup().await;
}
#[tokio::test]
async fn admin_origin_bucket_is_bounded_together_with_agents() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let admission = store(
registry.pool().clone(),
limits(1, 1, 1, 4),
Default::default(),
);
let held = permit(
admission
.admit(request(scope(ADMIN_ORIGIN_KEY, "op_leads", 1), 2_000))
.await
.expect("admission store is reachable"),
);
let started = Instant::now();
let outcome = admission
.admit(request(scope("agent_a", "op_leads", 1), 120))
.await
.expect("admission store is reachable");
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
assert!(started.elapsed() >= Duration::from_millis(100));
assert!(admission.release(&held).await.unwrap());
database.cleanup().await;
}
@@ -70,6 +70,8 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
(21, "prompts-v21"),
(22, "prompt-publications-v22"),
(23, "prompt-dependencies-v23"),
(24, "execution-admission-v24"),
(25, "durable-tasks-v25"),
];
assert_eq!(rows.len(), expected.len());
for (row, (version, name)) in rows.iter().zip(expected) {
@@ -180,7 +182,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
);
assert_eq!(
MigrationAuthority::preflight(first.pool()).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
#[tokio::test]
@@ -228,7 +230,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
.expect("the V18-local guard must be restorable");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restoring the exact V18 contract must return the current verdict"
);
}
@@ -354,7 +356,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -467,13 +469,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 2,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
let trace_column: bool = sqlx::query_scalar(
"select exists (
@@ -516,7 +518,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 3,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -576,7 +578,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 4,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -830,6 +832,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 23,
target: 25,
}
);
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
.unwrap();
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
for (drift, restore) in [
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"restore: {restore}"
);
}
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"a rejected primary-key nullability change must leave the schema current"
);
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"the baseline before the drift matrix must be the current V16 schema"
);
let mut observations: Vec<String> = Vec::new();
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
.await
.expect("restoring the exact definition must apply");
let restored = MigrationAuthority::preflight(&pool).await;
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 23 }));
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 }));
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
// мнением того же guard-а, который принимает неправильные определения (F4).
let restored_exact = catalog_definition_snapshot(&pool).await == before;
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"baseline before inserting the shadow schema"
);
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
// Тень не меняет вердикт guard-а по текущей схеме.
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"a same-named object in another schema must not change the verdict for the current schema"
);
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 },
MigrationPreflight::Current { version: 25 },
"the exact restoration must return the current V16 schema"
);
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
assert!(
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
let tables = sqlx::query(
"select table_name
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 13,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
assert_eq!(operation_count, 1);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
assert!(supports_elicitation);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 23,
target: 25,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
.fetch_one(&pool)
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 23,
target: 25,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 18, to: 23 }
MigrationApplyResult::Applied { from: 18, to: 25 }
);
let after: Vec<(i64, String, String)> =
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 23 }
MigrationPreflight::Current { version: 25 }
);
}
@@ -84,18 +84,18 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() {
let database_url = crank_test_support::postgres_schema_url("test_v19_future_version").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
// The same `version > CURRENT_VERSION` branch an old V23 binary takes when it
// meets a V24 database; a synthetic V24 row exercises it without a second binary.
// The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer
// database; a synthetic version above the current one exercises it without a second binary.
sqlx::query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values (24, 'future-v24', repeat('a', 64), 'expand', 'n-minus-one-readable')",
values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')",
)
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "future_version");
assert_eq!(error.version(), Some(24));
assert_eq!(error.version(), Some(26));
assert_eq!(error.recovery(), "install_matching_application");
assert!(
MigrationAuthority::require_current(&pool).await.is_err(),
@@ -155,7 +155,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 23,
target: 25,
}
);
}
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 23,
target: 25,
}
);
}
@@ -38,6 +38,7 @@ pub(crate) async fn remove_v4_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v5_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v6_schema(pool).await;
sqlx::raw_sql(
"drop index if exists invocation_logs_workspace_operation_version_idx;
@@ -154,6 +155,7 @@ pub(crate) async fn remove_v11_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v14_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v15_schema(pool).await;
sqlx::raw_sql(
"drop table if exists import_plan_applications;
@@ -224,6 +226,7 @@ pub(crate) async fn remove_v18_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v19_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v20_schema(pool).await;
sqlx::raw_sql(
"drop table if exists resource_security_revocations;
@@ -307,6 +310,7 @@ pub(crate) async fn remove_v20_schema(pool: &sqlx::PgPool) {
.unwrap();
}
pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
remove_v24_schema(pool).await;
remove_v14_schema(pool).await;
sqlx::raw_sql(
"drop index if exists import_jobs_openapi_source_idx;
@@ -323,3 +327,27 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
.await
.unwrap();
}
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
sqlx::raw_sql(
"drop table if exists tasks cascade;
delete from __crank_migrations where version = 25;",
)
.execute(pool)
.await
.unwrap();
}
/// Story 4.1: снять V24 (authoritative execution admission) перед возвратом к < 24.
pub(crate) async fn remove_v24_schema(pool: &sqlx::PgPool) {
remove_v25_schema(pool).await;
sqlx::raw_sql(
"drop table if exists execution_admission_entries;
drop table if exists execution_admission_controllers;
delete from __crank_migrations where version = 24;",
)
.execute(pool)
.await
.unwrap();
}
@@ -0,0 +1,682 @@
//! ATDD Story 4.2 (FR-22/FR-23): durable Task acceptance on a real PostgreSQL schema.
//!
//! These oracles drive the real store: Task + dispatch intent + queue slot in one transaction, replay
//! and typed conflict before any capacity mutation, bounded shared capacity, the Task-linked intent
//! guard of the legacy admission paths, one-winner terminal CAS with an immutable result, bounded
//! retention, owner/authorization-scope isolation and fail-closed health.
use super::common;
use crank_core::{
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms,
};
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore};
use serde_json::json;
use sqlx::{PgPool, Row};
const WORKSPACE: &str = "ws_default";
const AGENT: &str = "agent_a";
fn limits(queue: usize) -> AdmissionLimits {
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: queue,
}
}
fn store_with(pool: PgPool, queue: usize) -> PostgresTaskStore {
PostgresTaskStore::new(pool, limits(queue), Default::default()).expect("valid store config")
}
fn unsampled_support() -> TaskSupportIds {
TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00").unwrap(),
)
}
fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> TaskAcceptRequest {
let deadline = current_unix_ms() + deadline_in_ms;
TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: WORKSPACE.to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: agent.to_owned(),
authorization_scope: format!("credential:pk_{agent}"),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: deadline,
retain_until_unix_ms: deadline + 3_600_000,
approval_reference: Some("appr_1".to_owned()),
},
scoped_key: key.to_owned(),
input_fingerprint: fingerprint.to_owned(),
input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(),
support: unsampled_support(),
}
}
fn access(agent: &str) -> TaskAccess {
TaskAccess {
workspace_id: WORKSPACE.to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: agent.to_owned(),
authorization_scope: format!("credential:pk_{agent}"),
}
}
fn key(seed: &str) -> String {
format!("{seed:0<64}").chars().take(64).collect()
}
fn digest(seed: &str) -> String {
format!("{seed:0<64}").chars().take(64).collect()
}
async fn row_counts(pool: &PgPool) -> (i64, i64) {
let row = sqlx::query(
"select
(select count(*)::bigint from tasks) as tasks,
(select count(*)::bigint from execution_admission_entries where task_id is not null)
as intents",
)
.fetch_one(pool)
.await
.unwrap();
(row.get("tasks"), row.get("intents"))
}
#[tokio::test]
async fn acceptance_commits_task_intent_and_slot_together() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let request = request(&key("k1"), &digest("f1"), AGENT, 30_000);
let task_id = request.task_id;
let support = request.support.clone();
// Another connection holds both controller rows: the acceptance cannot commit yet.
let mut blocker = registry.pool().begin().await.unwrap();
sqlx::query(
"insert into execution_admission_controllers (workspace_id) values ('*'), ($1)
on conflict (workspace_id) do nothing",
)
.bind(WORKSPACE)
.execute(&mut *blocker)
.await
.unwrap();
sqlx::query("update execution_admission_controllers set updated_at = updated_at")
.execute(&mut *blocker)
.await
.unwrap();
let pool = registry.pool().clone();
let acceptance = tokio::spawn(async move { store.accept(request).await });
tokio::time::sleep(std::time::Duration::from_millis(150)).await;
// No partial Task, no intent and no accepted response exist before COMMIT.
assert!(!acceptance.is_finished());
assert_eq!(row_counts(&pool).await, (0, 0));
blocker.commit().await.unwrap();
let acceptance = acceptance.await.unwrap().expect("acceptance commits");
assert_eq!(acceptance.state, TaskState::Queued);
assert!(!acceptance.replayed);
assert_eq!(acceptance.task_id, task_id);
assert_eq!(acceptance.support, support);
assert_eq!(row_counts(registry.pool()).await, (1, 1));
// A fresh connection and store read the committed pins/identities back.
let reader = store_with(database.concurrent_registry(2).await.pool().clone(), 8);
let record = reader.read(&access(AGENT), task_id).await.unwrap();
assert_eq!(record.state, TaskState::Queued);
assert_eq!(record.pins.operation_version, 3);
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
assert_eq!(record.pins.approval_reference.as_deref(), Some("appr_1"));
assert_eq!(record.support.request_id, support.request_id);
assert_eq!(record.support.trace_id, support.trace_id);
assert_eq!(record.support.traceparent, support.traceparent);
assert!(!record.support.trace_sampled);
// Secret-bearing input is classified before persistence.
assert_eq!(record.input.value()["api_key"], json!("[redacted]"));
assert_eq!(record.input.value()["report"], json!("q3"));
assert!(record.result.is_none());
database.cleanup().await;
}
#[tokio::test]
async fn replay_returns_the_same_task_and_one_slot() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let first = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect("first acceptance");
let before = store.read(&access(AGENT), first.task_id).await.unwrap();
let replay = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect("replay returns the accepted Task");
assert_eq!(replay.task_id, first.task_id);
assert!(replay.replayed);
assert_eq!(replay.support.request_id, first.support.request_id);
assert_eq!(row_counts(registry.pool()).await, (1, 1));
// Replay never rewrites the accepted row or its audit evidence.
let after = store.read(&access(AGENT), first.task_id).await.unwrap();
assert_eq!(after.version, before.version);
assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms);
assert_eq!(after.created_at_unix_ms, before.created_at_unix_ms);
database.cleanup().await;
}
#[tokio::test]
async fn replay_stays_available_with_a_full_queue_and_past_execution_deadline() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 1);
// A short deadline: the Task is still retained after its execution deadline elapsed.
let first = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
.await
.expect("first acceptance");
sqlx::query(
"update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1",
)
.bind(first.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
let replayed = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
.await
.expect("replay works with a full queue and an elapsed execution deadline");
assert_eq!(replayed.task_id, first.task_id);
assert!(replayed.replayed);
database.cleanup().await;
}
#[tokio::test]
async fn a_different_fingerprint_on_the_same_key_is_a_typed_conflict() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let first = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect("first acceptance");
let before = store.read(&access(AGENT), first.task_id).await.unwrap();
let conflict = store
.accept(request(&key("k1"), &digest("f2"), AGENT, 30_000))
.await
.expect_err("different input is a conflict");
assert_eq!(conflict, TaskError::FingerprintConflict);
assert_eq!(row_counts(registry.pool()).await, (1, 1));
let after = store.read(&access(AGENT), first.task_id).await.unwrap();
assert_eq!(after.version, before.version);
assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms);
database.cleanup().await;
}
#[tokio::test]
async fn bounded_capacity_is_shared_and_overload_creates_nothing() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 1);
let admission =
PostgresExecutionAdmission::new(registry.pool().clone(), limits(1), Default::default())
.unwrap();
// A durable enqueue decision and a durable Task share the same bounded queue budget.
let waiting = admission
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: WORKSPACE.to_owned(),
agent_key: AGENT.to_owned(),
operation_id: "op_sync".to_owned(),
operation_version: 1,
},
mode: AdmissionMode::Background,
policy_mode: ToolExecutionMode::Background,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_sync".to_owned(),
expires_at_unix_ms: current_unix_ms() + 30_000,
})
.await
.unwrap();
assert!(matches!(waiting, AdmissionOutcome::Enqueued(_)));
let overloaded = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect_err("queue is full");
assert!(matches!(overloaded, TaskError::Overloaded { .. }));
assert_eq!(row_counts(registry.pool()).await, (0, 0));
assert_eq!(
admission.snapshot().await.unwrap().waiting,
1,
"the durable refusal changed no capacity"
);
database.cleanup().await;
}
#[tokio::test]
async fn legacy_cleanup_and_cancel_never_remove_a_task_linked_intent() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let admission =
PostgresExecutionAdmission::new(registry.pool().clone(), limits(8), Default::default())
.unwrap();
let accepted = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
.await
.expect("acceptance");
// The linked reserved row looks expired; the legacy TTL cleanup must ignore it.
sqlx::query("update execution_admission_entries set expires_at = now() - interval '1 hour'")
.execute(registry.pool())
.await
.unwrap();
let ticket: i64 =
sqlx::query("select ticket_id from execution_admission_entries where task_id = $1")
.bind(accepted.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap()
.get("ticket_id");
let owner_token: String =
sqlx::query("select owner_token from execution_admission_entries where ticket_id = $1")
.bind(ticket)
.fetch_one(registry.pool())
.await
.unwrap()
.get("owner_token");
// A legacy attempt runs the bounded expiry cleanup; a foreign/own legacy cancel is refused.
let _ = admission
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: WORKSPACE.to_owned(),
agent_key: "agent_b".to_owned(),
operation_id: "op_sync".to_owned(),
operation_version: 1,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_sync".to_owned(),
expires_at_unix_ms: current_unix_ms() + 50,
})
.await;
assert!(!admission.cancel(ticket as u64, &owner_token).await.unwrap());
assert_eq!(row_counts(registry.pool()).await, (1, 1));
assert_eq!(
store
.read(&access(AGENT), accepted.task_id)
.await
.unwrap()
.state,
TaskState::Queued
);
database.cleanup().await;
}
#[tokio::test]
async fn terminal_transition_has_one_winner_and_an_immutable_result() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let accepted = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
let queued = store.read(&access(AGENT), accepted.task_id).await.unwrap();
assert_eq!(queued.version, 1);
// Moving a queued Task straight to completed is not an allowed transition.
let skipped = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Queued,
expected_version: queued.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
};
assert_eq!(
store
.finish(skipped)
.await
.expect_err("transition rejected"),
TaskError::Invalid {
message: "task state transition is not allowed".to_owned()
}
);
let started = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Queued,
expected_version: queued.version,
target_state: TaskState::Working,
result: None,
error: None,
};
assert_eq!(
store
.finish(started)
.await
.expect_err("working is not terminal"),
TaskError::Invalid {
message: "task finish target is not terminal".to_owned()
}
);
sqlx::query("update tasks set state = 'working' where task_id = $1")
.bind(accepted.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let working = store.read(&access(AGENT), accepted.task_id).await.unwrap();
let mut finishers = Vec::new();
for marker in ["first", "second"] {
let store = store.clone();
let mut finish = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Working,
expected_version: working.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"marker": marker})).unwrap()),
error: None,
};
finish.target_state = TaskState::Completed;
finishers.push(tokio::spawn(async move { store.finish(finish).await }));
}
let first = finishers.remove(0).await.unwrap();
let second = finishers.remove(0).await.unwrap();
let winners = [&first, &second]
.iter()
.filter(|result| result.is_ok())
.count();
assert_eq!(winners, 1, "exactly one concurrent CAS wins");
let loser = if first.is_err() { &first } else { &second };
assert_eq!(
loser.as_ref().err().unwrap(),
&TaskError::StateConflict,
"the loser observes a rejected transition, not a rewrite"
);
let terminal = store.read(&access(AGENT), accepted.task_id).await.unwrap();
assert_eq!(terminal.state, TaskState::Completed);
assert_eq!(terminal.version, working.version + 1);
let marker = terminal.result.as_ref().unwrap().value()["marker"]
.as_str()
.unwrap()
.to_owned();
assert!(marker == "first" || marker == "second");
// A repeated terminal write and a terminal->terminal move are rejected.
let repeated = TaskFinishRequest {
access: access(AGENT),
task_id: accepted.task_id,
expected_state: TaskState::Completed,
expected_version: terminal.version,
target_state: TaskState::Failed,
result: None,
error: Some(TaskEnvelope::try_redacted(&json!({"reason": "x"})).unwrap()),
};
assert!(repeated.validate().is_err());
// Terminal Task and its dispatch intent left together.
assert_eq!(row_counts(registry.pool()).await, (1, 0));
database.cleanup().await;
}
#[tokio::test]
async fn retention_is_bounded_and_never_deletes_live_work() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let queued = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
let expired_queued = store
.accept(request(&key("k2"), &digest("f2"), AGENT, 1))
.await
.unwrap();
sqlx::query(
"update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1",
)
.bind(expired_queued.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
// A retained terminal Task whose retention has not expired yet and one that has.
let terminal = store
.accept(request(&key("k3"), &digest("f3"), AGENT, 30_000))
.await
.unwrap();
sqlx::query(
"update tasks set state = 'completed', result_envelope = '{\"ok\":true}'::jsonb,
retain_until = now() + interval '1 hour' where task_id = $1",
)
.bind(terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let expired_terminal = store
.accept(request(&key("k4"), &digest("f4"), AGENT, 30_000))
.await
.unwrap();
sqlx::query(
"update tasks set state = 'cancelled', execution_deadline = now() - interval '2 hours',
retain_until = now() - interval '1 hour'
where task_id = $1",
)
.bind(expired_terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(expired_terminal.task_id.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let report = store.purge_expired(64).await.unwrap();
assert_eq!(report.expired_queued, 1);
assert_eq!(report.expired_terminal, 1);
// Live queued work is never deleted or failed; the expired queued Task released its slot.
assert_eq!(
store
.read(&access(AGENT), queued.task_id)
.await
.unwrap()
.state,
TaskState::Queued
);
assert_eq!(
store
.read(&access(AGENT), terminal.task_id)
.await
.unwrap()
.state,
TaskState::Completed
);
assert!(matches!(
store
.read(&access(AGENT), expired_queued.task_id)
.await
.unwrap()
.state,
TaskState::Failed
));
assert!(
store
.read(&access(AGENT), expired_terminal.task_id)
.await
.is_err()
);
let remaining: i64 = sqlx::query("select count(*)::bigint as count from execution_admission_entries where task_id is not null")
.fetch_one(registry.pool())
.await
.unwrap()
.get("count");
assert_eq!(remaining, 1, "only the live queued Task keeps its intent");
database.cleanup().await;
}
#[tokio::test]
async fn foreign_scope_reads_are_not_found_and_never_leak_payload() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
let accepted = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.unwrap();
// Positive control: the owner reads its Task.
assert!(store.read(&access(AGENT), accepted.task_id).await.is_ok());
// Foreign owner, foreign authorization scope and foreign workspace are all not-found.
let mut foreign_owner = access(AGENT);
foreign_owner.owner_id = "agent_b".to_owned();
assert_eq!(
store
.read(&foreign_owner, accepted.task_id)
.await
.unwrap_err(),
TaskError::NotFound
);
let mut foreign_scope = access(AGENT);
foreign_scope.authorization_scope = "credential:pk_other".to_owned();
assert_eq!(
store
.read(&foreign_scope, accepted.task_id)
.await
.unwrap_err(),
TaskError::NotFound
);
let mut foreign_workspace = access(AGENT);
foreign_workspace.workspace_id = "ws_other".to_owned();
assert_eq!(
store
.read(&foreign_workspace, accepted.task_id)
.await
.unwrap_err(),
TaskError::NotFound
);
// The same caller key under a foreign owner creates its own Task, never a visible collision.
let other = store
.accept(request(&key("k1"), &digest("f1"), "agent_b", 30_000))
.await
.unwrap();
assert_ne!(other.task_id, accepted.task_id);
assert_eq!(row_counts(registry.pool()).await, (2, 2));
database.cleanup().await;
}
#[tokio::test]
async fn invalid_bounded_envelope_is_rejected_before_persistence() {
let database = common::TestDatabase::new().await;
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
let _migrated = database.registry().await;
let registry = database.concurrent_registry(4).await;
let store = store_with(registry.pool().clone(), 8);
assert!(
TaskEnvelope::try_redacted(&json!({"blob": "x".repeat(64 * 1024)})).is_err(),
"an oversized envelope is rejected before any accept"
);
let mut invalid = request(&key("k1"), &digest("f1"), AGENT, 30_000);
invalid.scoped_key = "short".to_owned();
assert!(matches!(invalid.validate(), Err(TaskError::Invalid { .. })));
assert_eq!(
store.accept(invalid).await.unwrap_err(),
TaskError::Invalid {
message: "task scoped key digest is invalid".to_owned()
}
);
assert_eq!(row_counts(registry.pool()).await, (0, 0));
database.cleanup().await;
}
#[tokio::test]
async fn store_failure_fails_closed_without_records() {
let database = common::TestDatabase::new().await;
// No migration on this schema: the durable Task state does not exist yet.
let unmigrated = database.raw_pool().await;
let store = store_with(unmigrated, 8);
assert!(store.health().await.is_err());
let error = store
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
.await
.expect_err("acceptance fails closed");
assert!(matches!(error, TaskError::Unavailable { .. }));
database.cleanup().await;
}
+255
View File
@@ -0,0 +1,255 @@
//! Common execution admission seam of the runtime (Story 4.1, FR-21/FR-24).
//!
//! Every real execution — synchronous, or a published `background`/`adaptive` policy that falls back
//! to synchronous because no durable Task transport exists yet — passes this one gate before any
//! adapter work. The gate asks the authoritative store for a scoped permit bounded by the published
//! Tool policy, the verified approval state of the exact Published Version/scope/input and the caller
//! deadline, and it releases the permit explicitly when the local execution future ends.
use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
AdmissionScope, CoordinationStateStore, ExecutionAdmission, NoTasksFallback, ToolExecutionMode,
current_unix_ms,
};
use crank_metrics::{
AdmissionModeLabel, RejectionReason, ScopeKind, record_admission_queue_wait,
record_admission_rejection,
};
use tracing::warn;
use crate::{RuntimeError, RuntimeOperation, RuntimeRequestContext};
/// Effective absolute deadline of one execution: the published policy deadline never extends the
/// caller deadline and always includes queue waiting time.
pub(crate) fn effective_deadline(
operation: &RuntimeOperation,
caller_deadline: Instant,
) -> Instant {
match operation.execution_config.execution_policy {
Some(policy) => {
caller_deadline.min(Instant::now() + Duration::from_millis(policy.deadline_ms))
}
None => caller_deadline,
}
}
/// Effective execution path for this deployment.
///
/// Story 4.1 has no Task transport, so a `background` or `adaptive` policy resolves through its
/// published `no_tasks_fallback`: either the same synchronous admission path, or a typed
/// unsupported refusal with no admission entry, no Task id and no outbound call.
pub(crate) fn effective_mode(
operation: &RuntimeOperation,
) -> Result<ToolExecutionMode, RuntimeError> {
let Some(policy) = operation.execution_config.execution_policy else {
return Ok(ToolExecutionMode::Synchronous);
};
match policy.mode {
ToolExecutionMode::Synchronous => Ok(ToolExecutionMode::Synchronous),
mode => match policy.no_tasks_fallback {
NoTasksFallback::Synchronous => Ok(ToolExecutionMode::Synchronous),
NoTasksFallback::Unsupported => Err(RuntimeError::ExecutionPolicyUnsupported {
operation_id: operation.operation_id.as_str().to_owned(),
mode,
}),
},
}
}
pub(crate) struct AdmissionLease {
store: Arc<dyn ExecutionAdmission>,
permit: Option<AdmissionPermit>,
runtime: tokio::runtime::Handle,
}
impl AdmissionLease {
fn new(store: Arc<dyn ExecutionAdmission>, permit: AdmissionPermit) -> Self {
Self {
store,
permit: Some(permit),
runtime: tokio::runtime::Handle::current(),
}
}
/// Explicit release on the normal/error path of the local execution. The `Drop` cleanup stays as
/// the cancellation path; both CAS on the same owner token, so only one of them can free it.
pub(crate) async fn release(&mut self) {
let Some(permit) = self.permit.take() else {
return;
};
if let Err(error) = self.store.release(&permit).await {
warn!(
name: "runtime.admission.release_failed",
error = %error,
"failed to release an admission permit; the reservation stays fail-closed"
);
}
}
}
impl Drop for AdmissionLease {
fn drop(&mut self) {
let Some(permit) = self.permit.take() else {
return;
};
let store = Arc::clone(&self.store);
// Bounded cleanup: the explicit release path already ran on success/error; cancellation
// reaches here and releases by owner token without ever freeing a foreign permit. A failed
// release keeps the reservation fail-closed (no TTL reuse of an active permit).
self.runtime.spawn(async move {
match store.release(&permit).await {
Ok(true) => {}
Ok(false) => warn!(
name: "runtime.admission.release_skipped",
"admission permit was already released"
),
Err(error) => warn!(
name: "runtime.admission.release_failed",
error = %error,
"failed to release an admission permit; the reservation stays fail-closed"
),
}
});
}
}
/// Acquires a scoped permit from the authoritative store. Returns `None` when this deployment has no
/// admission store wired (embedded/unit-test executors keep the pre-4.1 local limiter).
pub(crate) async fn acquire(
store: Option<&Arc<dyn ExecutionAdmission>>,
coordination_store: Option<&dyn CoordinationStateStore>,
request: &crate::RuntimeExecutionRequest<'_>,
deadline: Instant,
) -> Result<Option<AdmissionLease>, RuntimeError> {
let Some(store) = store else {
return Ok(None);
};
let operation = request.operation();
let input = request.input();
let request_context = request.request_context();
let operation_id = operation.operation_id.as_str().to_owned();
let policy_mode = effective_mode(operation)?;
// The one admission snapshot carries the authoritative approval state of this exact Published
// Version, scope, input and deadline. A confirmation-requiring request without a verified
// approval never takes capacity and never dispatches: the pending human decision is answered
// here with the existing challenge, and the single-use token is consumed only at dispatch.
let approval = crate::confirmation::approval_evidence(
coordination_store,
operation,
input,
Some(request_context),
)
.await?;
let agent_key = request.agent_id().map(|agent| agent.as_str());
let workspace_id = request.workspace_id().as_str().to_owned();
let request = AdmissionRequest {
scope: AdmissionScope {
workspace_id,
agent_key: agent_key.unwrap_or(ADMIN_ORIGIN_KEY).to_owned(),
operation_id: operation_id.clone(),
operation_version: operation.operation_version,
},
mode: AdmissionMode::Synchronous,
policy_mode,
no_tasks_fallback: operation
.execution_config
.execution_policy
.map(|policy| policy.no_tasks_fallback)
.unwrap_or(NoTasksFallback::Synchronous),
approval,
fingerprint: admission_fingerprint(request_context, operation),
expires_at_unix_ms: current_unix_ms().saturating_add(
deadline_remaining_ms(deadline)
.try_into()
.unwrap_or(i64::MAX),
),
};
let started = Instant::now();
let outcome = store.admit(request).await.map_err(|error| {
record_admission_rejection(RejectionReason::DependencyUnavailable, ScopeKind::Global);
RuntimeError::AdmissionStoreUnavailable {
operation_id: operation_id.clone(),
details: error.to_string(),
}
})?;
let waited_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or_default();
match outcome {
AdmissionOutcome::Permitted(permit) => {
record_admission_queue_wait(AdmissionModeLabel::Synchronous, waited_ms);
Ok(Some(AdmissionLease::new(Arc::clone(store), permit)))
}
AdmissionOutcome::Overloaded { retry_after_ms } => {
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
Err(RuntimeError::AdmissionOverloaded {
operation_id,
retry_after_ms,
})
}
AdmissionOutcome::DeadlineElapsed => {
// Queue wait exhausted the absolute deadline before a permit was granted: nothing was
// dispatched, so this is an admission refusal, not an upstream timeout.
Err(deadline_refusal(operation))
}
AdmissionOutcome::ApprovalRequired => {
// The store refused before reserving any capacity or queue entry. Answer with the
// existing single-use challenge so the caller can confirm out of band.
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
Err(crate::confirmation::challenge(
coordination_store,
operation,
input,
Some(request_context),
)
.await)
}
AdmissionOutcome::Unsupported | AdmissionOutcome::Enqueued(_) => {
// No Task transport exists in this deployment: an enqueue decision can never become a
// runnable execution here, so it must not be reported as accepted work.
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
Err(RuntimeError::ExecutionPolicyUnsupported {
operation_id,
mode: policy_mode,
})
}
}
}
/// Typed refusal for an admission that never granted a permit within the absolute deadline. Reuses the
/// existing `execution_overloaded` vocabulary (stage `admission`, retry guidance, certain outcome) so
/// no new client error code or API shape appears, while reporting zero dispatch truthfully.
pub(crate) fn deadline_refusal(operation: &RuntimeOperation) -> RuntimeError {
record_admission_rejection(RejectionReason::Deadline, ScopeKind::Global);
RuntimeError::AdmissionOverloaded {
operation_id: operation.operation_id.as_str().to_owned(),
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
}
}
fn deadline_remaining_ms(deadline: Instant) -> u64 {
u64::try_from(
deadline
.saturating_duration_since(Instant::now())
.as_millis(),
)
.unwrap_or_default()
.max(1)
}
/// Bounded request identity for correlating one admission entry. Never contains payload, headers or
/// credentials.
fn admission_fingerprint(
request_context: &RuntimeRequestContext,
operation: &RuntimeOperation,
) -> String {
let mut fingerprint = format!(
"{}:{}:{}",
operation.operation_id.as_str(),
operation.operation_version,
request_context.request_id
);
fingerprint.truncate(crank_core::ADMISSION_FINGERPRINT_MAX_BYTES);
fingerprint
}
+120 -23
View File
@@ -2,8 +2,8 @@ use std::time::Duration;
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod, OperationSafetyClass,
OperationSafetyPolicy, Target,
ApprovalEvidence, CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod,
OperationSafetyClass, OperationSafetyPolicy, Target,
};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
@@ -26,32 +26,14 @@ pub async fn confirm_operation(
}
let Some(store) = store else {
return Err(RuntimeError::ConfirmationStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
});
return Err(confirmation_store_unavailable(operation));
};
let scope = confirmation_scope(operation, request_context)?;
let input_hash = hash_json(input)?;
let provided_token = request_context
.and_then(RuntimeRequestContext::confirmation_token)
.or_else(|| confirmation_token_from_input(input));
let Some(provided_token) = provided_token else {
let token = issue_confirmation_token(
store,
operation.operation_id.as_str(),
&scope,
&input_hash,
&safety,
)
.await?;
return Err(RuntimeError::ConfirmationRequired {
operation_id: operation.operation_id.as_str().to_owned(),
safety_class: safety.class,
confirmation_token: token,
expires_in_ms: confirmation_ttl_ms(&safety),
});
let Some(provided_token) = provided_confirmation_token(request_context, input) else {
return Err(confirmation_required(store, operation, &scope, &input_hash, &safety).await);
};
consume_confirmation_token(
@@ -64,6 +46,121 @@ pub async fn confirm_operation(
.await
}
/// Read-only approval proof for the shared admission gate (Story 4.1, AC2).
///
/// It reports the authoritative approval state of this exact Published Version, scope and input and
/// never consumes the single-use confirmation token: the authoritative consumption stays in
/// [`confirm_operation`] immediately before the adapter call. A missing approval is `NotGranted`, so
/// admission refuses the request before taking execution capacity while the human decision is
/// pending; an unusable token or an unreachable safety store is a typed refusal too, never a permit.
pub(crate) async fn approval_evidence(
store: Option<&dyn CoordinationStateStore>,
operation: &RuntimeOperation,
input: &Value,
request_context: Option<&RuntimeRequestContext>,
) -> Result<ApprovalEvidence, RuntimeError> {
let safety = effective_safety_policy(operation);
if !safety.class.requires_confirmation() {
return Ok(ApprovalEvidence::NotRequired);
}
if request_context.is_some_and(RuntimeRequestContext::approval_granted) {
return Ok(ApprovalEvidence::Granted);
}
let Some(store) = store else {
return Err(confirmation_store_unavailable(operation));
};
let scope = confirmation_scope(operation, request_context)?;
let input_hash = hash_json(input)?;
let Some(token) = provided_confirmation_token(request_context, input) else {
return Ok(ApprovalEvidence::NotGranted);
};
let stored = store
.get_value(
CacheScope::Coordination,
&confirmation_cache_key(&scope, token),
)
.await
.map_err(|_| confirmation_store_unavailable(operation))?;
let verified = stored.is_some_and(|value| {
value.payload.get("scope").and_then(Value::as_str) == Some(scope.as_str())
&& value.payload.get("input_hash").and_then(Value::as_str) == Some(input_hash.as_str())
});
if verified {
Ok(ApprovalEvidence::Granted)
} else {
Err(RuntimeError::InvalidConfirmationToken {
operation_id: operation.operation_id.as_str().to_owned(),
})
}
}
/// Issues the existing single-use confirmation challenge and returns the typed refusal that carries
/// it. Admission calls this when no verified approval exists yet, so waiting for the human decision
/// never holds execution capacity.
pub(crate) async fn challenge(
store: Option<&dyn CoordinationStateStore>,
operation: &RuntimeOperation,
input: &Value,
request_context: Option<&RuntimeRequestContext>,
) -> RuntimeError {
let safety = effective_safety_policy(operation);
let Some(store) = store else {
return confirmation_store_unavailable(operation);
};
let scope = match confirmation_scope(operation, request_context) {
Ok(scope) => scope,
Err(error) => return error,
};
let input_hash = match hash_json(input) {
Ok(input_hash) => input_hash,
Err(error) => return error,
};
confirmation_required(store, operation, &scope, &input_hash, &safety).await
}
fn provided_confirmation_token<'a>(
request_context: Option<&'a RuntimeRequestContext>,
input: &'a Value,
) -> Option<&'a str> {
request_context
.and_then(RuntimeRequestContext::confirmation_token)
.or_else(|| confirmation_token_from_input(input))
}
fn confirmation_store_unavailable(operation: &RuntimeOperation) -> RuntimeError {
RuntimeError::ConfirmationStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
}
}
async fn confirmation_required(
store: &dyn CoordinationStateStore,
operation: &RuntimeOperation,
scope: &str,
input_hash: &str,
safety: &OperationSafetyPolicy,
) -> RuntimeError {
match issue_confirmation_token(
store,
operation.operation_id.as_str(),
scope,
input_hash,
safety,
)
.await
{
Ok(token) => RuntimeError::ConfirmationRequired {
operation_id: operation.operation_id.as_str().to_owned(),
safety_class: safety.class,
confirmation_token: token,
expires_in_ms: confirmation_ttl_ms(safety),
},
Err(error) => error,
}
}
pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool {
effective_safety_policy(operation)
.class
+21
View File
@@ -43,6 +43,27 @@ pub enum RuntimeError {
confirmation_token: String,
expires_in_ms: u64,
},
#[error(
"execution mode {mode:?} of operation {operation_id} is unavailable without a durable Task transport"
)]
ExecutionPolicyUnsupported {
operation_id: String,
mode: crank_core::ToolExecutionMode,
},
#[error(
"execution admission is overloaded for operation {operation_id}; retry after {retry_after_ms} ms"
)]
AdmissionOverloaded {
operation_id: String,
retry_after_ms: u64,
},
#[error("execution admission requires a verified approval for operation {operation_id}")]
AdmissionApprovalRequired { operation_id: String },
#[error("execution admission store is unavailable for operation {operation_id}: {details}")]
AdmissionStoreUnavailable {
operation_id: String,
details: String,
},
#[error("confirmation token for operation {operation_id} is invalid, expired, or already used")]
InvalidConfirmationToken { operation_id: String },
#[error("confirmation store is unavailable for operation {operation_id}")]
@@ -57,6 +57,11 @@ fn normalize_runtime_error_with_effect(
| RuntimeError::IdempotencyCompletionUnknown { .. } => {
failure = failure.with_dispatch_uncertainty();
}
RuntimeError::AdmissionOverloaded { retry_after_ms, .. } => {
if let Ok(with_retry) = failure.clone().try_with_retry_after_ms(*retry_after_ms) {
failure = with_retry;
}
}
RuntimeError::ConfirmationRequired {
confirmation_token,
expires_in_ms,
@@ -113,6 +118,14 @@ fn error_code(error: &RuntimeError) -> ExecutionErrorCode {
RuntimeError::UnsupportedExecutionMode { .. } => {
ExecutionErrorCode::ExecutionModeUnsupported
}
RuntimeError::ExecutionPolicyUnsupported { .. } => {
ExecutionErrorCode::ExecutionModeUnsupported
}
RuntimeError::AdmissionOverloaded { .. } => ExecutionErrorCode::ExecutionOverloaded,
RuntimeError::AdmissionApprovalRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
RuntimeError::AdmissionStoreUnavailable { .. } => {
ExecutionErrorCode::SafetyStoreUnavailable
}
RuntimeError::ConcurrencyLimitExceeded { .. } => ExecutionErrorCode::ExecutionOverloaded,
RuntimeError::InvalidPreparedRequest { .. } => ExecutionErrorCode::PreparedRequestInvalid,
RuntimeError::ConfirmationRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
@@ -220,6 +220,7 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+53 -10
View File
@@ -7,9 +7,9 @@ use std::time::{Duration, Instant};
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
use crank_core::{
AdapterRegistry, CoordinationStateStore, ExecutionMode, HttpMethod, InvocationSource,
InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
Target,
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
SharedProtocolAdapter, Target,
};
use crank_metrics::{
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
@@ -41,6 +41,7 @@ pub struct RuntimeExecutor {
unary_limiter: Arc<Semaphore>,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
@@ -139,6 +140,7 @@ impl RuntimeExecutor {
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
) -> Self {
Self {
@@ -147,6 +149,7 @@ impl RuntimeExecutor {
limits,
response_cache,
coordination_store,
admission,
metering_sink,
}
}
@@ -167,6 +170,11 @@ impl RuntimeExecutor {
self
}
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub(crate) async fn execute_request(
&self,
request: RuntimeExecutionRequest<'_>,
@@ -188,11 +196,16 @@ impl RuntimeExecutor {
request.agent_id()
);
let operation_may_mutate = operation_may_mutate(operation);
let execution_deadline =
crate::admission::effective_deadline(operation, request.deadline());
let runtime_span = Stage::RuntimeExecute.span();
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
log_runtime_event("unary.execute", operation, Some(request_context));
let started_at = Instant::now();
let dispatch_started = Arc::new(AtomicBool::new(false));
// True until the admission gate answered. A deadline that fires while the request is still
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
metric_invocation_source(Some(request_context)),
request_context
@@ -206,6 +219,17 @@ impl RuntimeExecutor {
.flatten(),
);
let execution = async {
// One admission gate for every executable path (Story 4.1): the published policy decides
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
// is held for the whole local execution future, including retries.
let mut lease = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
let _permit = self.acquire_unary_permit(operation)?;
let _inflight = InFlightGuard::runtime();
let mapping_span = Stage::RuntimeArgumentsMap.span();
@@ -215,7 +239,7 @@ impl RuntimeExecutor {
let prepared_request = prepared_request?;
let request_preview = sanitized_request_preview(&prepared_request);
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
let output = self
let result = self
.execute_prepared(
operation,
input,
@@ -223,19 +247,27 @@ impl RuntimeExecutor {
Some(request_context),
Arc::clone(&dispatch_started),
)
.await?;
.await;
if let Some(lease) = lease.as_mut() {
lease.release().await;
}
let output = result?;
Ok((output, request_preview))
};
let result = tokio::time::timeout_at(
request.deadline().into(),
execution_deadline.into(),
execution.instrument(runtime_span.clone()),
)
.await
.unwrap_or_else(|_| {
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
if admission_pending.load(Ordering::Acquire) {
Err(crate::admission::deadline_refusal(operation))
} else {
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
}
});
record_runtime_result(&runtime_span, &result);
drop(runtime_span);
@@ -952,6 +984,10 @@ fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
| RuntimeError::UnsupportedProtocol { .. }
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
RuntimeError::AdmissionOverloaded { .. }
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
RuntimeError::ConfirmationRequired { .. }
| RuntimeError::InvalidConfirmationToken { .. }
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
@@ -1008,6 +1044,13 @@ fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
// label domain stays fixed; the precise code lives in the execution failure/invocation log
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
+12 -2
View File
@@ -2,8 +2,8 @@ use std::sync::Arc;
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter};
use crank_core::{
AdapterRegistry, CoordinationStateStore, MeteringSink, NoopMeteringSink, ResponseCacheStore,
SharedMeteringSink, SharedProtocolAdapter,
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, MeteringSink, NoopMeteringSink,
ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
};
use crate::{RuntimeExecutor, RuntimeLimits};
@@ -13,6 +13,7 @@ pub struct RuntimeExecutorBuilder {
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
@@ -23,6 +24,7 @@ impl RuntimeExecutorBuilder {
adapters: AdapterRegistry::empty(),
response_cache: None,
coordination_store: None,
admission: None,
metering_sink: Arc::new(NoopMeteringSink) as Arc<dyn MeteringSink>,
}
}
@@ -47,6 +49,13 @@ impl RuntimeExecutorBuilder {
self
}
/// Wires the mandatory deployment-wide admission store (Story 4.1). Production processes wire
/// the PostgreSQL implementation; an executor without it keeps the pre-4.1 local limit.
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub fn with_metering_sink(mut self, sink: SharedMeteringSink) -> Self {
self.metering_sink = sink;
self
@@ -58,6 +67,7 @@ impl RuntimeExecutorBuilder {
self.adapters,
self.response_cache,
self.coordination_store,
self.admission,
self.metering_sink,
)
}
+2 -1
View File
@@ -314,7 +314,7 @@ fn request_fingerprint(input: &Value) -> Result<String, RuntimeError> {
Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded)))
}
fn canonical_json(value: &Value) -> Value {
pub(crate) fn canonical_json(value: &Value) -> Value {
match value {
Value::Object(object) => {
let mut entries = object.iter().collect::<Vec<_>>();
@@ -446,6 +446,7 @@ mod tests {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+3
View File
@@ -1,3 +1,4 @@
mod admission;
mod auth;
mod cache;
mod cache_factory;
@@ -14,6 +15,7 @@ mod rate_limit;
mod request_context;
mod request_preparation;
mod response_cache;
mod tasks;
pub fn validate_resource_template_parameter(
schema: &crank_schema::Schema,
value: &serde_json::Value,
@@ -113,3 +115,4 @@ pub use rate_limit::{
};
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
pub use secret_crypto::SecretCrypto;
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
+266
View File
@@ -0,0 +1,266 @@
//! Runtime application service of durable Tasks (Story 4.2, FR-22/FR-23).
//!
//! The service is the only place that turns an approved background request into the durable domain
//! request: bounded validated input, canonical input fingerprint, full-scope idempotency digest,
//! pinned Published Version/deadline/retention and the support identities of the request. It holds an
//! owned request, never `ResolvedAuth` or raw secrets, and it never dispatches: the durable store
//! commits Task + dispatch intent + queue slot in one transaction and only then produces an
//! acknowledgement.
//!
//! Reads go through the same owner-scoped access proof, so the durable row is only fetched for the
//! caller that owns it. There is no in-memory fallback: a store failure is a typed refusal.
use std::sync::Arc;
use std::time::Instant;
use crank_core::{
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds,
ToolExecutionMode, current_unix_ms, validate_task_envelope,
};
use sha2::{Digest, Sha256};
/// One owned background request, already authorized by the application layer.
#[derive(Clone, Debug)]
pub struct TaskAcceptDraft {
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
/// Server-derived authorization scope identity of the verified principal.
pub authorization_scope: String,
pub operation_id: String,
pub operation_version: u32,
pub policy_mode: ToolExecutionMode,
/// Caller key, bounded before it becomes a full-scope digest.
pub scoped_key: String,
/// Canonical executable input; it is validated, fingerprinted, then persisted redacted.
pub input: serde_json::Value,
pub correlation: CorrelationContext,
/// Absolute execution deadline of the accepted work.
pub deadline: Instant,
/// Retention after the execution deadline.
pub retention_ms: i64,
pub approval_reference: Option<String>,
}
/// Projection of the durable status/result read. `result`/`error` are only present once a terminal
/// transition committed; a non-terminal Task reports its state and inputs without inventing output.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskOutcome {
pub task_id: TaskId,
pub state: crank_core::TaskState,
pub result: Option<serde_json::Value>,
pub error: Option<serde_json::Value>,
}
impl TaskOutcome {
fn from_record(record: TaskRecord) -> Self {
Self {
task_id: record.task_id,
state: record.state,
result: record.result.map(|envelope| envelope.value().clone()),
error: record.error.map(|envelope| envelope.value().clone()),
}
}
}
#[derive(Clone)]
pub struct TaskService {
store: Arc<dyn TaskStore>,
}
impl TaskService {
pub fn new(store: Arc<dyn TaskStore>) -> Self {
Self { store }
}
/// Readiness of the mandatory durable Task state.
pub async fn health(&self) -> Result<(), TaskError> {
self.store.health().await
}
/// Durable acceptance: the returned identity exists only after the store committed it.
pub async fn accept(&self, draft: TaskAcceptDraft) -> Result<TaskAcceptance, TaskError> {
validate_task_envelope(&draft.input)?;
let (scoped_key, input_fingerprint) = (
scoped_key_digest(&draft)?,
canonical_input_fingerprint(&draft.input)?,
);
let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline);
let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: draft.workspace_id,
owner_kind: draft.owner_kind,
owner_id: draft.owner_id,
authorization_scope: draft.authorization_scope,
operation_id: draft.operation_id,
pins: TaskPins {
operation_version: draft.operation_version,
policy_mode: draft.policy_mode,
execution_deadline_unix_ms,
retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms),
approval_reference: draft.approval_reference,
},
scoped_key,
input_fingerprint,
input: TaskEnvelope::try_redacted(&draft.input)?,
support: TaskSupportIds::new(
draft.correlation.request_id().clone(),
draft.correlation.trace_context(),
),
};
self.store.accept(request).await
}
/// Owner-scoped status read. A foreign or absent Task is one typed not-found.
pub async fn status(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<TaskRecord, TaskError> {
self.store.read(access, task_id).await
}
/// Owner-scoped result read, projected from the same authorized row.
pub async fn result(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<TaskOutcome, TaskError> {
Ok(TaskOutcome::from_record(
self.store.read(access, task_id).await?,
))
}
/// Default retention pinned by the application layer when it does not choose one explicitly.
pub fn default_retention_ms() -> i64 {
TASK_DEFAULT_RETENTION_MS
}
/// Scoped lookup of an already accepted Task by the same full-scope identity, used to keep a
/// retry of an explicitly pinned version from being accepted as new work.
pub async fn lookup(&self, draft: &TaskAcceptDraft) -> Result<Option<TaskRecord>, TaskError> {
let scoped_key = scoped_key_digest(draft)?;
let access = TaskAccess {
workspace_id: draft.workspace_id.clone(),
owner_kind: draft.owner_kind,
owner_id: draft.owner_id.clone(),
authorization_scope: draft.authorization_scope.clone(),
};
self.store
.find(&access, &draft.operation_id, &scoped_key)
.await
}
/// Canonical digest of a validated input, exposed so the application layer can compare a stored
/// Task against the request it is retrying without duplicating the canonicalization.
pub fn input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
canonical_input_fingerprint(input)
}
}
fn deadline_unix_ms(deadline: Instant) -> i64 {
let remaining = u64::try_from(
deadline
.saturating_duration_since(Instant::now())
.as_millis(),
)
.unwrap_or_default();
current_unix_ms().saturating_add(i64::try_from(remaining.max(1)).unwrap_or(i64::MAX))
}
/// Digest of the caller key over the full Task identity. The scope is never abbreviated to a
/// connection, protocol revision or raw credential, so two callers cannot collide on one key.
fn scoped_key_digest(draft: &TaskAcceptDraft) -> Result<String, TaskError> {
let key = draft.scoped_key.as_str();
if key.is_empty()
|| key.len() > TASK_SCOPED_KEY_MAX_BYTES
|| key.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task idempotency key is invalid"));
}
let material = format!(
"{}\n{}\n{}\n{}\n{}\n{}",
draft.workspace_id,
draft.owner_kind.as_str(),
draft.owner_id,
draft.authorization_scope,
draft.operation_id,
key
);
Ok(hex_digest(material.as_bytes()))
}
/// Canonical digest of the validated executable input. Object key order never changes the digest,
/// while any material business or secret value does.
fn canonical_input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
let canonical = crate::idempotency::canonical_json(input);
let encoded = serde_json::to_vec(&canonical)
.map_err(|_| TaskError::invalid("task input is not canonicalizable"))?;
Ok(hex_digest(&encoded))
}
fn hex_digest(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn draft(key: &str, input: serde_json::Value) -> TaskAcceptDraft {
TaskAcceptDraft {
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
operation_id: "op_report".to_owned(),
operation_version: 2,
policy_mode: ToolExecutionMode::Background,
scoped_key: key.to_owned(),
input,
correlation: CorrelationContext::generate(),
deadline: Instant::now() + std::time::Duration::from_secs(30),
retention_ms: TASK_DEFAULT_RETENTION_MS,
approval_reference: None,
}
}
#[test]
fn canonical_fingerprint_ignores_key_order_and_tracks_values() {
let first = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 2}})).unwrap();
let reordered = canonical_input_fingerprint(&json!({"b": {"c": 2}, "a": 1})).unwrap();
assert_eq!(first, reordered);
assert_eq!(first.len(), 64);
let changed = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 3}})).unwrap();
assert_ne!(first, changed);
}
#[test]
fn scoped_key_digest_covers_the_whole_identity() {
let base = scoped_key_digest(&draft("key_1", json!({}))).unwrap();
let mut foreign = draft("key_1", json!({}));
foreign.owner_id = "agent_b".to_owned();
assert_ne!(base, scoped_key_digest(&foreign).unwrap());
let mut other_scope = draft("key_1", json!({}));
other_scope.authorization_scope = "credential:pk_2".to_owned();
assert_ne!(base, scoped_key_digest(&other_scope).unwrap());
assert!(scoped_key_digest(&draft("", json!({}))).is_err());
assert!(
scoped_key_digest(&draft(
&"k".repeat(TASK_SCOPED_KEY_MAX_BYTES + 1),
json!({})
))
.is_err()
);
}
}
@@ -333,6 +333,7 @@ fn runtime_operation(draft: RestImportCandidate) -> RuntimeOperation {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -96,6 +96,7 @@ paths:
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -334,6 +334,7 @@ fn destructive_delete_operation() -> Operation<Schema, MappingSet> {
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
}),
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -340,6 +340,7 @@ fn idempotent_post_operation() -> Operation<Schema, MappingSet> {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -109,6 +109,7 @@ fn no_input_get_operation() -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -573,6 +573,7 @@ fn operation(
idempotency,
safety,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+8 -1
View File
@@ -10,7 +10,7 @@ use std::{
use async_trait::async_trait;
use crank_core::{
AdapterResponse, ConfirmationPolicy, CorrelationContext, ExecutionConfig, ExecutionErrorCode,
ExecutionMode, ExecutionOrigin, HttpMethod, IdempotencyMode, IdempotencyPolicy,
ExecutionMode, ExecutionOrigin, ExecutionStage, HttpMethod, IdempotencyMode, IdempotencyPolicy,
InvocationSource, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy,
OperationSecurityLevel, OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter,
ProtocolAdapterError, RestTarget, RetryPolicy, Retryability, Target, ToolDescription,
@@ -64,6 +64,10 @@ async fn deadline_after_dispatch_is_normalized_as_unknown_outcome() {
failure.outcome_certainty(),
OutcomeCertainty::OutcomeUnknown
);
// A deadline that fires after the adapter was called stays an upstream timeout: the admission
// refusal vocabulary (Story 4.1) only covers deadlines reached before any dispatch.
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
@@ -98,6 +102,8 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
let failure = executor.execute_outcome(request).await.unwrap_err();
assert_eq!(failure.retryability(), Retryability::AfterDelay);
assert_eq!(failure.outcome_certainty(), OutcomeCertainty::Certain);
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
assert_eq!(failure.stage(), ExecutionStage::Upstream);
}
#[tokio::test]
@@ -632,6 +638,7 @@ fn operation() -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+3 -2
View File
@@ -18,8 +18,9 @@ feature jobs должны быть `skipped` для story и `success` для fe
У Crank и FreeHub полный Rust clippy/workspace test и тяжёлые feature/security
проверки выполняются только на feature. UI build и быстрые UI unit tests остаются
на story, Playwright E2E — на feature. Crank image build/scan/compose smoke — на
feature; фактический Deploy остаётся отдельным push-main job со своими build,
scan, health и требованием успешного SHA. Тег релиза по умолчанию запускает story
feature; фактический Deploy запускается только явным workflow_dispatch на main
с validation_level=feature и deploy_target=production после успешных feature gates.
Обычный merge/push не выкатывает production. Порядок: delivery-and-deploy.md. Тег релиза по умолчанию запускает story
ATDD, сборку бинарных/UI артефактов, security audit и image scan. Полный release
clippy/workspace/E2E доступен через ручной dispatch с `validation_level: feature`;
отчёт evidence прикладывается только если такой E2E был выполнен. FreeHub
+60
View File
@@ -0,0 +1,60 @@
# Delivery и production deploy
## Разделение merge и выкатки
Push в `main` запускает только source CI: story-проверки, UI build и
`Required Checks`. Он не запускает production deploy и не загружает секреты
OpenBao. PR, метка `feature-ready`, story dispatch и dispatch без явно выбранного
production также не выкатывают production.
Production deploy выполняет только job `Deploy` workflow `CI` и только при
одновременном выполнении условий:
- событие `workflow_dispatch`;
- ref `refs/heads/main`;
- `validation_level: feature`;
- `deploy_target: production`;
- успешный `Required Checks`, включая feature jobs `Rust Feature Validation`,
`Frontend E2E` и `Community Image Smoke`.
Любая другая комбинация оставляет deploy пропущенным, поэтому принятый merge в
`main` больше не блокируется ожиданием production-выкатки.
## Ручной запуск
В Gitea UI: `Actions` -> workflow `CI` -> `Run workflow`.
Указать точные входные значения:
| Input | Значение | Обязателен |
| --- | --- | --- |
| Branch / ref | `main` | да |
| `validation_level` | `feature` | да |
| `deploy_target` | `production` | да |
`validation_level: story`, `deploy_target: none` или отсутствие `deploy_target`
дают только source CI либо feature-проверки без deploy. Нельзя запускать
production deploy из PR, с другой ветки или через push в `main`.
## Approval и runbook
Сам по себе ручной dispatch не является автоматическим разрешением на
production. Перед запуском release owner или DevOps явно подтверждает:
1. SHA совпадает с принятым и проверенным коммитом в `main`.
2. `Required Checks` для этого SHA успешны, включая feature jobs.
3. Пройдены [production checklist](../production-checklist.md) и актуальный
[deploy and staging smoke](../deploy-and-staging-smoke.md); известен порядок
rollback и проверено наличие backup.
4. Нет незапланированных изменений секретов, миграций или инфраструктуры.
После deploy проверить health endpoints и authenticated product smoke, затем
зафиксировать результат в staging/production regression notes. Секреты OpenBao
загружаются только внутри уже разрешённого job `Deploy`; не выводить их в логи и
не передавать в PR или dispatch story.
## Ограничения
Workflow `Community Release` для тегов остаётся отдельным процессом публикации
release-образов и не является заменой этому ручному production deploy. Изменение
`ci.yml` не даёт постоянного разрешения на выкатку и не меняет приложение.
+78
View File
@@ -0,0 +1,78 @@
# Переход Crank на официальный MCP SDK rmcp
Статус: планирование одобрено владельцем 27.10.2026. Полная миграция и production-переключение этим планом не запускаются. Владелец планирования и очереди — hermes_tl; реализация — hermes_dev; независимая проверка — hermes_qa; Git/CI/релизы — hermes_devops. ARC подключается к конкретному техническому затруднению, не как обязательный новый этап.
## Результат
Стандартный протокольный слой MCP обслуживает официальный SDK. Crank сохраняет авторизацию, права Agent/workspace, опубликованные версии, REST mapping, admission и исполнение команд. Обе действующие версии протокола и функции Tools/Resources/Prompts остаются доступны. Снижается объём собственного протокольного кода; ускорение и производительность не объявляются измеренными заранее.
Исходная точка: принятый main после 4.1, известный merge cc612516a29d48f2831ba6bdf58c1dc777aa6abf. Перед исполнением получить актуальный main и убедиться, что в него вошла проверенная поставка. Плановый кандидат SDK — опубликованный rmcp 3.7.0; окончательный pin и lock проверяются на первом этапе, автоматического обновления на другую версию нет.
## 1. Проверить тонкий адаптер на реальном MCP-пути
Исполнитель: DEV, независимая QA. Риск: высокий — ошибка может нарушить доступ к инструментам.
Сделать изолированный исполняемый прототип SDK на существующей границе типизированных команд, сохранив авторизацию и динамический каталог. Публичный production endpoint пока обслуживает прежний путь.
Проверяемый результат:
- Динамические tools/list и tools/call работают через SDK с действующими сервисами Crank, включая запрет чужого Agent/неверных credentials и сохранение ошибок инструмента.
- Через настоящий HTTP-путь проверены обе revisions: legacy initialize/session и modern discover/stateless, JSON/SSE, headers, progress и cancellation.
- Отдельно проверены пригодность SessionManager для существующего хранения сессий, переподключение и владение отменой; наличие SDK-типа само по себе не считается доказательством.
- Установлены существенные gaps для Resources/Prompts/notifications/subscriptions и текущих execution modes. Наличие интерфейса не выдаётся за полную совместимость.
- На небольшом smoke-наборе проверено, что adapter не обходит действующие ограничения; замеченный риск памяти/очередей записан. Нагрузочная квалификация относится к завершённой функции, не превращается в условие этого прототипа. Существующие ceilings не меняются.
- Результат — работающий код и краткое решение TL: готов перенос либо конкретная несовместимость с минимальным вариантом решения. Если потребуется ослабить контракт, заменить хранилище или создать большой дополнительный framework — вопрос владельцу до продолжения.
Это один достаточный опыт, а не серия исследований SDK. Использовать существующие wire fixtures и точечные проверки, сохранять непроверенные пункты явно.
## 2. Перенести действующие MCP-функции на SDK
Зависит от положительного результата этапа 1 и решения TL о достаточности совместимости. Исполнитель: DEV, независимая QA. Риск: высокий — затрагивается общий клиентский интерфейс.
Подключить SDK к действующему пути и перенести стандартные envelopes, dispatch, lifecycle и transport. Порядок переноса устанавливается по результату первого опыта, без параллельных writers одних MCP-файлов.
Проверяемый результат:
- Через SDK проходят действующие Tools, Resources, Prompts, completion, уведомления и подписки в применимых revisions, а также progress/cancel и существующая logging policy.
- Сохранены авторизация/отзыв доступа, agent scope, advertised capabilities, admission, deadlines и семантика ошибок. SDK defaults не расширяют объявленные capabilities или список разрешённых revisions.
- Сессии, reconnect и ограничения памяти/очередей сохраняют действующий контракт; in-memory хранилище не заменяет существующее долговременное хранение без решения.
- Перенесённые участки собственного протокольного кода удаляются по мере принятия. Временный способ безопасного сравнения/отката не превращается во второй постоянный MCP runtime.
- Существующие affected ATDD подтверждают старое поведение; тесты меняются только там, где проверяли приватное устройство старого адаптера, а не нужный клиенту контракт.
Новые бизнес-возможности, переработка REST runtime, базы, модели или UI в эту работу не входят.
## 3. Завершить замену и поставить результат
Зависит от принятого этапа 2. Исполнители: DEV/QA, затем DevOps. Риск: высокий — финальный переход общего MCP-пути. Внешняя проверка относительно реализации: независимая QA на существующем клиентском/wire-наборе обеих revisions.
Проверяемый результат:
- Удалены ставшие ненужными собственные parser/dispatch/transport участки, временные переключатели и неиспользуемые зависимости. Поддержка legacy protocol при этом сохраняется средствами SDK.
- Один необходимый финальный feature-прогон подтверждает реальный клиентский путь обеих revisions, изоляцию доступа, streaming/cancel/session и применимые ограничения ресурсов. Полный набор не повторяется отдельно каждой ролью; обязательные CI-гейты сохраняются.
- DevOps выполнил commit/push/PR, protected merge и честный release с проверенными артефактами. Документация описывает активный SDK, поддерживаемые версии и реальные ограничения.
- План отката указывает предыдущую принятую версию и проверяет совместимость состояния. Production-переключение выполняется только отдельным разрешённым manual deployment по существующему runbook; эта карточка не даёт разрешения на выкатку.
## Порядок относительно текущей разработки
1. Закончить уже выполняемую проверку поставки 4.1, не открывать её разработку заново.
2. Планировать SDK-адаптер перед дальнейшим расширением заменяемого transport/protocol слоя. Этап 1 не ждёт завершения будущих 4.2–4.7 или всей нагрузочной кампании.
3. Доменные части будущих задач могут идти параллельно при отсутствии общего writer и явных интерфейсных зависимостях. Новую протокольную реализацию и миграцию одних файлов параллельно не вести.
4. TL сопоставляет существующие 4.2–4.7 с SDK, чтобы не реализовать вручную уже предоставленный механизм. Действующие продуктовые AC не сокращаются. Неизвестная совместимость — проверка этапа 1, а не повод считать будущие функции готовыми.
## Постановка и контроль
Достаточно трёх последовательных исполнений по этапам выше, с same-card QA и действующей приёмкой TL; не создавать отдельный новый бизнес-эпик, PRD, новый тестовый framework или цепочку архитектурных согласований.
На текущем шаге TL фиксирует эти этапы в существующем backlog/Gitea, зависимости и приоритет относительно текущей работы. Будущие исполнительские карточки оставить в triage/backlog; не запускать полную миграцию автоматически из задачи планирования. Сроки не выдумывать до первого опыта. Подтверждённая непригодность SDK эскалируется владельцу с фактами и вариантами, а не молчаливым возвращением к самописному MCP.
## Основания
Канонические пути ниже находятся на Hermes `192.168.1.127` (SSH-пользователь `codex`); относительные пути считаются от `/home/codex/code/crank`. Это доступный TL источник R, а не текущая локальная папка оператора. Локальная копия аудита у оператора: `/home/bsodfather/code/hermes/deploy/mcp-sdk-audit-20261027/REPORT.md`.
- Решение владельца 27.10.2026: запланировать переход на rmcp.
- Аудит: `/home/codex/.hermes/codex-operator/handoffs/crank-rmcp-audit-20261027.md`.
- Действующие AC: `_bmad-output/initiative-crank/epic-mcp-capabilities/story-mcp-revision-capabilities.md`, в том числе оценка SDK AT-3.1-14; остальные MCP AC остаются действующими.
- Официальный SDK: https://github.com/modelcontextprotocol/rust-sdk ; документация ServerHandler: https://docs.rs/rmcp/latest/rmcp/handler/server/trait.ServerHandler.html .
- Общий процесс: `/home/codex/.hermes/team/development-delivery.md`, `validation-policy.md`, `delivery-efficiency.md`.
Предыдущее default-правило «сохранять собственный transport» не является техническим доказательством отказа от SDK. Цель нового плана — проверить и принять SDK-путь; продуктовые и security-контракты сохраняются.
+16
View File
@@ -35,3 +35,19 @@ version, stage, stable error code, retryability и certainty. Для legacy v4
Пути Admin API, MCP methods и успешные response fields не изменены. Поля ошибки
добавлены обратно совместимо. Outbound DNS/redirect/retry/SSRF policy принадлежит
Story 1.9 и этой границей не считается завершённой.
## Durable background Task (Story 4.2)
Принятие фоновой работы идёт через `TaskApplication → crank-runtime::TaskService →
TaskStore`. PostgreSQL-реализация выполняет одну транзакцию: Task, dispatch intent
(`execution_admission_entries.task_id`) и queue slot коммитятся вместе, и только
после commit выдаётся ack с Task ID/поддержкой Request/Trace/parent TraceContext.
Повтор того же полного scoped key возвращает сохранённую Task, другой canonical
input fingerprint — типизированный конфликт; явная другая published-версия при том
же ключе не перезакрепляет уже принятую работу. Owner-скоуп (workspace, Agent/admin,
authorization scope) входит в каждый read; чужой Task неотличим от отсутствующего.
Границы этой story: публичный MCP wire Tasks/capabilities/activation остаётся
Story 4.7, worker/leases/recovery — Story 4.3. Существующий `tools/call` сохраняет
synchronous/unsupported поведение; durable acceptance не запускает ephemeral
execution и не рекламирует клиенту background capability.
+2
View File
@@ -178,6 +178,8 @@ scrape_configs:
- `crank_upstream_requests_total`,
`crank_upstream_request_duration_seconds`;
- `crank_runtime_inflight`, `crank_runtime_limit_rejections_total`,
`crank_runtime_admission_queue_wait_seconds`,
`crank_runtime_admission_oldest_waiting_seconds`,
`crank_runtime_cache_total`, `crank_idempotency_total`,
`crank_confirmation_total`;
- `crank_db_pool_connections`;
+1 -1
View File
@@ -1 +1 @@
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6285,"max_rendered_series":19515,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6381,"max_rendered_series":19611,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
+1 -1
View File
@@ -1 +1 @@
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
+23
View File
@@ -55,3 +55,26 @@ run_atdd "capability advertisement registry" -p crank-community-mcp --lib protoc
# smoke) runs against the real endpoint, a real published catalog and a real counted upstream.
run_atdd "tool call lifecycle bounds" -p crank-community-mcp --lib app::lifecycle -- --test-threads=1
run_atdd "tool call progress and cancellation" -p mcp-server --test integration tools_lifecycle -- --test-threads=1
# Story 4.1 (FR-21, FR-24): a published Tool execution policy and one authoritative admission gate.
# The policy/JSON contract and the fair-scheduler arithmetic are unit-level; the authoritative store
# (scoped permits, bounded queue, typed overload, durable decisions, version pinning, fail-closed
# health) and the runtime wiring (one shared global budget across executors, typed refusals with no
# outbound call, adaptive/synchronous fallback) run against a real PostgreSQL schema.
run_atdd "execution policy contract" -p crank-core --lib operation::tests::execution_policy -- --test-threads=1
run_atdd "admission scheduler bounds" -p crank-core --lib admission:: -- --test-threads=1
run_atdd "execution policy validation" -p admin-api --lib operation_validation::tests:: -- --test-threads=1
run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1
run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1
run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1
# Story 4.2 (FR-22, FR-23): durable background Tasks. The lifecycle/envelope bounds are unit-level; the
# authoritative store (one-transaction acceptance, replay/typed conflict, shared bounded capacity,
# Task-linked intent guard, terminal CAS, retention, scope isolation, fail-closed health) and the
# application entry (verified principal, exact publication pin, foreign-scope denial, readiness) run
# against a real PostgreSQL schema. The migration suite proves a fresh and an already-migrated v25.
run_atdd "durable task domain bounds" -p crank-core --lib task:: -- --test-threads=1
run_atdd "runtime task identity" -p crank-runtime --lib tasks:: -- --test-threads=1
run_atdd "durable task store" -p crank-registry --test integration tasks:: -- --test-threads=1
run_atdd "durable task application entry" -p mcp-server --test integration durable_tasks:: -- --test-threads=1
run_atdd "durable task migration" -p crank-registry --test integration migrations:: -- --test-threads=1
+201
View File
@@ -0,0 +1,201 @@
from __future__ import annotations
import re
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".gitea" / "workflows" / "ci.yml"
def workflow_text() -> str:
return WORKFLOW.read_text(encoding="utf-8")
def job_block(name: str) -> str:
lines = workflow_text().splitlines()
start = None
for index, line in enumerate(lines):
if line == f" {name}:":
start = index
break
if start is None:
raise AssertionError(f"job {name!r} not found")
end = len(lines)
for index in range(start + 1, len(lines)):
if re.match(r"^ [A-Za-z0-9_-]+:\s*$", lines[index]):
end = index
break
return "\n".join(lines[start:end])
def deploy_guard() -> str:
match = re.search(r"(?m)^ if:\s*(.+)$", job_block("deploy"))
if match is None:
raise AssertionError("deploy job has no guard")
return match.group(1).strip()
def guard_conditions() -> dict[str, str]:
expression = deploy_guard()
if expression.startswith("${{") and expression.endswith("}}"):
expression = expression[3:-2].strip()
conditions: dict[str, str] = {}
for part in expression.split("&&"):
match = re.fullmatch(
r"([A-Za-z0-9_.]+)\s*==\s*['\"]([^'\"]+)['\"]", part.strip()
)
if match is None:
raise AssertionError(f"unsupported deploy guard condition: {part!r}")
conditions[match.group(1)] = match.group(2)
return conditions
def deploys(context: dict[str, str | None]) -> bool:
return all(
context.get(condition) == expected
for condition, expected in guard_conditions().items()
)
def dispatch_input(name: str) -> str:
match = re.search(
rf"(?m)^ {re.escape(name)}:\n((?: .*\n)+)", workflow_text()
)
if match is None:
raise AssertionError(f"workflow_dispatch input {name!r} not found")
return match.group(0)
class DeployPolicyTests(unittest.TestCase):
def test_guard_allows_only_manual_production_feature_on_main(self):
self.assertEqual(
guard_conditions(),
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
},
)
def test_manual_production_feature_on_main_deploys(self):
self.assertTrue(
deploys(
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
}
)
)
def test_push_to_main_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "push",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
}
)
)
def test_pull_request_with_feature_label_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "pull_request",
"gitea.ref": "refs/pull/42/head",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
"gitea.event.pull_request.labels.*.name": "feature-ready",
}
)
)
def test_missing_or_none_deploy_target_does_not_deploy(self):
base = {
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "feature",
}
self.assertFalse(deploys(base))
self.assertFalse(deploys({**base, "inputs.deploy_target": "none"}))
def test_story_dispatch_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/main",
"inputs.validation_level": "story",
"inputs.deploy_target": "production",
}
)
)
def test_dispatch_on_other_ref_does_not_deploy(self):
self.assertFalse(
deploys(
{
"gitea.event_name": "workflow_dispatch",
"gitea.ref": "refs/heads/release",
"inputs.validation_level": "feature",
"inputs.deploy_target": "production",
}
)
)
def test_deploy_target_is_explicit_choice_defaulting_to_none(self):
block = dispatch_input("deploy_target")
self.assertIn("type: choice", block)
self.assertIn("options: [none, production]", block)
self.assertIn("default: none", block)
self.assertIn("required: true", block)
def test_push_main_source_ci_stays_enabled(self):
self.assertRegex(
workflow_text(),
r"(?m)^ push:\n branches:\n - main$",
)
def test_deploy_requires_successful_required_checks(self):
deploy = job_block("deploy")
self.assertRegex(deploy, r"(?ms)^ needs:\n - required-checks$")
def test_required_checks_gate_feature_jobs(self):
block = job_block("required-checks")
for job in ("rust-feature", "frontend-e2e", "deployment"):
self.assertIn(f" - {job}", block)
for variable in ("RUST_FEATURE_RESULT", "E2E_RESULT", "IMAGE_RESULT"):
self.assertIn(variable, block)
def test_only_deploy_loads_production_secrets(self):
text = workflow_text()
self.assertEqual(text.count("scripts/load-openbao-env.sh"), 1)
self.assertIn("scripts/load-openbao-env.sh", job_block("deploy"))
self.assertNotIn("openbao", job_block("deployment").lower())
def test_deploy_keeps_build_scan_backup_health_and_smoke_steps(self):
deploy = job_block("deploy")
for step in (
"Load deployment secrets from OpenBao",
"Build and push images",
"Save deploy image scan evidence",
"Sync deployment files to server",
"Write environment file",
"Validate required environment variables",
"Deploy with Docker Compose",
"Verify health endpoints",
"Run authenticated product smoke",
):
self.assertIn(f"name: {step}", deploy)
self.assertIn("scripts/scan-images.sh", deploy)
self.assertIn("deploy-community.sh", deploy)
if __name__ == "__main__":
unittest.main()