Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e80d30ddb0 | ||
|
|
44774883a0 | ||
|
|
a0365bb80b | ||
|
|
2499016dc7 | ||
|
|
f613250fd3 | ||
|
|
862cde8559 |
No files matched your search
Generated
+1
@@ -826,6 +826,7 @@ dependencies = [
|
||||
name = "crank-registry"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"crank-artifacts",
|
||||
"crank-core",
|
||||
"crank-mapping",
|
||||
|
||||
@@ -181,6 +181,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
|
||||
use admin_api::{
|
||||
app::build_app,
|
||||
auth::{AuthSettings, BootstrapAdminConfig},
|
||||
pool_metrics::spawn_postgres_pool_metrics,
|
||||
pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics},
|
||||
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
|
||||
service::AdminServiceBuilder,
|
||||
state::AppState,
|
||||
@@ -169,6 +169,7 @@ async fn run(
|
||||
};
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
spawn_admission_metrics(registry.pool().clone());
|
||||
}
|
||||
let base_url = config
|
||||
.runtime
|
||||
@@ -214,6 +215,16 @@ async fn run(
|
||||
.with_limits(runtime_limits)
|
||||
.with_response_cache(cache_stores.response.clone())
|
||||
.with_coordination_store(cache_stores.coordination.clone())
|
||||
.with_admission(std::sync::Arc::new(
|
||||
crank_registry::PostgresExecutionAdmission::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?,
|
||||
))
|
||||
.build();
|
||||
let identity_provider =
|
||||
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
|
||||
@@ -321,6 +332,12 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con
|
||||
config.runtime.max_concurrent_sessions,
|
||||
)
|
||||
.map_err(|_| invalid("runtime.limits"))?;
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: config.runtime.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
}
|
||||
.validate()
|
||||
.map_err(|_| invalid("runtime.admission"))?;
|
||||
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
|
||||
RequestRateLimitConfig::new(
|
||||
config.rate_limit.requests_per_second,
|
||||
|
||||
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
|
||||
/// to zero) without waiting for a new request.
|
||||
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
|
||||
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -37,13 +37,22 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
let execution_admission = if checks.execution_admission {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
if checks.is_ready() {
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
"checks": {
|
||||
"postgres": postgres,
|
||||
"artifact_storage": artifact_storage,
|
||||
"execution_admission": execution_admission
|
||||
}
|
||||
})),
|
||||
)
|
||||
} else {
|
||||
@@ -52,7 +61,11 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
"checks": {
|
||||
"postgres": postgres,
|
||||
"artifact_storage": artifact_storage,
|
||||
"execution_admission": execution_admission
|
||||
}
|
||||
})),
|
||||
)
|
||||
}
|
||||
@@ -67,6 +80,7 @@ mod tests {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: false,
|
||||
artifact_storage: true,
|
||||
execution_admission: true,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
@@ -92,6 +106,7 @@ mod tests {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: true,
|
||||
artifact_storage: false,
|
||||
execution_admission: true,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
@@ -111,4 +126,14 @@ mod tests {
|
||||
"not_ready"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readiness_checks_identify_an_admission_state_failure() {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: true,
|
||||
artifact_storage: true,
|
||||
execution_admission: false,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
}
|
||||
}
|
||||
@@ -55,8 +55,8 @@ use crate::{
|
||||
storage::LocalArtifactStorage,
|
||||
};
|
||||
use operation_validation::{
|
||||
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
|
||||
validate_protocol_target, validate_response_cache_policy,
|
||||
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
|
||||
validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -104,11 +104,12 @@ pub use crate::dto::*;
|
||||
pub struct ReadinessChecks {
|
||||
pub postgres: bool,
|
||||
pub artifact_storage: bool,
|
||||
pub execution_admission: bool,
|
||||
}
|
||||
|
||||
impl ReadinessChecks {
|
||||
pub const fn is_ready(self) -> bool {
|
||||
self.postgres && self.artifact_storage
|
||||
self.postgres && self.artifact_storage && self.execution_admission
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +151,8 @@ pub(super) struct CredentialAuditRecord<'a> {
|
||||
impl AdminService {
|
||||
pub async fn readiness(&self) -> ReadinessChecks {
|
||||
let postgres = self.registry.ping().await.is_ok();
|
||||
// Story 4.1: the mandatory admission state is a readiness dependency of every execution.
|
||||
let execution_admission = self.registry.execution_admission_health().await.is_ok();
|
||||
let store = Arc::clone(&self.artifact_store);
|
||||
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
|
||||
.await
|
||||
@@ -157,6 +160,7 @@ impl AdminService {
|
||||
ReadinessChecks {
|
||||
postgres,
|
||||
artifact_storage,
|
||||
execution_admission,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -725,6 +729,7 @@ impl AdminService {
|
||||
validate_response_cache_policy(&payload.target, &payload.execution_config)?;
|
||||
validate_idempotency_policy(&payload.target, &payload.execution_config)?;
|
||||
validate_approval_policy(&payload.execution_config)?;
|
||||
validate_execution_policy(&payload.execution_config)?;
|
||||
payload.input_mapping.validate_paths()?;
|
||||
payload.output_mapping.validate_paths()?;
|
||||
Ok(())
|
||||
@@ -738,6 +743,7 @@ impl AdminService {
|
||||
validate_response_cache_policy(&operation.target, &operation.execution_config)?;
|
||||
validate_idempotency_policy(&operation.target, &operation.execution_config)?;
|
||||
validate_approval_policy(&operation.execution_config)?;
|
||||
validate_execution_policy(&operation.execution_config)?;
|
||||
operation.input_mapping.validate_paths()?;
|
||||
operation.output_mapping.validate_paths()?;
|
||||
Ok(())
|
||||
|
||||
@@ -441,6 +441,7 @@ fn demo_rest_operation_payload() -> OperationPayload {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -635,6 +635,7 @@ impl AdminService {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -610,6 +610,7 @@ impl AdminService {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -160,6 +160,23 @@ pub(super) fn validate_approval_policy(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_execution_policy(
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<(), ApiError> {
|
||||
let Some(policy) = execution_config.execution_policy.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Err(error) = policy.validate() {
|
||||
return Err(ApiError::validation_with_context(
|
||||
error.to_string(),
|
||||
json!({
|
||||
"field": "execution_config.execution_policy.deadline_ms",
|
||||
}),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
@@ -171,8 +188,8 @@ mod tests {
|
||||
};
|
||||
|
||||
use super::{
|
||||
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
|
||||
validate_response_cache_policy,
|
||||
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
|
||||
validate_idempotency_policy, validate_response_cache_policy,
|
||||
};
|
||||
|
||||
fn cacheable_execution_config() -> ExecutionConfig {
|
||||
@@ -183,6 +200,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
@@ -201,6 +219,7 @@ mod tests {
|
||||
}),
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
@@ -365,4 +384,36 @@ mod tests {
|
||||
"approval ttl must be between 1 and 300 seconds"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_published_execution_policy_and_absent_legacy_policy() {
|
||||
let mut config = cacheable_execution_config();
|
||||
validate_execution_policy(&config).unwrap();
|
||||
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Adaptive,
|
||||
deadline_ms: 120_000,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
});
|
||||
validate_execution_policy(&config).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_and_overflowing_execution_policy_deadline() {
|
||||
let mut config = cacheable_execution_config();
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 0,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
|
||||
});
|
||||
let error = validate_execution_policy(&config).unwrap_err();
|
||||
assert!(matches!(error, crate::error::ApiError::Validation { .. }));
|
||||
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Background,
|
||||
deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
|
||||
});
|
||||
assert!(validate_execution_policy(&config).is_err());
|
||||
}
|
||||
}
|
||||
@@ -495,6 +495,7 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -14,13 +14,14 @@ use crank_observability::{
|
||||
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
|
||||
};
|
||||
use crank_registry::{
|
||||
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
|
||||
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
|
||||
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry,
|
||||
};
|
||||
use crank_runtime::{
|
||||
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
|
||||
RuntimeLimits, SecretCrypto,
|
||||
};
|
||||
use mcp_server::pool_metrics::spawn_postgres_pool_metrics;
|
||||
use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics};
|
||||
use sqlx::postgres::PgConnectOptions;
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::info;
|
||||
@@ -175,6 +176,7 @@ async fn run(
|
||||
};
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
spawn_admission_metrics(registry.pool().clone());
|
||||
}
|
||||
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
|
||||
let secret_crypto =
|
||||
@@ -190,6 +192,10 @@ async fn run(
|
||||
.with_limits(runtime_limits)
|
||||
.with_response_cache(cache_stores.response.clone())
|
||||
.with_coordination_store(cache_stores.coordination.clone())
|
||||
.with_admission(execution_admission(
|
||||
registry.pool().clone(),
|
||||
runtime_limits,
|
||||
)?)
|
||||
.build();
|
||||
// Published static Resources are read from the same immutable content-addressed root the
|
||||
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
|
||||
@@ -287,6 +293,21 @@ fn init_observability(
|
||||
)?)
|
||||
}
|
||||
|
||||
fn execution_admission(
|
||||
pool: sqlx::PgPool,
|
||||
runtime_limits: RuntimeLimits,
|
||||
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
|
||||
let store = PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..AdmissionLimits::default()
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?;
|
||||
Ok(std::sync::Arc::new(store))
|
||||
}
|
||||
|
||||
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
|
||||
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
|
||||
database_options(&config.database).map_err(|_| invalid("database.source"))?;
|
||||
@@ -307,6 +328,12 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi
|
||||
config.runtime.max_concurrent_sessions,
|
||||
)
|
||||
.map_err(|_| invalid("runtime.limits"))?;
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: config.runtime.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
}
|
||||
.validate()
|
||||
.map_err(|_| invalid("runtime.admission"))?;
|
||||
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
|
||||
RequestRateLimitConfig::new(
|
||||
config.rate_limit.requests_per_second,
|
||||
|
||||
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
|
||||
/// to zero) without waiting for a new request.
|
||||
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
|
||||
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,874 @@
|
||||
//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission.
|
||||
//!
|
||||
//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a
|
||||
//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported
|
||||
//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal.
|
||||
//!
|
||||
//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this
|
||||
//! app already wires both and this is the real deployment shape.
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence,
|
||||
ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
|
||||
ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation,
|
||||
OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
|
||||
OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget,
|
||||
RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy,
|
||||
WorkspaceId,
|
||||
};
|
||||
use crank_mapping::MappingSet;
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
|
||||
use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
struct CountingAdapter {
|
||||
calls: Arc<AtomicUsize>,
|
||||
current: Arc<AtomicUsize>,
|
||||
peak: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProtocolAdapter for CountingAdapter {
|
||||
fn protocol(&self) -> Protocol {
|
||||
Protocol::Rest
|
||||
}
|
||||
|
||||
fn supports_mode(&self, mode: ExecutionMode) -> bool {
|
||||
mode == ExecutionMode::Unary
|
||||
}
|
||||
|
||||
async fn invoke_unary(
|
||||
&self,
|
||||
_target: &Target,
|
||||
_prepared: &crank_core::PreparedRequest,
|
||||
context: &RuntimeRequestContext,
|
||||
) -> Result<AdapterResponse, crank_core::ProtocolAdapterError> {
|
||||
context.mark_dispatch_started();
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let current = self.current.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
self.peak.fetch_max(current, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
self.current.fetch_sub(1, Ordering::SeqCst);
|
||||
Ok(AdapterResponse {
|
||||
status_code: 200,
|
||||
headers: BTreeMap::new(),
|
||||
body: json!({"ok": true}),
|
||||
data: json!({"ok": true}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn schema(kind: SchemaKind) -> Schema {
|
||||
Schema {
|
||||
kind,
|
||||
format: None,
|
||||
description: None,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: None,
|
||||
fields: BTreeMap::new(),
|
||||
items: None,
|
||||
enum_values: Vec::new(),
|
||||
variants: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn operation(policy: Option<ToolExecutionPolicy>) -> Operation<Schema, MappingSet> {
|
||||
Operation {
|
||||
id: OperationId::new("op_admission_budget"),
|
||||
name: "admission_budget".to_owned(),
|
||||
display_name: "Admission budget".to_owned(),
|
||||
category: "general".to_owned(),
|
||||
protocol: Protocol::Rest,
|
||||
security_level: OperationSecurityLevel::Standard,
|
||||
status: OperationStatus::Published,
|
||||
version: 7,
|
||||
target: Target::Rest(RestTarget {
|
||||
base_url: "https://api.example.com".to_owned(),
|
||||
method: HttpMethod::Get,
|
||||
path_template: "/v1/items".to_owned(),
|
||||
static_headers: BTreeMap::new(),
|
||||
request_contract: None,
|
||||
response_contract: Vec::new(),
|
||||
multipart_field_mime_types: BTreeMap::new(),
|
||||
}),
|
||||
input_schema: schema(SchemaKind::Object),
|
||||
output_schema: schema(SchemaKind::Object),
|
||||
input_mapping: MappingSet::default(),
|
||||
output_mapping: MappingSet::default(),
|
||||
execution_config: ExecutionConfig {
|
||||
timeout_ms: 5_000,
|
||||
execution_policy: policy,
|
||||
..ExecutionConfig::default()
|
||||
},
|
||||
tool_description: ToolDescription {
|
||||
title: "Admission".to_owned(),
|
||||
description: "Admission budget probe".to_owned(),
|
||||
tags: Vec::new(),
|
||||
examples: Vec::new(),
|
||||
},
|
||||
samples: None,
|
||||
generated_draft: None,
|
||||
config_export: None,
|
||||
wizard_state: None,
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
updated_at: OffsetDateTime::now_utc(),
|
||||
published_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
executor: &RuntimeExecutor,
|
||||
operation: &crank_runtime::RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
|
||||
let workspace = WorkspaceId::new("ws_test");
|
||||
let context = crank_runtime::RuntimeRequestContext::from_correlation(
|
||||
&crank_core::CorrelationContext::generate(),
|
||||
)
|
||||
.with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun);
|
||||
execute_in(executor, operation, input, &context).await
|
||||
}
|
||||
|
||||
async fn execute_in(
|
||||
executor: &RuntimeExecutor,
|
||||
operation: &crank_runtime::RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
context: &crank_runtime::RuntimeRequestContext,
|
||||
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
|
||||
let workspace = context
|
||||
.metering_context()
|
||||
.map(|metering| metering.workspace_id.clone())
|
||||
.unwrap_or_else(|| WorkspaceId::new("ws_test"));
|
||||
let request = crank_runtime::RuntimeExecutionRequest::try_new(
|
||||
&workspace,
|
||||
ExecutionOrigin::AdminDraft,
|
||||
None,
|
||||
operation,
|
||||
input,
|
||||
crank_runtime::ExecutionAuthorization::Authorized,
|
||||
None,
|
||||
context,
|
||||
Instant::now() + Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| {
|
||||
crank_core::ExecutionFailure::new(
|
||||
ExecutionErrorCode::RuntimeInternal,
|
||||
crank_core::CorrelationContext::new(
|
||||
context.request_id.clone(),
|
||||
context.trace_context.clone(),
|
||||
),
|
||||
)
|
||||
})?;
|
||||
executor
|
||||
.execute_outcome(request)
|
||||
.await
|
||||
.map(|success| success.output)
|
||||
}
|
||||
|
||||
async fn schema_pool(schema_url: &str) -> PgPool {
|
||||
let pool = PgPool::connect(schema_url)
|
||||
.await
|
||||
.expect("admission schema pool");
|
||||
crank_registry::MigrationAuthority::apply(&pool)
|
||||
.await
|
||||
.expect("migration authority applies the admission schema");
|
||||
pool
|
||||
}
|
||||
|
||||
fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc<PostgresExecutionAdmission> {
|
||||
Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: global,
|
||||
agent_limit: 8,
|
||||
tool_limit: 8,
|
||||
queue_limit: queue,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)
|
||||
}
|
||||
|
||||
fn executor(
|
||||
store: Arc<PostgresExecutionAdmission>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
current: Arc<AtomicUsize>,
|
||||
peak: Arc<AtomicUsize>,
|
||||
) -> RuntimeExecutor {
|
||||
RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(CountingAdapter {
|
||||
calls,
|
||||
current,
|
||||
peak,
|
||||
}))
|
||||
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
|
||||
.with_admission(store)
|
||||
.build()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_executors_share_one_global_budget() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_budget").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool.clone(), 2, 8);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let first = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
let second = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(None).into();
|
||||
let mut tasks = Vec::new();
|
||||
for index in 0..4 {
|
||||
let executor = if index % 2 == 0 {
|
||||
first.clone()
|
||||
} else {
|
||||
second.clone()
|
||||
};
|
||||
let operation = operation.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
execute(&executor, &operation, &json!({})).await
|
||||
}));
|
||||
}
|
||||
let mut completed = 0;
|
||||
for task in tasks {
|
||||
match task.await.expect("execution task") {
|
||||
Ok(_) => completed += 1,
|
||||
Err(failure) => panic!("admission must grant every request: {failure:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(completed, 4);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 4);
|
||||
assert!(
|
||||
peak.load(Ordering::SeqCst) <= 2,
|
||||
"peak concurrency {} exceeded the shared global limit",
|
||||
peak.load(Ordering::SeqCst)
|
||||
);
|
||||
assert_eq!(store.snapshot().await.unwrap().executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturation_returns_typed_overload_without_an_outbound_call() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_overload").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 1, 1);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
// Another process already holds the only global permit.
|
||||
let held = match store
|
||||
.admit(AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: "ws_test".to_owned(),
|
||||
agent_key: "admin".to_owned(),
|
||||
operation_id: "op_admission_budget".to_owned(),
|
||||
operation_version: 7,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_holder".to_owned(),
|
||||
expires_at_unix_ms: crank_core::current_unix_ms() + 10_000,
|
||||
})
|
||||
.await
|
||||
.expect("admission store is reachable")
|
||||
{
|
||||
crank_core::AdmissionOutcome::Permitted(permit) => permit,
|
||||
other => panic!("expected a permit, got {other:?}"),
|
||||
};
|
||||
|
||||
let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 250,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}))
|
||||
.into();
|
||||
let waiting_runtime = runtime.clone();
|
||||
let first_waiting_operation = waiting_operation.clone();
|
||||
let waiting = tokio::spawn(async move {
|
||||
execute(&waiting_runtime, &first_waiting_operation, &json!({})).await
|
||||
});
|
||||
// Let the first request occupy the bounded queue before the next one arrives.
|
||||
while store.snapshot().await.unwrap().waiting == 0 {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
let overloaded = execute(&runtime, &waiting_operation, &json!({})).await;
|
||||
let failure = overloaded.expect_err("a full queue must refuse the request");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::ExecutionOverloaded
|
||||
);
|
||||
assert_eq!(failure.stage(), ExecutionStage::Admission);
|
||||
assert_eq!(failure.retry_after_ms(), Some(250));
|
||||
|
||||
// The queued request hits its absolute deadline instead of dispatching. That refusal happened
|
||||
// inside admission (nothing was ever dispatched), so the client must see an admission-stage
|
||||
// overload with retry guidance — not an `upstream_timeout`.
|
||||
let expired = waiting
|
||||
.await
|
||||
.expect("waiting task")
|
||||
.expect_err("deadline must refuse");
|
||||
assert_eq!(
|
||||
expired.error_code(),
|
||||
ExecutionErrorCode::ExecutionOverloaded,
|
||||
"queue waiting counts against the published deadline"
|
||||
);
|
||||
assert_eq!(expired.stage(), ExecutionStage::Admission);
|
||||
assert_eq!(expired.retry_after_ms(), Some(250));
|
||||
assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain);
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"no outbound call on refusal"
|
||||
);
|
||||
assert!(store.release(&held).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_policy_without_a_task_transport_is_typed_unsupported() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 2_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}))
|
||||
.into();
|
||||
let failure = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect_err("no Task transport exists in Story 4.1");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::ExecutionModeUnsupported
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
let snapshot = store.snapshot().await.unwrap();
|
||||
assert_eq!(snapshot.waiting, 0);
|
||||
assert_eq!(snapshot.executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Adaptive,
|
||||
deadline_ms: 2_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}))
|
||||
.into();
|
||||
let output = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect("adaptive resolves through the published synchronous fallback");
|
||||
// The response shape is unchanged by admission: an operation without output mapping still
|
||||
// returns the same mapped output object it returned before Story 4.1.
|
||||
assert_eq!(output, json!({}));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(store.snapshot().await.unwrap().executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admission_store_outage_fails_closed_before_dispatch() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_outage").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool.clone(), 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
// The mandatory state disappears after the store was built: every execution must fail closed.
|
||||
sqlx::query("drop table execution_admission_entries")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("drop the mandatory admission state");
|
||||
let operation: crank_runtime::RuntimeOperation = operation(None).into();
|
||||
let failure = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect_err("a missing mandatory store must fail closed");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert!(store.health().await.is_err());
|
||||
}
|
||||
|
||||
/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2).
|
||||
struct RecordingAdmission {
|
||||
inner: Arc<PostgresExecutionAdmission>,
|
||||
requests: Mutex<Vec<AdmissionRequest>>,
|
||||
permits: AtomicUsize,
|
||||
releases: AtomicUsize,
|
||||
}
|
||||
|
||||
impl RecordingAdmission {
|
||||
fn new(inner: Arc<PostgresExecutionAdmission>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
permits: AtomicUsize::new(0),
|
||||
releases: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn requests(&self) -> Vec<AdmissionRequest> {
|
||||
self.requests.lock().expect("admission log").clone()
|
||||
}
|
||||
|
||||
fn permits(&self) -> usize {
|
||||
self.permits.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn releases(&self) -> usize {
|
||||
self.releases.load(Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionAdmission for RecordingAdmission {
|
||||
async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> {
|
||||
self.inner.health().await
|
||||
}
|
||||
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<crank_core::AdmissionOutcome, crank_core::AdmissionStoreError> {
|
||||
self.requests
|
||||
.lock()
|
||||
.expect("admission log")
|
||||
.push(request.clone());
|
||||
let outcome = self.inner.admit(request).await?;
|
||||
if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) {
|
||||
self.permits.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn release(
|
||||
&self,
|
||||
permit: &crank_core::AdmissionPermit,
|
||||
) -> Result<bool, crank_core::AdmissionStoreError> {
|
||||
let released = self.inner.release(permit).await?;
|
||||
if released {
|
||||
self.releases.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(released)
|
||||
}
|
||||
|
||||
async fn cancel(
|
||||
&self,
|
||||
ticket_id: u64,
|
||||
owner_token: &str,
|
||||
) -> Result<bool, crank_core::AdmissionStoreError> {
|
||||
self.inner.cancel(ticket_id, owner_token).await
|
||||
}
|
||||
|
||||
async fn snapshot(
|
||||
&self,
|
||||
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
|
||||
self.inner.snapshot().await
|
||||
}
|
||||
}
|
||||
|
||||
fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation {
|
||||
let mut source = operation(None);
|
||||
source.id = OperationId::new("op_admission_confirmation");
|
||||
source.name = "admission_confirmation".to_owned();
|
||||
source.version = version;
|
||||
let Target::Rest(target) = &mut source.target;
|
||||
target.method = HttpMethod::Delete;
|
||||
source.execution_config.safety = Some(OperationSafetyPolicy {
|
||||
class: OperationSafetyClass::Destructive,
|
||||
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
|
||||
});
|
||||
source.into()
|
||||
}
|
||||
|
||||
fn confirmation_runtime(
|
||||
admission: Arc<RecordingAdmission>,
|
||||
coordination: Option<Arc<dyn crank_core::CoordinationStateStore>>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
) -> RuntimeExecutor {
|
||||
let store: Arc<dyn ExecutionAdmission> = admission;
|
||||
let builder = RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(CountingAdapter {
|
||||
calls,
|
||||
current: Arc::new(AtomicUsize::new(0)),
|
||||
peak: Arc::new(AtomicUsize::new(0)),
|
||||
}))
|
||||
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
|
||||
.with_admission(store);
|
||||
match coordination {
|
||||
Some(coordination) => builder.with_coordination_store(coordination).build(),
|
||||
None => builder.build(),
|
||||
}
|
||||
}
|
||||
|
||||
fn confirmation_context() -> crank_runtime::RuntimeRequestContext {
|
||||
crank_runtime::RuntimeRequestContext::from_correlation(
|
||||
&crank_core::CorrelationContext::generate(),
|
||||
)
|
||||
.with_response_cache_scope("ws_test", "agent_a")
|
||||
.with_metering_context(
|
||||
WorkspaceId::new("ws_test"),
|
||||
None,
|
||||
InvocationSource::AdminTestRun,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let recording = Arc::new(RecordingAdmission::new(Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)));
|
||||
let coordination: Arc<dyn crank_core::CoordinationStateStore> =
|
||||
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = confirmation_runtime(
|
||||
Arc::clone(&recording),
|
||||
Some(Arc::clone(&coordination)),
|
||||
Arc::clone(&calls),
|
||||
);
|
||||
let operation = confirmation_operation(3);
|
||||
let context = confirmation_context();
|
||||
let input = json!({ "reason": "remove the stale item" });
|
||||
|
||||
// 1. No approval yet: the existing challenge is returned and no execution capacity is taken.
|
||||
let challenge = execute_in(&runtime, &operation, &input, &context)
|
||||
.await
|
||||
.expect_err("an unapproved destructive call must only stage confirmation");
|
||||
assert_eq!(
|
||||
challenge.error_code(),
|
||||
ExecutionErrorCode::ConfirmationRequired
|
||||
);
|
||||
let token = challenge
|
||||
.confirmation()
|
||||
.expect("expected the confirmation challenge")
|
||||
.token()
|
||||
.to_owned();
|
||||
assert_eq!(
|
||||
recording.permits(),
|
||||
0,
|
||||
"an unapproved request must not take an execution permit"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(
|
||||
requests.len(),
|
||||
1,
|
||||
"the one admission snapshot is consulted once"
|
||||
);
|
||||
assert_eq!(
|
||||
requests[0].approval,
|
||||
ApprovalEvidence::NotGranted,
|
||||
"admission must see the real approval state, not a fixed NotRequired"
|
||||
);
|
||||
assert_eq!(requests[0].scope.operation_version, 3);
|
||||
let snapshot = recording.snapshot().await.expect("admission snapshot");
|
||||
assert_eq!((snapshot.executing, snapshot.waiting), (0, 0));
|
||||
|
||||
// 2. An approval issued for another published version never opens this execution.
|
||||
let other_version = confirmation_operation(4);
|
||||
let foreign = execute_in(
|
||||
&runtime,
|
||||
&other_version,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect_err("a foreign approval must not authorise this version");
|
||||
assert_eq!(
|
||||
foreign.error_code(),
|
||||
ExecutionErrorCode::ConfirmationInvalid
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
1,
|
||||
"an unusable approval is refused before any capacity request"
|
||||
);
|
||||
|
||||
// 3. An approval is bound to its exact input as well.
|
||||
let mismatched_input = json!({ "reason": "remove another item" });
|
||||
let mismatched = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&mismatched_input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect_err("a confirmation token must not authorise different input");
|
||||
assert_eq!(
|
||||
mismatched.error_code(),
|
||||
ExecutionErrorCode::ConfirmationInvalid
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(recording.requests().len(), 1);
|
||||
|
||||
// 4. The matching approval passes admission with truthful evidence and executes exactly once.
|
||||
let executed = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("a verified approval executes");
|
||||
assert!(executed.is_object());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(requests[1].approval, ApprovalEvidence::Granted);
|
||||
assert_eq!(requests[1].scope.operation_version, 3);
|
||||
assert_eq!(recording.permits(), 1);
|
||||
assert_eq!(
|
||||
recording.releases(),
|
||||
1,
|
||||
"the permit is released after execution"
|
||||
);
|
||||
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
|
||||
|
||||
// 5. Replay of the consumed confirmation is refused before capacity and without a side effect.
|
||||
let replay = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token),
|
||||
)
|
||||
.await
|
||||
.expect_err("a confirmation token is single-use");
|
||||
assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(recording.permits(), 1);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
2,
|
||||
"a replayed approval never reaches admission"
|
||||
);
|
||||
|
||||
// 6. The trusted approved-recovery path still passes the same gate with real evidence.
|
||||
let approved = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&json!({ "reason": "approved by an operator" }),
|
||||
&context.with_approval_granted(),
|
||||
)
|
||||
.await;
|
||||
assert!(approved.is_ok());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert_eq!(requests[2].approval, ApprovalEvidence::Granted);
|
||||
assert_eq!(recording.permits(), 2);
|
||||
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mandatory_safety_store_outage_fails_closed_before_admission() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let recording = Arc::new(RecordingAdmission::new(Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)));
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
// Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail
|
||||
// closed instead of reserving capacity and hoping for an in-memory fallback.
|
||||
let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls));
|
||||
let operation = confirmation_operation(3);
|
||||
|
||||
let failure = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&json!({ "reason": "no store" }),
|
||||
&confirmation_context(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a mandatory safety store outage must fail closed");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
let after_missing_store = recording.requests().len();
|
||||
|
||||
// The same Tool with a claimed approval and a *failing* safety store never reaches the capacity
|
||||
// store at all: the approval claim stays unverified, so no permit and no outbound call exist.
|
||||
let failing = confirmation_runtime(
|
||||
Arc::clone(&recording),
|
||||
Some(Arc::new(UnavailableSafetyStore)),
|
||||
Arc::clone(&calls),
|
||||
);
|
||||
let outage = execute_in(
|
||||
&failing,
|
||||
&operation,
|
||||
&json!({ "reason": "failing store" }),
|
||||
&confirmation_context().with_confirmation_token("ct_unavailable"),
|
||||
)
|
||||
.await
|
||||
.expect_err("an unreachable safety store must refuse the claimed approval");
|
||||
assert_eq!(
|
||||
outage.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
after_missing_store,
|
||||
"an unverified approval never becomes a capacity request"
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Coordination store whose mandatory backend is down: every operation fails closed.
|
||||
struct UnavailableSafetyStore;
|
||||
|
||||
impl UnavailableSafetyStore {
|
||||
fn error() -> crank_core::CacheStoreError {
|
||||
crank_core::CacheStoreError::Unavailable {
|
||||
message: "test backend unavailable".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crank_core::CoordinationStateStore for UnavailableSafetyStore {
|
||||
async fn get_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn put_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<(), crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn delete_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<(), crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn take_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn reserve_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<crank_core::CoordinationStateReservation, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn compare_and_set_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_expected: &crank_core::CoordinationStateValue,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<bool, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
}
|
||||
@@ -14,5 +14,6 @@ mod integration {
|
||||
mod revision_matrix;
|
||||
mod static_resources;
|
||||
mod tool_search;
|
||||
mod tools_lifecycle;
|
||||
mod transport_protocol;
|
||||
}
|
||||
@@ -60,6 +60,31 @@ impl SharedLogWriter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Story 3.4: a capturing writer plus the subscriber that records into it, so a test can prove that a
|
||||
/// request-scoped progress token never reaches the log projection. Install it with
|
||||
/// `tracing::subscriber::set_default` around the request on the `current_thread` runtime.
|
||||
pub(super) fn capturing_subscriber() -> (
|
||||
CapturedLogs,
|
||||
impl tracing::Subscriber + Send + Sync + 'static,
|
||||
) {
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(writer.clone())
|
||||
.with_max_level(LevelFilter::TRACE)
|
||||
.with_ansi(false)
|
||||
.finish();
|
||||
(CapturedLogs(writer), subscriber)
|
||||
}
|
||||
|
||||
/// The captured log text of one scoped subscriber.
|
||||
pub(super) struct CapturedLogs(SharedLogWriter);
|
||||
|
||||
impl CapturedLogs {
|
||||
pub(super) fn output(&self) -> String {
|
||||
self.0.output()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
@@ -596,6 +621,7 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation<Schema, Ma
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -20,7 +20,8 @@ use std::{
|
||||
use axum::http::header;
|
||||
use crank_artifacts::ArtifactStore;
|
||||
use crank_core::{
|
||||
Agent, AgentId, AgentStatus, AgentVersion, PlatformApiKeyScope, ToolSelectionPolicy,
|
||||
Agent, AgentId, AgentStatus, AgentVersion, PlatformApiKeyScope, ToolAccessMode, ToolGroup,
|
||||
ToolSearchSettings, ToolSelectionPolicy,
|
||||
ids::{PromptId, ResourceId},
|
||||
prompt::{PromptDefinition, PromptSensitivity},
|
||||
};
|
||||
@@ -47,10 +48,13 @@ const META_CLIENT_INFO: &str = "io.modelcontextprotocol/clientInfo";
|
||||
|
||||
const RESOURCE_LIST_CHANGED: &str = "notifications/resources/list_changed";
|
||||
const PROMPT_LIST_CHANGED: &str = "notifications/prompts/list_changed";
|
||||
const TOOL_LIST_CHANGED: &str = "notifications/tools/list_changed";
|
||||
const SUBSCRIPTIONS_ACKNOWLEDGED: &str = "notifications/subscriptions/acknowledged";
|
||||
const SUBSCRIPTION_ID_KEY: &str = "io.modelcontextprotocol/subscriptionId";
|
||||
|
||||
const AGENT_SLUG: &str = "sales-signals";
|
||||
/// A second published Agent of the same workspace: its catalog must never reach another Agent.
|
||||
const OTHER_AGENT_SLUG: &str = "sales-other";
|
||||
|
||||
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
@@ -254,6 +258,26 @@ impl SignalsEndpoint {
|
||||
tool_bindings: &[crank_core::AgentOperationBinding],
|
||||
resource_bindings: &[AgentResourceBinding],
|
||||
prompt_bindings: &[AgentPromptBinding],
|
||||
) {
|
||||
self.configure_draft_with_policy(
|
||||
version,
|
||||
tool_bindings,
|
||||
&ToolSelectionPolicy::default(),
|
||||
resource_bindings,
|
||||
prompt_bindings,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// The same transition with an explicit Tool selection policy (Story 3.5: the policy is part of
|
||||
/// the served `tools/list` projection).
|
||||
async fn configure_draft_with_policy(
|
||||
&self,
|
||||
version: u32,
|
||||
tool_bindings: &[crank_core::AgentOperationBinding],
|
||||
tool_selection_policy: &ToolSelectionPolicy,
|
||||
resource_bindings: &[AgentResourceBinding],
|
||||
prompt_bindings: &[AgentPromptBinding],
|
||||
) {
|
||||
self.registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
@@ -262,7 +286,7 @@ impl SignalsEndpoint {
|
||||
agent_id: &self.agent_id,
|
||||
agent_version: version,
|
||||
bindings: tool_bindings,
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
tool_selection_policy,
|
||||
resource_bindings,
|
||||
expected_state: None,
|
||||
},
|
||||
@@ -410,6 +434,169 @@ impl SignalsEndpoint {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// The reverse Tool-only commit: the second Tool binding leaves the published snapshot while the
|
||||
/// visible Resource/Prompt projection stays identical.
|
||||
async fn commit_tool_only_revert(&self) {
|
||||
let version = self.next_draft_version().await;
|
||||
self.open_draft(version).await;
|
||||
self.configure_draft(
|
||||
version,
|
||||
&self.tool_bindings(),
|
||||
&[self.notice_binding()],
|
||||
&self.extended_prompts(),
|
||||
)
|
||||
.await;
|
||||
self.publish_version(version).await;
|
||||
}
|
||||
|
||||
/// The search-mode Tool selection policy of this Agent: one group over the bound Tool.
|
||||
fn search_policy(&self) -> ToolSelectionPolicy {
|
||||
ToolSelectionPolicy {
|
||||
mode: ToolAccessMode::Search,
|
||||
groups: vec![ToolGroup {
|
||||
id: "crm".to_owned(),
|
||||
name: "CRM".to_owned(),
|
||||
description: "CRM operations".to_owned(),
|
||||
tool_names: vec![self.operation.name.clone()],
|
||||
}],
|
||||
search: ToolSearchSettings::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A commit that changes only the Tool selection policy: same Tool, Resource and Prompt
|
||||
/// bindings, direct mode -> search mode. The served `tools/list` changes from the bound Tool to
|
||||
/// `search_tools`/`call_tool`, so this is a visible Tool catalog change.
|
||||
async fn commit_policy_only_change(&self) {
|
||||
let version = self.next_draft_version().await;
|
||||
self.open_draft(version).await;
|
||||
self.configure_draft_with_policy(
|
||||
version,
|
||||
&self.tool_bindings(),
|
||||
&self.search_policy(),
|
||||
&[],
|
||||
&self.base_prompts(),
|
||||
)
|
||||
.await;
|
||||
self.publish_version(version).await;
|
||||
}
|
||||
|
||||
/// A search-mode commit that only adds a Tool binding: `tools/list` still serves exactly
|
||||
/// `search_tools`/`call_tool`, so it must not announce a Tools change.
|
||||
async fn commit_search_mode_binding_change(&self) {
|
||||
let version = self.next_draft_version().await;
|
||||
self.open_draft(version).await;
|
||||
self.configure_draft_with_policy(
|
||||
version,
|
||||
&self.both_tool_bindings(),
|
||||
&self.search_policy(),
|
||||
&[],
|
||||
&self.base_prompts(),
|
||||
)
|
||||
.await;
|
||||
self.publish_version(version).await;
|
||||
}
|
||||
|
||||
/// Publish an independent Agent of the same workspace, bound to the first fixture operation, and
|
||||
/// return its read-only MCP client key.
|
||||
async fn publish_other_agent(&self, slug: &str) -> String {
|
||||
let agent_id = test_agent_id(slug);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
self.registry
|
||||
.create_agent(CreateAgentRequest {
|
||||
agent: &Agent {
|
||||
id: agent_id.clone(),
|
||||
workspace_id: test_workspace_id(),
|
||||
slug: slug.to_owned(),
|
||||
display_name: slug.to_owned(),
|
||||
description: "Story 3.5 isolation fixture".to_owned(),
|
||||
status: AgentStatus::Draft,
|
||||
current_draft_version: 1,
|
||||
latest_published_version: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
published_at: None,
|
||||
},
|
||||
version: &AgentVersion {
|
||||
agent_id: agent_id.clone(),
|
||||
version: 1,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: now,
|
||||
},
|
||||
bindings: &[binding_for_operation(&self.operation)],
|
||||
})
|
||||
.await
|
||||
.expect("the second Agent is created");
|
||||
self.registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::now_utc(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.expect("the second Agent publishes");
|
||||
create_platform_api_key(
|
||||
&self.registry,
|
||||
slug,
|
||||
"mcp-signals-other",
|
||||
&[PlatformApiKeyScope::Read],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// A committed Tool-only change of the independent Agent: it binds the second fixture Tool.
|
||||
async fn commit_other_agent_tool_change(&self, slug: &str) {
|
||||
let agent_id = test_agent_id(slug);
|
||||
let version = AgentVersion {
|
||||
agent_id: agent_id.clone(),
|
||||
version: 2,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
};
|
||||
self.registry
|
||||
.create_agent_draft_version(CreateAgentDraftVersionRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: &version,
|
||||
bindings: &[],
|
||||
updated_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("a second draft version of the other Agent opens");
|
||||
self.registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
agent_version: 2,
|
||||
bindings: &self.both_tool_bindings(),
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[],
|
||||
expected_state: None,
|
||||
},
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.expect("the other Agent's second catalog config saves");
|
||||
self.registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: 2,
|
||||
published_at: &OffsetDateTime::now_utc(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.expect("the other Agent's second version publishes");
|
||||
}
|
||||
}
|
||||
|
||||
/// A live SSE response, read event by event.
|
||||
@@ -714,6 +901,17 @@ fn resource_uris(list: &Value) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn tool_names(list: &Value) -> Vec<String> {
|
||||
let mut names: Vec<String> = list["result"]["tools"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| panic!("tools/list must return a page: {list}"))
|
||||
.iter()
|
||||
.map(|tool| tool["name"].as_str().unwrap().to_owned())
|
||||
.collect();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
/// Write a stand file that only the current user may read (credential files live here).
|
||||
fn write_private(path: &PathBuf, contents: &str) {
|
||||
use std::io::Write as _;
|
||||
|
||||
@@ -127,12 +127,20 @@ async fn legacy_stream_announces_a_committed_catalog_change_after_the_commit() {
|
||||
"resources/list must serve the newly bound Resource: {resources}"
|
||||
);
|
||||
|
||||
// A Tool-only commit keeps the Resource/Prompt projection: no further signal.
|
||||
// A Tool-only commit keeps the Resource/Prompt projection: only a Tools frame follows (Story 3.5).
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let frame = stream
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
assert!(
|
||||
frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {frame}"
|
||||
);
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"signal for a Tool-only commit",
|
||||
"a Resource or Prompt signal for a Tool-only commit",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -162,7 +170,11 @@ async fn modern_listen_stream_acknowledges_then_announces_coalesced_changes() {
|
||||
);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "promptsListChanged": true, "resourcesListChanged": true }),
|
||||
json!({
|
||||
"promptsListChanged": true,
|
||||
"resourcesListChanged": true,
|
||||
"toolsListChanged": true
|
||||
}),
|
||||
"only the implemented list-change types are acknowledged: {acknowledgement}"
|
||||
);
|
||||
|
||||
@@ -173,7 +185,7 @@ async fn modern_listen_stream_acknowledges_then_announces_coalesced_changes() {
|
||||
endpoint.commit_tool_only_change().await;
|
||||
|
||||
let mut methods = Vec::new();
|
||||
for _ in 0..2 {
|
||||
for _ in 0..3 {
|
||||
let frame = stream
|
||||
.expect_frame("the coalesced list-change signal")
|
||||
.await;
|
||||
@@ -185,7 +197,8 @@ async fn modern_listen_stream_acknowledges_then_announces_coalesced_changes() {
|
||||
methods,
|
||||
vec![
|
||||
PROMPT_LIST_CHANGED.to_owned(),
|
||||
RESOURCE_LIST_CHANGED.to_owned()
|
||||
RESOURCE_LIST_CHANGED.to_owned(),
|
||||
TOOL_LIST_CHANGED.to_owned()
|
||||
]
|
||||
);
|
||||
stream
|
||||
@@ -228,8 +241,8 @@ async fn modern_listen_delivers_only_the_negotiated_signal_types() {
|
||||
);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "promptsListChanged": true }),
|
||||
"only the implemented and requested type is acknowledged: {acknowledgement}"
|
||||
json!({ "promptsListChanged": true, "toolsListChanged": true }),
|
||||
"only the implemented and requested types are acknowledged: {acknowledgement}"
|
||||
);
|
||||
|
||||
// The watcher takes its baseline before it can report anything.
|
||||
@@ -256,7 +269,7 @@ async fn modern_listen_delivers_only_the_negotiated_signal_types() {
|
||||
json!("prompts-only")
|
||||
);
|
||||
|
||||
// Neither `toolsListChanged` nor `resourceSubscriptions` were honoured, so nothing else follows.
|
||||
// No Tool commit happened, so no Tools frame follows; `resourceSubscriptions` was not honoured.
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
@@ -354,6 +367,11 @@ async fn a_lost_notification_is_recovered_through_the_authoritative_methods() {
|
||||
discover["result"]["capabilities"]["resources"]["listChanged"],
|
||||
json!(true)
|
||||
);
|
||||
assert_eq!(
|
||||
discover["result"]["capabilities"]["tools"]["listChanged"],
|
||||
json!(true),
|
||||
"the Tools list-change capability is advertised: {discover}"
|
||||
);
|
||||
let resources = endpoint.modern("resources/list", json!({})).await;
|
||||
assert!(
|
||||
resource_uris(&resources)
|
||||
@@ -414,3 +432,360 @@ async fn a_lost_notification_is_recovered_through_the_authoritative_methods() {
|
||||
"re-initialization serves the authoritative catalog: {prompts}"
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Story 3.5 (FR-14): Tool catalog change signals. A client that negotiated the Tools list-changed
|
||||
// notification learns about a *committed* Tool catalog change, and `tools/list` stays the
|
||||
// authoritative source for a client without the notification or after a reconnect.
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
/// AT-3.5-01/02 (modern): the Tools signal is delivered only to the stream that negotiated it, once
|
||||
/// per committed change, and `tools/list` serves the same committed snapshot.
|
||||
#[tokio::test]
|
||||
async fn modern_tools_list_changed_is_delivered_only_to_a_client_that_asks_for_it() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
let mut tools = endpoint
|
||||
.listen(json!(51), json!({ "toolsListChanged": true }))
|
||||
.await;
|
||||
let mut prompts = endpoint
|
||||
.listen(json!(52), json!({ "promptsListChanged": true }))
|
||||
.await;
|
||||
|
||||
let acknowledgement = tools.expect_frame("the Tools acknowledgment").await;
|
||||
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "toolsListChanged": true }),
|
||||
"the Tools list-change type must be honoured and acknowledged: {acknowledgement}"
|
||||
);
|
||||
let other = prompts.expect_frame("the Prompts acknowledgment").await;
|
||||
assert_eq!(
|
||||
other["params"]["notifications"],
|
||||
json!({ "promptsListChanged": true }),
|
||||
"a stream that did not ask for Tools is acknowledged without it: {other}"
|
||||
);
|
||||
|
||||
// The watchers take their baseline before they can report anything.
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
|
||||
// A draft-only edit is not a commit: nothing is announced.
|
||||
endpoint.edit_draft_without_publishing().await;
|
||||
tools
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"Tools signal for a draft-only edit",
|
||||
)
|
||||
.await;
|
||||
|
||||
// A committed Resource/Prompt change is not a Tools change: the Tools stream stays silent while
|
||||
// the Prompts stream receives exactly its own type.
|
||||
endpoint.commit_resource_and_prompt_change().await;
|
||||
let prompt_frame = prompts
|
||||
.expect_frame("the committed Prompt list-change signal")
|
||||
.await;
|
||||
assert_eq!(prompt_frame["method"], PROMPT_LIST_CHANGED);
|
||||
tools
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"Tools signal for a Resource/Prompt commit",
|
||||
)
|
||||
.await;
|
||||
|
||||
// One committed Tool catalog change is announced exactly once.
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let frame = tools
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["jsonrpc"], json!("2.0"));
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
assert_eq!(frame["params"]["_meta"][SUBSCRIPTION_ID_KEY], json!(51));
|
||||
tools
|
||||
.expect_silence(Duration::from_millis(2_500), "a repeated Tools signal")
|
||||
.await;
|
||||
|
||||
// The prompts-only stream was never told about Tools, even though the commit happened after it
|
||||
// opened.
|
||||
prompts
|
||||
.expect_silence(Duration::from_millis(1_200), "unrequested Tools signal")
|
||||
.await;
|
||||
|
||||
// `tools/list` is authoritative for the same committed state, notification or not.
|
||||
let list = endpoint.modern("tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()]
|
||||
);
|
||||
}
|
||||
|
||||
/// AT-3.5-01/02 (legacy): the legacy `GET` stream carries the Tools signal after the commit; a
|
||||
/// Resource/Prompt commit produces no Tools frame; a committed change of another Agent is never
|
||||
/// disclosed on this stream, and `tools/list` stays authoritative.
|
||||
#[tokio::test]
|
||||
async fn legacy_stream_announces_a_committed_tools_change_and_never_another_agents() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
let other_key = endpoint.publish_other_agent(OTHER_AGENT_SLUG).await;
|
||||
let session = endpoint.legacy_session().await;
|
||||
|
||||
let mut stream = endpoint.legacy_stream(&session).await;
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
stream
|
||||
.expect_silence(Duration::from_millis(1_200), "signal before any commit")
|
||||
.await;
|
||||
|
||||
// A committed Resource/Prompt change keeps the Tool binding set: no Tools frame.
|
||||
endpoint.commit_resource_and_prompt_change().await;
|
||||
let mut methods = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let frame = stream.expect_frame("the committed catalog signals").await;
|
||||
assert!(
|
||||
frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {frame}"
|
||||
);
|
||||
methods.push(frame["method"].as_str().unwrap().to_owned());
|
||||
}
|
||||
methods.sort();
|
||||
assert_eq!(
|
||||
methods,
|
||||
vec![
|
||||
PROMPT_LIST_CHANGED.to_owned(),
|
||||
RESOURCE_LIST_CHANGED.to_owned()
|
||||
],
|
||||
"a Resource/Prompt commit must not announce Tools"
|
||||
);
|
||||
|
||||
// A committed Tool change of another Agent of the same workspace never reaches this stream or
|
||||
// this Agent's `tools/list`.
|
||||
endpoint
|
||||
.commit_other_agent_tool_change(OTHER_AGENT_SLUG)
|
||||
.await;
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"Tools signal for another Agent's commit",
|
||||
)
|
||||
.await;
|
||||
let isolated = endpoint.legacy(&session, "tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&isolated),
|
||||
vec!["crm_create_lead".to_owned()],
|
||||
"another Agent's commit must not change this Agent's catalog: {isolated}"
|
||||
);
|
||||
// The commit really happened: the other Agent serves its own two-Tool catalog.
|
||||
let other_url = agent_mcp_url(&endpoint.base_url, OTHER_AGENT_SLUG);
|
||||
let other_session = initialize_session(&endpoint.client, &other_url, &other_key).await;
|
||||
let other_list = endpoint
|
||||
.client
|
||||
.post(&other_url)
|
||||
.header(header::ACCEPT, "application/json, text/event-stream")
|
||||
.header(header::AUTHORIZATION, format!("Bearer {other_key}"))
|
||||
.header("MCP-Protocol-Version", LEGACY_REVISION)
|
||||
.header("MCP-Session-Id", &other_session)
|
||||
.json(&json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 7,
|
||||
"method": "tools/list",
|
||||
"params": {}
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let other_list: Value = serde_json::from_str(&other_list.text().await.unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
tool_names(&other_list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
|
||||
"the other Agent's committed catalog changed: {other_list}"
|
||||
);
|
||||
|
||||
// This Agent's committed Tool change is announced after the commit, exactly once.
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let frame = stream
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
assert!(
|
||||
frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {frame}"
|
||||
);
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a repeated or unrequested signal",
|
||||
)
|
||||
.await;
|
||||
|
||||
let list = endpoint.legacy(&session, "tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
|
||||
"tools/list must serve the committed catalog without the notification: {list}"
|
||||
);
|
||||
}
|
||||
|
||||
/// AT-3.5-04: the Tools signal follows the same projection `tools/list` serves, so a committed
|
||||
/// publication that only changes the Tool selection policy is announced on both revisions, while a
|
||||
/// committed binding change that leaves `tools/list` unchanged is silent.
|
||||
#[tokio::test]
|
||||
async fn a_policy_only_commit_is_announced_on_both_revisions_and_an_invisible_change_is_silent() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
let direct_projection = vec!["crm_create_lead".to_owned()];
|
||||
// `tool_names` sorts the served page.
|
||||
let search_projection = vec!["call_tool".to_owned(), "search_tools".to_owned()];
|
||||
|
||||
// Both revisions negotiate the Tools signal and take their baseline before anything can arrive.
|
||||
let mut modern = endpoint
|
||||
.listen(json!(61), json!({ "toolsListChanged": true }))
|
||||
.await;
|
||||
let acknowledgement = modern.expect_frame("the Tools acknowledgment").await;
|
||||
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "toolsListChanged": true })
|
||||
);
|
||||
let session = endpoint.legacy_session().await;
|
||||
let mut legacy = endpoint.legacy_stream(&session).await;
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
|
||||
// The served projection before the commit is the direct-mode binding itself.
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.modern("tools/list", json!({})).await),
|
||||
direct_projection
|
||||
);
|
||||
|
||||
// A policy-only commit: identical Tool/Resource/Prompt bindings, direct -> search mode. The
|
||||
// served list changes, so each negotiated stream receives exactly one Tools frame.
|
||||
endpoint.commit_policy_only_change().await;
|
||||
let modern_frame = modern
|
||||
.expect_frame("the committed Tools list-change signal (modern)")
|
||||
.await;
|
||||
assert_eq!(modern_frame["method"], TOOL_LIST_CHANGED);
|
||||
assert_eq!(
|
||||
modern_frame["params"]["_meta"][SUBSCRIPTION_ID_KEY],
|
||||
json!(61)
|
||||
);
|
||||
let legacy_frame = legacy
|
||||
.expect_frame("the committed Tools list-change signal (legacy)")
|
||||
.await;
|
||||
assert_eq!(legacy_frame["method"], TOOL_LIST_CHANGED);
|
||||
assert!(
|
||||
legacy_frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {legacy_frame}"
|
||||
);
|
||||
modern
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a repeated modern Tools signal",
|
||||
)
|
||||
.await;
|
||||
legacy
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a repeated legacy Tools signal",
|
||||
)
|
||||
.await;
|
||||
|
||||
// `tools/list` is authoritative for the new projection on both revisions.
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.modern("tools/list", json!({})).await),
|
||||
search_projection
|
||||
);
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.legacy(&session, "tools/list", json!({})).await),
|
||||
search_projection
|
||||
);
|
||||
|
||||
// The inverse direction: in search mode a binding-only commit does not change `tools/list`, so
|
||||
// neither stream may claim a visible Tool change.
|
||||
endpoint.commit_search_mode_binding_change().await;
|
||||
modern
|
||||
.expect_silence(
|
||||
Duration::from_millis(3_500),
|
||||
"a Tools signal for an unchanged modern tools/list",
|
||||
)
|
||||
.await;
|
||||
legacy
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a Tools signal for an unchanged legacy tools/list",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.modern("tools/list", json!({})).await),
|
||||
search_projection
|
||||
);
|
||||
}
|
||||
|
||||
/// AT-3.5-02/03: a client that missed the notification recovers Tool discovery from the authoritative
|
||||
/// state — the modern stateless client after a reconnect, and the legacy client after re-initializing
|
||||
/// a retired session — and a reconnected stream never replays the missed change.
|
||||
#[tokio::test]
|
||||
async fn a_reconnected_client_recovers_tools_discovery_from_the_authoritative_state() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
|
||||
// Modern: the commit happens while no stream is open at all, then a client reconnects.
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let mut reconnected = endpoint
|
||||
.listen(
|
||||
json!("after-reconnect"),
|
||||
json!({ "toolsListChanged": true }),
|
||||
)
|
||||
.await;
|
||||
let acknowledgement = reconnected.expect_frame("the Tools acknowledgment").await;
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "toolsListChanged": true })
|
||||
);
|
||||
// The baseline of a new stream is the committed state: the missed change is not replayed.
|
||||
reconnected
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a replayed Tools signal after reconnect",
|
||||
)
|
||||
.await;
|
||||
let list = endpoint.modern("tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
|
||||
"a client without the notification recovers the committed catalog: {list}"
|
||||
);
|
||||
|
||||
// Legacy: one more committed Tool change is announced on the live stream, then the session is
|
||||
// retired and the client re-initializes.
|
||||
let session = endpoint.legacy_session().await;
|
||||
let mut stale = endpoint.legacy_stream(&session).await;
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
endpoint.commit_tool_only_revert().await;
|
||||
let frame = stale
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
|
||||
let deleted = endpoint
|
||||
.client
|
||||
.delete(&endpoint.mcp_url)
|
||||
.header(
|
||||
header::AUTHORIZATION,
|
||||
format!("Bearer {}", endpoint.api_key),
|
||||
)
|
||||
.header("MCP-Protocol-Version", LEGACY_REVISION)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(deleted.status(), reqwest::StatusCode::NO_CONTENT);
|
||||
|
||||
let fresh = endpoint.legacy_session().await;
|
||||
let list = endpoint.legacy(&fresh, "tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned()],
|
||||
"re-initialization serves the committed catalog: {list}"
|
||||
);
|
||||
// A stream reopened after the reconnect starts from the committed baseline, not from a replay.
|
||||
let mut reopened = endpoint.legacy_stream(&fresh).await;
|
||||
reopened
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a replayed Tools signal on a reconnected stream",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,868 @@
|
||||
//! Story 3.4 (FR-14) ATDD: progress and cancellation of one synchronous `tools/call`.
|
||||
//!
|
||||
//! Every case drives the real HTTP endpoint of the real application against a real (counted, slow)
|
||||
//! upstream and real published catalog/registry state; nothing here is a fake client of the server
|
||||
//! internals. The oracles are the pinned MCP shapes: `notifications/progress` (monotonic `progress`,
|
||||
//! the request's own `progressToken`, no invented `total`) and the terminal `tools/call` result.
|
||||
//!
|
||||
//! Cancellation is revision-correct: `2025-11-25` cancels with `notifications/cancelled`, and
|
||||
//! `2026-07-28` cancels by closing the request's own response stream. Both are proven to stop the
|
||||
//! work, to release the call, and to never produce a `completed` record.
|
||||
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{Json, Router, routing::post};
|
||||
use crank_core::{InvocationStatus, OperationId, PlatformApiKeyScope};
|
||||
use crank_registry::{ListInvocationLogsQuery, PostgresRegistry, PublishRequest};
|
||||
use serde_json::{Value, json};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use super::common::{
|
||||
agent_mcp_url, build_test_app, capturing_subscriber, create_platform_api_key,
|
||||
initialize_session, post_jsonrpc, post_jsonrpc_with_headers, publish_agent_for_operation,
|
||||
spawn_mcp_server, test_operation, test_registry, test_workspace_id,
|
||||
};
|
||||
|
||||
const LEGACY: &str = "2025-11-25";
|
||||
const MODERN: &str = "2026-07-28";
|
||||
const META_PROTOCOL_VERSION: &str = "io.modelcontextprotocol/protocolVersion";
|
||||
const META_CLIENT_CAPABILITIES: &str = "io.modelcontextprotocol/clientCapabilities";
|
||||
const META_CLIENT_INFO: &str = "io.modelcontextprotocol/clientInfo";
|
||||
|
||||
const TOOL: &str = "crm_lifecycle_lead";
|
||||
/// A call that is long enough for the client to observe real progress, short enough for a suite.
|
||||
const HOLD_MS: u64 = 250;
|
||||
/// A call that is still running when the client cancels it.
|
||||
const LONG_HOLD_MS: u64 = 1_500;
|
||||
/// The grace period after a cancellation. It must exceed [`LONG_HOLD_MS`]: a call that was *not*
|
||||
/// stopped completes its upstream request and records the outcome inside this window, so the checks
|
||||
/// after it prove that the cancellation stopped the call rather than only that it had not finished
|
||||
/// yet.
|
||||
const CANCEL_GRACE_MS: u64 = 2_500;
|
||||
/// The suite depends on that order: the constant must not silently drop back below the hold.
|
||||
const _: () = assert!(CANCEL_GRACE_MS > LONG_HOLD_MS);
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Revision {
|
||||
Legacy,
|
||||
Modern,
|
||||
}
|
||||
|
||||
impl Revision {
|
||||
const fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Legacy => LEGACY,
|
||||
Self::Modern => MODERN,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Upstream {
|
||||
base_url: String,
|
||||
started: Arc<AtomicUsize>,
|
||||
completed: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
/// A real upstream that records when a request starts and when it finishes. A cancelled call must
|
||||
/// increment `started` exactly once and never `completed`: the request really was dispatched (so the
|
||||
/// outcome is not `certain`-not-sent) and its response really was abandoned (so it is not retried).
|
||||
async fn spawn_counted_slow_upstream(hold_ms: u64) -> Upstream {
|
||||
let started = Arc::new(AtomicUsize::new(0));
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let handler_started = Arc::clone(&started);
|
||||
let handler_completed = Arc::clone(&completed);
|
||||
let app = Router::new().route(
|
||||
"/crm/lifecycle-leads",
|
||||
post(move |Json(payload): Json<Value>| {
|
||||
let started = Arc::clone(&handler_started);
|
||||
let completed = Arc::clone(&handler_completed);
|
||||
async move {
|
||||
started.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(hold_ms)).await;
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
Json(json!({ "id": "lead_123", "email": payload["email"] }))
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Upstream {
|
||||
base_url: format!("http://{address}"),
|
||||
started,
|
||||
completed,
|
||||
}
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
client: reqwest::Client,
|
||||
mcp_url: String,
|
||||
api_key: String,
|
||||
other_api_key: String,
|
||||
registry: PostgresRegistry,
|
||||
operation_id: OperationId,
|
||||
upstream: Upstream,
|
||||
}
|
||||
|
||||
async fn fixture(agent_slug: &str, hold_ms: u64, timeout_ms: u64) -> Fixture {
|
||||
let upstream = spawn_counted_slow_upstream(hold_ms).await;
|
||||
let registry = test_registry().await;
|
||||
let mut operation = test_operation(&upstream.base_url, TOOL);
|
||||
let crank_core::Target::Rest(target) = &mut operation.target;
|
||||
target.path_template = "/crm/lifecycle-leads".to_owned();
|
||||
operation.execution_config.timeout_ms = timeout_ms;
|
||||
registry
|
||||
.create_operation(&test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent_for_operation(®istry, &operation, agent_slug).await;
|
||||
let scopes = [PlatformApiKeyScope::Read, PlatformApiKeyScope::Write];
|
||||
let api_key = create_platform_api_key(®istry, agent_slug, "mcp-lifecycle", &scopes).await;
|
||||
let other_api_key =
|
||||
create_platform_api_key(®istry, agent_slug, "mcp-lifecycle-other", &scopes).await;
|
||||
let base_url = spawn_mcp_server(build_test_app(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Some("https://crank.example.com".to_owned()),
|
||||
))
|
||||
.await;
|
||||
|
||||
Fixture {
|
||||
client: reqwest::Client::new(),
|
||||
mcp_url: agent_mcp_url(&base_url, agent_slug),
|
||||
api_key,
|
||||
other_api_key,
|
||||
registry,
|
||||
operation_id: operation.id,
|
||||
upstream,
|
||||
}
|
||||
}
|
||||
|
||||
/// One parsed answer: the HTTP status, the negotiated media type and every JSON-RPC frame in order.
|
||||
struct Reply {
|
||||
status: reqwest::StatusCode,
|
||||
content_type: String,
|
||||
frames: Vec<Value>,
|
||||
raw: String,
|
||||
}
|
||||
|
||||
impl Reply {
|
||||
fn parse(status: reqwest::StatusCode, content_type: String, body: String) -> Self {
|
||||
let mut frames = Vec::new();
|
||||
for block in body.split("\n\n") {
|
||||
for line in block.lines() {
|
||||
let Some(data) = line.strip_prefix("data:") else {
|
||||
continue;
|
||||
};
|
||||
let data = data.trim_start();
|
||||
if data.is_empty() {
|
||||
continue;
|
||||
}
|
||||
frames.push(
|
||||
serde_json::from_str(data)
|
||||
.unwrap_or_else(|error| panic!("every frame is JSON: {data:?} {error}")),
|
||||
);
|
||||
}
|
||||
}
|
||||
if frames.is_empty() && !body.trim().is_empty() {
|
||||
frames.push(
|
||||
serde_json::from_str(body.trim()).unwrap_or_else(|error| {
|
||||
panic!("expected a JSON-RPC body, got {body:?}: {error}")
|
||||
}),
|
||||
);
|
||||
}
|
||||
Self {
|
||||
status,
|
||||
content_type,
|
||||
frames,
|
||||
raw: body,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_stream(&self) {
|
||||
assert_eq!(self.status, reqwest::StatusCode::OK);
|
||||
assert!(
|
||||
self.content_type.starts_with("text/event-stream"),
|
||||
"a progress answer must be the request's SSE stream, got {:?}: {}",
|
||||
self.content_type,
|
||||
self.raw
|
||||
);
|
||||
}
|
||||
|
||||
fn progress_frames(&self) -> Vec<&Value> {
|
||||
self.frames
|
||||
.iter()
|
||||
.filter(|frame| frame["method"] == json!("notifications/progress"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn terminal(&self) -> &Value {
|
||||
let terminal = self
|
||||
.frames
|
||||
.last()
|
||||
.unwrap_or_else(|| panic!("the stream must carry a terminal frame: {}", self.raw));
|
||||
assert!(
|
||||
terminal.get("method").is_none(),
|
||||
"the terminal frame must be the JSON-RPC response, got {terminal}"
|
||||
);
|
||||
terminal
|
||||
}
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn call_params(&self, token: Option<Value>, revision: Revision) -> Value {
|
||||
let mut params = json!({
|
||||
"name": TOOL,
|
||||
"arguments": { "email": "lifecycle@example.com" },
|
||||
});
|
||||
let mut meta = match revision {
|
||||
Revision::Modern => json!({
|
||||
META_PROTOCOL_VERSION: MODERN,
|
||||
META_CLIENT_CAPABILITIES: {},
|
||||
META_CLIENT_INFO: { "name": "crank-3.4-atdd", "version": "0.1.0" },
|
||||
}),
|
||||
Revision::Legacy => json!({}),
|
||||
};
|
||||
if let Some(token) = token {
|
||||
meta["progressToken"] = token;
|
||||
}
|
||||
if meta.as_object().is_some_and(|object| !object.is_empty()) {
|
||||
params["_meta"] = meta;
|
||||
}
|
||||
params
|
||||
}
|
||||
|
||||
fn headers(&self, revision: Revision, session: Option<&str>) -> Vec<(&'static str, String)> {
|
||||
let mut headers = vec![("MCP-Protocol-Version", revision.label().to_owned())];
|
||||
match revision {
|
||||
Revision::Legacy => {
|
||||
if let Some(session) = session {
|
||||
headers.push(("MCP-Session-Id", session.to_owned()));
|
||||
}
|
||||
}
|
||||
Revision::Modern => {
|
||||
headers.push(("Mcp-Method", "tools/call".to_owned()));
|
||||
headers.push(("Mcp-Name", TOOL.to_owned()));
|
||||
}
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
fn request(
|
||||
&self,
|
||||
accept: &'static str,
|
||||
key: &str,
|
||||
headers: &[(&'static str, String)],
|
||||
payload: Value,
|
||||
) -> reqwest::RequestBuilder {
|
||||
let mut request = self
|
||||
.client
|
||||
.post(&self.mcp_url)
|
||||
.header(reqwest::header::ACCEPT, accept)
|
||||
.header(reqwest::header::AUTHORIZATION, format!("Bearer {key}"));
|
||||
for (name, value) in headers {
|
||||
request = request.header(*name, value);
|
||||
}
|
||||
request.json(&payload)
|
||||
}
|
||||
|
||||
/// One `tools/call` whose answer is negotiated as the request's own SSE stream.
|
||||
async fn call_stream(
|
||||
&self,
|
||||
revision: Revision,
|
||||
id: u64,
|
||||
params: Value,
|
||||
session: Option<&str>,
|
||||
request_id: &str,
|
||||
) -> Reply {
|
||||
self.call_stream_with_key(revision, &self.api_key, id, params, session, request_id)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn call_stream_with_key(
|
||||
&self,
|
||||
revision: Revision,
|
||||
key: &str,
|
||||
id: u64,
|
||||
params: Value,
|
||||
session: Option<&str>,
|
||||
request_id: &str,
|
||||
) -> Reply {
|
||||
self.send(
|
||||
"text/event-stream",
|
||||
key,
|
||||
revision,
|
||||
id,
|
||||
params,
|
||||
session,
|
||||
Some(request_id),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// One `tools/call` whose answer is negotiated as a single JSON body.
|
||||
async fn call_json(
|
||||
&self,
|
||||
revision: Revision,
|
||||
id: u64,
|
||||
params: Value,
|
||||
session: Option<&str>,
|
||||
) -> Reply {
|
||||
self.send(
|
||||
"application/json",
|
||||
&self.api_key,
|
||||
revision,
|
||||
id,
|
||||
params,
|
||||
session,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn send(
|
||||
&self,
|
||||
accept: &'static str,
|
||||
key: &str,
|
||||
revision: Revision,
|
||||
id: u64,
|
||||
params: Value,
|
||||
session: Option<&str>,
|
||||
request_id: Option<&str>,
|
||||
) -> Reply {
|
||||
let headers = self.headers(revision, session);
|
||||
let payload =
|
||||
json!({ "jsonrpc": "2.0", "id": id, "method": "tools/call", "params": params });
|
||||
let mut request = self.request(accept, key, &headers, payload);
|
||||
if let Some(request_id) = request_id {
|
||||
request = request.header("x-request-id", request_id);
|
||||
}
|
||||
let response = request.send().await.expect("the MCP endpoint answers");
|
||||
let status = response.status();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
let body = response.text().await.expect("the body is readable");
|
||||
Reply::parse(status, content_type, body)
|
||||
}
|
||||
|
||||
async fn cancel(&self, key: &str, session: &str, request_id: Value) -> reqwest::StatusCode {
|
||||
let response = self
|
||||
.request(
|
||||
"application/json",
|
||||
key,
|
||||
&[
|
||||
("MCP-Protocol-Version", LEGACY.to_owned()),
|
||||
("MCP-Session-Id", session.to_owned()),
|
||||
],
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/cancelled",
|
||||
"params": { "requestId": request_id, "reason": "user requested cancellation" }
|
||||
}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("the cancellation is acknowledged");
|
||||
response.status()
|
||||
}
|
||||
|
||||
/// Every Invocation History row of this operation and request id, oldest first.
|
||||
async fn history(&self, request_id: &str) -> Vec<crank_core::InvocationLog> {
|
||||
self.registry
|
||||
.list_invocation_logs(ListInvocationLogsQuery {
|
||||
workspace_id: &test_workspace_id(),
|
||||
level: None,
|
||||
status: None,
|
||||
outcome_group: None,
|
||||
search_text: None,
|
||||
source: Some(crank_core::InvocationSource::AgentToolCall),
|
||||
operation_id: Some(&self.operation_id),
|
||||
agent_id: None,
|
||||
created_after: None,
|
||||
created_before: None,
|
||||
cursor_created_at: None,
|
||||
cursor_id: None,
|
||||
limit: 50,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|record| record.log)
|
||||
.filter(|log| log.request_id.as_deref() == Some(request_id))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Wait for the counted upstream to be entered, so a cancellation happens while it really runs.
|
||||
async fn wait_until_dispatched(&self) {
|
||||
for _ in 0..200 {
|
||||
if self.upstream.started.load(Ordering::SeqCst) > 0 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
panic!("the upstream was never dispatched");
|
||||
}
|
||||
}
|
||||
|
||||
fn progress_values(frames: &[&Value]) -> Vec<u64> {
|
||||
frames
|
||||
.iter()
|
||||
.map(|frame| {
|
||||
frame["params"]["progress"]
|
||||
.as_u64()
|
||||
.unwrap_or_else(|| panic!("progress must be a number, got {frame}"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// AT-3.4-01: a negotiated stream carries bounded monotonic progress of the real execution, and a
|
||||
// missing token (or a missing carrier) never changes the execution result.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_01_legacy_progress_stream_is_monotonic_and_ends_with_the_real_result() {
|
||||
let fixture = fixture("lifecycle-legacy-progress", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
|
||||
let reply = fixture
|
||||
.call_stream(
|
||||
Revision::Legacy,
|
||||
501,
|
||||
fixture.call_params(Some(json!("tok-legacy-progress")), Revision::Legacy),
|
||||
Some(&session),
|
||||
"req-legacy-progress",
|
||||
)
|
||||
.await;
|
||||
reply.assert_stream();
|
||||
|
||||
let progress = reply.progress_frames();
|
||||
assert!(
|
||||
!progress.is_empty(),
|
||||
"a token with a stream carrier must produce progress, got {}",
|
||||
reply.raw
|
||||
);
|
||||
for frame in &progress {
|
||||
assert_eq!(frame["jsonrpc"], json!("2.0"));
|
||||
assert_eq!(
|
||||
frame["params"]["progressToken"],
|
||||
json!("tok-legacy-progress")
|
||||
);
|
||||
assert!(
|
||||
frame["params"].get("total").is_none(),
|
||||
"this server knows no total and must not invent one: {frame}"
|
||||
);
|
||||
assert!(
|
||||
frame["params"]["message"].is_string(),
|
||||
"every real step names itself: {frame}"
|
||||
);
|
||||
}
|
||||
let values = progress_values(&progress);
|
||||
assert!(
|
||||
values.windows(2).all(|pair| pair[0] < pair[1]),
|
||||
"progress must be strictly monotonic, got {values:?}"
|
||||
);
|
||||
// One event per real execution step this server actually crosses: resolved, authorized,
|
||||
// dispatched. Never a timer-driven percentage and never an unbounded stream.
|
||||
assert_eq!(values, vec![1, 2, 3], "got {values:?}");
|
||||
|
||||
let terminal = reply.terminal();
|
||||
assert_eq!(terminal["id"], json!(501));
|
||||
assert_eq!(terminal["error"], Value::Null);
|
||||
assert_eq!(terminal["result"]["isError"], json!(false));
|
||||
assert_eq!(
|
||||
terminal["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" })
|
||||
);
|
||||
assert_eq!(fixture.upstream.started.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(fixture.upstream.completed.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_01_modern_progress_stream_is_monotonic_and_ends_with_the_real_result() {
|
||||
let fixture = fixture("lifecycle-modern-progress", HOLD_MS, 5_000).await;
|
||||
|
||||
let reply = fixture
|
||||
.call_stream(
|
||||
Revision::Modern,
|
||||
502,
|
||||
fixture.call_params(Some(json!(77)), Revision::Modern),
|
||||
None,
|
||||
"req-modern-progress",
|
||||
)
|
||||
.await;
|
||||
reply.assert_stream();
|
||||
|
||||
let progress = reply.progress_frames();
|
||||
assert!(!progress.is_empty(), "got {}", reply.raw);
|
||||
for frame in &progress {
|
||||
assert_eq!(frame["params"]["progressToken"], json!(77));
|
||||
}
|
||||
let values = progress_values(&progress);
|
||||
assert!(
|
||||
values.windows(2).all(|pair| pair[0] < pair[1]),
|
||||
"{values:?}"
|
||||
);
|
||||
|
||||
let terminal = reply.terminal();
|
||||
assert_eq!(terminal["result"]["isError"], json!(false));
|
||||
assert_eq!(
|
||||
terminal["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" })
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// AT-3.4-01 (TL correction): the official SDK sends the dual `Accept` JSON-first. A `tools/call`
|
||||
// that carries a validated token and offers `text/event-stream` in that same header must still be
|
||||
// answered on its own stream, wherever the media types are ordered.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn at_3_4_01_json_first_dual_accept_keeps_progress_for_a_valid_token() {
|
||||
/// Exactly what the official MCP SDK sends when it can read a stream.
|
||||
const SDK_ACCEPT: &str = "application/json, text/event-stream";
|
||||
let fixture = fixture("lifecycle-json-first", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
|
||||
for revision in [Revision::Legacy, Revision::Modern] {
|
||||
let carrier = match revision {
|
||||
Revision::Legacy => Some(session.as_str()),
|
||||
Revision::Modern => None,
|
||||
};
|
||||
let token = json!("tok-json-first");
|
||||
let reply = fixture
|
||||
.send(
|
||||
SDK_ACCEPT,
|
||||
&fixture.api_key,
|
||||
revision,
|
||||
550,
|
||||
fixture.call_params(Some(token.clone()), revision),
|
||||
carrier,
|
||||
Some("req-json-first"),
|
||||
)
|
||||
.await;
|
||||
reply.assert_stream();
|
||||
|
||||
let progress = reply.progress_frames();
|
||||
assert!(
|
||||
!progress.is_empty(),
|
||||
"{revision:?}: a valid token with an acceptable stream must produce progress, got {}",
|
||||
reply.raw
|
||||
);
|
||||
for frame in &progress {
|
||||
assert_eq!(frame["params"]["progressToken"], token, "{frame}");
|
||||
assert!(
|
||||
frame["params"].get("total").is_none(),
|
||||
"this server knows no total: {frame}"
|
||||
);
|
||||
}
|
||||
let values = progress_values(&progress);
|
||||
assert!(
|
||||
values.windows(2).all(|pair| pair[0] < pair[1]),
|
||||
"{revision:?}: progress must be strictly monotonic, got {values:?}"
|
||||
);
|
||||
assert_eq!(values, vec![1, 2, 3], "{revision:?}: got {values:?}");
|
||||
|
||||
let terminal = reply.terminal();
|
||||
assert_eq!(terminal["id"], json!(550), "{revision:?}");
|
||||
assert_eq!(terminal["result"]["isError"], json!(false), "{revision:?}");
|
||||
assert_eq!(
|
||||
terminal["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" }),
|
||||
"{revision:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// The correction is scoped to a call that can actually carry progress: the same JSON-first
|
||||
// header without a token, and with an unusable token, keeps the negotiated single JSON body.
|
||||
for token in [None, Some(json!("x".repeat(4096)))] {
|
||||
let reply = fixture
|
||||
.send(
|
||||
SDK_ACCEPT,
|
||||
&fixture.api_key,
|
||||
Revision::Legacy,
|
||||
551,
|
||||
fixture.call_params(token.clone(), Revision::Legacy),
|
||||
Some(&session),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(reply.status, reqwest::StatusCode::OK);
|
||||
assert!(
|
||||
reply.content_type.starts_with("application/json"),
|
||||
"token {token:?} must keep the negotiated JSON body, got {:?}: {}",
|
||||
reply.content_type,
|
||||
reply.raw
|
||||
);
|
||||
assert!(reply.progress_frames().is_empty(), "{token:?}");
|
||||
assert_eq!(
|
||||
reply.terminal()["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" }),
|
||||
"token {token:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_01_no_token_and_no_carrier_keep_the_result_identical() {
|
||||
let fixture = fixture("lifecycle-no-token", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
|
||||
// Without a token the very same call is one plain JSON body: no notification exists.
|
||||
let without_token = fixture
|
||||
.call_json(
|
||||
Revision::Legacy,
|
||||
510,
|
||||
fixture.call_params(None, Revision::Legacy),
|
||||
Some(&session),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(without_token.status, reqwest::StatusCode::OK);
|
||||
assert!(without_token.progress_frames().is_empty());
|
||||
let plain = without_token.terminal()["result"].clone();
|
||||
assert_eq!(plain["isError"], json!(false));
|
||||
|
||||
// With a token but only the `application/json` carrier the result is the same and progress is
|
||||
// simply not sent (the receiver is never obligated to send it).
|
||||
let token_without_carrier = fixture
|
||||
.call_json(
|
||||
Revision::Legacy,
|
||||
511,
|
||||
fixture.call_params(Some(json!("tok-no-carrier")), Revision::Legacy),
|
||||
Some(&session),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(token_without_carrier.status, reqwest::StatusCode::OK);
|
||||
assert!(token_without_carrier.progress_frames().is_empty());
|
||||
assert_eq!(token_without_carrier.terminal()["result"], plain);
|
||||
|
||||
// In the stream mode without a token the only frame is the terminal response itself.
|
||||
let stream_without_token = fixture
|
||||
.call_stream(
|
||||
Revision::Legacy,
|
||||
512,
|
||||
fixture.call_params(None, Revision::Legacy),
|
||||
Some(&session),
|
||||
"req-no-token-stream",
|
||||
)
|
||||
.await;
|
||||
assert!(stream_without_token.progress_frames().is_empty());
|
||||
assert_eq!(stream_without_token.frames.len(), 1);
|
||||
assert_eq!(stream_without_token.terminal()["result"], plain);
|
||||
}
|
||||
|
||||
/// The streamed answer keeps the Story 3.18 projection: a client that opted in to logging gets the
|
||||
/// same single closed `notifications/message` of this call before the terminal frame, next to the
|
||||
/// progress events of Story 3.4.
|
||||
#[tokio::test]
|
||||
async fn at_3_4_01_a_progress_stream_keeps_the_logging_projection() {
|
||||
let fixture = fixture("lifecycle-progress-logging", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
let set_level = fixture
|
||||
.request(
|
||||
"application/json",
|
||||
&fixture.api_key,
|
||||
&[
|
||||
("MCP-Protocol-Version", LEGACY.to_owned()),
|
||||
("MCP-Session-Id", session.clone()),
|
||||
],
|
||||
json!({
|
||||
"jsonrpc": "2.0", "id": 900, "method": "logging/setLevel",
|
||||
"params": { "level": "debug" }
|
||||
}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("the level is accepted");
|
||||
assert_eq!(set_level.status(), reqwest::StatusCode::OK);
|
||||
|
||||
let reply = fixture
|
||||
.call_stream(
|
||||
Revision::Legacy,
|
||||
901,
|
||||
fixture.call_params(Some(json!("tok-logging")), Revision::Legacy),
|
||||
Some(&session),
|
||||
"req-progress-logging",
|
||||
)
|
||||
.await;
|
||||
reply.assert_stream();
|
||||
assert!(!reply.progress_frames().is_empty(), "got {}", reply.raw);
|
||||
|
||||
let logged: Vec<&Value> = reply
|
||||
.frames
|
||||
.iter()
|
||||
.filter(|frame| frame["method"] == json!("notifications/message"))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
logged.len(),
|
||||
1,
|
||||
"exactly one closed log event per response, got {}",
|
||||
reply.raw
|
||||
);
|
||||
assert_eq!(
|
||||
logged[0]["params"]["data"]["event"],
|
||||
json!("mcp.request.served")
|
||||
);
|
||||
assert_eq!(logged[0]["params"]["data"]["outcome"], json!("ok"));
|
||||
assert_eq!(reply.terminal()["result"]["isError"], json!(false));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// AT-3.4-02: a malformed, reused-in-another-scope or oversized token disables progress, and the
|
||||
// token never becomes a metric label, a log field or an authorization scope.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
#[tokio::test]
|
||||
async fn at_3_4_02_malformed_and_oversized_tokens_disable_progress_without_changing_the_result() {
|
||||
let fixture = fixture("lifecycle-bad-token", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
|
||||
// Both supported revisions read the token from the same pinned `RequestMetaObject` field, so an
|
||||
// unusable token must behave identically in each of them: no progress, unchanged result.
|
||||
for revision in [Revision::Legacy, Revision::Modern] {
|
||||
let carrier = match revision {
|
||||
Revision::Legacy => Some(session.as_str()),
|
||||
Revision::Modern => None,
|
||||
};
|
||||
let tokens = [
|
||||
json!(""),
|
||||
json!({ "not": "a token" }),
|
||||
json!([1, 2]),
|
||||
json!(true),
|
||||
json!("x".repeat(4096)),
|
||||
];
|
||||
let mut observed = Vec::new();
|
||||
for token in tokens {
|
||||
let reply = fixture
|
||||
.call_stream(
|
||||
revision,
|
||||
520,
|
||||
fixture.call_params(Some(token.clone()), revision),
|
||||
carrier,
|
||||
"req-bad-token",
|
||||
)
|
||||
.await;
|
||||
observed.push((token, reply));
|
||||
}
|
||||
|
||||
for (token, reply) in observed {
|
||||
assert!(
|
||||
reply.progress_frames().is_empty(),
|
||||
"{revision:?}: token {token} must not produce progress, got {}",
|
||||
reply.raw
|
||||
);
|
||||
let terminal = reply.terminal();
|
||||
assert_eq!(
|
||||
terminal["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" }),
|
||||
"{revision:?}: token {token} must not change the execution result, got {terminal}"
|
||||
);
|
||||
assert_eq!(terminal["result"]["isError"], json!(false));
|
||||
assert!(terminal["error"].is_null() || terminal.get("error").is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_02_a_reused_token_is_scoped_to_its_own_request() {
|
||||
let fixture = fixture("lifecycle-reused-token", HOLD_MS, 5_000).await;
|
||||
let first = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
let second = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
let token = json!("tok-shared");
|
||||
|
||||
let (first_reply, second_reply) = tokio::join!(
|
||||
fixture.call_stream(
|
||||
Revision::Legacy,
|
||||
530,
|
||||
fixture.call_params(Some(token.clone()), Revision::Legacy),
|
||||
Some(&first),
|
||||
"req-first",
|
||||
),
|
||||
fixture.call_stream(
|
||||
Revision::Legacy,
|
||||
531,
|
||||
fixture.call_params(Some(token.clone()), Revision::Legacy),
|
||||
Some(&second),
|
||||
"req-second",
|
||||
)
|
||||
);
|
||||
|
||||
for (reply, id) in [(&first_reply, 530), (&second_reply, 531)] {
|
||||
reply.assert_stream();
|
||||
let values = progress_values(&reply.progress_frames());
|
||||
// Each request counts its own real steps: a reused token never inherits another call's count.
|
||||
assert_eq!(values, vec![1, 2, 3], "request {id} got {values:?}");
|
||||
assert_eq!(reply.terminal()["id"], json!(id));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_02_a_progress_token_never_reaches_logs_or_invocation_history() {
|
||||
const CANARY: &str = "crank-progress-canary-4f1c2d";
|
||||
let fixture = fixture("lifecycle-token-leak", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
let (writer, subscriber) = capturing_subscriber();
|
||||
|
||||
let reply = {
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
fixture
|
||||
.call_stream(
|
||||
Revision::Legacy,
|
||||
540,
|
||||
fixture.call_params(Some(json!(CANARY)), Revision::Legacy),
|
||||
Some(&session),
|
||||
"req-token-leak",
|
||||
)
|
||||
.await
|
||||
};
|
||||
reply.assert_stream();
|
||||
|
||||
let logs = writer.output();
|
||||
assert!(
|
||||
logs.contains("mcp request received"),
|
||||
"the capture must really see the endpoint's own log projection, got {logs}"
|
||||
);
|
||||
assert!(
|
||||
!logs.contains(CANARY),
|
||||
"the progress token is client data and must never be logged, got {logs}"
|
||||
);
|
||||
|
||||
let history = fixture.history("req-token-leak").await;
|
||||
assert_eq!(history.len(), 1, "got {history:?}");
|
||||
let row = &history[0];
|
||||
let serialized = serde_json::to_string(row).unwrap();
|
||||
assert!(
|
||||
!serialized.contains(CANARY),
|
||||
"the progress token must never reach Invocation History, got {serialized}"
|
||||
);
|
||||
assert_eq!(row.status, InvocationStatus::Ok);
|
||||
}
|
||||
|
||||
// AT-3.4-03: a cancellation stops a call that can still be stopped, releases it and never yields a
|
||||
// `completed` result or record.
|
||||
mod cancellation;
|
||||
// AT-3.4-05: the client-visible smoke of both supported revisions.
|
||||
mod smoke;
|
||||
@@ -0,0 +1,294 @@
|
||||
//! AT-3.4-03: a cancellation stops a call that can still be stopped, releases it, and never yields a
|
||||
//! `completed` result or history row. The harness of the suite lives in `super`.
|
||||
|
||||
use super::*;
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// AT-3.4-03: a cancellation stops a call that can still be stopped, releases it, and never yields a
|
||||
// `completed` result or history row.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_03_legacy_cancellation_stops_the_call_and_never_completes_it() {
|
||||
let fixture = fixture("lifecycle-legacy-cancel", LONG_HOLD_MS, 30_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
|
||||
let call_session = session.clone();
|
||||
let call = tokio::spawn({
|
||||
let fixture_client = fixture.client.clone();
|
||||
let mcp_url = fixture.mcp_url.clone();
|
||||
let api_key = fixture.api_key.clone();
|
||||
let params = fixture.call_params(Some(json!("tok-cancel")), Revision::Legacy);
|
||||
async move {
|
||||
let response = fixture_client
|
||||
.post(mcp_url)
|
||||
.header(reqwest::header::ACCEPT, "text/event-stream")
|
||||
.header(reqwest::header::AUTHORIZATION, format!("Bearer {api_key}"))
|
||||
.header("MCP-Protocol-Version", LEGACY)
|
||||
.header("MCP-Session-Id", call_session)
|
||||
.header("x-request-id", "req-legacy-cancel")
|
||||
.json(&json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 601,
|
||||
"method": "tools/call",
|
||||
"params": params,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
(response.status(), response.text().await.unwrap())
|
||||
}
|
||||
});
|
||||
|
||||
fixture.wait_until_dispatched().await;
|
||||
assert_eq!(
|
||||
fixture.cancel(&fixture.api_key, &session, json!(601)).await,
|
||||
reqwest::StatusCode::ACCEPTED,
|
||||
"an accepted cancellation is acknowledged without a body"
|
||||
);
|
||||
|
||||
let (status, body) = tokio::time::timeout(Duration::from_secs(10), call)
|
||||
.await
|
||||
.expect("the cancelled call is released, it does not hang")
|
||||
.expect("the request task joins");
|
||||
assert_eq!(status, reqwest::StatusCode::OK);
|
||||
let reply = Reply::parse(status, String::new(), body);
|
||||
assert!(
|
||||
reply
|
||||
.frames
|
||||
.iter()
|
||||
.all(|frame| frame.get("method").is_some()),
|
||||
"a cancelled request gets no terminal message, only the progress it already saw: {}",
|
||||
reply.raw
|
||||
);
|
||||
assert_eq!(
|
||||
reply.progress_frames().len(),
|
||||
3,
|
||||
"the client really saw the real steps before the stop: {}",
|
||||
reply.raw
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(CANCEL_GRACE_MS)).await;
|
||||
assert_eq!(
|
||||
fixture.upstream.completed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"the abandoned upstream response must never complete the call"
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.upstream.started.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the write really was dispatched and must not be retried automatically"
|
||||
);
|
||||
|
||||
let history = fixture.history("req-legacy-cancel").await;
|
||||
assert!(
|
||||
history.iter().all(|row| row.status != InvocationStatus::Ok),
|
||||
"a cancelled call must never be recorded as completed, got {history:?}"
|
||||
);
|
||||
// The record contract admits a non-`ok` row only with a closed execution error code, and a
|
||||
// cancellation is not one of them: the honest terminal state of this call is "no record" instead
|
||||
// of an invented error code. Either way nothing claims that the call completed.
|
||||
assert!(
|
||||
history.is_empty(),
|
||||
"a stopped call has no terminal outcome to record, got {history:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_03_a_foreign_or_unknown_cancellation_never_touches_the_active_call() {
|
||||
let fixture = fixture("lifecycle-foreign-cancel", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
let foreign_session =
|
||||
initialize_session(&fixture.client, &fixture.mcp_url, &fixture.other_api_key).await;
|
||||
|
||||
let call = tokio::spawn({
|
||||
let client = fixture.client.clone();
|
||||
let mcp_url = fixture.mcp_url.clone();
|
||||
let api_key = fixture.api_key.clone();
|
||||
let call_session = session.clone();
|
||||
let params = fixture.call_params(Some(json!("tok-owner")), Revision::Legacy);
|
||||
async move {
|
||||
let response = client
|
||||
.post(mcp_url)
|
||||
.header(reqwest::header::ACCEPT, "text/event-stream")
|
||||
.header(reqwest::header::AUTHORIZATION, format!("Bearer {api_key}"))
|
||||
.header("MCP-Protocol-Version", LEGACY)
|
||||
.header("MCP-Session-Id", call_session)
|
||||
.header("x-request-id", "req-owner")
|
||||
.json(&json!({
|
||||
"jsonrpc": "2.0", "id": 602, "method": "tools/call", "params": params,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
response.text().await.unwrap()
|
||||
}
|
||||
});
|
||||
|
||||
fixture.wait_until_dispatched().await;
|
||||
// A foreign credential of the same agent, and an unknown request id: neither may cancel this call.
|
||||
assert_eq!(
|
||||
fixture
|
||||
.cancel(&fixture.other_api_key, &session, json!(602))
|
||||
.await,
|
||||
reqwest::StatusCode::ACCEPTED
|
||||
);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.cancel(&fixture.api_key, &session, json!(999_999))
|
||||
.await,
|
||||
reqwest::StatusCode::ACCEPTED
|
||||
);
|
||||
// A foreign session of the same credential cannot reach this call either.
|
||||
assert_eq!(
|
||||
fixture
|
||||
.cancel(&fixture.api_key, &foreign_session, json!(602))
|
||||
.await,
|
||||
reqwest::StatusCode::ACCEPTED
|
||||
);
|
||||
|
||||
let body = tokio::time::timeout(Duration::from_secs(10), call)
|
||||
.await
|
||||
.expect("the untouched call completes")
|
||||
.expect("the request task joins");
|
||||
let reply = Reply::parse(reqwest::StatusCode::OK, String::new(), body);
|
||||
assert_eq!(
|
||||
reply.terminal()["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" }),
|
||||
"a foreign cancellation must not stop the owner's call, got {}",
|
||||
reply.raw
|
||||
);
|
||||
let history = fixture.history("req-owner").await;
|
||||
assert_eq!(history.len(), 1, "got {history:?}");
|
||||
assert_eq!(history[0].status, InvocationStatus::Ok);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_03_a_late_cancellation_after_the_result_is_ignored() {
|
||||
let fixture = fixture("lifecycle-late-cancel", HOLD_MS, 5_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
|
||||
let reply = fixture
|
||||
.call_stream(
|
||||
Revision::Legacy,
|
||||
603,
|
||||
fixture.call_params(Some(json!("tok-late")), Revision::Legacy),
|
||||
Some(&session),
|
||||
"req-late-cancel",
|
||||
)
|
||||
.await;
|
||||
reply.assert_stream();
|
||||
assert_eq!(reply.terminal()["result"]["isError"], json!(false));
|
||||
|
||||
assert_eq!(
|
||||
fixture.cancel(&fixture.api_key, &session, json!(603)).await,
|
||||
reqwest::StatusCode::ACCEPTED
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
|
||||
let history = fixture.history("req-late-cancel").await;
|
||||
assert_eq!(
|
||||
history.len(),
|
||||
1,
|
||||
"a cancellation of a finished call must not add a second record, got {history:?}"
|
||||
);
|
||||
assert_eq!(history[0].status, InvocationStatus::Ok);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_03_a_plain_json_call_is_cancellable_too() {
|
||||
let fixture = fixture("lifecycle-json-cancel", LONG_HOLD_MS, 30_000).await;
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
|
||||
let call = tokio::spawn({
|
||||
let client = fixture.client.clone();
|
||||
let mcp_url = fixture.mcp_url.clone();
|
||||
let api_key = fixture.api_key.clone();
|
||||
let call_session = session.clone();
|
||||
let params = fixture.call_params(None, Revision::Legacy);
|
||||
async move {
|
||||
let response = client
|
||||
.post(mcp_url)
|
||||
.header(reqwest::header::ACCEPT, "application/json")
|
||||
.header(reqwest::header::AUTHORIZATION, format!("Bearer {api_key}"))
|
||||
.header("MCP-Protocol-Version", LEGACY)
|
||||
.header("MCP-Session-Id", call_session)
|
||||
.header("x-request-id", "req-json-cancel")
|
||||
.json(&json!({
|
||||
"jsonrpc": "2.0", "id": 605, "method": "tools/call", "params": params,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
(status, response.text().await.unwrap())
|
||||
}
|
||||
});
|
||||
|
||||
fixture.wait_until_dispatched().await;
|
||||
assert_eq!(
|
||||
fixture.cancel(&fixture.api_key, &session, json!(605)).await,
|
||||
reqwest::StatusCode::ACCEPTED
|
||||
);
|
||||
|
||||
let (status, body) = tokio::time::timeout(Duration::from_secs(10), call)
|
||||
.await
|
||||
.expect("the cancelled call is released")
|
||||
.expect("the request task joins");
|
||||
assert_eq!(status, reqwest::StatusCode::OK);
|
||||
let reply = Reply::parse(status, String::new(), body);
|
||||
assert!(
|
||||
reply.frames.is_empty(),
|
||||
"a cancelled request gets no terminal message, got {}",
|
||||
reply.raw
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(CANCEL_GRACE_MS)).await;
|
||||
assert_eq!(fixture.upstream.completed.load(Ordering::SeqCst), 0);
|
||||
let history = fixture.history("req-json-cancel").await;
|
||||
assert!(
|
||||
history.iter().all(|row| row.status != InvocationStatus::Ok),
|
||||
"a cancelled call must never be recorded as completed, got {history:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_03_closing_the_modern_response_stream_cancels_the_call() {
|
||||
let fixture = fixture("lifecycle-modern-disconnect", LONG_HOLD_MS, 30_000).await;
|
||||
|
||||
let response = fixture
|
||||
.request(
|
||||
"text/event-stream",
|
||||
&fixture.api_key,
|
||||
&fixture.headers(Revision::Modern, None),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 604,
|
||||
"method": "tools/call",
|
||||
"params": fixture.call_params(Some(json!("tok-disconnect")), Revision::Modern),
|
||||
}),
|
||||
)
|
||||
.header("x-request-id", "req-modern-disconnect")
|
||||
.send()
|
||||
.await
|
||||
.expect("the stream opens before the call finishes");
|
||||
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||
fixture.wait_until_dispatched().await;
|
||||
|
||||
// The pinned `2026-07-28` cancellation signal: closing the request's own response stream.
|
||||
drop(response);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(CANCEL_GRACE_MS)).await;
|
||||
assert_eq!(
|
||||
fixture.upstream.completed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"the closed stream must stop the dispatched call instead of letting it finish"
|
||||
);
|
||||
assert_eq!(fixture.upstream.started.load(Ordering::SeqCst), 1);
|
||||
|
||||
let history = fixture.history("req-modern-disconnect").await;
|
||||
assert!(
|
||||
history.iter().all(|row| row.status != InvocationStatus::Ok),
|
||||
"a cancelled call must never be recorded as completed, got {history:?}"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
//! AT-3.4-05: the client-visible smoke of both supported revisions.
|
||||
//!
|
||||
//! A representative client of each revision performs the whole flow a real MCP client performs —
|
||||
//! discovery, then a `tools/call` that asks for progress and reads the request's own stream to the
|
||||
//! terminal response — against the real endpoint. The harness of the suite lives in `super`.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn at_3_4_05_representative_client_smoke_of_both_revisions() {
|
||||
let fixture = fixture("lifecycle-client-smoke", HOLD_MS, 5_000).await;
|
||||
|
||||
// Legacy client: initialize → tools/list → tools/call with progress.
|
||||
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
|
||||
let tools = post_jsonrpc(
|
||||
&fixture.client,
|
||||
&fixture.mcp_url,
|
||||
&fixture.api_key,
|
||||
Some(&session),
|
||||
json!({ "jsonrpc": "2.0", "id": 700, "method": "tools/list", "params": {} }),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(tools["result"]["tools"][0]["name"], json!(TOOL));
|
||||
|
||||
let legacy_call = fixture
|
||||
.call_stream(
|
||||
Revision::Legacy,
|
||||
701,
|
||||
fixture.call_params(Some(json!("tok-smoke-legacy")), Revision::Legacy),
|
||||
Some(&session),
|
||||
"req-smoke-legacy",
|
||||
)
|
||||
.await;
|
||||
legacy_call.assert_stream();
|
||||
assert_eq!(
|
||||
legacy_call.terminal()["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" })
|
||||
);
|
||||
|
||||
// Modern client: `tools/list` and `tools/call` of the stateless revision, progress on the stream.
|
||||
let modern_list = post_jsonrpc_with_headers(
|
||||
&fixture.client,
|
||||
&fixture.mcp_url,
|
||||
&fixture.api_key,
|
||||
&[
|
||||
("MCP-Protocol-Version", MODERN),
|
||||
("Mcp-Method", "tools/list"),
|
||||
],
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 710,
|
||||
"method": "tools/list",
|
||||
"params": { "_meta": {
|
||||
META_PROTOCOL_VERSION: MODERN,
|
||||
META_CLIENT_CAPABILITIES: {},
|
||||
META_CLIENT_INFO: { "name": "crank-3.4-smoke", "version": "0.1.0" },
|
||||
}}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let modern_list = modern_list.json::<Value>().await.unwrap();
|
||||
assert_eq!(modern_list["result"]["tools"][0]["name"], json!(TOOL));
|
||||
|
||||
let modern_call = fixture
|
||||
.call_stream(
|
||||
Revision::Modern,
|
||||
711,
|
||||
fixture.call_params(Some(json!("tok-smoke-modern")), Revision::Modern),
|
||||
None,
|
||||
"req-smoke-modern",
|
||||
)
|
||||
.await;
|
||||
modern_call.assert_stream();
|
||||
assert!(!modern_call.progress_frames().is_empty());
|
||||
assert_eq!(
|
||||
modern_call.terminal()["result"]["structuredContent"],
|
||||
json!({ "id": "lead_123" })
|
||||
);
|
||||
}
|
||||
@@ -1036,6 +1036,7 @@ var TRANSLATIONS = {
|
||||
'wizard.error.secret_name_required': 'Secret name is required.',
|
||||
'wizard.error.secret_value_required': 'Secret value is required.',
|
||||
'wizard.error.save_upstream_first': 'Save the API host after creating a new authorization profile before continuing.',
|
||||
'wizard.error.execution_policy_invalid': 'Execution policy is invalid: mode, deadline from 1 to 3600000 ms and no-Tasks fallback are required.',
|
||||
'wizard.toast.auth_profile_created_title': 'Authorization profile created',
|
||||
'wizard.toast.auth_profile_created_body': 'New authorization profile is ready and selected.',
|
||||
'wizard.toast.auth_profile_error_title': 'Authorization profile creation failed',
|
||||
@@ -2353,6 +2354,7 @@ var TRANSLATIONS = {
|
||||
'wizard.error.secret_name_required': 'Нужно указать имя секрета.',
|
||||
'wizard.error.secret_value_required': 'Нужно указать значение секрета.',
|
||||
'wizard.error.save_upstream_first': 'Сначала сохраните API-хост после создания нового профиля авторизации.',
|
||||
'wizard.error.execution_policy_invalid': 'Политика выполнения некорректна: нужны режим, deadline от 1 до 3600000 мс и поведение без Tasks.',
|
||||
'wizard.toast.auth_profile_created_title': 'Профиль авторизации создан',
|
||||
'wizard.toast.auth_profile_created_body': 'Новый профиль авторизации готов и уже выбран.',
|
||||
'wizard.toast.auth_profile_error_title': 'Не удалось создать профиль авторизации',
|
||||
|
||||
@@ -214,10 +214,33 @@ function parseExecutionConfig(text) {
|
||||
auth_profile_ref: authProfileRef,
|
||||
headers: headers,
|
||||
};
|
||||
var executionPolicy = parseExecutionPolicy(value.execution_policy);
|
||||
if (executionPolicy) {
|
||||
config.execution_policy = executionPolicy;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
// Story 4.1: the published execution policy must survive a wizard save. The editor is a free-form
|
||||
// document, so unknown/legacy shapes stay untouched: only a fully valid policy is projected, and an
|
||||
// explicit invalid value is reported instead of being silently replaced by defaults.
|
||||
function parseExecutionPolicy(raw) {
|
||||
if (raw === undefined || raw === null) return null;
|
||||
if (typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
throw new Error(tKey('wizard.error.execution_policy_invalid'));
|
||||
}
|
||||
var mode = raw.mode === undefined ? 'synchronous' : raw.mode;
|
||||
var fallback = raw.no_tasks_fallback === undefined ? 'synchronous' : raw.no_tasks_fallback;
|
||||
var deadline = Number(raw.deadline_ms);
|
||||
if (['synchronous', 'background', 'adaptive'].indexOf(mode) < 0
|
||||
|| ['synchronous', 'unsupported'].indexOf(fallback) < 0
|
||||
|| !Number.isInteger(deadline) || deadline < 1 || deadline > 3600000) {
|
||||
throw new Error(tKey('wizard.error.execution_policy_invalid'));
|
||||
}
|
||||
return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback };
|
||||
}
|
||||
|
||||
function currentUpstream() {
|
||||
var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; });
|
||||
if (selected) return selected;
|
||||
@@ -415,6 +438,13 @@ function executionConfigToEditorValue(config) {
|
||||
var value = {
|
||||
timeout_ms: config.timeout_ms,
|
||||
};
|
||||
if (config.execution_policy) {
|
||||
value.execution_policy = {
|
||||
mode: config.execution_policy.mode,
|
||||
deadline_ms: config.execution_policy.deadline_ms,
|
||||
no_tasks_fallback: config.execution_policy.no_tasks_fallback,
|
||||
};
|
||||
}
|
||||
if (config.retry_policy) {
|
||||
value.retry = { max_attempts: config.retry_policy.max_attempts };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
const { test, expect } = require('@playwright/test');
|
||||
const { getCurrentWorkspace, login } = require('./helpers');
|
||||
|
||||
// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a
|
||||
// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit
|
||||
// invalid value must be reported instead of being silently replaced by defaults.
|
||||
function draftSnapshot(operationId) {
|
||||
return {
|
||||
id: operationId,
|
||||
name: operationId,
|
||||
display_name: 'Policy rates',
|
||||
category: 'general',
|
||||
protocol: 'rest',
|
||||
target: {
|
||||
kind: 'rest',
|
||||
base_url: 'https://api.example.test',
|
||||
method: 'GET',
|
||||
path_template: '/rates',
|
||||
static_headers: {},
|
||||
},
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
description: null,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: null,
|
||||
fields: {},
|
||||
items: null,
|
||||
enum_values: [],
|
||||
variants: [],
|
||||
},
|
||||
output_schema: {
|
||||
type: 'object',
|
||||
description: null,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: null,
|
||||
fields: {},
|
||||
items: null,
|
||||
enum_values: [],
|
||||
variants: [],
|
||||
},
|
||||
input_mapping: { rules: [] },
|
||||
output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] },
|
||||
execution_config: {
|
||||
timeout_ms: 10000,
|
||||
retry_policy: null,
|
||||
auth_profile_ref: null,
|
||||
headers: {},
|
||||
approval_policy: null,
|
||||
execution_policy: {
|
||||
mode: 'adaptive',
|
||||
deadline_ms: 45000,
|
||||
no_tasks_fallback: 'unsupported',
|
||||
},
|
||||
},
|
||||
tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] },
|
||||
wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] },
|
||||
};
|
||||
}
|
||||
|
||||
test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => {
|
||||
await login(page);
|
||||
const workspace = await getCurrentWorkspace(page);
|
||||
const operationId = 'policy_rates_41';
|
||||
const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`;
|
||||
const savedPayloads = [];
|
||||
|
||||
await page.route(operationPath, async (route) => {
|
||||
if (route.request().method() === 'PATCH') {
|
||||
savedPayloads.push(route.request().postDataJSON());
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
id: operationId,
|
||||
workspace_id: workspace.id,
|
||||
name: operationId,
|
||||
display_name: 'Policy rates',
|
||||
category: 'general',
|
||||
protocol: 'rest',
|
||||
status: 'draft',
|
||||
current_draft_version: 1,
|
||||
draft_version_ref: { version: 1, status: 'draft' },
|
||||
published_version_ref: null,
|
||||
agent_refs: [],
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route(`${operationPath}/versions/*`, async (route) => {
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
operation_id: operationId,
|
||||
workspace_id: workspace.id,
|
||||
version: 1,
|
||||
status: 'draft',
|
||||
snapshot: draftSnapshot(operationId),
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route('**/operations/analyze-quality', async (route) => {
|
||||
await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) });
|
||||
});
|
||||
page.on('dialog', (dialog) => dialog.accept());
|
||||
|
||||
await page.goto(`/wizard/?mode=edit&operationId=${operationId}`);
|
||||
await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates');
|
||||
|
||||
const editorValue = await page.locator('#tool-exec-config').inputValue();
|
||||
expect(editorValue).toContain('execution_policy');
|
||||
expect(editorValue).toContain('adaptive');
|
||||
expect(editorValue).toContain('45000');
|
||||
expect(editorValue).toContain('unsupported');
|
||||
|
||||
// A real edit still saves the pinned policy unchanged.
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(4));
|
||||
await page.locator('#tool-display-name').fill('Policy rates edited');
|
||||
await page.locator('.btn-save-draft').click();
|
||||
await expect.poll(() => savedPayloads.length).toBe(1);
|
||||
expect(savedPayloads[0].execution_config.execution_policy).toEqual({
|
||||
mode: 'adaptive',
|
||||
deadline_ms: 45000,
|
||||
no_tasks_fallback: 'unsupported',
|
||||
});
|
||||
|
||||
// An explicit invalid policy is refused instead of being silently rewritten to defaults.
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
|
||||
await expect(page.locator('#tool-exec-config')).toBeVisible();
|
||||
await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n');
|
||||
await page.locator('.btn-save-draft').click();
|
||||
await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/);
|
||||
expect(savedPayloads.length).toBe(1);
|
||||
});
|
||||
@@ -30,6 +30,7 @@ mod approval_policy;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod dispatch_observer;
|
||||
mod invocation_history;
|
||||
pub(crate) mod lifecycle;
|
||||
mod metrics;
|
||||
mod request;
|
||||
mod response;
|
||||
@@ -89,6 +90,8 @@ pub(super) struct AppState {
|
||||
/// Story 3.19: the bounded table of the legacy session that currently owns the notification
|
||||
/// stream, so one session never broadcasts the same list-change signal into two streams.
|
||||
pub(super) signal_streams: Arc<crate::protocol::StreamRegistry>,
|
||||
/// Story 3.4: the bounded table of the `tools/call` requests a legacy client can still cancel.
|
||||
pub(super) in_flight: Arc<self::lifecycle::InFlightCalls>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -257,6 +260,9 @@ fn build_app_inner(
|
||||
allowed_origins: AllowedOrigins::new(public_base_url),
|
||||
logging_levels: Arc::new(crate::protocol::LogLevelStore::new()),
|
||||
signal_streams: Arc::new(crate::protocol::StreamRegistry::new()),
|
||||
in_flight: Arc::new(lifecycle::InFlightCalls::new(
|
||||
lifecycle::MAX_IN_FLIGHT_CALLS,
|
||||
)),
|
||||
});
|
||||
if start_background_workers {
|
||||
spawn_approval_recovery(Arc::clone(&state));
|
||||
@@ -305,24 +311,41 @@ async fn health() -> Json<Value> {
|
||||
}
|
||||
|
||||
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
|
||||
match state.registry.ping().await {
|
||||
Ok(()) => Json(json!({
|
||||
let postgres_ready = state.registry.ping().await.is_ok();
|
||||
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
|
||||
// reflect it rather than reporting ready on a bare `select 1`.
|
||||
let admission_ready = state.registry.execution_admission_health().await.is_ok();
|
||||
if postgres_ready && admission_ready {
|
||||
return Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": "ready" }
|
||||
"checks": { "postgres": "ready", "execution_admission": "ready" }
|
||||
}))
|
||||
.into_response(),
|
||||
Err(_) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": "not_ready" },
|
||||
"error": "database is unavailable"
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
.into_response();
|
||||
}
|
||||
let mut checks = serde_json::Map::new();
|
||||
checks.insert(
|
||||
"postgres".to_owned(),
|
||||
json!(if postgres_ready { "ready" } else { "not_ready" }),
|
||||
);
|
||||
checks.insert(
|
||||
"execution_admission".to_owned(),
|
||||
json!(if admission_ready {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
}),
|
||||
);
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "not_ready",
|
||||
"checks": checks,
|
||||
"error": "a mandatory dependency is unavailable"
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -39,8 +39,7 @@ use time::OffsetDateTime;
|
||||
use super::response::{
|
||||
completion_result, internal_jsonrpc_error, prompt_arguments_error_response, prompt_get_result,
|
||||
prompts_list_result, resource_error_response, resource_read_result,
|
||||
resource_templates_list_result, resources_list_result, tool_command_response,
|
||||
tools_list_result,
|
||||
resource_templates_list_result, resources_list_result, tools_list_result,
|
||||
};
|
||||
use super::{AgentRoutePath, AppState, TRANSPORT_SESSION_TTL_MS};
|
||||
use crate::protocol::{ResourceTemplatesListError, list_resource_templates};
|
||||
@@ -156,6 +155,8 @@ pub(super) async fn handle_modern_request(
|
||||
locale: execution_locale(message),
|
||||
},
|
||||
ToolsCommand::List(list_command),
|
||||
// A list command has no progress or cancellation lifecycle (Story 3.4).
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
match outcome {
|
||||
@@ -220,30 +221,35 @@ pub(super) async fn handle_modern_request(
|
||||
// session-less, so its calls cannot advertise elicitation support.
|
||||
let mut arguments = tool_call_params.arguments.unwrap_or_else(|| json!({}));
|
||||
let confirmation_token = take_confirmation_token(&mut arguments);
|
||||
let outcome = handle_tools_command(
|
||||
state,
|
||||
ServiceContext {
|
||||
workspace_slug: &path.workspace_slug,
|
||||
agent_slug: &path.agent_slug,
|
||||
credential,
|
||||
correlation: transport_correlation,
|
||||
// Story 3.4: the client's own `_meta.progressToken` of this request, validated and bounded.
|
||||
let progress = crate::app::lifecycle::token_from_meta(tool_call_params.meta.as_ref());
|
||||
super::lifecycle::run_tool_call(
|
||||
Arc::clone(state),
|
||||
crate::app::lifecycle::ToolCallRequest {
|
||||
workspace_slug: path.workspace_slug.clone(),
|
||||
agent_slug: path.agent_slug.clone(),
|
||||
credential: credential.clone(),
|
||||
correlation: transport_correlation.clone(),
|
||||
continuation: ContinuationSupport::ElicitationUnsupported,
|
||||
locale: execution_locale(message),
|
||||
},
|
||||
ToolsCommand::Call(ToolCallCommand {
|
||||
crate::app::lifecycle::ToolCallWire {
|
||||
message: message.clone(),
|
||||
protocol_version: version.to_owned(),
|
||||
response_mode,
|
||||
progress,
|
||||
session_id: None,
|
||||
// The modern revision cancels by closing this request's response stream, so no
|
||||
// session-scoped cancellation table is involved.
|
||||
in_flight: None,
|
||||
},
|
||||
ToolCallCommand {
|
||||
tool_name: tool_call_params.name,
|
||||
arguments,
|
||||
confirmation_token,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
tool_command_response(
|
||||
message,
|
||||
response_mode,
|
||||
version,
|
||||
transport_correlation,
|
||||
outcome,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
DispatchDecision::Allowed {
|
||||
handler: MethodHandler::ResourceList,
|
||||
|
||||
@@ -17,8 +17,8 @@ use crate::{
|
||||
rate_limit::{rate_limited_jsonrpc_response, rate_limited_status_response},
|
||||
request_context::RequestContext,
|
||||
transport::{
|
||||
accepts_event_stream, negotiate_post_response_mode, session_id_from_headers, sse_response,
|
||||
transport_response, validate_get_accept_header, validate_origin,
|
||||
ResponseMode, accepts_event_stream, negotiate_post_response_mode, session_id_from_headers,
|
||||
sse_response, transport_response, validate_get_accept_header, validate_origin,
|
||||
validate_session_protocol_version, with_request_id_header,
|
||||
},
|
||||
};
|
||||
@@ -103,7 +103,7 @@ pub(super) async fn mcp_get(
|
||||
};
|
||||
|
||||
// Story 3.19: the pinned legacy transport forbids broadcasting one JSON-RPC message across the
|
||||
// SSE streams of a session, so exactly one admitted `GET` receives the Resource/Prompt
|
||||
// SSE streams of a session, so exactly one admitted `GET` receives the Resource/Prompt/Tool
|
||||
// list-change signals. A newer `GET` takes them over and the older stream simply ends; a
|
||||
// `DELETE` retires the producer, so a stale generation never writes into a reconnect.
|
||||
let Some(generation) = state.signal_streams.admit(&session_id) else {
|
||||
@@ -115,7 +115,7 @@ pub(super) async fn mcp_get(
|
||||
crate::protocol::Watch {
|
||||
workspace_slug: path.workspace_slug.clone(),
|
||||
agent_slug: path.agent_slug.clone(),
|
||||
signals: crate::protocol::Signals::BOTH,
|
||||
signals: crate::protocol::Signals::ALL,
|
||||
wire: crate::protocol::Wire::Legacy,
|
||||
legacy_stream: Some((
|
||||
Arc::clone(&state.signal_streams),
|
||||
@@ -256,7 +256,7 @@ async fn mcp_post_response(
|
||||
return status.into_response();
|
||||
}
|
||||
|
||||
let response_mode = match negotiate_post_response_mode(headers) {
|
||||
let mut response_mode = match negotiate_post_response_mode(headers) {
|
||||
Ok(mode) => mode,
|
||||
Err(status) => return status.into_response(),
|
||||
};
|
||||
@@ -365,6 +365,19 @@ async fn mcp_post_response(
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
|
||||
// Story 3.4 correction: the official SDK lists the media types JSON-first. For a `tools/call`
|
||||
// that carries a validated progress token and offered `text/event-stream`, the stream is the only
|
||||
// carrier that can deliver the progress the client asked for, so the media-type order is not a
|
||||
// preference for the JSON body here. Every other request — another method, no token, an unusable
|
||||
// token, a JSON-only `Accept` — keeps the negotiated mode, and no admission check moved.
|
||||
if matches!(response_mode, ResponseMode::Json)
|
||||
&& method_name(message) == Some("tools/call")
|
||||
&& accepts_event_stream(headers)
|
||||
&& crate::app::lifecycle::token_from_meta(message.pointer("/params/_meta")).is_some()
|
||||
{
|
||||
response_mode = ResponseMode::Sse;
|
||||
}
|
||||
|
||||
if matches!(identity.era, RequestEra::Modern) {
|
||||
// ERA_MODERN_ENTER (spec section 7.2): this request entered the modern era-adapter.
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::response::{
|
||||
completion_result, internal_jsonrpc_error, prompt_arguments_error_response, prompt_get_result,
|
||||
prompts_list_result, resource_error_response, resource_read_result,
|
||||
resource_templates_list_result, resources_list_result, take_confirmation_token,
|
||||
tool_command_response, tools_list_result,
|
||||
tools_list_result,
|
||||
};
|
||||
use super::{AgentRoutePath, AppState};
|
||||
use crate::{
|
||||
@@ -41,7 +41,7 @@ use crate::{
|
||||
handle_tools_command,
|
||||
},
|
||||
tool_error::execution_locale,
|
||||
transport::{ResponseMode, transport_response},
|
||||
transport::{ResponseMode, session_id_from_headers, transport_response},
|
||||
};
|
||||
|
||||
// The transport invariants of one request travel together here; a parameter struct would only
|
||||
@@ -138,6 +138,8 @@ pub(super) async fn handle_legacy_request(
|
||||
locale: execution_locale(message),
|
||||
},
|
||||
ToolsCommand::List(list_command),
|
||||
// A list command has no progress or cancellation lifecycle (Story 3.4).
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
match outcome {
|
||||
@@ -192,30 +194,43 @@ pub(super) async fn handle_legacy_request(
|
||||
// response mode or the protocol revision (spec section 6.2).
|
||||
let mut arguments = tool_call_params.arguments.unwrap_or_else(|| json!({}));
|
||||
let confirmation_token = take_confirmation_token(&mut arguments);
|
||||
let outcome = handle_tools_command(
|
||||
&state,
|
||||
ServiceContext {
|
||||
workspace_slug: &session.workspace_slug,
|
||||
agent_slug: &session.agent_slug,
|
||||
credential,
|
||||
correlation: transport_correlation,
|
||||
// Story 3.4: the client's own `_meta.progressToken`, validated and bounded, and the
|
||||
// session/credential ownership of the in-flight cancellation table.
|
||||
let progress = crate::app::lifecycle::token_from_meta(tool_call_params.meta.as_ref());
|
||||
super::lifecycle::run_tool_call(
|
||||
Arc::clone(&state),
|
||||
crate::app::lifecycle::ToolCallRequest {
|
||||
workspace_slug: session.workspace_slug.clone(),
|
||||
agent_slug: session.agent_slug.clone(),
|
||||
credential: credential.clone(),
|
||||
correlation: transport_correlation.clone(),
|
||||
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
|
||||
locale: execution_locale(message),
|
||||
},
|
||||
ToolsCommand::Call(ToolCallCommand {
|
||||
crate::app::lifecycle::ToolCallWire {
|
||||
message: message.clone(),
|
||||
protocol_version: session.protocol_version.clone(),
|
||||
response_mode,
|
||||
progress,
|
||||
session_id: Some(session.id.clone()),
|
||||
in_flight: Some(crate::app::lifecycle::InFlightScope {
|
||||
session_id: session.id.clone(),
|
||||
owner: in_flight_owner(credential),
|
||||
}),
|
||||
},
|
||||
ToolCallCommand {
|
||||
tool_name: tool_call_params.name,
|
||||
arguments,
|
||||
confirmation_token,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
tool_command_response(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
transport_correlation,
|
||||
outcome,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
DispatchDecision::Allowed {
|
||||
handler: MethodHandler::Cancelled,
|
||||
kind: Kind::Notification,
|
||||
} if is_notification(message) => {
|
||||
handle_cancelled_notification(&state, path, headers, message, credential).await
|
||||
}
|
||||
DispatchDecision::Allowed {
|
||||
handler: MethodHandler::ResourceList,
|
||||
@@ -729,3 +744,51 @@ pub(super) async fn handle_legacy_request(
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// The credential half of the cancellation ownership of one call (Story 3.4). The progress token is
|
||||
/// deliberately not part of it: a token never authorizes a cancellation.
|
||||
fn in_flight_owner(credential: &VerifiedMachineCredential) -> String {
|
||||
credential
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map_or_else(String::new, |id| id.as_str().to_owned())
|
||||
}
|
||||
|
||||
/// Route one legacy `notifications/cancelled` to the call it names, then acknowledge it.
|
||||
///
|
||||
/// The pinned rules of a cancelled request are all "ignore": an unknown or already finished request,
|
||||
/// a foreign session or credential, a malformed `requestId` and an unknown session are acknowledged
|
||||
/// without any effect. That is also why this handler never fails the transport — a client that
|
||||
/// cancels must not learn anything about another client's calls.
|
||||
async fn handle_cancelled_notification(
|
||||
state: &Arc<AppState>,
|
||||
path: &AgentRoutePath,
|
||||
headers: &HeaderMap,
|
||||
message: &Value,
|
||||
credential: &VerifiedMachineCredential,
|
||||
) -> Response {
|
||||
let request_id = params(message).get("requestId").cloned();
|
||||
let session_id = session_id_from_headers(headers).ok().flatten();
|
||||
if let (Some(request_id), Some(session_id)) = (request_id, session_id) {
|
||||
// Only a string or a number is a pinned `RequestId`; anything else cannot name a call.
|
||||
if matches!(request_id, Value::String(_) | Value::Number(_)) {
|
||||
let owned_by_session = match state.sessions.get(&session_id).await {
|
||||
Ok(Some(session)) => {
|
||||
session.workspace_slug == path.workspace_slug
|
||||
&& session.agent_slug == path.agent_slug
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if owned_by_session {
|
||||
state.in_flight.cancel(
|
||||
&crate::app::lifecycle::InFlightScope {
|
||||
session_id,
|
||||
owner: in_flight_owner(credential),
|
||||
},
|
||||
&request_id,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
StatusCode::ACCEPTED.into_response()
|
||||
}
|
||||
@@ -0,0 +1,566 @@
|
||||
//! Story 3.4 (FR-14): request-scoped progress and cancellation of one synchronous `tools/call`.
|
||||
//!
|
||||
//! The pinned `RequestMetaObject` of *both* supported revisions carries an optional opaque
|
||||
//! `progressToken`; a request that has one asks for out-of-band `notifications/progress` messages
|
||||
//! about that request. Such a message only has a carrier on the request's own SSE response stream, so
|
||||
//! a token is honoured only when the client negotiated `text/event-stream`. Without a token — or
|
||||
//! without the stream — the call runs exactly as before: the receiver of a progress token is never
|
||||
//! obligated to send progress, so the execution result cannot change.
|
||||
//!
|
||||
//! Cancellation follows the revision-specific rule of the pinned specification:
|
||||
//! * `2025-11-25`: the client POSTs `notifications/cancelled` with the id of the request it issued.
|
||||
//! The bounded in-flight table routes it to that call. Ownership is the session *and* the platform
|
||||
//! credential of the call, never the progress token: a foreign session or key cancels nothing.
|
||||
//! * `2026-07-28`: closing the request's SSE response stream *is* the cancellation signal, so there is
|
||||
//! no notification to route; dropping the response stream drops the call future.
|
||||
//!
|
||||
//! Either way a cancelled call never becomes a false `completed`: no further message is sent for it
|
||||
//! (the pinned rule for a cancelled request), and the call never reaches its terminal outcome, so no
|
||||
//! Invocation History row claims a completion. The record contract only admits a *closed* execution
|
||||
//! error code for a non-`ok` row and a cancellation is not one of them, so the honest state of a
|
||||
//! stopped call is "no terminal record" rather than an invented error code; a dedicated cancellation
|
||||
//! code would be a schema/vocabulary decision of its own.
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response, sse::Event},
|
||||
};
|
||||
use crank_core::{CorrelationContext, ExecutionLocale, VerifiedMachineCredential};
|
||||
use futures_util::stream;
|
||||
use serde_json::{Number, Value, json};
|
||||
use tokio::sync::{mpsc, watch};
|
||||
|
||||
use super::{
|
||||
AppState,
|
||||
response::{ToolCommandPayload, tool_command_payload, tool_command_response},
|
||||
};
|
||||
use crate::{
|
||||
jsonrpc::request_id,
|
||||
protocol::{
|
||||
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsOutcome,
|
||||
handle_tools_command,
|
||||
},
|
||||
transport::{ResponseMode, sse_response},
|
||||
};
|
||||
|
||||
/// The bounded progress queue of one call. A client that reads its stream slowly can never block the
|
||||
/// call: a full queue drops the event instead of waiting, and the remaining events stay monotonic.
|
||||
pub(crate) const PROGRESS_CAPACITY: usize = 8;
|
||||
/// The pinned `ProgressToken` bound this server accepts. A longer token is not rejected as an error:
|
||||
/// progress is simply disabled for that request, exactly as for a malformed one.
|
||||
pub(crate) const MAX_PROGRESS_TOKEN_LEN: usize = 128;
|
||||
/// The bounded table of calls that can still be cancelled. A call that finds the table full runs
|
||||
/// without cancellation instead of growing the table.
|
||||
pub(crate) const MAX_IN_FLIGHT_CALLS: usize = 256;
|
||||
|
||||
/// A validated `RequestMetaObject.progressToken`: the pinned opaque token is a string or a number.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ProgressToken {
|
||||
Text(String),
|
||||
Number(Number),
|
||||
}
|
||||
|
||||
impl ProgressToken {
|
||||
/// `None` means "disable progress", never "fail the call": the pinned schema declares the token
|
||||
/// as an opaque client value the receiver may simply ignore.
|
||||
fn parse(value: &Value) -> Option<Self> {
|
||||
match value {
|
||||
Value::String(text)
|
||||
if !text.is_empty()
|
||||
&& text.len() <= MAX_PROGRESS_TOKEN_LEN
|
||||
&& !text.chars().any(char::is_control) =>
|
||||
{
|
||||
Some(Self::Text(text.clone()))
|
||||
}
|
||||
Value::Number(number) => Some(Self::Number(number.clone())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn as_value(&self) -> Value {
|
||||
match self {
|
||||
Self::Text(text) => Value::String(text.clone()),
|
||||
Self::Number(number) => Value::Number(number.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The progress token of one request parameters object, already validated and bounded. The adapters
|
||||
/// read the `_meta` object they already parsed ([`token_from_meta`]); this is the test seam of the
|
||||
/// same rule.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn progress_token(params: &Value) -> Option<ProgressToken> {
|
||||
token_from_meta(params.get("_meta"))
|
||||
}
|
||||
|
||||
/// The validated token of one `RequestMetaObject`. A non-object `_meta` has no token.
|
||||
pub(crate) fn token_from_meta(meta: Option<&Value>) -> Option<ProgressToken> {
|
||||
meta.and_then(|meta| meta.get("progressToken"))
|
||||
.and_then(ProgressToken::parse)
|
||||
}
|
||||
|
||||
/// One real, already reached execution step: the shared Tools service emits these at the stage
|
||||
/// boundaries it actually crosses, so no percentage is invented and no timer drives them.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum Phase {
|
||||
Resolved,
|
||||
Authorized,
|
||||
Dispatching,
|
||||
}
|
||||
|
||||
impl Phase {
|
||||
const fn message(self) -> &'static str {
|
||||
match self {
|
||||
Self::Resolved => "tool_resolved",
|
||||
Self::Authorized => "authorization_resolved",
|
||||
Self::Dispatching => "execution_dispatched",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ProgressEvent {
|
||||
progress: u64,
|
||||
message: &'static str,
|
||||
}
|
||||
|
||||
/// The pinned `notifications/progress` frame. Both supported revisions declare the same parameters,
|
||||
/// so the projection is shared; the token is echoed verbatim and never validated as a credential.
|
||||
fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/progress",
|
||||
"params": {
|
||||
"progressToken": token.as_value(),
|
||||
"progress": event.progress,
|
||||
"message": event.message,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ProgressSink {
|
||||
events: mpsc::Sender<ProgressEvent>,
|
||||
produced: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl ProgressSink {
|
||||
fn emit(&self, message: &'static str) {
|
||||
let progress = self
|
||||
.produced
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
|
||||
+ 1;
|
||||
let _ = self.events.try_send(ProgressEvent { progress, message });
|
||||
}
|
||||
}
|
||||
|
||||
/// A cooperative cancellation signal of one call. It is set at most once; a signal whose only sender
|
||||
/// is gone simply never fires.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CancelSignal {
|
||||
receiver: watch::Receiver<bool>,
|
||||
}
|
||||
|
||||
impl CancelSignal {
|
||||
fn never() -> Self {
|
||||
let (_sender, receiver) = watch::channel(false);
|
||||
Self { receiver }
|
||||
}
|
||||
|
||||
pub(crate) async fn cancelled(&mut self) {
|
||||
if self
|
||||
.receiver
|
||||
.wait_for(|cancelled| *cancelled)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
// The owning call is gone, so nothing can cancel it any more.
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The request-scoped lifecycle of one `tools/call`: its progress sink (when the client asked for
|
||||
/// progress *and* offered the stream carrier) and its cancellation signal.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ToolCallLifecycle {
|
||||
progress: Option<ProgressSink>,
|
||||
cancel: CancelSignal,
|
||||
}
|
||||
|
||||
impl ToolCallLifecycle {
|
||||
pub(crate) fn new(
|
||||
token: Option<ProgressToken>,
|
||||
) -> (Self, Option<mpsc::Receiver<ProgressEvent>>) {
|
||||
match token {
|
||||
Some(_) => {
|
||||
let (sender, receiver) = mpsc::channel(PROGRESS_CAPACITY);
|
||||
(
|
||||
Self {
|
||||
progress: Some(ProgressSink {
|
||||
events: sender,
|
||||
produced: Arc::new(std::sync::atomic::AtomicU64::new(0)),
|
||||
}),
|
||||
cancel: CancelSignal::never(),
|
||||
},
|
||||
Some(receiver),
|
||||
)
|
||||
}
|
||||
None => (
|
||||
Self {
|
||||
progress: None,
|
||||
cancel: CancelSignal::never(),
|
||||
},
|
||||
None,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_cancel(mut self, cancel: CancelSignal) -> Self {
|
||||
self.cancel = cancel;
|
||||
self
|
||||
}
|
||||
|
||||
/// Record one real execution step. Without a progress sink this is a no-op, so the shared service
|
||||
/// does not branch on the presence of a token.
|
||||
pub(crate) fn advance(&self, phase: Phase) {
|
||||
if let Some(progress) = &self.progress {
|
||||
progress.emit(phase.message());
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn cancelled(&mut self) -> impl Future<Output = ()> + '_ {
|
||||
self.cancel.cancelled()
|
||||
}
|
||||
}
|
||||
|
||||
/// The owned identity of one call, so the call future can be driven by the response stream itself.
|
||||
pub(crate) struct ToolCallRequest {
|
||||
pub(crate) workspace_slug: String,
|
||||
pub(crate) agent_slug: String,
|
||||
pub(crate) credential: VerifiedMachineCredential,
|
||||
pub(crate) correlation: CorrelationContext,
|
||||
pub(crate) continuation: ContinuationSupport,
|
||||
pub(crate) locale: ExecutionLocale,
|
||||
}
|
||||
|
||||
/// The wire facts the adapter owns: the envelope, the negotiated revision and response mode, the
|
||||
/// validated token and — for the legacy revision — the session-scoped in-flight ownership.
|
||||
pub(crate) struct ToolCallWire {
|
||||
pub(crate) message: Value,
|
||||
pub(crate) protocol_version: String,
|
||||
pub(crate) response_mode: ResponseMode,
|
||||
pub(crate) progress: Option<ProgressToken>,
|
||||
pub(crate) session_id: Option<String>,
|
||||
pub(crate) in_flight: Option<InFlightScope>,
|
||||
}
|
||||
|
||||
/// Who owns a cancellable call. The session and the platform credential are the ownership of the
|
||||
/// legacy revision; the progress token is deliberately not part of it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct InFlightScope {
|
||||
pub(crate) session_id: String,
|
||||
pub(crate) owner: String,
|
||||
}
|
||||
|
||||
/// The bounded table of the calls a legacy client can still cancel.
|
||||
pub(crate) struct InFlightCalls {
|
||||
entries: Mutex<HashMap<String, watch::Sender<bool>>>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl InFlightCalls {
|
||||
pub(crate) fn new(capacity: usize) -> Self {
|
||||
Self {
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
fn entries(&self) -> std::sync::MutexGuard<'_, HashMap<String, watch::Sender<bool>>> {
|
||||
// A panic of another request must not make the cancellation table unusable.
|
||||
self.entries
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
fn key(scope: &InFlightScope, request_id: &Value) -> String {
|
||||
// The pinned `RequestId` is a string or a number; the two must not collapse into one key.
|
||||
format!(
|
||||
"{}\u{1f}{}\u{1f}{request_id}",
|
||||
scope.session_id, scope.owner
|
||||
)
|
||||
}
|
||||
|
||||
/// Register one call as cancellable. `None` means the table is full (or the id is already live),
|
||||
/// and the call then runs without cancellation instead of evicting someone else's entry.
|
||||
pub(crate) fn register(
|
||||
self: &Arc<Self>,
|
||||
scope: &InFlightScope,
|
||||
request_id: &Value,
|
||||
) -> Option<InFlightEntry> {
|
||||
let key = Self::key(scope, request_id);
|
||||
let (sender, receiver) = watch::channel(false);
|
||||
{
|
||||
let mut entries = self.entries();
|
||||
if entries.len() >= self.capacity || entries.contains_key(&key) {
|
||||
return None;
|
||||
}
|
||||
entries.insert(key.clone(), sender);
|
||||
}
|
||||
Some(InFlightEntry {
|
||||
registry: Arc::clone(self),
|
||||
key,
|
||||
cancel: CancelSignal { receiver },
|
||||
})
|
||||
}
|
||||
|
||||
/// Route one `notifications/cancelled` to the call it names. `false` means there is no such
|
||||
/// in-flight call of this owner, which the specification requires to be ignored.
|
||||
pub(crate) fn cancel(&self, scope: &InFlightScope, request_id: &Value) -> bool {
|
||||
let entries = self.entries();
|
||||
entries
|
||||
.get(&Self::key(scope, request_id))
|
||||
.is_some_and(|sender| sender.send(true).is_ok())
|
||||
}
|
||||
}
|
||||
|
||||
/// The live registration of one call: dropping it (terminal response, cancellation or a dropped
|
||||
/// response stream) releases the table slot.
|
||||
pub(crate) struct InFlightEntry {
|
||||
registry: Arc<InFlightCalls>,
|
||||
key: String,
|
||||
cancel: CancelSignal,
|
||||
}
|
||||
|
||||
impl InFlightEntry {
|
||||
pub(crate) fn cancel_signal(&self) -> CancelSignal {
|
||||
self.cancel.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InFlightEntry {
|
||||
fn drop(&mut self) {
|
||||
self.registry.entries().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one `tools/call` through the shared Tools service with the lifecycle of this request.
|
||||
///
|
||||
/// Without a progress token — or without the SSE carrier the client negotiated — the response is the
|
||||
/// unchanged single JSON (or single-event SSE) response. With both, the response is the request's own
|
||||
/// bounded stream: the real progress events of this call, then the terminal response as the last
|
||||
/// event. Dropping that stream (a `2026-07-28` cancellation) or setting the signal of a legacy
|
||||
/// `notifications/cancelled` drops the call future, which is the cooperative stop.
|
||||
pub(crate) async fn run_tool_call(
|
||||
state: Arc<AppState>,
|
||||
request: ToolCallRequest,
|
||||
wire: ToolCallWire,
|
||||
command: ToolCallCommand,
|
||||
) -> Response {
|
||||
// A token is only honoured when the client offered the only carrier a notification has: the
|
||||
// request's own SSE stream. Otherwise the call runs unchanged (a receiver is never obligated).
|
||||
let token = match (&wire.response_mode, &wire.progress) {
|
||||
(ResponseMode::Sse, Some(token)) => Some(token.clone()),
|
||||
_ => None,
|
||||
};
|
||||
let (mut lifecycle, progress_events) = ToolCallLifecycle::new(token.clone());
|
||||
let in_flight = wire.in_flight.as_ref().and_then(|scope| {
|
||||
let entry = state.in_flight.register(scope, &request_id(&wire.message));
|
||||
if let Some(entry) = &entry {
|
||||
lifecycle = lifecycle.clone().with_cancel(entry.cancel_signal());
|
||||
}
|
||||
entry
|
||||
});
|
||||
|
||||
let message = wire.message.clone();
|
||||
let correlation = request.correlation.clone();
|
||||
let protocol_version = wire.protocol_version.clone();
|
||||
let response_mode = wire.response_mode;
|
||||
|
||||
let mut call = Box::pin(call_future(
|
||||
Arc::clone(&state),
|
||||
request,
|
||||
lifecycle.clone(),
|
||||
command,
|
||||
));
|
||||
|
||||
let Some(progress_events) = progress_events else {
|
||||
// No notification carrier for this call: run it as before, still cancellable.
|
||||
let outcome = tokio::select! {
|
||||
biased;
|
||||
outcome = &mut call => outcome,
|
||||
_ = lifecycle.cancelled() => return cancelled_response(),
|
||||
};
|
||||
return tool_command_response(
|
||||
&message,
|
||||
response_mode,
|
||||
&protocol_version,
|
||||
&correlation,
|
||||
outcome,
|
||||
);
|
||||
};
|
||||
|
||||
debug_assert!(matches!(response_mode, ResponseMode::Sse));
|
||||
let token = token.expect("a progress receiver implies a validated token");
|
||||
let stream = stream::unfold(
|
||||
CallStream {
|
||||
call,
|
||||
events: progress_events,
|
||||
cancel: lifecycle.cancel.clone(),
|
||||
token,
|
||||
message,
|
||||
protocol_version: protocol_version.clone(),
|
||||
correlation,
|
||||
pending: VecDeque::new(),
|
||||
finished: false,
|
||||
logging: crate::protocol::carry_scope(),
|
||||
_in_flight: in_flight,
|
||||
},
|
||||
next_event,
|
||||
);
|
||||
sse_response(
|
||||
StatusCode::OK,
|
||||
stream,
|
||||
wire.session_id.as_deref(),
|
||||
Some(&protocol_version),
|
||||
)
|
||||
}
|
||||
|
||||
async fn call_future(
|
||||
state: Arc<AppState>,
|
||||
request: ToolCallRequest,
|
||||
lifecycle: ToolCallLifecycle,
|
||||
command: ToolCallCommand,
|
||||
) -> ToolsOutcome {
|
||||
handle_tools_command(
|
||||
&state,
|
||||
ServiceContext {
|
||||
workspace_slug: &request.workspace_slug,
|
||||
agent_slug: &request.agent_slug,
|
||||
credential: &request.credential,
|
||||
correlation: &request.correlation,
|
||||
continuation: request.continuation,
|
||||
locale: request.locale,
|
||||
},
|
||||
ToolsCommand::Call(command),
|
||||
Some(lifecycle),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// A cancelled call is not answered: the pinned rule for a cancelled request is that no further
|
||||
/// message is sent for it, and the client that cancelled it ignores a late response anyway. Nothing
|
||||
/// here claims that an outbound side effect did or did not happen — and nothing else claims the
|
||||
/// opposite either: the call is stopped before its terminal outcome, so its Invocation History holds
|
||||
/// no row for this call at all (see the module docs for why there is no invented cancellation code).
|
||||
fn cancelled_response() -> Response {
|
||||
StatusCode::OK.into_response()
|
||||
}
|
||||
|
||||
struct CallStream {
|
||||
call: Pin<Box<dyn Future<Output = ToolsOutcome> + Send>>,
|
||||
events: mpsc::Receiver<ProgressEvent>,
|
||||
cancel: CancelSignal,
|
||||
token: ProgressToken,
|
||||
message: Value,
|
||||
protocol_version: String,
|
||||
correlation: CorrelationContext,
|
||||
pending: VecDeque<Event>,
|
||||
finished: bool,
|
||||
/// The request's logging context, so the terminal event of the stream is projected exactly as the
|
||||
/// single response of the same call would be (Story 3.18 carried across the stream, Story 3.4).
|
||||
logging: crate::protocol::CarriedScope,
|
||||
/// Held for the lifetime of the stream: the slot is released when the call is over, however it ends.
|
||||
_in_flight: Option<InFlightEntry>,
|
||||
}
|
||||
|
||||
enum Step {
|
||||
Progress(ProgressEvent),
|
||||
Outcome(ToolsOutcome),
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// One stream item: a queued frame, the next real progress event, the terminal response, or the end of
|
||||
/// a cancelled stream. A cancelled call sends no terminal frame at all.
|
||||
async fn next_event(
|
||||
call: CallStream,
|
||||
) -> Option<(Result<Event, std::convert::Infallible>, CallStream)> {
|
||||
let scope = call.logging;
|
||||
crate::protocol::in_carried_scope(scope, next_event_in_scope(call)).await
|
||||
}
|
||||
|
||||
async fn next_event_in_scope(
|
||||
mut call: CallStream,
|
||||
) -> Option<(Result<Event, std::convert::Infallible>, CallStream)> {
|
||||
if let Some(frame) = call.pending.pop_front() {
|
||||
return Some((Ok(frame), call));
|
||||
}
|
||||
if call.finished {
|
||||
return None;
|
||||
}
|
||||
|
||||
let step = {
|
||||
let CallStream {
|
||||
call,
|
||||
events,
|
||||
cancel,
|
||||
..
|
||||
} = &mut call;
|
||||
tokio::select! {
|
||||
biased;
|
||||
Some(event) = events.recv() => Step::Progress(event),
|
||||
outcome = call => Step::Outcome(outcome),
|
||||
_ = cancel.cancelled() => Step::Cancelled,
|
||||
}
|
||||
};
|
||||
|
||||
match step {
|
||||
Step::Progress(event) => {
|
||||
let frame = progress_frame(&call.token, &event);
|
||||
Some((Ok(Event::default().data(frame.to_string())), call))
|
||||
}
|
||||
Step::Outcome(outcome) => {
|
||||
call.finished = true;
|
||||
let payload = match tool_command_payload(
|
||||
&call.message,
|
||||
&call.protocol_version,
|
||||
&call.correlation,
|
||||
outcome,
|
||||
) {
|
||||
ToolCommandPayload::Ok(payload) => payload,
|
||||
ToolCommandPayload::Internal(_) => jsonrpc_internal_error(&call.message),
|
||||
};
|
||||
// The same projection point the single response uses, so a streamed answer delivers the
|
||||
// same closed `notifications/message` (Story 3.18) before its terminal frame.
|
||||
crate::transport::note_logging_outcome(&payload);
|
||||
for notification in crate::protocol::take_notifications() {
|
||||
call.pending
|
||||
.push_back(Event::default().data(notification.to_string()));
|
||||
}
|
||||
call.pending
|
||||
.push_back(Event::default().data(payload.to_string()));
|
||||
Some((Ok(call.pending.pop_front().expect("queued frame")), call))
|
||||
}
|
||||
Step::Cancelled => {
|
||||
call.finished = true;
|
||||
call.pending.clear();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn jsonrpc_internal_error(message: &Value) -> Value {
|
||||
crate::jsonrpc::jsonrpc_error(request_id(message), -32603, "internal server error")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "lifecycle_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,170 @@
|
||||
//! Story 3.4 bounds that do not need the HTTP surface: token validation, the monotonic bounded
|
||||
//! progress queue, the pinned frame shape and the ownership of the in-flight cancellation table.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn token(value: Value) -> Option<ProgressToken> {
|
||||
progress_token(&json!({ "_meta": { "progressToken": value } }))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pinned_progress_token_is_a_bounded_string_or_a_number() {
|
||||
assert_eq!(
|
||||
token(json!("tok-1")),
|
||||
Some(ProgressToken::Text("tok-1".to_owned()))
|
||||
);
|
||||
assert!(matches!(token(json!(7)), Some(ProgressToken::Number(_))));
|
||||
assert!(matches!(token(json!(1.5)), Some(ProgressToken::Number(_))));
|
||||
|
||||
// Malformed and out-of-bound tokens disable progress instead of failing the call.
|
||||
assert_eq!(token(json!("")), None);
|
||||
assert_eq!(token(json!("x".repeat(MAX_PROGRESS_TOKEN_LEN + 1))), None);
|
||||
assert_eq!(token(json!("line\nbreak")), None);
|
||||
assert_eq!(token(json!(null)), None);
|
||||
assert_eq!(token(json!(true)), None);
|
||||
assert_eq!(token(json!({ "nested": 1 })), None);
|
||||
assert_eq!(token(json!([1, 2])), None);
|
||||
// A token at the bound is still accepted.
|
||||
assert!(token(json!("x".repeat(MAX_PROGRESS_TOKEN_LEN))).is_some());
|
||||
// Absent `_meta`, absent token and a non-object `_meta` are all "no progress".
|
||||
assert_eq!(progress_token(&json!({})), None);
|
||||
assert_eq!(progress_token(&json!({ "_meta": { "other": 1 } })), None);
|
||||
assert_eq!(progress_token(&json!({ "_meta": "not-an-object" })), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_steps_produce_strictly_increasing_bounded_progress_events() {
|
||||
let (lifecycle, events) = ToolCallLifecycle::new(token(json!("tok-monotonic")));
|
||||
let mut events = events.expect("a validated token opens the progress queue");
|
||||
for phase in [Phase::Resolved, Phase::Authorized, Phase::Dispatching] {
|
||||
lifecycle.advance(phase);
|
||||
}
|
||||
drop(lifecycle);
|
||||
|
||||
let mut observed = Vec::new();
|
||||
while let Some(event) = events.recv().await {
|
||||
observed.push((event.progress, event.message));
|
||||
}
|
||||
assert_eq!(
|
||||
observed,
|
||||
vec![
|
||||
(1, "tool_resolved"),
|
||||
(2, "authorization_resolved"),
|
||||
(3, "execution_dispatched"),
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
observed.windows(2).all(|pair| pair[0].0 < pair[1].0),
|
||||
"progress must increase monotonically, got {observed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_full_progress_queue_drops_events_instead_of_blocking_the_call() {
|
||||
let (lifecycle, events) = ToolCallLifecycle::new(token(json!(9)));
|
||||
let mut events = events.expect("a validated token opens the progress queue");
|
||||
// Many more steps than the queue holds: the call must never wait for the reader.
|
||||
for _ in 0..(PROGRESS_CAPACITY * 4) {
|
||||
lifecycle.advance(Phase::Resolved);
|
||||
}
|
||||
let mut delivered = 0;
|
||||
while events.try_recv().is_ok() {
|
||||
delivered += 1;
|
||||
}
|
||||
assert_eq!(delivered, PROGRESS_CAPACITY);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn without_a_token_no_progress_is_produced_at_all() {
|
||||
let (lifecycle, events) = ToolCallLifecycle::new(None);
|
||||
assert!(events.is_none());
|
||||
lifecycle.advance(Phase::Resolved);
|
||||
lifecycle.advance(Phase::Dispatching);
|
||||
}
|
||||
|
||||
/// The pinned `notifications/progress` frame (schema `ProgressNotification`, both supported
|
||||
/// revisions): `progressToken` echoed verbatim, `progress` increasing, `message` optional and
|
||||
/// `total` absent because this server never invents one.
|
||||
#[test]
|
||||
fn the_progress_frame_matches_the_pinned_notification_shape() {
|
||||
let event = ProgressEvent {
|
||||
progress: 2,
|
||||
message: Phase::Authorized.message(),
|
||||
};
|
||||
assert_eq!(
|
||||
progress_frame(&ProgressToken::Text("tok-2".to_owned()), &event),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/progress",
|
||||
"params": {
|
||||
"progressToken": "tok-2",
|
||||
"progress": 2,
|
||||
"message": "authorization_resolved"
|
||||
}
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
progress_frame(&ProgressToken::Number(7.into()), &event)["params"]["progressToken"],
|
||||
json!(7)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_cancellation_only_reaches_the_call_of_the_same_session_and_credential() {
|
||||
let registry = Arc::new(InFlightCalls::new(MAX_IN_FLIGHT_CALLS));
|
||||
let scope = InFlightScope {
|
||||
session_id: "sess-1".to_owned(),
|
||||
owner: "pk_a".to_owned(),
|
||||
};
|
||||
let entry = registry
|
||||
.register(&scope, &json!(42))
|
||||
.expect("the table has room");
|
||||
let mut signal = entry.cancel_signal();
|
||||
|
||||
// A foreign credential, a foreign session and a different request id cancel nothing.
|
||||
for foreign in [
|
||||
InFlightScope {
|
||||
session_id: "sess-1".to_owned(),
|
||||
owner: "pk_b".to_owned(),
|
||||
},
|
||||
InFlightScope {
|
||||
session_id: "sess-2".to_owned(),
|
||||
owner: "pk_a".to_owned(),
|
||||
},
|
||||
] {
|
||||
assert!(!registry.cancel(&foreign, &json!(42)));
|
||||
}
|
||||
assert!(!registry.cancel(&scope, &json!(43)));
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(50), signal.cancelled())
|
||||
.await
|
||||
.is_err(),
|
||||
"a foreign cancellation must not fire the signal"
|
||||
);
|
||||
|
||||
assert!(registry.cancel(&scope, &json!(42)));
|
||||
tokio::time::timeout(std::time::Duration::from_millis(50), signal.cancelled())
|
||||
.await
|
||||
.expect("the owner's cancellation fires the signal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_released_call_releases_its_slot_and_the_table_is_bounded() {
|
||||
let registry = Arc::new(InFlightCalls::new(1));
|
||||
let scope = InFlightScope {
|
||||
session_id: "sess-1".to_owned(),
|
||||
owner: "pk_a".to_owned(),
|
||||
};
|
||||
let entry = registry.register(&scope, &json!(1)).expect("first slot");
|
||||
assert!(
|
||||
registry.register(&scope, &json!(2)).is_none(),
|
||||
"the table refuses a call beyond its bound instead of evicting the live one"
|
||||
);
|
||||
assert!(registry.cancel(&scope, &json!(1)));
|
||||
drop(entry);
|
||||
assert!(
|
||||
!registry.cancel(&scope, &json!(1)),
|
||||
"a terminal call is no longer cancellable"
|
||||
);
|
||||
assert!(registry.register(&scope, &json!(2)).is_some());
|
||||
}
|
||||
@@ -14,6 +14,11 @@ pub(super) struct ToolCallParams {
|
||||
pub(super) name: String,
|
||||
#[serde(default, deserialize_with = "deserialize_arguments_object")]
|
||||
pub(super) arguments: Option<Value>,
|
||||
/// Story 3.4: the pinned `RequestMetaObject` of this request. Only the validated
|
||||
/// `progressToken` of it is read, and only to decide whether the client asked for out-of-band
|
||||
/// progress; a malformed or foreign `_meta` never fails the call.
|
||||
#[serde(default, rename = "_meta")]
|
||||
pub(super) meta: Option<Value>,
|
||||
}
|
||||
|
||||
/// The pinned `CallToolRequest.params.arguments` is an **optional object**. Omitting it stays valid
|
||||
|
||||
@@ -23,71 +23,75 @@ pub(crate) fn take_confirmation_token(arguments: &mut Value) -> Option<String> {
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn success_tool_response(
|
||||
/// The era-neutral success payload of one `tools/call` (Story 3.4: the streamed response needs the
|
||||
/// payload itself, because the terminal response is the last event of the request's stream).
|
||||
pub(super) fn success_tool_payload(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
output: Value,
|
||||
) -> Response {
|
||||
transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
project_modern_result(
|
||||
protocol_version,
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
|
||||
}
|
||||
],
|
||||
"structuredContent": output,
|
||||
"isError": false
|
||||
}),
|
||||
),
|
||||
) -> Value {
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
project_modern_result(
|
||||
protocol_version,
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
|
||||
}
|
||||
],
|
||||
"structuredContent": output,
|
||||
"isError": false
|
||||
}),
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn tool_error_response(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
error: ToolErrorContract,
|
||||
) -> Response {
|
||||
let error_message = tool_error_text(&error);
|
||||
let error_value = tool_error_value(&error);
|
||||
|
||||
transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
project_modern_result(
|
||||
protocol_version,
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": error_message
|
||||
}
|
||||
],
|
||||
"structuredContent": {
|
||||
"error": error_value
|
||||
},
|
||||
"isError": true
|
||||
}),
|
||||
),
|
||||
),
|
||||
tool_error_payload(message, protocol_version, error),
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn tool_error_payload(
|
||||
message: &Value,
|
||||
protocol_version: &str,
|
||||
error: ToolErrorContract,
|
||||
) -> Value {
|
||||
let error_message = tool_error_text(&error);
|
||||
let error_value = tool_error_value(&error);
|
||||
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
project_modern_result(
|
||||
protocol_version,
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": error_message
|
||||
}
|
||||
],
|
||||
"structuredContent": {
|
||||
"error": error_value
|
||||
},
|
||||
"isError": true
|
||||
}),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// The era-neutral `tools/list` result shape built from the shared command outcome. Object keys end
|
||||
/// up byte-lexicographically ordered, exactly as before the extraction.
|
||||
pub(super) fn tools_list_result(tools: Vec<Value>, next_cursor: Option<String>) -> Value {
|
||||
@@ -193,28 +197,35 @@ pub(super) fn prompt_arguments_error_response(
|
||||
)
|
||||
}
|
||||
|
||||
/// Map one shared Tools-command outcome back to the era-specific wire response (spec section 6.2).
|
||||
/// One `tools/call` wire frame, or the adapter's own internal-error detail.
|
||||
///
|
||||
/// This adapter function is the only place that turns a neutral outcome into a response: the shared
|
||||
/// Story 3.4: the same projection serves the single response and the terminal event of a streamed
|
||||
/// response, so both forms of one outcome can never drift apart.
|
||||
pub(super) enum ToolCommandPayload {
|
||||
Ok(Value),
|
||||
Internal(&'static str),
|
||||
}
|
||||
|
||||
/// Map one shared Tools-command outcome back to its era-specific wire payload (spec section 6.2).
|
||||
///
|
||||
/// This adapter function is the only place that turns a neutral outcome into a wire value: the shared
|
||||
/// service never sees `ResponseMode`, the protocol revision, the request envelope or an
|
||||
/// `axum::Response`. `ApprovalPending` is a success for the client (it still has to answer), while
|
||||
/// an unavailable elicitation and a failing call keep their era-correct tool-error shape.
|
||||
pub(super) fn tool_command_response(
|
||||
pub(super) fn tool_command_payload(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
correlation: &CorrelationContext,
|
||||
outcome: ToolsOutcome,
|
||||
) -> Response {
|
||||
) -> ToolCommandPayload {
|
||||
match outcome {
|
||||
ToolsOutcome::Call(ToolCallOutcome::Completed(output))
|
||||
| ToolsOutcome::Call(ToolCallOutcome::ApprovalPending(output)) => {
|
||||
success_tool_response(message, response_mode, protocol_version, output)
|
||||
ToolCommandPayload::Ok(success_tool_payload(message, protocol_version, output))
|
||||
}
|
||||
ToolsOutcome::Call(ToolCallOutcome::ApprovalUnavailable(control_error)) => {
|
||||
tool_error_response(
|
||||
ToolCommandPayload::Ok(tool_error_payload(
|
||||
message,
|
||||
response_mode,
|
||||
protocol_version,
|
||||
control_tool_error_contract(
|
||||
control_error,
|
||||
@@ -222,34 +233,49 @@ pub(super) fn tool_command_response(
|
||||
correlation.trace_id().as_str(),
|
||||
execution_locale(message),
|
||||
),
|
||||
)
|
||||
))
|
||||
}
|
||||
ToolsOutcome::Call(ToolCallOutcome::InvalidParams(detail)) => {
|
||||
ToolCommandPayload::Ok(jsonrpc_error(request_id(message), -32602, detail))
|
||||
}
|
||||
ToolsOutcome::Call(ToolCallOutcome::InvalidParams(detail)) => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_error(request_id(message), -32602, detail),
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
),
|
||||
ToolsOutcome::Call(ToolCallOutcome::Error(contract)) => {
|
||||
tool_error_response(message, response_mode, protocol_version, contract)
|
||||
ToolCommandPayload::Ok(tool_error_payload(message, protocol_version, contract))
|
||||
}
|
||||
// The pending approval could not be persisted: the adapter answers its internal error shape,
|
||||
// as the previous wire path did.
|
||||
ToolsOutcome::Call(ToolCallOutcome::InternalFailure) => internal_jsonrpc_error(
|
||||
message,
|
||||
"the pending approval request could not be persisted",
|
||||
),
|
||||
ToolsOutcome::Call(ToolCallOutcome::InternalFailure) => {
|
||||
ToolCommandPayload::Internal("the pending approval request could not be persisted")
|
||||
}
|
||||
// A call command never produces these variants; they are answered with the adapter's
|
||||
// internal error instead of being silently ignored.
|
||||
ToolsOutcome::List { .. }
|
||||
| ToolsOutcome::InvalidParams(_)
|
||||
| ToolsOutcome::CatalogUnavailable => {
|
||||
internal_jsonrpc_error(message, "unexpected shared tools outcome for a tools/call")
|
||||
ToolCommandPayload::Internal("unexpected shared tools outcome for a tools/call")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The single-response form of [`tool_command_payload`].
|
||||
pub(super) fn tool_command_response(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
correlation: &CorrelationContext,
|
||||
outcome: ToolsOutcome,
|
||||
) -> Response {
|
||||
match tool_command_payload(message, protocol_version, correlation, outcome) {
|
||||
ToolCommandPayload::Ok(payload) => transport_response(
|
||||
StatusCode::OK,
|
||||
payload,
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
),
|
||||
ToolCommandPayload::Internal(detail) => internal_jsonrpc_error(message, detail),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resource_error_response(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
|
||||
@@ -550,6 +550,7 @@ pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -33,6 +33,7 @@ use crate::{
|
||||
access::credential_allows_security_level,
|
||||
app::{
|
||||
AppState, ApprovalPolicyContext, ApprovalPolicyDecision, InvocationRecord,
|
||||
lifecycle::{Phase, ToolCallLifecycle},
|
||||
maybe_handle_approval_policy, persist_invocation_for_key, resolve_generated_tool,
|
||||
resolve_operation_auth, runtime_operation, take_confirmation_token,
|
||||
},
|
||||
@@ -140,10 +141,15 @@ pub(crate) enum ToolCallOutcome {
|
||||
|
||||
/// Handle one shared Tools command. No wire envelope, `ResponseMode`, `axum::Response`, session or
|
||||
/// protocol revision crosses this boundary.
|
||||
///
|
||||
/// `lifecycle` is the request-scoped progress/cancellation handle of a `tools/call` (Story 3.4);
|
||||
/// a command that cannot have one (`tools/list`) passes `None`, and the service then simply never
|
||||
/// produces a progress event.
|
||||
pub(crate) async fn handle_tools_command(
|
||||
state: &Arc<AppState>,
|
||||
context: ServiceContext<'_>,
|
||||
command: ToolsCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
// One marker per entered owning Tools service/handler (Story 3.1 section 7.2). It is a
|
||||
// `#[cfg(test)]`-only, request-scoped counter and has no production effect.
|
||||
@@ -166,9 +172,9 @@ pub(crate) async fn handle_tools_command(
|
||||
Err(message) => ToolsOutcome::InvalidParams(message.to_owned()),
|
||||
}
|
||||
}
|
||||
ToolsCommand::Call(call) => {
|
||||
ToolsOutcome::Call(execute_tool_command(state, &context, call, &catalog).await)
|
||||
}
|
||||
ToolsCommand::Call(call) => ToolsOutcome::Call(
|
||||
execute_tool_command(state, &context, call, &catalog, lifecycle.as_ref()).await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,6 +184,7 @@ async fn execute_tool_command(
|
||||
context: &ServiceContext<'_>,
|
||||
command: ToolCallCommand,
|
||||
catalog: &PublishedAgentCatalog,
|
||||
lifecycle: Option<&ToolCallLifecycle>,
|
||||
) -> ToolCallOutcome {
|
||||
match catalog.tool_selection_policy.mode {
|
||||
ToolAccessMode::Direct => {
|
||||
@@ -188,6 +195,7 @@ async fn execute_tool_command(
|
||||
&command.tool_name,
|
||||
command.arguments,
|
||||
command.confirmation_token,
|
||||
lifecycle,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -222,6 +230,7 @@ async fn execute_tool_command(
|
||||
&proxy.name,
|
||||
arguments,
|
||||
confirmation_token,
|
||||
lifecycle,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -240,6 +249,7 @@ async fn execute_resolved_tool(
|
||||
tool_name: &str,
|
||||
arguments: Value,
|
||||
confirmation_token: Option<String>,
|
||||
lifecycle: Option<&ToolCallLifecycle>,
|
||||
) -> ToolCallOutcome {
|
||||
let correlation = context.correlation;
|
||||
let transport_request_id = correlation.request_id().as_str();
|
||||
@@ -250,6 +260,10 @@ async fn execute_resolved_tool(
|
||||
Some(resolved) => {
|
||||
StageOutcome::Success.record(&resolve_span);
|
||||
drop(resolve_span);
|
||||
// First real execution step: the addressed tool of the published catalog is resolved.
|
||||
if let Some(lifecycle) = lifecycle {
|
||||
lifecycle.advance(Phase::Resolved);
|
||||
}
|
||||
resolved.tool
|
||||
}
|
||||
None => {
|
||||
@@ -349,8 +363,14 @@ async fn execute_resolved_tool(
|
||||
let resolved_auth =
|
||||
resolve_operation_auth(state, &tool.workspace_id, &operation.execution_config).await;
|
||||
|
||||
// A cancellation can only stop the call before it reaches its terminal outcome: nothing is
|
||||
// persisted for a stopped call, so Invocation History can never claim a false `completed`.
|
||||
let result = match resolved_auth {
|
||||
Ok(resolved_auth) => {
|
||||
// Second real step: the operation's own authorization is resolved, so the runtime may run.
|
||||
if let Some(lifecycle) = lifecycle {
|
||||
lifecycle.advance(Phase::Authorized);
|
||||
}
|
||||
match RuntimeExecutionRequest::try_new(
|
||||
&tool.workspace_id,
|
||||
ExecutionOrigin::AgentSnapshot,
|
||||
@@ -365,7 +385,15 @@ async fn execute_resolved_tool(
|
||||
operation.execution_config.timeout_ms.max(1),
|
||||
),
|
||||
) {
|
||||
Ok(request) => state.runtime.execute_outcome(request).await,
|
||||
Ok(request) => {
|
||||
// Third real step: the request is about to enter the runtime, so from here on an
|
||||
// outbound side effect may happen and a cancellation can no longer promise that
|
||||
// nothing was sent.
|
||||
if let Some(lifecycle) = lifecycle {
|
||||
lifecycle.advance(Phase::Dispatching);
|
||||
}
|
||||
state.runtime.execute_outcome(request).await
|
||||
}
|
||||
Err(_) => Err(ExecutionFailure::new(
|
||||
ExecutionErrorCode::RuntimeInternal,
|
||||
correlation.clone(),
|
||||
|
||||
@@ -192,6 +192,46 @@ pub(crate) fn set_active_level(level: Option<LogLevel>) {
|
||||
});
|
||||
}
|
||||
|
||||
/// The request's logging context, carried into a streamed response body (Story 3.4).
|
||||
///
|
||||
/// A streamed `tools/call` response is assembled after the request handler returned, so its terminal
|
||||
/// event would be projected outside the task-local scope of the request. This snapshot carries the
|
||||
/// only two facts the projection needs — the level the client opted in to and whether the request
|
||||
/// offered the SSE carrier — and never a request value.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub(crate) struct CarriedScope {
|
||||
active: Option<LogLevel>,
|
||||
sse_acceptable: bool,
|
||||
}
|
||||
|
||||
/// Snapshot the scope of the request that is being served now. Outside a scope the snapshot is
|
||||
/// "not opted in", which delivers nothing.
|
||||
pub(crate) fn carry_scope() -> CarriedScope {
|
||||
REQUEST_LOG
|
||||
.try_with(|log| {
|
||||
let log = log.borrow();
|
||||
CarriedScope {
|
||||
active: log.active,
|
||||
sse_acceptable: log.sse_acceptable,
|
||||
}
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Run one step of the streamed response inside a scope carrying the request's logging context.
|
||||
pub(crate) async fn in_carried_scope<F: Future>(scope: CarriedScope, future: F) -> F::Output {
|
||||
REQUEST_LOG
|
||||
.scope(
|
||||
RefCell::new(RequestLog {
|
||||
active: scope.active,
|
||||
sse_acceptable: scope.sse_acceptable,
|
||||
pending: Vec::new(),
|
||||
}),
|
||||
future,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Remember that this request offered `text/event-stream`, the only carrier of a notification.
|
||||
pub(crate) fn set_sse_acceptable() {
|
||||
let _ = REQUEST_LOG.try_with(|log| {
|
||||
|
||||
@@ -38,9 +38,10 @@ pub(crate) use command::{
|
||||
};
|
||||
pub(crate) use completion::{CompletionCommand, CompletionOutcome, handle_completion_command};
|
||||
pub(crate) use logging::{
|
||||
EVENT_FAILED, EVENT_LEVEL_APPLIED, EVENT_REJECTED, EVENT_SERVED, INVALID_LEVEL, LogLevel,
|
||||
LogLevelStore, LoggingOutcome, SetLevelCommand, handle_logging_command, modern_request_level,
|
||||
note, scoped as logging_scope, set_active_level, set_sse_acceptable, take_notifications,
|
||||
CarriedScope, EVENT_FAILED, EVENT_LEVEL_APPLIED, EVENT_REJECTED, EVENT_SERVED, INVALID_LEVEL,
|
||||
LogLevel, LogLevelStore, LoggingOutcome, SetLevelCommand, carry_scope, handle_logging_command,
|
||||
in_carried_scope, modern_request_level, note, scoped as logging_scope, set_active_level,
|
||||
set_sse_acceptable, take_notifications,
|
||||
};
|
||||
pub(crate) use prompts::{
|
||||
PromptGetCommand, PromptsCommand, PromptsListCommand, PromptsOutcome, handle_prompts_command,
|
||||
|
||||
@@ -105,6 +105,10 @@ pub(crate) enum MethodHandler {
|
||||
Ping,
|
||||
Discover,
|
||||
Lifecycle,
|
||||
/// `notifications/cancelled`: the legacy client cancels a `tools/call` it already issued
|
||||
/// (Story 3.4). The handler only routes the id to the owner's in-flight call; it never widens
|
||||
/// what may be cancelled and never authorizes by the progress token.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
@@ -369,7 +373,9 @@ pub(crate) const CAPABILITY_DESCRIPTORS: &[CapabilityIdDescriptor] = &[
|
||||
extension: None,
|
||||
deprecated: None,
|
||||
sub_features: FeatureSet(&["listChanged"]),
|
||||
advertised: Some(("tools", &[("listChanged", false)])),
|
||||
// Story 3.5: the adapter emits `notifications/tools/list_changed` from the committed
|
||||
// published Tool bindings of the Agent snapshot, so the declaration is now true.
|
||||
advertised: Some(("tools", &[("listChanged", true)])),
|
||||
revisions: MODERN_AND_LEGACY,
|
||||
},
|
||||
CapabilityIdDescriptor {
|
||||
@@ -599,6 +605,17 @@ pub(crate) const METHOD_DESCRIPTORS: &[MethodDescriptor] = &[
|
||||
required_headers: HEADERS_NONE,
|
||||
handler: MethodHandler::Lifecycle,
|
||||
},
|
||||
MethodDescriptor {
|
||||
// Story 3.4: the legacy revision's client-side cancellation of a request it issued. The
|
||||
// pinned modern revision has no client-to-server notification over Streamable HTTP (closing
|
||||
// the response stream is its cancellation signal), so this name exists in the legacy era
|
||||
// only. It is a dispatch-only notification registration and advertises nothing.
|
||||
name: "notifications/cancelled",
|
||||
capability: CapabilityId::LegacyLifecycle,
|
||||
kind: Kind::Notification,
|
||||
required_headers: HEADERS_NONE,
|
||||
handler: MethodHandler::Cancelled,
|
||||
},
|
||||
];
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -39,7 +39,7 @@ fn static_registration_is_unique_and_advertises_only_the_registered_capability()
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
}),
|
||||
"the registered advertisement must be the same reviewed object in {revision}"
|
||||
);
|
||||
@@ -50,7 +50,7 @@ fn static_registration_is_unique_and_advertises_only_the_registered_capability()
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
}),
|
||||
"the production advertisement entry point must serve {revision}"
|
||||
);
|
||||
@@ -264,7 +264,7 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -275,7 +275,7 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -286,7 +286,7 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
|
||||
@@ -470,7 +470,7 @@ fn a_single_era_descriptor_never_leaks_into_the_other_era_advertisement() {
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
}),
|
||||
"the other era advertisement stays the reviewed object"
|
||||
);
|
||||
@@ -761,7 +761,7 @@ fn adding_and_removing_a_method_descriptor_changes_the_dispatch_decision() {
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
//! Story 3.19: Resource and Prompt list-change signals (FR-19).
|
||||
//! Story 3.19: Resource and Prompt list-change signals (FR-19); Story 3.5 adds the Tools signal of
|
||||
//! the same seam (FR-14).
|
||||
//!
|
||||
//! The positive result of the story is that a client which asked for the list-change signal of its
|
||||
//! revision learns that the *committed* Resource or Prompt catalog changed, and that the ordinary
|
||||
//! `resources/list`, `resources/read`, `resources/templates/list`, `prompts/list` and `prompts/get`
|
||||
//! stay authoritative for a client that misses the notification.
|
||||
//! revision learns that the *committed* Resource, Prompt or Tool catalog changed, and that the
|
||||
//! ordinary `resources/list`, `resources/read`, `resources/templates/list`, `prompts/list`,
|
||||
//! `prompts/get` and `tools/list` stay authoritative for a client that misses the notification.
|
||||
//!
|
||||
//! Shape of the chosen seam (ARC decision, kanban t_c12863fb): a **stream-local watcher**. One open
|
||||
//! notification stream owns one bounded periodic read of the published Agent catalog the stream is
|
||||
//! scoped to; a change of the visible Resource/Prompt binding projection produces the revision's
|
||||
//! scoped to; a change of the visible Resource/Prompt/Tool projection of that snapshot (the Resource
|
||||
//! and Prompt bindings plus the exact Tool definitions `tools/list` serves) produces the revision's
|
||||
//! list-change notification. There is no post-commit hook, no cross-process bus and no durable
|
||||
//! queue: the committed catalog is the source of truth, so a signal can never be produced before its
|
||||
//! commit, and a failed read can never roll the commit back — it only skips that tick and stays
|
||||
@@ -55,6 +57,8 @@ pub(crate) const SUBSCRIPTION_ID_KEY: &str = "io.modelcontextprotocol/subscripti
|
||||
pub(crate) const RESOURCE_LIST_CHANGED: &str = "notifications/resources/list_changed";
|
||||
/// `notifications/prompts/list_changed` (both approved revisions).
|
||||
pub(crate) const PROMPT_LIST_CHANGED: &str = "notifications/prompts/list_changed";
|
||||
/// `notifications/tools/list_changed` (both approved revisions, Story 3.5).
|
||||
pub(crate) const TOOL_LIST_CHANGED: &str = "notifications/tools/list_changed";
|
||||
/// `notifications/subscriptions/acknowledged` (modern revision only).
|
||||
pub(crate) const SUBSCRIPTIONS_ACKNOWLEDGED: &str = "notifications/subscriptions/acknowledged";
|
||||
|
||||
@@ -79,29 +83,32 @@ pub(crate) const MAX_FILTER_URI_BYTES: usize = 2_048;
|
||||
/// The one safe detail of a malformed or oversized `subscriptions/listen` filter.
|
||||
pub(crate) const INVALID_LISTEN_FILTER: &str = "invalid subscriptions/listen notification filter";
|
||||
|
||||
/// The two list-change signal types this server implements.
|
||||
/// The three list-change signal types this server implements.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct Signals {
|
||||
pub(crate) resources: bool,
|
||||
pub(crate) prompts: bool,
|
||||
pub(crate) tools: bool,
|
||||
}
|
||||
|
||||
impl Signals {
|
||||
/// Both types: the legacy revision defines no client opt-in, so its stream carries both.
|
||||
pub(crate) const BOTH: Self = Self {
|
||||
/// All types: the legacy revision defines no client opt-in, so its stream carries every type.
|
||||
pub(crate) const ALL: Self = Self {
|
||||
resources: true,
|
||||
prompts: true,
|
||||
tools: true,
|
||||
};
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))] // read by the unit oracle of the projection
|
||||
pub(crate) fn any(self) -> bool {
|
||||
self.resources || self.prompts
|
||||
self.resources || self.prompts || self.tools
|
||||
}
|
||||
|
||||
fn intersect(self, other: Self) -> Self {
|
||||
Self {
|
||||
resources: self.resources && other.resources,
|
||||
prompts: self.prompts && other.prompts,
|
||||
tools: self.tools && other.tools,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,8 +121,6 @@ impl Signals {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct ListenFilter {
|
||||
#[serde(default)]
|
||||
/// `toolsListChanged` is not implemented in this scope and is therefore never acknowledged.
|
||||
#[allow(dead_code)]
|
||||
tools_list_changed: bool,
|
||||
#[serde(default)]
|
||||
prompts_list_changed: bool,
|
||||
@@ -145,9 +150,9 @@ pub(crate) enum FilterOutcome {
|
||||
impl ListenFilter {
|
||||
/// Validate the bounded inputs and keep only the honoured types.
|
||||
///
|
||||
/// `toolsListChanged` and `resourceSubscriptions` are not implemented in the current scope, which
|
||||
/// is what the acknowledgment reports: they are simply not honoured, and the pinned schema
|
||||
/// forbids the server from sending a type it did not agree to.
|
||||
/// `resourceSubscriptions` is not implemented in the current scope, which is what the
|
||||
/// acknowledgment reports: it is simply not honoured, and the pinned schema forbids the server
|
||||
/// from sending a type it did not agree to.
|
||||
pub(crate) fn accepted(&self) -> FilterOutcome {
|
||||
if self.resource_subscriptions.len() > MAX_FILTER_URIS
|
||||
|| self
|
||||
@@ -160,6 +165,7 @@ impl ListenFilter {
|
||||
FilterOutcome::Accepted(Signals {
|
||||
resources: self.resources_list_changed,
|
||||
prompts: self.prompts_list_changed,
|
||||
tools: self.tools_list_changed,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -173,16 +179,23 @@ fn acknowledged_filter_wire(accepted: Signals) -> Value {
|
||||
if accepted.resources {
|
||||
filter.insert("resourcesListChanged".to_owned(), Value::Bool(true));
|
||||
}
|
||||
if accepted.tools {
|
||||
filter.insert("toolsListChanged".to_owned(), Value::Bool(true));
|
||||
}
|
||||
Value::Object(filter)
|
||||
}
|
||||
|
||||
/// The exact bound Resource/Prompt projection of one published Agent catalog.
|
||||
/// The exact bound Resource/Prompt/Tool projection of one published Agent catalog.
|
||||
///
|
||||
/// The projection is derived from the same committed catalog read that already feeds
|
||||
/// `resources/list`, `resources/templates/list`, `prompts/list` and `prompts/get`: the exact Resource
|
||||
/// and Prompt bindings of the published Agent snapshot, in deterministic binding order.
|
||||
/// `catalog_revision` alone is *not* the comparison key — a commit that only changes Tools must not
|
||||
/// produce a Resource/Prompt signal — so the two binding vectors are compared on their own.
|
||||
/// `resources/list`, `resources/templates/list`, `prompts/list`, `prompts/get` and `tools/list`: the
|
||||
/// exact Resource and Prompt bindings of the published Agent snapshot, in deterministic binding
|
||||
/// order, and the exact Tool definitions that `tools/list` would serve. The Tool part is taken from
|
||||
/// [`crate::manifest::catalog_tool_definitions`] rather than from the bindings, because that is the
|
||||
/// projection the client sees: it also depends on the Tool selection policy (mode, groups, search
|
||||
/// settings), and in `Search` mode it does not depend on the bindings at all. `catalog_revision`
|
||||
/// alone is *not* the comparison key — a commit that only changes Tools must not produce a Resource
|
||||
/// or Prompt signal, and vice versa — so the three projections are compared on their own.
|
||||
///
|
||||
/// shortcut: a Resource revocation invalidates a bound publication without a new Agent snapshot, so
|
||||
/// it does not change this projection and produces no list-change signal; the revocation gate at
|
||||
@@ -192,40 +205,45 @@ fn acknowledged_filter_wire(accepted: Signals) -> Value {
|
||||
pub(crate) struct CatalogFingerprint {
|
||||
resources: Vec<(String, u32)>,
|
||||
prompts: Vec<(String, u32)>,
|
||||
/// Canonical JSON of every definition `tools/list` serves, sorted so a re-ordered read is not a
|
||||
/// false change.
|
||||
tools: Vec<String>,
|
||||
}
|
||||
|
||||
impl CatalogFingerprint {
|
||||
pub(crate) fn of(catalog: &PublishedAgentCatalog) -> Self {
|
||||
let mut resources = catalog
|
||||
.resources
|
||||
.iter()
|
||||
.map(|binding| {
|
||||
fn sorted<T: Ord>(values: impl Iterator<Item = T>) -> Vec<T> {
|
||||
let mut sorted = values.collect::<Vec<_>>();
|
||||
sorted.sort_unstable();
|
||||
sorted
|
||||
}
|
||||
Self {
|
||||
resources: sorted(catalog.resources.iter().map(|binding| {
|
||||
(
|
||||
binding.resource_id.as_str().to_owned(),
|
||||
binding.resource_version,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut prompts = catalog
|
||||
.prompts
|
||||
.iter()
|
||||
.map(|binding| {
|
||||
})),
|
||||
prompts: sorted(catalog.prompts.iter().map(|binding| {
|
||||
(
|
||||
binding.prompt_id.as_str().to_owned(),
|
||||
binding.prompt_version,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
resources.sort();
|
||||
prompts.sort();
|
||||
Self { resources, prompts }
|
||||
})),
|
||||
tools: sorted(
|
||||
crate::manifest::catalog_tool_definitions(catalog)
|
||||
.into_iter()
|
||||
.map(|definition| definition.to_string()),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// The list-change types whose visible binding projection differs between `previous` and `self`.
|
||||
/// The list-change types whose visible projection differs between `previous` and `self`.
|
||||
pub(crate) fn changes_since(&self, previous: &Self) -> Signals {
|
||||
Signals {
|
||||
resources: self.resources != previous.resources,
|
||||
prompts: self.prompts != previous.prompts,
|
||||
tools: self.tools != previous.tools,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,10 +259,11 @@ pub(crate) enum Wire {
|
||||
|
||||
impl Wire {
|
||||
fn signal_frames(&self, signals: Signals) -> Vec<String> {
|
||||
let mut frames = Vec::with_capacity(2);
|
||||
let mut frames = Vec::with_capacity(3);
|
||||
for (changed, method) in [
|
||||
(signals.resources, RESOURCE_LIST_CHANGED),
|
||||
(signals.prompts, PROMPT_LIST_CHANGED),
|
||||
(signals.tools, TOOL_LIST_CHANGED),
|
||||
] {
|
||||
if !changed {
|
||||
continue;
|
||||
|
||||
@@ -2,35 +2,59 @@
|
||||
//! wire frames of both approved revisions and the bounded legacy stream ownership.
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use crank_core::ToolSelectionPolicy;
|
||||
use crank_core::ids::{PromptId, ResourceId};
|
||||
use crank_core::{
|
||||
ExecutionConfig, HttpMethod, Operation, OperationId, OperationSecurityLevel, OperationStatus,
|
||||
Protocol, RestTarget, Target, ToolAccessMode, ToolDescription, ToolGroup, ToolSearchSettings,
|
||||
ToolSelectionPolicy, WorkspaceId,
|
||||
};
|
||||
use crank_mapping::MappingSet;
|
||||
use crank_registry::PublishedAgentCatalog;
|
||||
use crank_registry::model::{AgentPromptBinding, AgentResourceBinding};
|
||||
use crank_registry::model::{AgentPromptBinding, AgentResourceBinding, PublishedAgentTool};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use serde_json::{Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::{
|
||||
CatalogFingerprint, FilterOutcome, INVALID_LISTEN_FILTER, ListenFilter, MAX_FILTER_URIS,
|
||||
MAX_SUBSCRIPTION_ID_BYTES, PROMPT_LIST_CHANGED, RESOURCE_LIST_CHANGED, ReadError, Reader,
|
||||
SIGNAL_QUEUE_CAPACITY, SUBSCRIPTION_ID_KEY, SUBSCRIPTIONS_ACKNOWLEDGED, Signals,
|
||||
StreamRegistry, Watch, Wire, acknowledged_frame, run, subscription_id,
|
||||
StreamRegistry, TOOL_LIST_CHANGED, Watch, Wire, acknowledged_frame, run, subscription_id,
|
||||
};
|
||||
|
||||
fn catalog(
|
||||
resources: &[(&str, u32)],
|
||||
prompts: &[(&str, u32)],
|
||||
catalog_revision: &str,
|
||||
) -> PublishedAgentCatalog {
|
||||
catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
Vec::new(),
|
||||
resources,
|
||||
prompts,
|
||||
catalog_revision,
|
||||
)
|
||||
}
|
||||
|
||||
fn catalog_with(
|
||||
tool_selection_policy: ToolSelectionPolicy,
|
||||
tools: Vec<PublishedAgentTool>,
|
||||
resources: &[(&str, u32)],
|
||||
prompts: &[(&str, u32)],
|
||||
catalog_revision: &str,
|
||||
) -> PublishedAgentCatalog {
|
||||
PublishedAgentCatalog {
|
||||
agent_version: 1,
|
||||
catalog_revision: catalog_revision.to_owned(),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
tools: Vec::new(),
|
||||
tool_selection_policy,
|
||||
tools,
|
||||
resources: resources
|
||||
.iter()
|
||||
.map(|(id, version)| AgentResourceBinding {
|
||||
@@ -48,6 +72,94 @@ fn catalog(
|
||||
}
|
||||
}
|
||||
|
||||
fn search_policy(tool_names: &[&str]) -> ToolSelectionPolicy {
|
||||
ToolSelectionPolicy {
|
||||
mode: ToolAccessMode::Search,
|
||||
groups: vec![ToolGroup {
|
||||
id: "crm".to_owned(),
|
||||
name: "CRM".to_owned(),
|
||||
description: "CRM operations".to_owned(),
|
||||
tool_names: tool_names.iter().map(|name| (*name).to_owned()).collect(),
|
||||
}],
|
||||
search: ToolSearchSettings::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_object_schema() -> Schema {
|
||||
Schema {
|
||||
kind: SchemaKind::Object,
|
||||
format: None,
|
||||
description: None,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: None,
|
||||
fields: BTreeMap::new(),
|
||||
items: None,
|
||||
enum_values: Vec::new(),
|
||||
variants: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// One published Tool binding whose served definition is exactly `title`/`description`.
|
||||
fn published_tool(name: &str, title: &str, description: &str, version: u32) -> PublishedAgentTool {
|
||||
PublishedAgentTool {
|
||||
workspace_id: WorkspaceId::new("ws_test"),
|
||||
workspace_slug: "ws-test".to_owned(),
|
||||
agent_id: "agent_test".into(),
|
||||
agent_slug: "agent".to_owned(),
|
||||
operation: Operation {
|
||||
id: OperationId::new(format!("op_{name}")),
|
||||
name: name.to_owned(),
|
||||
display_name: title.to_owned(),
|
||||
category: "test".to_owned(),
|
||||
protocol: Protocol::Rest,
|
||||
security_level: OperationSecurityLevel::Standard,
|
||||
status: OperationStatus::Published,
|
||||
version,
|
||||
target: Target::Rest(RestTarget {
|
||||
base_url: "https://example.test".to_owned(),
|
||||
method: HttpMethod::Get,
|
||||
path_template: "/leads".to_owned(),
|
||||
static_headers: BTreeMap::new(),
|
||||
request_contract: None,
|
||||
response_contract: Vec::new(),
|
||||
multipart_field_mime_types: BTreeMap::new(),
|
||||
}),
|
||||
input_schema: empty_object_schema(),
|
||||
output_schema: empty_object_schema(),
|
||||
input_mapping: MappingSet { rules: Vec::new() },
|
||||
output_mapping: MappingSet { rules: Vec::new() },
|
||||
execution_config: ExecutionConfig {
|
||||
timeout_ms: 10_000,
|
||||
retry_policy: None,
|
||||
response_cache: None,
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
tool_description: ToolDescription {
|
||||
title: title.to_owned(),
|
||||
description: description.to_owned(),
|
||||
tags: Vec::new(),
|
||||
examples: Vec::new(),
|
||||
},
|
||||
samples: None,
|
||||
generated_draft: None,
|
||||
config_export: None,
|
||||
wizard_state: None,
|
||||
created_at: OffsetDateTime::UNIX_EPOCH,
|
||||
updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
published_at: None,
|
||||
},
|
||||
tool_name: name.to_owned(),
|
||||
tool_title: title.to_owned(),
|
||||
tool_description: description.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_committed_resource_binding_is_a_resource_change_only() {
|
||||
let before = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-1"));
|
||||
@@ -61,7 +173,8 @@ fn a_committed_resource_binding_is_a_resource_change_only() {
|
||||
after.changes_since(&before),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false
|
||||
prompts: false,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
assert!(!before.changes_since(&after).prompts);
|
||||
@@ -69,7 +182,8 @@ fn a_committed_resource_binding_is_a_resource_change_only() {
|
||||
before.changes_since(&after),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false
|
||||
prompts: false,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -83,7 +197,8 @@ fn a_committed_prompt_binding_is_a_prompt_change_only() {
|
||||
after.changes_since(&before),
|
||||
Signals {
|
||||
resources: false,
|
||||
prompts: true
|
||||
prompts: true,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -97,15 +212,219 @@ fn a_pinned_version_bump_is_a_change_even_when_the_binding_names_are_stable() {
|
||||
after.changes_since(&before),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false
|
||||
prompts: false,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tool_only_catalog_commit_is_silent() {
|
||||
// `catalog_revision` moved, but the visible Resource/Prompt projection did not: a Tool-only
|
||||
// commit must not produce a Resource or Prompt list-change signal.
|
||||
fn a_committed_tool_definition_change_is_a_tool_change_only() {
|
||||
let lead = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![lead()],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let added = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![
|
||||
lead(),
|
||||
published_tool(
|
||||
"crm_create_second",
|
||||
"Create Second",
|
||||
"Creates a second lead",
|
||||
1,
|
||||
),
|
||||
],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
let retitled = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create a lead",
|
||||
"Creates a lead",
|
||||
1,
|
||||
)],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
let tools_only = Signals {
|
||||
resources: false,
|
||||
prompts: false,
|
||||
tools: true,
|
||||
};
|
||||
assert_eq!(added.changes_since(&before), tools_only);
|
||||
assert_eq!(
|
||||
retitled.changes_since(&before),
|
||||
tools_only,
|
||||
"a retitled binding changes the visible Tool projection without a new Operation version"
|
||||
);
|
||||
// The reverse direction is a Tool change as well: a removed binding is a real change.
|
||||
assert!(before.changes_since(&added).tools);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_republished_operation_version_with_an_identical_definition_is_silent() {
|
||||
// `tools/list` serves the definition, not the Operation version: a version bump that leaves the
|
||||
// served definition unchanged is not a client-visible Tool catalog change.
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create Lead",
|
||||
"Creates a lead",
|
||||
1,
|
||||
)],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let after = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create Lead",
|
||||
"Creates a lead",
|
||||
2,
|
||||
)],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
assert!(!after.changes_since(&before).any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_policy_only_commit_that_changes_tools_list_is_a_tool_change_only() {
|
||||
// The Tool selection policy is part of the served `tools/list` projection: switching the same
|
||||
// bindings from direct to search mode replaces the served tools with `search_tools`/`call_tool`.
|
||||
let tool = published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let direct = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool.clone()],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let search = CatalogFingerprint::of(&catalog_with(
|
||||
search_policy(&["crm_create_lead"]),
|
||||
vec![tool],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
let tools_only = Signals {
|
||||
resources: false,
|
||||
prompts: false,
|
||||
tools: true,
|
||||
};
|
||||
assert_eq!(search.changes_since(&direct), tools_only);
|
||||
assert_eq!(direct.changes_since(&search), tools_only);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_policy_field_invisible_to_tools_list_is_silent() {
|
||||
// `search.max_results` is only served inside the search-mode definitions: in direct mode it
|
||||
// changes nothing the client receives.
|
||||
let tool = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool()],
|
||||
&[],
|
||||
&[],
|
||||
"rev-1",
|
||||
));
|
||||
let after = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy {
|
||||
search: ToolSearchSettings { max_results: 5 },
|
||||
..ToolSelectionPolicy::default()
|
||||
},
|
||||
vec![tool()],
|
||||
&[],
|
||||
&[],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
assert!(!after.changes_since(&before).any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_search_mode_binding_change_is_silent_when_tools_list_is_unchanged() {
|
||||
// In search mode the served list is `search_tools`/`call_tool`; the bindings behind it are not
|
||||
// part of `tools/list`, so a binding-only commit must not claim a visible Tool change.
|
||||
let policy = search_policy(&["crm_create_lead"]);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
policy.clone(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create Lead",
|
||||
"Creates a lead",
|
||||
1,
|
||||
)],
|
||||
&[],
|
||||
&[],
|
||||
"rev-1",
|
||||
));
|
||||
let after = CatalogFingerprint::of(&catalog_with(
|
||||
policy,
|
||||
vec![
|
||||
published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1),
|
||||
published_tool(
|
||||
"crm_create_second",
|
||||
"Create Second",
|
||||
"Creates a second lead",
|
||||
1,
|
||||
),
|
||||
],
|
||||
&[],
|
||||
&[],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
assert!(!after.changes_since(&before).any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resource_or_prompt_change_is_not_a_tool_change() {
|
||||
let tool = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool()],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let changed = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool()],
|
||||
&[("res_a", 2)],
|
||||
&[("prm_a", 1), ("prm_b", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
changed.changes_since(&before),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: true,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_catalog_revision_bump_without_a_visible_change_is_silent() {
|
||||
// `catalog_revision` moved, but no visible Resource/Prompt/Tool projection did: no signal.
|
||||
let before = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-1"));
|
||||
let after = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-2"));
|
||||
|
||||
@@ -130,13 +449,14 @@ fn the_projection_is_order_independent_and_has_no_duplicate_growth() {
|
||||
|
||||
#[test]
|
||||
fn the_legacy_frames_carry_a_closed_method_name_and_no_parameters() {
|
||||
let frames = Wire::Legacy.signal_frames(Signals::BOTH);
|
||||
let frames = Wire::Legacy.signal_frames(Signals::ALL);
|
||||
|
||||
assert_eq!(
|
||||
frames,
|
||||
vec![
|
||||
json!({"jsonrpc": "2.0", "method": RESOURCE_LIST_CHANGED}).to_string(),
|
||||
json!({"jsonrpc": "2.0", "method": PROMPT_LIST_CHANGED}).to_string(),
|
||||
json!({"jsonrpc": "2.0", "method": TOOL_LIST_CHANGED}).to_string(),
|
||||
]
|
||||
);
|
||||
// No URI, name or payload of a changed object is part of a list-change notification.
|
||||
@@ -146,7 +466,7 @@ fn the_legacy_frames_carry_a_closed_method_name_and_no_parameters() {
|
||||
#[test]
|
||||
fn the_modern_frames_carry_the_subscription_id_of_their_listen_request() {
|
||||
let id = json!(41);
|
||||
let frames = Wire::Modern(id.clone()).signal_frames(Signals::BOTH);
|
||||
let frames = Wire::Modern(id.clone()).signal_frames(Signals::ALL);
|
||||
|
||||
assert_eq!(
|
||||
frames,
|
||||
@@ -163,6 +483,12 @@ fn the_modern_frames_carry_the_subscription_id_of_their_listen_request() {
|
||||
"params": { "_meta": { SUBSCRIPTION_ID_KEY: 41 } }
|
||||
})
|
||||
.to_string(),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": TOOL_LIST_CHANGED,
|
||||
"params": { "_meta": { SUBSCRIPTION_ID_KEY: 41 } }
|
||||
})
|
||||
.to_string(),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -176,6 +502,7 @@ fn only_the_requested_and_honoured_types_are_encoded() {
|
||||
let frames = Wire::Modern(json!(1)).signal_frames(Signals {
|
||||
resources: false,
|
||||
prompts: true,
|
||||
tools: false,
|
||||
});
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
@@ -189,6 +516,7 @@ fn the_acknowledgment_reports_exactly_the_honoured_subset() {
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false,
|
||||
tools: true,
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
@@ -197,7 +525,7 @@ fn the_acknowledgment_reports_exactly_the_honoured_subset() {
|
||||
assert_eq!(frame["params"]["_meta"][SUBSCRIPTION_ID_KEY], json!(7));
|
||||
assert_eq!(
|
||||
frame["params"]["notifications"],
|
||||
json!({ "resourcesListChanged": true })
|
||||
json!({ "resourcesListChanged": true, "toolsListChanged": true })
|
||||
);
|
||||
}
|
||||
|
||||
@@ -224,8 +552,10 @@ fn the_filter_keeps_only_the_implemented_list_change_types() {
|
||||
signals,
|
||||
Signals {
|
||||
resources: false,
|
||||
prompts: true
|
||||
}
|
||||
prompts: true,
|
||||
tools: true
|
||||
},
|
||||
"the honoured set is the requested list-change subset: {signals:?}"
|
||||
),
|
||||
FilterOutcome::Invalid(detail) => panic!("a bounded filter must be accepted: {detail}"),
|
||||
}
|
||||
@@ -372,7 +702,7 @@ async fn a_failed_catalog_read_is_observable_and_loses_no_committed_change() {
|
||||
Watch {
|
||||
workspace_slug: "workspace-canary".to_owned(),
|
||||
agent_slug: "agent-canary".to_owned(),
|
||||
signals: Signals::BOTH,
|
||||
signals: Signals::ALL,
|
||||
wire: Wire::Legacy,
|
||||
legacy_stream: None,
|
||||
deadline: Some(Instant::now() + Duration::from_millis(2_100)),
|
||||
|
||||
@@ -313,7 +313,7 @@ pub(super) fn transport_response(
|
||||
/// Record the closed safe event of one response: served, rejected as invalid, failed, or a tool-level
|
||||
/// error. Every label is a fixed string of the closed vocabulary; nothing from the payload or the
|
||||
/// request is copied into it.
|
||||
fn note_logging_outcome(payload: &Value) {
|
||||
pub(super) fn note_logging_outcome(payload: &Value) {
|
||||
use crate::protocol::{EVENT_FAILED, EVENT_REJECTED, EVENT_SERVED, LogLevel, note};
|
||||
if payload.get("error").is_some() {
|
||||
match payload.pointer("/error/code").and_then(Value::as_i64) {
|
||||
|
||||
@@ -284,6 +284,7 @@ fn operation() -> RegistryOperation {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
//! Typed execution-admission seam (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Admission is the single gate in front of every real Tool execution: it decides whether the
|
||||
//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide
|
||||
//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives
|
||||
//! in PostgreSQL (`crank-registry`), so several processes share one budget.
|
||||
//!
|
||||
//! Deliberately absent from this contract: credentials, connection or protocol revision (they must
|
||||
//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2).
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{NoTasksFallback, ToolExecutionMode};
|
||||
|
||||
pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128;
|
||||
pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64;
|
||||
pub const MAX_RETRY_AFTER_MS: u64 = 60_000;
|
||||
/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for
|
||||
/// the store and every runtime so the client contract stays stable.
|
||||
pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250;
|
||||
|
||||
/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms).
|
||||
pub fn current_unix_ms() -> i64 {
|
||||
i64::try_from(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|elapsed| elapsed.as_millis())
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
.unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin
|
||||
/// bucket that is still bounded by the global limit.
|
||||
pub const ADMIN_ORIGIN_KEY: &str = "admin";
|
||||
|
||||
/// Exact execution identity pinned at admission time.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AdmissionScope {
|
||||
pub workspace_id: String,
|
||||
/// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution.
|
||||
pub agent_key: String,
|
||||
pub operation_id: String,
|
||||
/// Quota keys exclude the version; the version is pinned for execution semantics only.
|
||||
pub operation_version: u32,
|
||||
}
|
||||
|
||||
impl AdmissionScope {
|
||||
/// Quota key of the Agent bucket: `(workspace, agent)`.
|
||||
pub fn agent_bucket(&self) -> String {
|
||||
format!("{}:{}", self.workspace_id, self.agent_key)
|
||||
}
|
||||
|
||||
/// Quota key of the Tool bucket: `(workspace, operation)` without version.
|
||||
pub fn tool_bucket(&self) -> String {
|
||||
format!("{}:{}", self.workspace_id, self.operation_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AdmissionMode {
|
||||
/// The caller waits for a scoped permit and dispatches inside it.
|
||||
Synchronous,
|
||||
/// The store commits a durable enqueue decision and reserves queue capacity.
|
||||
Background,
|
||||
}
|
||||
|
||||
impl AdmissionMode {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Synchronous => "synchronous",
|
||||
Self::Background => "background",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-consuming approval evidence gathered by the runtime before admission.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ApprovalEvidence {
|
||||
/// The operation does not require confirmation.
|
||||
NotRequired,
|
||||
/// A verified approval exists for this exact scope and input.
|
||||
Granted,
|
||||
/// Confirmation is required and no verified approval exists yet.
|
||||
NotGranted,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionRequest {
|
||||
pub scope: AdmissionScope,
|
||||
pub mode: AdmissionMode,
|
||||
/// Published policy mode that produced this request.
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
pub no_tasks_fallback: NoTasksFallback,
|
||||
pub approval: ApprovalEvidence,
|
||||
/// Bounded request identity used only to correlate one admission entry.
|
||||
pub fingerprint: String,
|
||||
/// Absolute deadline (unix milliseconds), already including queue waiting time.
|
||||
pub expires_at_unix_ms: i64,
|
||||
}
|
||||
|
||||
impl AdmissionRequest {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
if self.fingerprint.is_empty()
|
||||
|| self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES
|
||||
|| self
|
||||
.fingerprint
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_control() || byte == b' ')
|
||||
{
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission fingerprint is outside its allowed bounds",
|
||||
));
|
||||
}
|
||||
if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission workspace is invalid",
|
||||
));
|
||||
}
|
||||
if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid("admission agent is invalid"));
|
||||
}
|
||||
if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission operation is invalid",
|
||||
));
|
||||
}
|
||||
if self.scope.operation_version == 0 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission operation version must be positive",
|
||||
));
|
||||
}
|
||||
if self.expires_at_unix_ms <= 0 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission deadline is invalid",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Scoped permit. Held for the whole local execution future (including retries) and released
|
||||
/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionPermit {
|
||||
pub ticket_id: u64,
|
||||
pub owner_token: String,
|
||||
pub scope: AdmissionScope,
|
||||
pub mode: AdmissionMode,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
}
|
||||
|
||||
/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics.
|
||||
/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionDecision {
|
||||
pub ticket_id: u64,
|
||||
pub scope: AdmissionScope,
|
||||
pub expires_at_unix_ms: i64,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AdmissionOutcome {
|
||||
Permitted(AdmissionPermit),
|
||||
Enqueued(AdmissionDecision),
|
||||
/// Capacity or queue bounds were exhausted before any unbounded allocation.
|
||||
Overloaded {
|
||||
retry_after_ms: u64,
|
||||
},
|
||||
/// The queue wait reached the absolute deadline; nothing was dispatched.
|
||||
DeadlineElapsed,
|
||||
/// Confirmation is mandatory for this operation and no verified approval exists.
|
||||
ApprovalRequired,
|
||||
/// The published policy does not advertise durable Tasks and refuses the fallback.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct AdmissionQueueSnapshot {
|
||||
pub waiting: u64,
|
||||
pub executing: u64,
|
||||
/// Age of the oldest waiting request in the authoritative queue, in milliseconds.
|
||||
pub oldest_waiting_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum AdmissionStoreError {
|
||||
/// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed.
|
||||
#[error("execution admission store is unavailable: {message}")]
|
||||
Unavailable { message: String },
|
||||
/// The store rejected a malformed admission request.
|
||||
#[error("execution admission request is invalid: {message}")]
|
||||
Invalid { message: String },
|
||||
}
|
||||
|
||||
impl AdmissionStoreError {
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::Unavailable {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn invalid(message: impl Into<String>) -> Self {
|
||||
Self::Invalid {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ExecutionAdmission: Send + Sync {
|
||||
/// Proves the mandatory admission state is usable (schema and operations reachable).
|
||||
async fn health(&self) -> Result<(), AdmissionStoreError>;
|
||||
|
||||
/// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue
|
||||
/// decision for a background request, or returns a typed refusal.
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<AdmissionOutcome, AdmissionStoreError>;
|
||||
|
||||
/// Releases the caller's own permit. Idempotent: a repeated release returns `false`.
|
||||
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError>;
|
||||
|
||||
/// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket.
|
||||
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError>;
|
||||
|
||||
/// Bounded authoritative queue readings for low-cardinality metrics.
|
||||
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError>;
|
||||
}
|
||||
|
||||
/// One waiting entry as seen by the fairness scheduler.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionQueueEntry {
|
||||
pub ticket_id: u64,
|
||||
/// Authoritative insertion order inside `(Agent, Tool)`; never rewritten.
|
||||
pub sequence: u64,
|
||||
pub agent_key: String,
|
||||
pub operation_id: String,
|
||||
}
|
||||
|
||||
/// Weighted round-robin state persisted with the authoritative queue.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct AdmissionCursor {
|
||||
pub agent: Option<String>,
|
||||
pub agent_credit: u32,
|
||||
pub tool: Option<String>,
|
||||
pub tool_credit: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionSchedulerConfig {
|
||||
pub agent_weight: u32,
|
||||
pub tool_weight: u32,
|
||||
}
|
||||
|
||||
impl Default for AdmissionSchedulerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
agent_weight: 1,
|
||||
tool_weight: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdmissionSchedulerConfig {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission weights must be between 1 and 64",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the
|
||||
/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its
|
||||
/// turn instead of being skipped by newer arrivals.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionSelection {
|
||||
pub index: usize,
|
||||
pub cursor: AdmissionCursor,
|
||||
}
|
||||
|
||||
/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside
|
||||
/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit.
|
||||
pub fn select_next_grant(
|
||||
queue: &[AdmissionQueueEntry],
|
||||
cursor: &AdmissionCursor,
|
||||
config: AdmissionSchedulerConfig,
|
||||
) -> Option<AdmissionSelection> {
|
||||
if queue.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let agent_weight = config.agent_weight.max(1);
|
||||
let tool_weight = config.tool_weight.max(1);
|
||||
let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect();
|
||||
agents.sort_unstable();
|
||||
agents.dedup();
|
||||
|
||||
let mut next = cursor.clone();
|
||||
let agent = match cursor.agent.as_deref() {
|
||||
Some(current) if cursor.agent_credit > 0 && agents.contains(¤t) => current.to_owned(),
|
||||
_ => {
|
||||
let chosen = match cursor.agent.as_deref() {
|
||||
None => agents[0].to_owned(),
|
||||
Some(current) => agents
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| *candidate > current)
|
||||
.unwrap_or(agents[0])
|
||||
.to_owned(),
|
||||
};
|
||||
if cursor.agent.as_deref() != Some(chosen.as_str()) {
|
||||
next.tool = None;
|
||||
next.tool_credit = 0;
|
||||
}
|
||||
next.agent = Some(chosen.clone());
|
||||
next.agent_credit = agent_weight;
|
||||
chosen
|
||||
}
|
||||
};
|
||||
|
||||
let mut tools: Vec<&str> = queue
|
||||
.iter()
|
||||
.filter(|entry| entry.agent_key == agent)
|
||||
.map(|entry| entry.operation_id.as_str())
|
||||
.collect();
|
||||
tools.sort_unstable();
|
||||
tools.dedup();
|
||||
|
||||
let tool = match cursor.tool.as_deref() {
|
||||
Some(current) if cursor.tool_credit > 0 && tools.contains(¤t) => current.to_owned(),
|
||||
_ => {
|
||||
let chosen = match cursor.tool.as_deref() {
|
||||
None => tools[0].to_owned(),
|
||||
Some(current) => tools
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| *candidate > current)
|
||||
.unwrap_or(tools[0])
|
||||
.to_owned(),
|
||||
};
|
||||
next.tool = Some(chosen.clone());
|
||||
next.tool_credit = tool_weight;
|
||||
chosen
|
||||
}
|
||||
};
|
||||
|
||||
let index = queue
|
||||
.iter()
|
||||
.position(|entry| entry.agent_key == agent && entry.operation_id == tool)?;
|
||||
next.agent_credit = next.agent_credit.saturating_sub(1);
|
||||
next.tool_credit = next.tool_credit.saturating_sub(1);
|
||||
Some(AdmissionSelection {
|
||||
index,
|
||||
cursor: next,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest,
|
||||
AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence,
|
||||
select_next_grant,
|
||||
};
|
||||
use crate::{NoTasksFallback, ToolExecutionMode};
|
||||
|
||||
fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry {
|
||||
super::AdmissionQueueEntry {
|
||||
ticket_id,
|
||||
sequence: ticket_id,
|
||||
agent_key: agent.to_owned(),
|
||||
operation_id: tool.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn request() -> AdmissionRequest {
|
||||
AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
agent_key: ADMIN_ORIGIN_KEY.to_owned(),
|
||||
operation_id: "op_leads".to_owned(),
|
||||
operation_version: 3,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_0123456789".to_owned(),
|
||||
expires_at_unix_ms: 1_790_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_keys_exclude_version_and_keep_agent_bucket() {
|
||||
let scope = request().scope;
|
||||
assert_eq!(scope.agent_bucket(), "ws_default:admin");
|
||||
assert_eq!(scope.tool_bucket(), "ws_default:op_leads");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() {
|
||||
let mut invalid = request();
|
||||
invalid.expires_at_unix_ms = 0;
|
||||
assert!(matches!(
|
||||
invalid.validate(),
|
||||
Err(AdmissionStoreError::Invalid { .. })
|
||||
));
|
||||
|
||||
let mut invalid = request();
|
||||
invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1);
|
||||
assert!(invalid.validate().is_err());
|
||||
|
||||
let mut invalid = request();
|
||||
invalid.scope.operation_version = 0;
|
||||
assert!(invalid.validate().is_err());
|
||||
|
||||
assert!(request().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weights_are_bounded_positive_configuration() {
|
||||
assert!(
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 0,
|
||||
tool_weight: 1
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 1,
|
||||
tool_weight: 65
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(AdmissionSchedulerConfig::default().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weighted_round_robin_serves_each_agent_its_quantum() {
|
||||
let mut queue = vec![
|
||||
entry(1, "agent_a", "op_1"),
|
||||
entry(2, "agent_a", "op_1"),
|
||||
entry(3, "agent_a", "op_1"),
|
||||
entry(4, "agent_b", "op_1"),
|
||||
entry(5, "agent_b", "op_1"),
|
||||
];
|
||||
let config = AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
};
|
||||
let mut cursor = AdmissionCursor::default();
|
||||
let mut order = Vec::new();
|
||||
while !queue.is_empty() {
|
||||
let selection = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
order.push(queue.remove(selection.index).agent_key.clone());
|
||||
cursor = selection.cursor;
|
||||
}
|
||||
assert_eq!(
|
||||
order,
|
||||
vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_inside_one_tool_and_round_robin_between_tools() {
|
||||
let mut queue = vec![
|
||||
entry(1, "agent_a", "op_1"),
|
||||
entry(2, "agent_a", "op_1"),
|
||||
entry(3, "agent_a", "op_2"),
|
||||
];
|
||||
let config = AdmissionSchedulerConfig::default();
|
||||
let mut cursor = AdmissionCursor::default();
|
||||
let mut order = Vec::new();
|
||||
while !queue.is_empty() {
|
||||
let selection = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
// A granted ticket leaves the waiting queue: granted entries are never re-granted.
|
||||
order.push(queue.remove(selection.index).ticket_id);
|
||||
cursor = selection.cursor;
|
||||
}
|
||||
// One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue.
|
||||
assert_eq!(order, vec![1, 3, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() {
|
||||
let queue = vec![entry(1, "agent_a", "op_1")];
|
||||
let config = AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
};
|
||||
let cursor = AdmissionCursor::default();
|
||||
let first = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
assert_eq!(first.cursor.agent.as_deref(), Some("agent_a"));
|
||||
assert_eq!(first.cursor.agent_credit, 1);
|
||||
|
||||
// A later Agent joins; the cursor keeps its remaining quantum for `agent_a`.
|
||||
let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")];
|
||||
let second = select_next_grant(&queue, &first.cursor, config).unwrap();
|
||||
assert_eq!(queue[second.index].agent_key, "agent_a");
|
||||
assert_eq!(second.cursor.agent_credit, 0);
|
||||
|
||||
let third = select_next_grant(&queue, &second.cursor, config).unwrap();
|
||||
assert_eq!(queue[third.index].agent_key, "agent_b");
|
||||
assert_eq!(third.cursor.agent_credit, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_queue_keeps_cursor_untouched() {
|
||||
let cursor = AdmissionCursor {
|
||||
agent: Some("agent_b".to_owned()),
|
||||
agent_credit: 1,
|
||||
tool: Some("op_1".to_owned()),
|
||||
tool_credit: 0,
|
||||
};
|
||||
assert_eq!(
|
||||
select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()),
|
||||
None
|
||||
);
|
||||
assert_eq!(cursor.agent.as_deref(), Some("agent_b"));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod access;
|
||||
pub mod admission;
|
||||
pub mod agent;
|
||||
pub mod approval;
|
||||
pub mod auth;
|
||||
@@ -60,10 +61,11 @@ pub mod domain {
|
||||
pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
|
||||
pub use crate::operation::{
|
||||
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
|
||||
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
|
||||
IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode,
|
||||
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
|
||||
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy,
|
||||
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState,
|
||||
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
|
||||
ToolExecutionPolicy, WizardState,
|
||||
};
|
||||
pub use crate::product_event::{
|
||||
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
|
||||
@@ -114,6 +116,13 @@ pub use access::{
|
||||
InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
|
||||
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
|
||||
};
|
||||
pub use admission::{
|
||||
ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
|
||||
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
|
||||
AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission,
|
||||
MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant,
|
||||
};
|
||||
pub use agent::{
|
||||
Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
|
||||
ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
|
||||
@@ -168,13 +177,14 @@ pub use observability::{
|
||||
pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
|
||||
pub use operation::{
|
||||
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
|
||||
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
|
||||
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
|
||||
OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError,
|
||||
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState,
|
||||
ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor,
|
||||
RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample,
|
||||
WizardState,
|
||||
IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation,
|
||||
OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
|
||||
OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle,
|
||||
OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy,
|
||||
OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract,
|
||||
RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget,
|
||||
RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
|
||||
ToolExecutionPolicy, ToolExecutionPolicyError, WizardState,
|
||||
};
|
||||
pub use product_event::{
|
||||
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
|
||||
|
||||
@@ -270,6 +270,72 @@ pub struct OperationApprovalPolicy {
|
||||
pub elicitation_message: Option<String>,
|
||||
}
|
||||
|
||||
/// Published execution mode of one Tool (Story 4.1, FR-21).
|
||||
///
|
||||
/// This is a scheduling policy of the Published Tool version. It is deliberately separate from
|
||||
/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolExecutionMode {
|
||||
#[default]
|
||||
Synchronous,
|
||||
Background,
|
||||
Adaptive,
|
||||
}
|
||||
|
||||
/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NoTasksFallback {
|
||||
/// Execute through the same synchronous admission path.
|
||||
#[default]
|
||||
Synchronous,
|
||||
/// Reject with a typed unsupported-execution-mode failure; never a Task ID.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000;
|
||||
|
||||
/// Execution policy pinned by a Published Tool version.
|
||||
///
|
||||
/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by
|
||||
/// `ExecutionConfig::timeout_ms`, with no background path.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ToolExecutionPolicy {
|
||||
#[serde(default)]
|
||||
pub mode: ToolExecutionMode,
|
||||
/// Absolute budget of one execution, counted from the start of admission.
|
||||
pub deadline_ms: u64,
|
||||
#[serde(default)]
|
||||
pub no_tasks_fallback: NoTasksFallback,
|
||||
}
|
||||
|
||||
impl Default for ToolExecutionPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolExecutionPolicy {
|
||||
pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> {
|
||||
if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) {
|
||||
return Err(ToolExecutionPolicyError::DeadlineOutOfRange);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ToolExecutionPolicyError {
|
||||
#[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")]
|
||||
DeadlineOutOfRange,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ExecutionConfig {
|
||||
@@ -285,6 +351,8 @@ pub struct ExecutionConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub approval_policy: Option<OperationApprovalPolicy>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub execution_policy: Option<ToolExecutionPolicy>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub auth_profile_ref: Option<AuthProfileId>,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub headers: BTreeMap<String, String>,
|
||||
@@ -444,9 +512,10 @@ mod tests {
|
||||
edition::OperationSecurityLevel,
|
||||
ids::{AuthProfileId, OperationId},
|
||||
operation::{
|
||||
ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle,
|
||||
OperationLifecycleAction, OperationLifecycleError, OperationStatus,
|
||||
ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability,
|
||||
OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus,
|
||||
OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample,
|
||||
ToolExecutionMode, ToolExecutionPolicy,
|
||||
},
|
||||
protocol::{AuthKind, ExportMode, HttpMethod, Protocol},
|
||||
};
|
||||
@@ -664,6 +733,7 @@ updated_at: 2026-03-25T08:10:00Z
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: Some(AuthProfileId::new("auth_01")),
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
@@ -684,4 +754,76 @@ updated_at: 2026-03-25T08:10:00Z
|
||||
published_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_round_trips_all_modes_and_fallbacks() {
|
||||
for (mode, fallback) in [
|
||||
(ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous),
|
||||
(ToolExecutionMode::Background, NoTasksFallback::Synchronous),
|
||||
(ToolExecutionMode::Background, NoTasksFallback::Unsupported),
|
||||
(ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported),
|
||||
] {
|
||||
let config = ExecutionConfig {
|
||||
execution_policy: Some(ToolExecutionPolicy {
|
||||
mode,
|
||||
deadline_ms: 45_000,
|
||||
no_tasks_fallback: fallback,
|
||||
}),
|
||||
..ExecutionConfig::default()
|
||||
};
|
||||
let encoded = serde_json::to_value(&config).unwrap();
|
||||
assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000));
|
||||
let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap();
|
||||
assert_eq!(decoded, config);
|
||||
assert_eq!(
|
||||
decoded.execution_policy.unwrap().mode,
|
||||
mode,
|
||||
"the published mode must survive the JSON round-trip"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_execution_config_without_policy_stays_synchronous() {
|
||||
let legacy: ExecutionConfig =
|
||||
serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap();
|
||||
assert_eq!(legacy.execution_policy, None);
|
||||
assert_eq!(legacy.timeout_ms, 12_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_rejects_explicit_invalid_deadline() {
|
||||
assert!(
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 0,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Adaptive,
|
||||
deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(ToolExecutionPolicy::default().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_rejects_unknown_fields() {
|
||||
assert!(
|
||||
serde_json::from_value::<ToolExecutionPolicy>(json!({
|
||||
"mode": "background",
|
||||
"deadline_ms": 1000,
|
||||
"no_tasks_fallback": "synchronous",
|
||||
"task_queue": "default"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -231,6 +231,7 @@ fn default_execution_config() -> ExecutionConfig {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
|
||||
@@ -564,6 +564,7 @@ fn default_execution_config() -> ExecutionConfig {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
|
||||
@@ -217,6 +217,34 @@ string_enum! {
|
||||
Concurrency => "concurrency",
|
||||
RateLimit => "rate_limit",
|
||||
McpStream => "mcp_stream",
|
||||
Admission => "admission",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum RejectionReason {
|
||||
Limit => "limit",
|
||||
Capacity => "capacity",
|
||||
QueueFull => "queue_full",
|
||||
Deadline => "deadline",
|
||||
Unsupported => "unsupported",
|
||||
DependencyUnavailable => "dependency_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum ScopeKind {
|
||||
Process => "process",
|
||||
Global => "global",
|
||||
Agent => "agent",
|
||||
Tool => "tool",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum AdmissionModeLabel {
|
||||
Synchronous => "synchronous",
|
||||
Background => "background",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,14 +16,15 @@ mod schema;
|
||||
|
||||
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
|
||||
pub use labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
|
||||
UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
pub use record::{
|
||||
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service,
|
||||
bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome,
|
||||
bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting,
|
||||
record_admission_queue_wait, record_admission_rejection, record_cache_outcome,
|
||||
record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
|
||||
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
|
||||
|
||||
@@ -5,10 +5,10 @@ use metrics::Gauge;
|
||||
|
||||
use crate::schema::{MetricDefinition, metric_schema};
|
||||
use crate::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter,
|
||||
HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage,
|
||||
McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason,
|
||||
ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
|
||||
static BINDING: OnceLock<MetricService> = OnceLock::new();
|
||||
@@ -352,11 +352,47 @@ pub fn record_limit_rejection(stage: LimitStage) {
|
||||
}
|
||||
metrics::counter!(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
"stage" => stage.as_str()
|
||||
"stage" => stage.as_str(),
|
||||
"reason" => RejectionReason::Limit.as_str(),
|
||||
"scope_kind" => ScopeKind::Process.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool,
|
||||
/// request id, fingerprint or arbitrary error text ever becomes a label.
|
||||
pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) {
|
||||
if bound_family("crank_runtime_limit_rejections_total").is_none() {
|
||||
return;
|
||||
}
|
||||
metrics::counter!(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
"stage" => LimitStage::Admission.as_str(),
|
||||
"reason" => reason.as_str(),
|
||||
"scope_kind" => scope_kind.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) {
|
||||
if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() {
|
||||
return;
|
||||
}
|
||||
metrics::gauge!(
|
||||
"crank_runtime_admission_queue_wait_seconds",
|
||||
"mode" => mode.as_str()
|
||||
)
|
||||
.set(waited_ms as f64 / 1000.0);
|
||||
}
|
||||
|
||||
pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option<u64>) {
|
||||
if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() {
|
||||
return;
|
||||
}
|
||||
let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0;
|
||||
metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds);
|
||||
}
|
||||
|
||||
pub fn record_cache_outcome(outcome: CacheOutcome) {
|
||||
if bound_family("crank_runtime_cache_total").is_none() {
|
||||
return;
|
||||
|
||||
@@ -3,16 +3,17 @@
|
||||
|
||||
use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP};
|
||||
use crate::labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
|
||||
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
|
||||
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome,
|
||||
UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
|
||||
pub const METRIC_SCHEMA_VERSION: u32 = 2;
|
||||
pub const MAX_METRIC_FAMILIES: usize = 128;
|
||||
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611;
|
||||
pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348;
|
||||
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
@@ -179,6 +180,9 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
|
||||
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
|
||||
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
|
||||
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
|
||||
const REJECTION_REASONS: &[&str] = RejectionReason::VALUES;
|
||||
const SCOPE_KINDS: &[&str] = ScopeKind::VALUES;
|
||||
const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES;
|
||||
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
|
||||
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
|
||||
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
|
||||
@@ -403,13 +407,39 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[
|
||||
"crank_runtime_limit_rejections_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["stage"],
|
||||
&[domain("stage", LIMIT_STAGES)],
|
||||
&["stage", "reason", "scope_kind"],
|
||||
&[
|
||||
domain("stage", LIMIT_STAGES),
|
||||
domain("reason", REJECTION_REASONS),
|
||||
domain("scope_kind", SCOPE_KINDS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
3,
|
||||
96,
|
||||
"Runtime executions rejected by a bounded limit.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_admission_queue_wait_seconds",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Seconds,
|
||||
&["mode"],
|
||||
&[domain("mode", ADMISSION_MODES)],
|
||||
&[],
|
||||
BOTH,
|
||||
2,
|
||||
"Wait of the most recently admitted execution before it received a scoped permit.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_admission_oldest_waiting_seconds",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Seconds,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"Age of the oldest waiting admission request.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_cache_total",
|
||||
MetricKind::Counter,
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
|
||||
use crank_metrics::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics,
|
||||
bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome,
|
||||
record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage,
|
||||
McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType,
|
||||
ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema,
|
||||
record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection,
|
||||
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
|
||||
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
|
||||
set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
|
||||
@@ -71,6 +72,9 @@ fn schema_names_are_unique_and_labels_are_closed() {
|
||||
"state",
|
||||
"signal_type",
|
||||
"exporter",
|
||||
"reason",
|
||||
"scope_kind",
|
||||
"mode",
|
||||
];
|
||||
for metric in schema {
|
||||
assert!(
|
||||
@@ -265,6 +269,9 @@ fn every_recording_api_matches_the_declared_schema() {
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
record_limit_rejection(LimitStage::Concurrency);
|
||||
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
|
||||
record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5);
|
||||
record_admission_oldest_waiting(Some(5));
|
||||
record_cache_outcome(CacheOutcome::Hit);
|
||||
record_idempotency_outcome(IdempotencyOutcome::Replay);
|
||||
record_confirmation_outcome(ConfirmationOutcome::Required);
|
||||
|
||||
@@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
|
||||
|
||||
#[test]
|
||||
fn frozen_budget_edges_are_exact() {
|
||||
assert!(validate_budget(6_284, 19_514).is_ok());
|
||||
assert!(validate_budget(6_285, 19_515).is_ok());
|
||||
assert!(validate_budget(6_286, 16_000).is_err());
|
||||
assert!(validate_budget(6_000, 19_516).is_err());
|
||||
assert!(validate_budget(6_380, 19_610).is_ok());
|
||||
assert!(validate_budget(6_381, 19_611).is_ok());
|
||||
assert!(validate_budget(6_382, 19_000).is_err());
|
||||
assert!(validate_budget(6_000, 19_612).is_err());
|
||||
assert_eq!(
|
||||
crank_metrics::max_exemplar_slots(MetricService::AdminApi),
|
||||
12_348
|
||||
@@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() {
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::AdminApi),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 6_285,
|
||||
rendered_series: 19_515
|
||||
logical_series: 6_381,
|
||||
rendered_series: 19_611
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::McpServer),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 1_083,
|
||||
rendered_series: 3_558
|
||||
logical_series: 1_179,
|
||||
rendered_series: 3_654
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() {
|
||||
"root обязан объявлять ровно один ключ budgets"
|
||||
);
|
||||
assert!(rendered.ends_with(
|
||||
",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\
|
||||
\"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
|
||||
\"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n"
|
||||
",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\
|
||||
\"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
|
||||
\"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n"
|
||||
));
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::AdminApi),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 6_285,
|
||||
rendered_series: 19_515
|
||||
logical_series: 6_381,
|
||||
rendered_series: 19_611
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::McpServer),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 1_083,
|
||||
rendered_series: 3_558
|
||||
logical_series: 1_179,
|
||||
rendered_series: 3_654
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() {
|
||||
/// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны.
|
||||
#[test]
|
||||
fn frozen_caps_and_edges_are_exact() {
|
||||
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285);
|
||||
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515);
|
||||
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381);
|
||||
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611);
|
||||
assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348);
|
||||
assert!(validate_budget(6_285, 19_515).is_ok());
|
||||
assert!(validate_budget(6_286, 19_515).is_err());
|
||||
assert!(validate_budget(6_381, 19_611).is_ok());
|
||||
assert!(validate_budget(6_382, 19_611).is_err());
|
||||
assert!(validate_exemplar_budget(12_347).is_ok());
|
||||
assert!(validate_exemplar_budget(12_348).is_ok());
|
||||
assert!(validate_exemplar_budget(12_349).is_err());
|
||||
|
||||
@@ -13,7 +13,7 @@ mod schema;
|
||||
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
|
||||
pub use crank_metrics::{
|
||||
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit,
|
||||
bind_metric_service, bound_metric_service, metric_schema,
|
||||
bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting,
|
||||
};
|
||||
pub use error_reporting::{
|
||||
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
|
||||
|
||||
@@ -7,6 +7,7 @@ publish.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1"
|
||||
crank-artifacts = { path = "../crank-artifacts" }
|
||||
crank-core = { path = "../crank-core" }
|
||||
crank-mapping = { path = "../crank-mapping" }
|
||||
|
||||
@@ -116,6 +116,7 @@ pub use model::{
|
||||
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
|
||||
};
|
||||
pub use postgres::{
|
||||
AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError,
|
||||
PostgresRegistry,
|
||||
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
|
||||
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, admission_health,
|
||||
admission_snapshot,
|
||||
};
|
||||
@@ -6,6 +6,7 @@ mod artifact_cleanup_indexes_v13;
|
||||
mod artifact_metadata_v12;
|
||||
mod authority;
|
||||
mod baseline_v1;
|
||||
mod execution_admission_v24;
|
||||
mod execution_outcome_v5;
|
||||
mod import_plan_v14;
|
||||
mod master_key_identity_v7;
|
||||
@@ -35,6 +36,7 @@ mod schema_guard_v19;
|
||||
mod schema_guard_v20;
|
||||
mod schema_guard_v22;
|
||||
mod schema_guard_v23;
|
||||
mod schema_guard_v24;
|
||||
mod schema_guard_v7;
|
||||
mod schema_guard_v8;
|
||||
mod schema_guard_v9;
|
||||
|
||||
@@ -4,6 +4,7 @@ use super::agent_resource_bindings_v18;
|
||||
use super::approval_side_effects_v10;
|
||||
use super::artifact_cleanup_indexes_v13;
|
||||
use super::artifact_metadata_v12;
|
||||
use super::execution_admission_v24;
|
||||
use super::execution_outcome_v5;
|
||||
use super::import_plan_v14;
|
||||
use super::master_key_identity_v7;
|
||||
@@ -39,9 +40,9 @@ use preflight::*;
|
||||
use contract::baseline_source_digest;
|
||||
use contract::validate_descriptors;
|
||||
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
|
||||
const CURRENT_VERSION: i64 = 23;
|
||||
const CURRENT_VERSION: i64 = 24;
|
||||
const IMPLEMENTED_VERSIONS: &[i64] = &[
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
||||
];
|
||||
const BASELINE_SOURCE_SHA256: &str =
|
||||
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
|
||||
@@ -330,6 +331,12 @@ impl MigrationAuthority {
|
||||
prompt_dependencies_v23::SOURCE_SHA256,
|
||||
22,
|
||||
),
|
||||
expand_descriptor(
|
||||
24,
|
||||
execution_admission_v24::NAME,
|
||||
execution_admission_v24::SOURCE_SHA256,
|
||||
23,
|
||||
),
|
||||
]
|
||||
}
|
||||
pub fn validate_sequence() -> Result<(), MigrationError> {
|
||||
@@ -489,6 +496,9 @@ impl MigrationAuthority {
|
||||
if from < 23 {
|
||||
prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?;
|
||||
}
|
||||
if from < 24 {
|
||||
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
|
||||
}
|
||||
match inspect(&mut transaction).await? {
|
||||
MigrationPreflight::Current {
|
||||
version: CURRENT_VERSION,
|
||||
|
||||
@@ -140,6 +140,8 @@ pub(super) fn validate_descriptors(
|
||||
!= agent_resource_bindings_v18::SOURCE_SHA256
|
||||
|| sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes())
|
||||
!= resource_security_revocations_v19::SOURCE_SHA256
|
||||
|| sha256_hex(execution_admission_v24::SOURCE.as_bytes())
|
||||
!= execution_admission_v24::SOURCE_SHA256
|
||||
{
|
||||
return Err(MigrationError::new(
|
||||
"invalid_contract",
|
||||
|
||||
@@ -148,6 +148,7 @@ pub(super) async fn inspect(
|
||||
owned_relations::PROMPTS,
|
||||
owned_relations::PROMPT_PUBLICATIONS,
|
||||
owned_relations::PROMPT_DEPENDENCIES,
|
||||
owned_relations::EXECUTION_ADMISSION,
|
||||
] {
|
||||
validate_required_relations(connection, relations, CURRENT_VERSION).await?;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "execution-admission-v24";
|
||||
pub(super) const SOURCE: &str = include_str!("execution_admission_v24.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.execution_admission_v24",
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.canonical_ledger",
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
-- V24: authoritative execution admission state (Story 4.1, FR-21, FR-24).
|
||||
--
|
||||
-- Один deployment-wide admission: PostgreSQL хранит cursor/fairness state, ожидающие тикеты,
|
||||
-- выданные permits и durable enqueue decisions, поэтому несколько процессов делят общий бюджет.
|
||||
--
|
||||
-- `execution_admission_entries` — bounded набор: waiting (ожидает capacity), executing (scoped
|
||||
-- permit выдан, держится до явного release) и reserved (durable enqueue decision с зарезервированным
|
||||
-- queue slot; не исполняется и не рекламируется как Task до Story 4.2).
|
||||
--
|
||||
-- Строка не хранит ни credentials, ни connection/session id, ни prepared headers, ни тело запроса:
|
||||
-- quota key — (workspace, agent) и (workspace, operation) без version. `operation_version` пишется
|
||||
-- только как pin исполняемых semantics и не участвует в квотах.
|
||||
--
|
||||
-- `expires_at` — абсолютный deadline. Queue decisions без dispatch можно expire по deadline; активный
|
||||
-- permit НЕ освобождается по TTL (иначе paused/crashed owner обойдёт global limit): его снимает только
|
||||
-- явный release/cancel владельца.
|
||||
|
||||
create table execution_admission_controllers (
|
||||
workspace_id text not null,
|
||||
cursor_agent text,
|
||||
cursor_agent_credit integer not null default 0,
|
||||
cursor_tool text,
|
||||
cursor_tool_credit integer not null default 0,
|
||||
updated_at timestamptz not null default now(),
|
||||
constraint execution_admission_controllers_pkey primary key (workspace_id),
|
||||
constraint execution_admission_controllers_workspace_check check (
|
||||
octet_length(workspace_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_controllers_agent_credit_check check (
|
||||
cursor_agent_credit between 0 and 64
|
||||
),
|
||||
constraint execution_admission_controllers_tool_credit_check check (
|
||||
cursor_tool_credit between 0 and 64
|
||||
)
|
||||
);
|
||||
|
||||
create table execution_admission_entries (
|
||||
ticket_id bigint generated always as identity,
|
||||
workspace_id text not null,
|
||||
agent_key text not null,
|
||||
operation_id text not null,
|
||||
operation_version integer not null,
|
||||
mode text not null,
|
||||
policy_mode text not null,
|
||||
state text not null,
|
||||
owner_token text not null,
|
||||
fingerprint text not null,
|
||||
created_at timestamptz not null default now(),
|
||||
expires_at timestamptz not null,
|
||||
granted_at timestamptz,
|
||||
constraint execution_admission_entries_pkey primary key (ticket_id),
|
||||
constraint execution_admission_entries_workspace_check check (
|
||||
octet_length(workspace_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_agent_check check (
|
||||
octet_length(agent_key) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_operation_check check (
|
||||
octet_length(operation_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_version_check check (operation_version >= 1),
|
||||
constraint execution_admission_entries_mode_check check (
|
||||
mode in ('synchronous', 'background')
|
||||
),
|
||||
constraint execution_admission_entries_policy_mode_check check (
|
||||
policy_mode in ('synchronous', 'background', 'adaptive')
|
||||
),
|
||||
constraint execution_admission_entries_state_check check (
|
||||
state in ('waiting', 'executing', 'reserved')
|
||||
),
|
||||
constraint execution_admission_entries_owner_check check (
|
||||
octet_length(owner_token) between 1 and 64
|
||||
),
|
||||
constraint execution_admission_entries_fingerprint_check check (
|
||||
octet_length(fingerprint) between 1 and 128
|
||||
),
|
||||
constraint execution_admission_entries_reserved_mode_check check (
|
||||
state <> 'reserved' or mode = 'background'
|
||||
)
|
||||
);
|
||||
|
||||
create unique index execution_admission_entries_owner_uidx
|
||||
on execution_admission_entries (owner_token);
|
||||
|
||||
create index execution_admission_entries_state_idx
|
||||
on execution_admission_entries (state, workspace_id, ticket_id);
|
||||
|
||||
create index execution_admission_entries_agent_idx
|
||||
on execution_admission_entries (state, workspace_id, agent_key);
|
||||
|
||||
create index execution_admission_entries_operation_idx
|
||||
on execution_admission_entries (state, workspace_id, operation_id);
|
||||
@@ -55,6 +55,11 @@ pub(super) const PROMPT_PUBLICATIONS: &[&str] = &["prompt_publications", "agent_
|
||||
pub(super) const PROMPT_DEPENDENCIES: &[&str] =
|
||||
&["prompt_dependencies", "prompt_message_resource_refs"];
|
||||
|
||||
pub(super) const EXECUTION_ADMISSION: &[&str] = &[
|
||||
"execution_admission_controllers",
|
||||
"execution_admission_entries",
|
||||
];
|
||||
|
||||
pub(super) const ARTIFACT_METADATA: &[&str] = &["artifact_blobs", "artifact_sources"];
|
||||
|
||||
pub(super) const IMPORT_PLAN: &[&str] = &["import_plans", "import_plan_applications"];
|
||||
|
||||
@@ -468,6 +468,13 @@ pub(super) async fn validate_schema_fingerprint(
|
||||
} else {
|
||||
super::schema_guard_v23::validate_v23_prompt_dependencies(connection).await?;
|
||||
}
|
||||
if current_version < 24 {
|
||||
if !super::schema_guard_v24::validate_v24_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v24::validate_v24_execution_admission(connection).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,8 @@ pub(crate) const OWNED_RELATIONS: &[&str] = &[
|
||||
"prompt_messages",
|
||||
"prompt_dependencies",
|
||||
"prompt_message_resource_refs",
|
||||
"execution_admission_controllers",
|
||||
"execution_admission_entries",
|
||||
];
|
||||
|
||||
pub(crate) const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
//! V24 schema guard: authoritative execution admission state (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Схема V24 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
|
||||
//! ожидаемой форме: controller-row с weighted round-robin cursor и bounded entries с закрытым
|
||||
//! словарём mode/policy_mode/state, уникальным owner token и индексами квот.
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const CONTROLLERS: &str = "execution_admission_controllers";
|
||||
const ENTRIES: &str = "execution_admission_entries";
|
||||
|
||||
const OWNED_RELATIONS: &[&str] = &[CONTROLLERS, ENTRIES];
|
||||
|
||||
const CONTROLLER_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("workspace_id", "text", false),
|
||||
("cursor_agent", "text", true),
|
||||
("cursor_agent_credit", "integer", false),
|
||||
("cursor_tool", "text", true),
|
||||
("cursor_tool_credit", "integer", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
];
|
||||
|
||||
const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("ticket_id", "bigint", false),
|
||||
("workspace_id", "text", false),
|
||||
("agent_key", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("mode", "text", false),
|
||||
("policy_mode", "text", false),
|
||||
("state", "text", false),
|
||||
("owner_token", "text", false),
|
||||
("fingerprint", "text", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("expires_at", "timestamp with time zone", false),
|
||||
("granted_at", "timestamp with time zone", true),
|
||||
];
|
||||
|
||||
const CONSTRAINT_COUNT: &[(&str, usize)] = &[(CONTROLLERS, 4), (ENTRIES, 11)];
|
||||
|
||||
const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"];
|
||||
|
||||
fn v24_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
/// Схема V24 отсутствует целиком (для current_version < 24).
|
||||
pub(super) async fn validate_v24_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
for relation in OWNED_RELATIONS {
|
||||
if relation_exists(connection, relation).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
for index in UNIQUE_INDEXES {
|
||||
if index_exists(connection, index).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24).
|
||||
pub(super) async fn validate_v24_execution_admission(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
for relation in OWNED_RELATIONS {
|
||||
if !relation_exists(connection, relation).await? {
|
||||
return Err(v24_error("v24.relations"));
|
||||
}
|
||||
}
|
||||
for (table, columns) in [(CONTROLLERS, CONTROLLER_COLUMNS), (ENTRIES, ENTRY_COLUMNS)] {
|
||||
validate_columns(connection, table, columns).await?;
|
||||
}
|
||||
for (table, expected) in CONSTRAINT_COUNT {
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1",
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(*expected).unwrap_or_default() {
|
||||
return Err(v24_error("v24.constraints"));
|
||||
}
|
||||
}
|
||||
for (table, constraint) in [
|
||||
(CONTROLLERS, "execution_admission_controllers_pkey"),
|
||||
(ENTRIES, "execution_admission_entries_pkey"),
|
||||
(ENTRIES, "execution_admission_entries_state_check"),
|
||||
] {
|
||||
if !constraint_exists(connection, table, constraint).await? {
|
||||
return Err(v24_error("v24.constraint_missing"));
|
||||
}
|
||||
}
|
||||
if !normalized_expression(
|
||||
connection,
|
||||
ENTRIES,
|
||||
"execution_admission_entries_state_check",
|
||||
)
|
||||
.await?
|
||||
.is_some_and(|expression| {
|
||||
super::schema_guard::enum_constraint_matches(
|
||||
&expression,
|
||||
"state",
|
||||
&["waiting", "executing", "reserved"],
|
||||
)
|
||||
}) {
|
||||
return Err(v24_error("v24.state_vocabulary"));
|
||||
}
|
||||
for index in UNIQUE_INDEXES {
|
||||
let unique = index_unique(connection, index).await?;
|
||||
if unique != Some(true) {
|
||||
return Err(v24_error("v24.indexes"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_columns(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
columns: &[(&str, &str, bool)],
|
||||
) -> Result<(), MigrationError> {
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(columns.len()).unwrap_or_default() {
|
||||
return Err(v24_error("v24.columns"));
|
||||
}
|
||||
for (column, data_type, nullable) in columns {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v24_error("v24.columns"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn constraint_exists(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
) as present",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn normalized_expression(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<Option<String>, MigrationError> {
|
||||
let row = query(
|
||||
"select pg_get_expr(c.conbin, c.conrelid) as expression
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
and c.contype = 'c'",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
|
||||
}
|
||||
|
||||
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_class idx
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1
|
||||
) as present",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn index_unique(
|
||||
connection: &mut PgConnection,
|
||||
index: &str,
|
||||
) -> Result<Option<bool>, MigrationError> {
|
||||
let row = query(
|
||||
"select i.indisunique, i.indisvalid
|
||||
from pg_catalog.pg_index i
|
||||
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| {
|
||||
let unique = row.try_get::<bool, _>("indisunique").ok()?;
|
||||
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
|
||||
Some(unique && valid)
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,618 @@
|
||||
//! PostgreSQL implementation of the mandatory execution-admission store (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Source of truth for capacity and durable enqueue decisions of every process: a short transaction
|
||||
//! locks one deployment controller row and one workspace controller row in a fixed order, then reads
|
||||
//! bounded counters, inserts/transitions at most one entry and persists the weighted round-robin
|
||||
//! cursor. No network call happens under the transaction and no connection is held while a waiter is
|
||||
//! idle between attempts.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
|
||||
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
|
||||
AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, select_next_grant,
|
||||
};
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEPLOYMENT_CONTROLLER: &str = "*";
|
||||
const OVERLOAD_RETRY_AFTER_MS: u64 = crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS;
|
||||
const MAX_WAIT_BACKOFF_MS: u64 = 250;
|
||||
const INITIAL_WAIT_BACKOFF_MS: u64 = 5;
|
||||
const EXPIRED_CLEANUP_BATCH: i64 = 128;
|
||||
const QUEUE_READ_LIMIT: i64 = 1024;
|
||||
|
||||
/// Deployment-wide bounds of one admission store. They come from validated deployment
|
||||
/// configuration, never from a request.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionLimits {
|
||||
pub global_limit: usize,
|
||||
pub agent_limit: usize,
|
||||
pub tool_limit: usize,
|
||||
pub queue_limit: usize,
|
||||
}
|
||||
|
||||
impl Default for AdmissionLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
global_limit: 64,
|
||||
agent_limit: 16,
|
||||
tool_limit: 8,
|
||||
queue_limit: 32,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdmissionLimits {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
for (field, value) in [
|
||||
("global_limit", self.global_limit),
|
||||
("agent_limit", self.agent_limit),
|
||||
("tool_limit", self.tool_limit),
|
||||
] {
|
||||
if !(1..=65_535).contains(&value) {
|
||||
return Err(AdmissionStoreError::invalid(format!(
|
||||
"admission {field} is outside its allowed bounds"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if !(1..=1_024).contains(&self.queue_limit) {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission queue_limit is outside its allowed bounds",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PostgresExecutionAdmission {
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
}
|
||||
|
||||
impl PostgresExecutionAdmission {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
) -> Result<Self, AdmissionStoreError> {
|
||||
limits.validate()?;
|
||||
scheduler.validate()?;
|
||||
Ok(Self {
|
||||
pool,
|
||||
limits,
|
||||
scheduler,
|
||||
})
|
||||
}
|
||||
|
||||
fn unavailable(error: impl std::fmt::Display) -> AdmissionStoreError {
|
||||
AdmissionStoreError::unavailable(format!(
|
||||
"execution admission store is unavailable: {error}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
struct AttemptContext<'a> {
|
||||
request: &'a AdmissionRequest,
|
||||
owner_token: &'a str,
|
||||
}
|
||||
|
||||
enum Attempt {
|
||||
Granted,
|
||||
Enqueued(u64),
|
||||
Waiting,
|
||||
Overloaded,
|
||||
Expired,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionAdmission for PostgresExecutionAdmission {
|
||||
async fn health(&self) -> Result<(), AdmissionStoreError> {
|
||||
admission_health(&self.pool).await
|
||||
}
|
||||
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<AdmissionOutcome, AdmissionStoreError> {
|
||||
request.validate()?;
|
||||
if request.approval == ApprovalEvidence::NotGranted {
|
||||
return Ok(AdmissionOutcome::ApprovalRequired);
|
||||
}
|
||||
if request.mode == AdmissionMode::Background
|
||||
&& request.no_tasks_fallback == NoTasksFallback::Unsupported
|
||||
{
|
||||
return Ok(AdmissionOutcome::Unsupported);
|
||||
}
|
||||
|
||||
let owner_token = format!("adm_{}", Uuid::now_v7().simple());
|
||||
let mut ticket: Option<u64> = None;
|
||||
let mut backoff = INITIAL_WAIT_BACKOFF_MS;
|
||||
loop {
|
||||
if crank_core::current_unix_ms() >= request.expires_at_unix_ms {
|
||||
if let Some(ticket_id) = ticket {
|
||||
self.cancel(ticket_id, &owner_token).await?;
|
||||
}
|
||||
return Ok(AdmissionOutcome::DeadlineElapsed);
|
||||
}
|
||||
let attempt = self
|
||||
.attempt(
|
||||
AttemptContext {
|
||||
request: &request,
|
||||
owner_token: &owner_token,
|
||||
},
|
||||
&mut ticket,
|
||||
)
|
||||
.await?;
|
||||
match attempt {
|
||||
Attempt::Granted => {
|
||||
return Ok(AdmissionOutcome::Permitted(AdmissionPermit {
|
||||
ticket_id: ticket.unwrap_or_default(),
|
||||
owner_token,
|
||||
scope: request.scope.clone(),
|
||||
mode: request.mode,
|
||||
policy_mode: request.policy_mode,
|
||||
}));
|
||||
}
|
||||
Attempt::Enqueued(ticket_id) => {
|
||||
return Ok(AdmissionOutcome::Enqueued(AdmissionDecision {
|
||||
ticket_id,
|
||||
scope: request.scope.clone(),
|
||||
expires_at_unix_ms: request.expires_at_unix_ms,
|
||||
policy_mode: request.policy_mode,
|
||||
}));
|
||||
}
|
||||
Attempt::Expired => {
|
||||
return Ok(AdmissionOutcome::DeadlineElapsed);
|
||||
}
|
||||
Attempt::Overloaded => {
|
||||
return Ok(AdmissionOutcome::Overloaded {
|
||||
retry_after_ms: OVERLOAD_RETRY_AFTER_MS,
|
||||
});
|
||||
}
|
||||
Attempt::Waiting => {
|
||||
let remaining = request
|
||||
.expires_at_unix_ms
|
||||
.saturating_sub(crank_core::current_unix_ms())
|
||||
.max(1);
|
||||
let sleep_ms = i64::try_from(backoff).unwrap_or(remaining).min(remaining);
|
||||
tokio::time::sleep(Duration::from_millis(
|
||||
u64::try_from(sleep_ms).unwrap_or(backoff),
|
||||
))
|
||||
.await;
|
||||
backoff = (backoff * 2).min(MAX_WAIT_BACKOFF_MS);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError> {
|
||||
let result = sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id = $1 and owner_token = $2 and state = 'executing'",
|
||||
)
|
||||
.bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX))
|
||||
.bind(&permit.owner_token)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError> {
|
||||
let result = sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')",
|
||||
)
|
||||
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
|
||||
.bind(owner_token)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
|
||||
admission_snapshot(&self.pool).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness probe of the mandatory admission state: both tables must be reachable and readable.
|
||||
pub async fn admission_health(pool: &PgPool) -> Result<(), AdmissionStoreError> {
|
||||
sqlx::query(
|
||||
"select count(*)::bigint from execution_admission_controllers where workspace_id = $1",
|
||||
)
|
||||
.bind(DEPLOYMENT_CONTROLLER)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
sqlx::query("select count(*)::bigint from execution_admission_entries where false")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Authoritative queue readings of the deployment-wide admission state.
|
||||
pub async fn admission_snapshot(
|
||||
pool: &PgPool,
|
||||
) -> Result<AdmissionQueueSnapshot, AdmissionStoreError> {
|
||||
let row = sqlx::query(
|
||||
"select
|
||||
count(*) filter (where state in ('waiting', 'reserved')) as queued,
|
||||
count(*) filter (where state = 'executing') as executing,
|
||||
coalesce(
|
||||
(extract(epoch from (now() - min(created_at) filter (where state in ('waiting', 'reserved')))) * 1000)::bigint,
|
||||
-1
|
||||
) as oldest_waiting_ms
|
||||
from execution_admission_entries",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
let queued: i64 = row
|
||||
.try_get("queued")
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
let executing: i64 = row
|
||||
.try_get("executing")
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
let oldest: i64 = row
|
||||
.try_get("oldest_waiting_ms")
|
||||
.map_err(PostgresExecutionAdmission::unavailable)?;
|
||||
Ok(AdmissionQueueSnapshot {
|
||||
waiting: u64::try_from(queued).unwrap_or_default(),
|
||||
executing: u64::try_from(executing).unwrap_or_default(),
|
||||
oldest_waiting_ms: (oldest >= 0).then(|| u64::try_from(oldest).unwrap_or_default()),
|
||||
})
|
||||
}
|
||||
|
||||
impl PostgresExecutionAdmission {
|
||||
async fn attempt(
|
||||
&self,
|
||||
context: AttemptContext<'_>,
|
||||
ticket: &mut Option<u64>,
|
||||
) -> Result<Attempt, AdmissionStoreError> {
|
||||
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local lock_timeout = '3000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local statement_timeout = '5000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
|
||||
match self.attempt_locked(&mut transaction, context, ticket).await {
|
||||
Ok(attempt) => {
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(attempt)
|
||||
}
|
||||
Err(AttemptError::Contended) => {
|
||||
transaction.rollback().await.ok();
|
||||
Ok(Attempt::Waiting)
|
||||
}
|
||||
Err(AttemptError::Overloaded) => {
|
||||
transaction.rollback().await.ok();
|
||||
Ok(Attempt::Overloaded)
|
||||
}
|
||||
Err(AttemptError::Failure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn attempt_locked(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
context: AttemptContext<'_>,
|
||||
ticket: &mut Option<u64>,
|
||||
) -> Result<Attempt, AttemptError> {
|
||||
let workspace_id = context.request.scope.workspace_id.as_str();
|
||||
sqlx::query(
|
||||
"insert into execution_admission_controllers (workspace_id)
|
||||
values ($1), ($2)
|
||||
on conflict (workspace_id) do nothing",
|
||||
)
|
||||
.bind(DEPLOYMENT_CONTROLLER)
|
||||
.bind(workspace_id)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
// Fixed lock order: deployment controller first, then the workspace controller.
|
||||
lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?;
|
||||
let cursor = lock_controller(transaction, workspace_id).await?;
|
||||
|
||||
// Expired queue decisions and waiting tickets leave the bounded queue; held permits never do.
|
||||
sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id in (
|
||||
select ticket_id from execution_admission_entries
|
||||
where state in ('waiting', 'reserved') and expires_at <= now()
|
||||
order by ticket_id
|
||||
limit $1
|
||||
)",
|
||||
)
|
||||
.bind(EXPIRED_CLEANUP_BATCH)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
if ticket.is_none() {
|
||||
let queued: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries
|
||||
where state in ('waiting', 'reserved')",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
if u64::try_from(queued).unwrap_or(u64::MAX)
|
||||
>= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX)
|
||||
{
|
||||
return Err(AttemptError::Overloaded);
|
||||
}
|
||||
let state = match context.request.mode {
|
||||
AdmissionMode::Synchronous => "waiting",
|
||||
AdmissionMode::Background => "reserved",
|
||||
};
|
||||
let row = sqlx::query(
|
||||
"insert into execution_admission_entries (
|
||||
workspace_id, agent_key, operation_id, operation_version, mode, policy_mode,
|
||||
state, owner_token, fingerprint, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
to_timestamp($10::double precision / 1000.0))
|
||||
returning ticket_id",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(&context.request.scope.agent_key)
|
||||
.bind(&context.request.scope.operation_id)
|
||||
.bind(i32::try_from(context.request.scope.operation_version).unwrap_or(i32::MAX))
|
||||
.bind(context.request.mode.as_str())
|
||||
.bind(policy_mode_str(context.request))
|
||||
.bind(state)
|
||||
.bind(context.owner_token)
|
||||
.bind(&context.request.fingerprint)
|
||||
.bind(context.request.expires_at_unix_ms)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
|
||||
*ticket = Some(u64::try_from(ticket_id).unwrap_or_default());
|
||||
if context.request.mode == AdmissionMode::Background {
|
||||
return Ok(Attempt::Enqueued(
|
||||
u64::try_from(ticket_id).unwrap_or_default(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
self.grant_next(transaction, workspace_id, &cursor).await?;
|
||||
|
||||
let Some(ticket_id) = *ticket else {
|
||||
return Ok(Attempt::Waiting);
|
||||
};
|
||||
let state: Option<String> =
|
||||
sqlx::query("select state from execution_admission_entries where ticket_id = $1")
|
||||
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.map(|row| row.try_get("state"))
|
||||
.transpose()
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
match state.as_deref() {
|
||||
Some("executing") => Ok(Attempt::Granted),
|
||||
Some(_) => Ok(Attempt::Waiting),
|
||||
// The ticket expired inside the store (or was removed by another cleanup).
|
||||
None => Ok(Attempt::Expired),
|
||||
}
|
||||
}
|
||||
|
||||
/// Grants at most one waiting ticket per transaction, following the persisted weighted
|
||||
/// round-robin cursor. A blocked head keeps its turn: nothing later is granted in its place.
|
||||
async fn grant_next(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
cursor: &AdmissionCursor,
|
||||
) -> Result<(), AttemptError> {
|
||||
let rows = sqlx::query(
|
||||
"select ticket_id, agent_key, operation_id from execution_admission_entries
|
||||
where workspace_id = $1 and state = 'waiting'
|
||||
order by ticket_id
|
||||
limit $2",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(QUEUE_READ_LIMIT)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let queue = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
Ok(AdmissionQueueEntry {
|
||||
ticket_id: u64::try_from(
|
||||
row.try_get::<i64, _>("ticket_id")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
sequence: 0,
|
||||
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
|
||||
operation_id: row
|
||||
.try_get("operation_id")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, AttemptError>>()?;
|
||||
let Some(selection) = select_next_grant(&queue, cursor, self.scheduler) else {
|
||||
return Ok(());
|
||||
};
|
||||
let head = &queue[selection.index];
|
||||
|
||||
let global_executing: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries
|
||||
where state = 'executing'",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let agent_counts = scoped_counts(transaction, workspace_id, "agent_key").await?;
|
||||
let tool_counts = scoped_counts(transaction, workspace_id, "operation_id").await?;
|
||||
if u64::try_from(global_executing).unwrap_or(u64::MAX)
|
||||
>= u64::try_from(self.limits.global_limit).unwrap_or(u64::MAX)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let agent_executing = agent_counts
|
||||
.get(head.agent_key.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if agent_executing >= u64::try_from(self.limits.agent_limit).unwrap_or(u64::MAX) {
|
||||
return Ok(());
|
||||
}
|
||||
let tool_executing = tool_counts
|
||||
.get(head.operation_id.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if tool_executing >= u64::try_from(self.limits.tool_limit).unwrap_or(u64::MAX) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"update execution_admission_entries
|
||||
set state = 'executing', granted_at = now()
|
||||
where ticket_id = $1 and state = 'waiting'",
|
||||
)
|
||||
.bind(i64::try_from(head.ticket_id).unwrap_or(i64::MAX))
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
sqlx::query(
|
||||
"update execution_admission_controllers
|
||||
set cursor_agent = $2,
|
||||
cursor_agent_credit = $3,
|
||||
cursor_tool = $4,
|
||||
cursor_tool_credit = $5,
|
||||
updated_at = now()
|
||||
where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(selection.cursor.agent.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
|
||||
.bind(selection.cursor.tool.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
enum AttemptError {
|
||||
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
|
||||
Contended,
|
||||
/// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait.
|
||||
Overloaded,
|
||||
Failure(AdmissionStoreError),
|
||||
}
|
||||
|
||||
impl AttemptError {
|
||||
fn from_sql(error: sqlx::Error) -> Self {
|
||||
if let Some(database) = error.as_database_error()
|
||||
&& matches!(database.code().as_deref(), Some("55P03" | "57014"))
|
||||
{
|
||||
return Self::Contended;
|
||||
}
|
||||
Self::Failure(PostgresExecutionAdmission::unavailable(error))
|
||||
}
|
||||
}
|
||||
|
||||
async fn lock_controller(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
) -> Result<AdmissionCursor, AttemptError> {
|
||||
sqlx::query(
|
||||
"update execution_admission_controllers set updated_at = updated_at
|
||||
where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let row = sqlx::query(
|
||||
"select cursor_agent, cursor_agent_credit, cursor_tool, cursor_tool_credit
|
||||
from execution_admission_controllers where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
Ok(AdmissionCursor {
|
||||
agent: row
|
||||
.try_get("cursor_agent")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
agent_credit: u32::try_from(
|
||||
row.try_get::<i32, _>("cursor_agent_credit")
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.max(0),
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
tool: row.try_get("cursor_tool").map_err(AttemptError::from_sql)?,
|
||||
tool_credit: u32::try_from(
|
||||
row.try_get::<i32, _>("cursor_tool_credit")
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.max(0),
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn scoped_counts(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
column: &str,
|
||||
) -> Result<BTreeMap<String, u64>, AttemptError> {
|
||||
let statement = if column == "agent_key" {
|
||||
"select agent_key as key, count(*)::bigint as count
|
||||
from execution_admission_entries
|
||||
where workspace_id = $1 and state = 'executing'
|
||||
group by agent_key"
|
||||
} else {
|
||||
"select operation_id as key, count(*)::bigint as count
|
||||
from execution_admission_entries
|
||||
where workspace_id = $1 and state = 'executing'
|
||||
group by operation_id"
|
||||
};
|
||||
let rows = sqlx::query(statement)
|
||||
.bind(workspace_id)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let mut counts = BTreeMap::new();
|
||||
for row in rows {
|
||||
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
|
||||
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
|
||||
counts.insert(key, u64::try_from(count).unwrap_or_default());
|
||||
}
|
||||
Ok(counts)
|
||||
}
|
||||
|
||||
fn policy_mode_str(request: &AdmissionRequest) -> &'static str {
|
||||
match request.policy_mode {
|
||||
crank_core::ToolExecutionMode::Synchronous => "synchronous",
|
||||
crank_core::ToolExecutionMode::Background => "background",
|
||||
crank_core::ToolExecutionMode::Adaptive => "adaptive",
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,19 @@ impl PostgresRegistry {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Readiness probe of the mandatory execution-admission state (Story 4.1). A missing or
|
||||
/// unreadable admission schema is a dependency failure, never a degraded-but-ready state.
|
||||
pub async fn execution_admission_health(&self) -> Result<(), crank_core::AdmissionStoreError> {
|
||||
super::admission::admission_health(&self.pool).await
|
||||
}
|
||||
|
||||
/// Authoritative admission queue readings for low-cardinality metrics.
|
||||
pub async fn execution_admission_snapshot(
|
||||
&self,
|
||||
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
|
||||
super::admission::admission_snapshot(&self.pool).await
|
||||
}
|
||||
|
||||
async fn connect_in_schema(
|
||||
database_url: &str,
|
||||
schema: Option<&str>,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod admission;
|
||||
mod agent;
|
||||
mod agent_catalog;
|
||||
mod agent_prompt_bindings;
|
||||
@@ -36,6 +37,9 @@ use serde_json::Value;
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow, types::Json};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
pub use admission::{
|
||||
AdmissionLimits, PostgresExecutionAdmission, admission_health, admission_snapshot,
|
||||
};
|
||||
pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError};
|
||||
pub use resource::{AuthorizedResourceRead, BoundResourceAdmission};
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod integration {
|
||||
mod admission;
|
||||
mod agent_resource_bindings;
|
||||
mod agents_usage;
|
||||
mod approval;
|
||||
|
||||
@@ -0,0 +1,496 @@
|
||||
//! ATDD Story 4.1 (FR-21/FR-24): authoritative PostgreSQL execution admission.
|
||||
//!
|
||||
//! These oracles drive the real store against a real PostgreSQL schema: scoped permits, bounded
|
||||
//! queue, typed overload, durable enqueue decisions, deadline propagation, quota keys that ignore
|
||||
//! the operation version, fairness order and fail-closed health.
|
||||
|
||||
use super::common;
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
|
||||
AdmissionSchedulerConfig, AdmissionScope, ApprovalEvidence, ExecutionAdmission,
|
||||
NoTasksFallback, ToolExecutionMode, current_unix_ms,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
fn limits(global: usize, agent: usize, tool: usize, queue: usize) -> AdmissionLimits {
|
||||
AdmissionLimits {
|
||||
global_limit: global,
|
||||
agent_limit: agent,
|
||||
tool_limit: tool,
|
||||
queue_limit: queue,
|
||||
}
|
||||
}
|
||||
|
||||
fn store(
|
||||
pool: sqlx::PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
) -> PostgresExecutionAdmission {
|
||||
PostgresExecutionAdmission::new(pool, limits, scheduler).expect("valid admission store")
|
||||
}
|
||||
|
||||
fn scope(agent: &str, operation: &str, version: u32) -> AdmissionScope {
|
||||
AdmissionScope {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
agent_key: agent.to_owned(),
|
||||
operation_id: operation.to_owned(),
|
||||
operation_version: version,
|
||||
}
|
||||
}
|
||||
|
||||
fn request(scope: AdmissionScope, deadline_in_ms: i64) -> AdmissionRequest {
|
||||
AdmissionRequest {
|
||||
scope,
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: format!("fp_{}", current_unix_ms()),
|
||||
expires_at_unix_ms: current_unix_ms() + deadline_in_ms,
|
||||
}
|
||||
}
|
||||
|
||||
fn background(scope: AdmissionScope, fallback: NoTasksFallback) -> AdmissionRequest {
|
||||
AdmissionRequest {
|
||||
mode: AdmissionMode::Background,
|
||||
no_tasks_fallback: fallback,
|
||||
..request(scope, 30_000)
|
||||
}
|
||||
}
|
||||
|
||||
fn permit(outcome: AdmissionOutcome) -> AdmissionPermit {
|
||||
match outcome {
|
||||
AdmissionOutcome::Permitted(permit) => permit,
|
||||
other => panic!("expected a scoped permit, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_metric_poll() {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn permits_are_scoped_released_once_and_respect_the_agent_budget() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 1, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let first = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
assert_eq!(first.scope.agent_key, "agent_a");
|
||||
|
||||
// The same Agent is already at its scoped budget: the second request waits and expires instead of
|
||||
// over-admitting, and it never dispatches.
|
||||
let waiting = admission
|
||||
.admit(request(scope("agent_a", "op_leads", 2), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(waiting, AdmissionOutcome::DeadlineElapsed);
|
||||
|
||||
// Another Agent still has the global capacity it is entitled to.
|
||||
let other = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_b", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
assert_eq!(other.scope.agent_key, "agent_b");
|
||||
assert!(admission.release(&other).await.unwrap());
|
||||
// Repeated release is idempotent and never frees a foreign or already released permit.
|
||||
assert!(!admission.release(&other).await.unwrap());
|
||||
|
||||
assert!(admission.release(&first).await.unwrap());
|
||||
let snapshot = admission.snapshot().await.unwrap();
|
||||
assert_eq!(snapshot.executing, 0);
|
||||
assert_eq!(snapshot.waiting, 0);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_queue_returns_a_typed_overload_with_retry_guidance() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 1, 1, 2),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 5_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
|
||||
// Two bounded waiting entries fill the queue.
|
||||
let mut waiters = Vec::new();
|
||||
for version in 1..=2 {
|
||||
let admission = admission.clone();
|
||||
waiters.push(tokio::spawn(async move {
|
||||
admission
|
||||
.admit(request(scope("agent_b", "op_leads", version), 5_000))
|
||||
.await
|
||||
}));
|
||||
}
|
||||
while admission.snapshot().await.unwrap().waiting < 2 {
|
||||
wait_for_metric_poll().await;
|
||||
}
|
||||
|
||||
let rejected = admission
|
||||
.admit(request(scope("agent_c", "op_leads", 1), 5_000))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
match rejected {
|
||||
AdmissionOutcome::Overloaded { retry_after_ms } => {
|
||||
assert!((1..=crank_core::MAX_RETRY_AFTER_MS).contains(&retry_after_ms));
|
||||
}
|
||||
other => panic!("expected typed overload, got {other:?}"),
|
||||
}
|
||||
|
||||
// Cancelling the held permit lets the queued waiters make progress instead of leaking capacity.
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
for waiter in waiters.drain(..) {
|
||||
let outcome = waiter.await.expect("waiter task");
|
||||
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
|
||||
let _ = admission.release(&permit).await;
|
||||
}
|
||||
}
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_decisions_are_durable_pinned_and_released_once() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let decision = match admission
|
||||
.admit(background(
|
||||
scope("agent_a", "op_report", 3),
|
||||
NoTasksFallback::Synchronous,
|
||||
))
|
||||
.await
|
||||
.expect("admission store is reachable")
|
||||
{
|
||||
AdmissionOutcome::Enqueued(decision) => decision,
|
||||
other => panic!("expected a durable enqueue decision, got {other:?}"),
|
||||
};
|
||||
assert_eq!(decision.scope.operation_version, 3);
|
||||
assert_eq!(decision.policy_mode, ToolExecutionMode::Synchronous);
|
||||
|
||||
// A second process (fresh pool and store over the same schema) must read the committed decision:
|
||||
// it occupies bounded queue capacity and is not a runnable execution.
|
||||
let reconnected = store(
|
||||
database.concurrent_registry(2).await.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
let snapshot = reconnected.snapshot().await.unwrap();
|
||||
assert_eq!(snapshot.waiting, 1);
|
||||
assert_eq!(snapshot.executing, 0);
|
||||
|
||||
assert!(
|
||||
reconnected
|
||||
.cancel(decision.ticket_id, "not_the_owner")
|
||||
.await
|
||||
.unwrap()
|
||||
.eq(&false),
|
||||
"a foreign cancel must never release the decision"
|
||||
);
|
||||
assert!(
|
||||
reconnected.snapshot().await.unwrap().waiting.eq(&1),
|
||||
"the foreign cancel must leave the decision intact"
|
||||
);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_without_a_task_transport_is_typed_unsupported() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let outcome = admission
|
||||
.admit(background(
|
||||
scope("agent_a", "op_report", 1),
|
||||
NoTasksFallback::Unsupported,
|
||||
))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(outcome, AdmissionOutcome::Unsupported);
|
||||
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unverified_approval_never_produces_a_decision() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let mut denied = background(
|
||||
scope("agent_a", "op_report", 1),
|
||||
NoTasksFallback::Synchronous,
|
||||
);
|
||||
denied.approval = ApprovalEvidence::NotGranted;
|
||||
assert_eq!(
|
||||
admission.admit(denied).await.unwrap(),
|
||||
AdmissionOutcome::ApprovalRequired
|
||||
);
|
||||
|
||||
let mut denied_sync = request(scope("agent_a", "op_report", 1), 1_000);
|
||||
denied_sync.approval = ApprovalEvidence::NotGranted;
|
||||
assert_eq!(
|
||||
admission.admit(denied_sync).await.unwrap(),
|
||||
AdmissionOutcome::ApprovalRequired
|
||||
);
|
||||
assert_eq!(admission.snapshot().await.unwrap().waiting, 0);
|
||||
assert_eq!(admission.snapshot().await.unwrap().executing, 0);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_keys_ignore_the_operation_version() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(4, 4, 1, 8),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
// A different Published Version of the same Tool shares the same per-Tool budget.
|
||||
let outcome = admission
|
||||
.admit(request(scope("agent_a", "op_leads", 2), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn weighted_agents_are_served_their_quantum_in_order() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 4, 4, 8),
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
},
|
||||
);
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_z", "op_leads", 1), 5_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
|
||||
let (sender, mut receiver) = mpsc::channel(3);
|
||||
let mut tasks = Vec::new();
|
||||
for (agent, version) in [("agent_a", 1), ("agent_a", 2), ("agent_b", 1)] {
|
||||
let admission = admission.clone();
|
||||
let sender = sender.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let outcome = admission
|
||||
.admit(request(scope(agent, "op_leads", version), 5_000))
|
||||
.await;
|
||||
if let Ok(AdmissionOutcome::Permitted(permit)) = outcome {
|
||||
let _ = sender.send(permit.scope.agent_key.clone()).await;
|
||||
// Hold briefly so the grant order, not task scheduling, decides the sequence.
|
||||
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||
let _ = admission.release(&permit).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
drop(sender);
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
|
||||
let mut order = Vec::new();
|
||||
while let Some(agent) = receiver.recv().await {
|
||||
order.push(agent);
|
||||
if order.len() == 3 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for task in tasks {
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), task).await;
|
||||
}
|
||||
// agent_a carries weight 2 and therefore receives two consecutive grants before agent_b.
|
||||
assert_eq!(order, vec!["agent_a", "agent_a", "agent_b"]);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mandatory_state_failure_fails_closed_and_reports_not_ready() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// No migration on this schema: the mandatory admission state does not exist yet.
|
||||
let unmigrated = database.raw_pool().await;
|
||||
let admission = store(unmigrated, limits(4, 4, 4, 8), Default::default());
|
||||
assert!(admission.health().await.is_err());
|
||||
assert!(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 1_000))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn held_permits_survive_a_new_process_view() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
|
||||
let reconnected = store(
|
||||
database.concurrent_registry(2).await.pool().clone(),
|
||||
limits(1, 4, 4, 8),
|
||||
Default::default(),
|
||||
);
|
||||
assert_eq!(reconnected.snapshot().await.unwrap().executing, 1);
|
||||
let outcome = reconnected
|
||||
.admit(request(scope("agent_b", "op_leads", 1), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(
|
||||
outcome,
|
||||
AdmissionOutcome::DeadlineElapsed,
|
||||
"a held permit is not silently reused by another process"
|
||||
);
|
||||
assert!(reconnected.release(&held).await.unwrap());
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn published_execution_policy_is_pinned_across_a_new_draft_version() {
|
||||
use crank_core::{NoTasksFallback, OperationStatus, ToolExecutionMode, ToolExecutionPolicy};
|
||||
use crank_registry::{CreateVersionRequest, PublishRequest};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let workspace = common::test_workspace_id();
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 45_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
let mut version_one = common::test_operation("op_policy", 1, OperationStatus::Draft);
|
||||
version_one.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&workspace, &version_one, None)
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &workspace,
|
||||
operation_id: &version_one.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse("2026-03-25T12:10:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let published = registry
|
||||
.get_published_operation(&version_one.id)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("published operation");
|
||||
assert_eq!(published.execution_config.execution_policy, Some(policy));
|
||||
|
||||
// A later Draft with another policy never rewrites the already accepted publication.
|
||||
let mut version_two = common::test_operation("op_policy", 2, OperationStatus::Draft);
|
||||
version_two.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 1_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
});
|
||||
registry
|
||||
.create_version(CreateVersionRequest {
|
||||
workspace_id: &workspace,
|
||||
snapshot: &version_two,
|
||||
change_note: Some("next draft"),
|
||||
created_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let unchanged = registry
|
||||
.get_published_operation(&version_one.id)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("published operation");
|
||||
assert_eq!(unchanged.version, 1);
|
||||
assert_eq!(unchanged.execution_config.execution_policy, Some(policy));
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_origin_bucket_is_bounded_together_with_agents() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let admission = store(
|
||||
registry.pool().clone(),
|
||||
limits(1, 1, 1, 4),
|
||||
Default::default(),
|
||||
);
|
||||
let held = permit(
|
||||
admission
|
||||
.admit(request(scope(ADMIN_ORIGIN_KEY, "op_leads", 1), 2_000))
|
||||
.await
|
||||
.expect("admission store is reachable"),
|
||||
);
|
||||
let started = Instant::now();
|
||||
let outcome = admission
|
||||
.admit(request(scope("agent_a", "op_leads", 1), 120))
|
||||
.await
|
||||
.expect("admission store is reachable");
|
||||
assert_eq!(outcome, AdmissionOutcome::DeadlineElapsed);
|
||||
assert!(started.elapsed() >= Duration::from_millis(100));
|
||||
assert!(admission.release(&held).await.unwrap());
|
||||
database.cleanup().await;
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
//! Common execution admission seam of the runtime (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Every real execution — synchronous, or a published `background`/`adaptive` policy that falls back
|
||||
//! to synchronous because no durable Task transport exists yet — passes this one gate before any
|
||||
//! adapter work. The gate asks the authoritative store for a scoped permit bounded by the published
|
||||
//! Tool policy, the verified approval state of the exact Published Version/scope/input and the caller
|
||||
//! deadline, and it releases the permit explicitly when the local execution future ends.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
ADMIN_ORIGIN_KEY, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionRequest,
|
||||
AdmissionScope, CoordinationStateStore, ExecutionAdmission, NoTasksFallback, ToolExecutionMode,
|
||||
current_unix_ms,
|
||||
};
|
||||
use crank_metrics::{
|
||||
AdmissionModeLabel, RejectionReason, ScopeKind, record_admission_queue_wait,
|
||||
record_admission_rejection,
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::{RuntimeError, RuntimeOperation, RuntimeRequestContext};
|
||||
|
||||
/// Effective absolute deadline of one execution: the published policy deadline never extends the
|
||||
/// caller deadline and always includes queue waiting time.
|
||||
pub(crate) fn effective_deadline(
|
||||
operation: &RuntimeOperation,
|
||||
caller_deadline: Instant,
|
||||
) -> Instant {
|
||||
match operation.execution_config.execution_policy {
|
||||
Some(policy) => {
|
||||
caller_deadline.min(Instant::now() + Duration::from_millis(policy.deadline_ms))
|
||||
}
|
||||
None => caller_deadline,
|
||||
}
|
||||
}
|
||||
|
||||
/// Effective execution path for this deployment.
|
||||
///
|
||||
/// Story 4.1 has no Task transport, so a `background` or `adaptive` policy resolves through its
|
||||
/// published `no_tasks_fallback`: either the same synchronous admission path, or a typed
|
||||
/// unsupported refusal with no admission entry, no Task id and no outbound call.
|
||||
pub(crate) fn effective_mode(
|
||||
operation: &RuntimeOperation,
|
||||
) -> Result<ToolExecutionMode, RuntimeError> {
|
||||
let Some(policy) = operation.execution_config.execution_policy else {
|
||||
return Ok(ToolExecutionMode::Synchronous);
|
||||
};
|
||||
match policy.mode {
|
||||
ToolExecutionMode::Synchronous => Ok(ToolExecutionMode::Synchronous),
|
||||
mode => match policy.no_tasks_fallback {
|
||||
NoTasksFallback::Synchronous => Ok(ToolExecutionMode::Synchronous),
|
||||
NoTasksFallback::Unsupported => Err(RuntimeError::ExecutionPolicyUnsupported {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
mode,
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct AdmissionLease {
|
||||
store: Arc<dyn ExecutionAdmission>,
|
||||
permit: Option<AdmissionPermit>,
|
||||
runtime: tokio::runtime::Handle,
|
||||
}
|
||||
|
||||
impl AdmissionLease {
|
||||
fn new(store: Arc<dyn ExecutionAdmission>, permit: AdmissionPermit) -> Self {
|
||||
Self {
|
||||
store,
|
||||
permit: Some(permit),
|
||||
runtime: tokio::runtime::Handle::current(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit release on the normal/error path of the local execution. The `Drop` cleanup stays as
|
||||
/// the cancellation path; both CAS on the same owner token, so only one of them can free it.
|
||||
pub(crate) async fn release(&mut self) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
if let Err(error) = self.store.release(&permit).await {
|
||||
warn!(
|
||||
name: "runtime.admission.release_failed",
|
||||
error = %error,
|
||||
"failed to release an admission permit; the reservation stays fail-closed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AdmissionLease {
|
||||
fn drop(&mut self) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
let store = Arc::clone(&self.store);
|
||||
// Bounded cleanup: the explicit release path already ran on success/error; cancellation
|
||||
// reaches here and releases by owner token without ever freeing a foreign permit. A failed
|
||||
// release keeps the reservation fail-closed (no TTL reuse of an active permit).
|
||||
self.runtime.spawn(async move {
|
||||
match store.release(&permit).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => warn!(
|
||||
name: "runtime.admission.release_skipped",
|
||||
"admission permit was already released"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
name: "runtime.admission.release_failed",
|
||||
error = %error,
|
||||
"failed to release an admission permit; the reservation stays fail-closed"
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquires a scoped permit from the authoritative store. Returns `None` when this deployment has no
|
||||
/// admission store wired (embedded/unit-test executors keep the pre-4.1 local limiter).
|
||||
pub(crate) async fn acquire(
|
||||
store: Option<&Arc<dyn ExecutionAdmission>>,
|
||||
coordination_store: Option<&dyn CoordinationStateStore>,
|
||||
request: &crate::RuntimeExecutionRequest<'_>,
|
||||
deadline: Instant,
|
||||
) -> Result<Option<AdmissionLease>, RuntimeError> {
|
||||
let Some(store) = store else {
|
||||
return Ok(None);
|
||||
};
|
||||
let operation = request.operation();
|
||||
let input = request.input();
|
||||
let request_context = request.request_context();
|
||||
let operation_id = operation.operation_id.as_str().to_owned();
|
||||
let policy_mode = effective_mode(operation)?;
|
||||
// The one admission snapshot carries the authoritative approval state of this exact Published
|
||||
// Version, scope, input and deadline. A confirmation-requiring request without a verified
|
||||
// approval never takes capacity and never dispatches: the pending human decision is answered
|
||||
// here with the existing challenge, and the single-use token is consumed only at dispatch.
|
||||
let approval = crate::confirmation::approval_evidence(
|
||||
coordination_store,
|
||||
operation,
|
||||
input,
|
||||
Some(request_context),
|
||||
)
|
||||
.await?;
|
||||
let agent_key = request.agent_id().map(|agent| agent.as_str());
|
||||
let workspace_id = request.workspace_id().as_str().to_owned();
|
||||
let request = AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id,
|
||||
agent_key: agent_key.unwrap_or(ADMIN_ORIGIN_KEY).to_owned(),
|
||||
operation_id: operation_id.clone(),
|
||||
operation_version: operation.operation_version,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode,
|
||||
no_tasks_fallback: operation
|
||||
.execution_config
|
||||
.execution_policy
|
||||
.map(|policy| policy.no_tasks_fallback)
|
||||
.unwrap_or(NoTasksFallback::Synchronous),
|
||||
approval,
|
||||
fingerprint: admission_fingerprint(request_context, operation),
|
||||
expires_at_unix_ms: current_unix_ms().saturating_add(
|
||||
deadline_remaining_ms(deadline)
|
||||
.try_into()
|
||||
.unwrap_or(i64::MAX),
|
||||
),
|
||||
};
|
||||
let started = Instant::now();
|
||||
let outcome = store.admit(request).await.map_err(|error| {
|
||||
record_admission_rejection(RejectionReason::DependencyUnavailable, ScopeKind::Global);
|
||||
RuntimeError::AdmissionStoreUnavailable {
|
||||
operation_id: operation_id.clone(),
|
||||
details: error.to_string(),
|
||||
}
|
||||
})?;
|
||||
let waited_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or_default();
|
||||
match outcome {
|
||||
AdmissionOutcome::Permitted(permit) => {
|
||||
record_admission_queue_wait(AdmissionModeLabel::Synchronous, waited_ms);
|
||||
Ok(Some(AdmissionLease::new(Arc::clone(store), permit)))
|
||||
}
|
||||
AdmissionOutcome::Overloaded { retry_after_ms } => {
|
||||
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
|
||||
Err(RuntimeError::AdmissionOverloaded {
|
||||
operation_id,
|
||||
retry_after_ms,
|
||||
})
|
||||
}
|
||||
AdmissionOutcome::DeadlineElapsed => {
|
||||
// Queue wait exhausted the absolute deadline before a permit was granted: nothing was
|
||||
// dispatched, so this is an admission refusal, not an upstream timeout.
|
||||
Err(deadline_refusal(operation))
|
||||
}
|
||||
AdmissionOutcome::ApprovalRequired => {
|
||||
// The store refused before reserving any capacity or queue entry. Answer with the
|
||||
// existing single-use challenge so the caller can confirm out of band.
|
||||
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
|
||||
Err(crate::confirmation::challenge(
|
||||
coordination_store,
|
||||
operation,
|
||||
input,
|
||||
Some(request_context),
|
||||
)
|
||||
.await)
|
||||
}
|
||||
AdmissionOutcome::Unsupported | AdmissionOutcome::Enqueued(_) => {
|
||||
// No Task transport exists in this deployment: an enqueue decision can never become a
|
||||
// runnable execution here, so it must not be reported as accepted work.
|
||||
record_admission_rejection(RejectionReason::Unsupported, ScopeKind::Global);
|
||||
Err(RuntimeError::ExecutionPolicyUnsupported {
|
||||
operation_id,
|
||||
mode: policy_mode,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed refusal for an admission that never granted a permit within the absolute deadline. Reuses the
|
||||
/// existing `execution_overloaded` vocabulary (stage `admission`, retry guidance, certain outcome) so
|
||||
/// no new client error code or API shape appears, while reporting zero dispatch truthfully.
|
||||
pub(crate) fn deadline_refusal(operation: &RuntimeOperation) -> RuntimeError {
|
||||
record_admission_rejection(RejectionReason::Deadline, ScopeKind::Global);
|
||||
RuntimeError::AdmissionOverloaded {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
}
|
||||
}
|
||||
|
||||
fn deadline_remaining_ms(deadline: Instant) -> u64 {
|
||||
u64::try_from(
|
||||
deadline
|
||||
.saturating_duration_since(Instant::now())
|
||||
.as_millis(),
|
||||
)
|
||||
.unwrap_or_default()
|
||||
.max(1)
|
||||
}
|
||||
|
||||
/// Bounded request identity for correlating one admission entry. Never contains payload, headers or
|
||||
/// credentials.
|
||||
fn admission_fingerprint(
|
||||
request_context: &RuntimeRequestContext,
|
||||
operation: &RuntimeOperation,
|
||||
) -> String {
|
||||
let mut fingerprint = format!(
|
||||
"{}:{}:{}",
|
||||
operation.operation_id.as_str(),
|
||||
operation.operation_version,
|
||||
request_context.request_id
|
||||
);
|
||||
fingerprint.truncate(crank_core::ADMISSION_FINGERPRINT_MAX_BYTES);
|
||||
fingerprint
|
||||
}
|
||||
@@ -2,8 +2,8 @@ use std::time::Duration;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use crank_core::{
|
||||
CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod, OperationSafetyClass,
|
||||
OperationSafetyPolicy, Target,
|
||||
ApprovalEvidence, CacheScope, CoordinationStateStore, CoordinationStateValue, HttpMethod,
|
||||
OperationSafetyClass, OperationSafetyPolicy, Target,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -26,32 +26,14 @@ pub async fn confirm_operation(
|
||||
}
|
||||
|
||||
let Some(store) = store else {
|
||||
return Err(RuntimeError::ConfirmationStoreUnavailable {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
});
|
||||
return Err(confirmation_store_unavailable(operation));
|
||||
};
|
||||
|
||||
let scope = confirmation_scope(operation, request_context)?;
|
||||
let input_hash = hash_json(input)?;
|
||||
let provided_token = request_context
|
||||
.and_then(RuntimeRequestContext::confirmation_token)
|
||||
.or_else(|| confirmation_token_from_input(input));
|
||||
|
||||
let Some(provided_token) = provided_token else {
|
||||
let token = issue_confirmation_token(
|
||||
store,
|
||||
operation.operation_id.as_str(),
|
||||
&scope,
|
||||
&input_hash,
|
||||
&safety,
|
||||
)
|
||||
.await?;
|
||||
return Err(RuntimeError::ConfirmationRequired {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
safety_class: safety.class,
|
||||
confirmation_token: token,
|
||||
expires_in_ms: confirmation_ttl_ms(&safety),
|
||||
});
|
||||
let Some(provided_token) = provided_confirmation_token(request_context, input) else {
|
||||
return Err(confirmation_required(store, operation, &scope, &input_hash, &safety).await);
|
||||
};
|
||||
|
||||
consume_confirmation_token(
|
||||
@@ -64,6 +46,121 @@ pub async fn confirm_operation(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Read-only approval proof for the shared admission gate (Story 4.1, AC2).
|
||||
///
|
||||
/// It reports the authoritative approval state of this exact Published Version, scope and input and
|
||||
/// never consumes the single-use confirmation token: the authoritative consumption stays in
|
||||
/// [`confirm_operation`] immediately before the adapter call. A missing approval is `NotGranted`, so
|
||||
/// admission refuses the request before taking execution capacity while the human decision is
|
||||
/// pending; an unusable token or an unreachable safety store is a typed refusal too, never a permit.
|
||||
pub(crate) async fn approval_evidence(
|
||||
store: Option<&dyn CoordinationStateStore>,
|
||||
operation: &RuntimeOperation,
|
||||
input: &Value,
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
) -> Result<ApprovalEvidence, RuntimeError> {
|
||||
let safety = effective_safety_policy(operation);
|
||||
if !safety.class.requires_confirmation() {
|
||||
return Ok(ApprovalEvidence::NotRequired);
|
||||
}
|
||||
if request_context.is_some_and(RuntimeRequestContext::approval_granted) {
|
||||
return Ok(ApprovalEvidence::Granted);
|
||||
}
|
||||
let Some(store) = store else {
|
||||
return Err(confirmation_store_unavailable(operation));
|
||||
};
|
||||
let scope = confirmation_scope(operation, request_context)?;
|
||||
let input_hash = hash_json(input)?;
|
||||
|
||||
let Some(token) = provided_confirmation_token(request_context, input) else {
|
||||
return Ok(ApprovalEvidence::NotGranted);
|
||||
};
|
||||
|
||||
let stored = store
|
||||
.get_value(
|
||||
CacheScope::Coordination,
|
||||
&confirmation_cache_key(&scope, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| confirmation_store_unavailable(operation))?;
|
||||
let verified = stored.is_some_and(|value| {
|
||||
value.payload.get("scope").and_then(Value::as_str) == Some(scope.as_str())
|
||||
&& value.payload.get("input_hash").and_then(Value::as_str) == Some(input_hash.as_str())
|
||||
});
|
||||
if verified {
|
||||
Ok(ApprovalEvidence::Granted)
|
||||
} else {
|
||||
Err(RuntimeError::InvalidConfirmationToken {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Issues the existing single-use confirmation challenge and returns the typed refusal that carries
|
||||
/// it. Admission calls this when no verified approval exists yet, so waiting for the human decision
|
||||
/// never holds execution capacity.
|
||||
pub(crate) async fn challenge(
|
||||
store: Option<&dyn CoordinationStateStore>,
|
||||
operation: &RuntimeOperation,
|
||||
input: &Value,
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
) -> RuntimeError {
|
||||
let safety = effective_safety_policy(operation);
|
||||
let Some(store) = store else {
|
||||
return confirmation_store_unavailable(operation);
|
||||
};
|
||||
let scope = match confirmation_scope(operation, request_context) {
|
||||
Ok(scope) => scope,
|
||||
Err(error) => return error,
|
||||
};
|
||||
let input_hash = match hash_json(input) {
|
||||
Ok(input_hash) => input_hash,
|
||||
Err(error) => return error,
|
||||
};
|
||||
confirmation_required(store, operation, &scope, &input_hash, &safety).await
|
||||
}
|
||||
|
||||
fn provided_confirmation_token<'a>(
|
||||
request_context: Option<&'a RuntimeRequestContext>,
|
||||
input: &'a Value,
|
||||
) -> Option<&'a str> {
|
||||
request_context
|
||||
.and_then(RuntimeRequestContext::confirmation_token)
|
||||
.or_else(|| confirmation_token_from_input(input))
|
||||
}
|
||||
|
||||
fn confirmation_store_unavailable(operation: &RuntimeOperation) -> RuntimeError {
|
||||
RuntimeError::ConfirmationStoreUnavailable {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn confirmation_required(
|
||||
store: &dyn CoordinationStateStore,
|
||||
operation: &RuntimeOperation,
|
||||
scope: &str,
|
||||
input_hash: &str,
|
||||
safety: &OperationSafetyPolicy,
|
||||
) -> RuntimeError {
|
||||
match issue_confirmation_token(
|
||||
store,
|
||||
operation.operation_id.as_str(),
|
||||
scope,
|
||||
input_hash,
|
||||
safety,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(token) => RuntimeError::ConfirmationRequired {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
safety_class: safety.class,
|
||||
confirmation_token: token,
|
||||
expires_in_ms: confirmation_ttl_ms(safety),
|
||||
},
|
||||
Err(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool {
|
||||
effective_safety_policy(operation)
|
||||
.class
|
||||
|
||||
@@ -43,6 +43,27 @@ pub enum RuntimeError {
|
||||
confirmation_token: String,
|
||||
expires_in_ms: u64,
|
||||
},
|
||||
#[error(
|
||||
"execution mode {mode:?} of operation {operation_id} is unavailable without a durable Task transport"
|
||||
)]
|
||||
ExecutionPolicyUnsupported {
|
||||
operation_id: String,
|
||||
mode: crank_core::ToolExecutionMode,
|
||||
},
|
||||
#[error(
|
||||
"execution admission is overloaded for operation {operation_id}; retry after {retry_after_ms} ms"
|
||||
)]
|
||||
AdmissionOverloaded {
|
||||
operation_id: String,
|
||||
retry_after_ms: u64,
|
||||
},
|
||||
#[error("execution admission requires a verified approval for operation {operation_id}")]
|
||||
AdmissionApprovalRequired { operation_id: String },
|
||||
#[error("execution admission store is unavailable for operation {operation_id}: {details}")]
|
||||
AdmissionStoreUnavailable {
|
||||
operation_id: String,
|
||||
details: String,
|
||||
},
|
||||
#[error("confirmation token for operation {operation_id} is invalid, expired, or already used")]
|
||||
InvalidConfirmationToken { operation_id: String },
|
||||
#[error("confirmation store is unavailable for operation {operation_id}")]
|
||||
|
||||
@@ -57,6 +57,11 @@ fn normalize_runtime_error_with_effect(
|
||||
| RuntimeError::IdempotencyCompletionUnknown { .. } => {
|
||||
failure = failure.with_dispatch_uncertainty();
|
||||
}
|
||||
RuntimeError::AdmissionOverloaded { retry_after_ms, .. } => {
|
||||
if let Ok(with_retry) = failure.clone().try_with_retry_after_ms(*retry_after_ms) {
|
||||
failure = with_retry;
|
||||
}
|
||||
}
|
||||
RuntimeError::ConfirmationRequired {
|
||||
confirmation_token,
|
||||
expires_in_ms,
|
||||
@@ -113,6 +118,14 @@ fn error_code(error: &RuntimeError) -> ExecutionErrorCode {
|
||||
RuntimeError::UnsupportedExecutionMode { .. } => {
|
||||
ExecutionErrorCode::ExecutionModeUnsupported
|
||||
}
|
||||
RuntimeError::ExecutionPolicyUnsupported { .. } => {
|
||||
ExecutionErrorCode::ExecutionModeUnsupported
|
||||
}
|
||||
RuntimeError::AdmissionOverloaded { .. } => ExecutionErrorCode::ExecutionOverloaded,
|
||||
RuntimeError::AdmissionApprovalRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
|
||||
RuntimeError::AdmissionStoreUnavailable { .. } => {
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
}
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => ExecutionErrorCode::ExecutionOverloaded,
|
||||
RuntimeError::InvalidPreparedRequest { .. } => ExecutionErrorCode::PreparedRequestInvalid,
|
||||
RuntimeError::ConfirmationRequired { .. } => ExecutionErrorCode::ConfirmationRequired,
|
||||
|
||||
@@ -220,6 +220,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -7,9 +7,9 @@ use std::time::{Duration, Instant};
|
||||
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
use crank_core::{
|
||||
AdapterRegistry, CoordinationStateStore, ExecutionMode, HttpMethod, InvocationSource,
|
||||
InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
|
||||
Target,
|
||||
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
|
||||
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
|
||||
SharedProtocolAdapter, Target,
|
||||
};
|
||||
use crank_metrics::{
|
||||
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
|
||||
@@ -41,6 +41,7 @@ pub struct RuntimeExecutor {
|
||||
unary_limiter: Arc<Semaphore>,
|
||||
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
||||
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
||||
admission: Option<Arc<dyn ExecutionAdmission>>,
|
||||
metering_sink: SharedMeteringSink,
|
||||
}
|
||||
|
||||
@@ -139,6 +140,7 @@ impl RuntimeExecutor {
|
||||
adapters: AdapterRegistry,
|
||||
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
||||
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
||||
admission: Option<Arc<dyn ExecutionAdmission>>,
|
||||
metering_sink: SharedMeteringSink,
|
||||
) -> Self {
|
||||
Self {
|
||||
@@ -147,6 +149,7 @@ impl RuntimeExecutor {
|
||||
limits,
|
||||
response_cache,
|
||||
coordination_store,
|
||||
admission,
|
||||
metering_sink,
|
||||
}
|
||||
}
|
||||
@@ -167,6 +170,11 @@ impl RuntimeExecutor {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
|
||||
self.admission = Some(admission);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_request(
|
||||
&self,
|
||||
request: RuntimeExecutionRequest<'_>,
|
||||
@@ -188,11 +196,16 @@ impl RuntimeExecutor {
|
||||
request.agent_id()
|
||||
);
|
||||
let operation_may_mutate = operation_may_mutate(operation);
|
||||
let execution_deadline =
|
||||
crate::admission::effective_deadline(operation, request.deadline());
|
||||
let runtime_span = Stage::RuntimeExecute.span();
|
||||
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
|
||||
log_runtime_event("unary.execute", operation, Some(request_context));
|
||||
let started_at = Instant::now();
|
||||
let dispatch_started = Arc::new(AtomicBool::new(false));
|
||||
// True until the admission gate answered. A deadline that fires while the request is still
|
||||
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
|
||||
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
|
||||
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
|
||||
metric_invocation_source(Some(request_context)),
|
||||
request_context
|
||||
@@ -206,6 +219,17 @@ impl RuntimeExecutor {
|
||||
.flatten(),
|
||||
);
|
||||
let execution = async {
|
||||
// One admission gate for every executable path (Story 4.1): the published policy decides
|
||||
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
|
||||
// is held for the whole local execution future, including retries.
|
||||
let mut lease = crate::admission::acquire(
|
||||
self.admission.as_ref(),
|
||||
self.coordination_store.as_deref(),
|
||||
&request,
|
||||
execution_deadline,
|
||||
)
|
||||
.await?;
|
||||
admission_pending.store(false, Ordering::Release);
|
||||
let _permit = self.acquire_unary_permit(operation)?;
|
||||
let _inflight = InFlightGuard::runtime();
|
||||
let mapping_span = Stage::RuntimeArgumentsMap.span();
|
||||
@@ -215,7 +239,7 @@ impl RuntimeExecutor {
|
||||
let prepared_request = prepared_request?;
|
||||
let request_preview = sanitized_request_preview(&prepared_request);
|
||||
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
|
||||
let output = self
|
||||
let result = self
|
||||
.execute_prepared(
|
||||
operation,
|
||||
input,
|
||||
@@ -223,19 +247,27 @@ impl RuntimeExecutor {
|
||||
Some(request_context),
|
||||
Arc::clone(&dispatch_started),
|
||||
)
|
||||
.await?;
|
||||
.await;
|
||||
if let Some(lease) = lease.as_mut() {
|
||||
lease.release().await;
|
||||
}
|
||||
let output = result?;
|
||||
Ok((output, request_preview))
|
||||
};
|
||||
let result = tokio::time::timeout_at(
|
||||
request.deadline().into(),
|
||||
execution_deadline.into(),
|
||||
execution.instrument(runtime_span.clone()),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
Err(RuntimeError::ExecutionDeadlineElapsed {
|
||||
may_have_dispatched: operation_may_mutate
|
||||
&& dispatch_started.load(Ordering::Acquire),
|
||||
})
|
||||
if admission_pending.load(Ordering::Acquire) {
|
||||
Err(crate::admission::deadline_refusal(operation))
|
||||
} else {
|
||||
Err(RuntimeError::ExecutionDeadlineElapsed {
|
||||
may_have_dispatched: operation_may_mutate
|
||||
&& dispatch_started.load(Ordering::Acquire),
|
||||
})
|
||||
}
|
||||
});
|
||||
record_runtime_result(&runtime_span, &result);
|
||||
drop(runtime_span);
|
||||
@@ -952,6 +984,10 @@ fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
|
||||
| RuntimeError::UnsupportedProtocol { .. }
|
||||
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
|
||||
RuntimeError::AdmissionOverloaded { .. }
|
||||
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
|
||||
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
|
||||
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
|
||||
RuntimeError::ConfirmationRequired { .. }
|
||||
| RuntimeError::InvalidConfirmationToken { .. }
|
||||
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
|
||||
@@ -1008,6 +1044,13 @@ fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
|
||||
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
|
||||
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
|
||||
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
|
||||
// label domain stays fixed; the precise code lives in the execution failure/invocation log
|
||||
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
|
||||
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
|
||||
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
|
||||
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
|
||||
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
|
||||
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
|
||||
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
|
||||
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
|
||||
|
||||
@@ -2,8 +2,8 @@ use std::sync::Arc;
|
||||
|
||||
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter};
|
||||
use crank_core::{
|
||||
AdapterRegistry, CoordinationStateStore, MeteringSink, NoopMeteringSink, ResponseCacheStore,
|
||||
SharedMeteringSink, SharedProtocolAdapter,
|
||||
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, MeteringSink, NoopMeteringSink,
|
||||
ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
|
||||
};
|
||||
|
||||
use crate::{RuntimeExecutor, RuntimeLimits};
|
||||
@@ -13,6 +13,7 @@ pub struct RuntimeExecutorBuilder {
|
||||
adapters: AdapterRegistry,
|
||||
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
||||
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
||||
admission: Option<Arc<dyn ExecutionAdmission>>,
|
||||
metering_sink: SharedMeteringSink,
|
||||
}
|
||||
|
||||
@@ -23,6 +24,7 @@ impl RuntimeExecutorBuilder {
|
||||
adapters: AdapterRegistry::empty(),
|
||||
response_cache: None,
|
||||
coordination_store: None,
|
||||
admission: None,
|
||||
metering_sink: Arc::new(NoopMeteringSink) as Arc<dyn MeteringSink>,
|
||||
}
|
||||
}
|
||||
@@ -47,6 +49,13 @@ impl RuntimeExecutorBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Wires the mandatory deployment-wide admission store (Story 4.1). Production processes wire
|
||||
/// the PostgreSQL implementation; an executor without it keeps the pre-4.1 local limit.
|
||||
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
|
||||
self.admission = Some(admission);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_metering_sink(mut self, sink: SharedMeteringSink) -> Self {
|
||||
self.metering_sink = sink;
|
||||
self
|
||||
@@ -58,6 +67,7 @@ impl RuntimeExecutorBuilder {
|
||||
self.adapters,
|
||||
self.response_cache,
|
||||
self.coordination_store,
|
||||
self.admission,
|
||||
self.metering_sink,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -446,6 +446,7 @@ mod tests {
|
||||
}),
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod admission;
|
||||
mod auth;
|
||||
mod cache;
|
||||
mod cache_factory;
|
||||
|
||||
@@ -333,6 +333,7 @@ fn runtime_operation(draft: RestImportCandidate) -> RuntimeOperation {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -96,6 +96,7 @@ paths:
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -334,6 +334,7 @@ fn destructive_delete_operation() -> Operation<Schema, MappingSet> {
|
||||
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
|
||||
}),
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -340,6 +340,7 @@ fn idempotent_post_operation() -> Operation<Schema, MappingSet> {
|
||||
}),
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -109,6 +109,7 @@ fn no_input_get_operation() -> Operation<Schema, MappingSet> {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -573,6 +573,7 @@ fn operation(
|
||||
idempotency,
|
||||
safety,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -10,7 +10,7 @@ use std::{
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdapterResponse, ConfirmationPolicy, CorrelationContext, ExecutionConfig, ExecutionErrorCode,
|
||||
ExecutionMode, ExecutionOrigin, HttpMethod, IdempotencyMode, IdempotencyPolicy,
|
||||
ExecutionMode, ExecutionOrigin, ExecutionStage, HttpMethod, IdempotencyMode, IdempotencyPolicy,
|
||||
InvocationSource, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy,
|
||||
OperationSecurityLevel, OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter,
|
||||
ProtocolAdapterError, RestTarget, RetryPolicy, Retryability, Target, ToolDescription,
|
||||
@@ -64,6 +64,10 @@ async fn deadline_after_dispatch_is_normalized_as_unknown_outcome() {
|
||||
failure.outcome_certainty(),
|
||||
OutcomeCertainty::OutcomeUnknown
|
||||
);
|
||||
// A deadline that fires after the adapter was called stays an upstream timeout: the admission
|
||||
// refusal vocabulary (Story 4.1) only covers deadlines reached before any dispatch.
|
||||
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
|
||||
assert_eq!(failure.stage(), ExecutionStage::Upstream);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -98,6 +102,8 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
|
||||
let failure = executor.execute_outcome(request).await.unwrap_err();
|
||||
assert_eq!(failure.retryability(), Retryability::AfterDelay);
|
||||
assert_eq!(failure.outcome_certainty(), OutcomeCertainty::Certain);
|
||||
assert_eq!(failure.error_code(), ExecutionErrorCode::UpstreamTimeout);
|
||||
assert_eq!(failure.stage(), ExecutionStage::Upstream);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -632,6 +638,7 @@ fn operation() -> Operation<Schema, MappingSet> {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
{
|
||||
"kind": "inventory",
|
||||
"path": "docs/capability-inventory.json",
|
||||
"sha256": "10ca3538aa1b6b25868f80e19932c7ef918398c4817c52bc1aae52f18c31522b"
|
||||
"sha256": "98b1f3a7f3c8e9ca1bc15bef8b73b360fbd47a42acaef2e0dffb149e2f46687d"
|
||||
},
|
||||
{
|
||||
"kind": "required_surfaces",
|
||||
|
||||
@@ -984,7 +984,7 @@
|
||||
{
|
||||
"advertised": {
|
||||
"tools": {
|
||||
"listChanged": false
|
||||
"listChanged": true
|
||||
}
|
||||
},
|
||||
"applicability": "stable",
|
||||
@@ -993,6 +993,7 @@
|
||||
"kind": "capability",
|
||||
"open_subfeatures": false,
|
||||
"owner": "community-release",
|
||||
"reason": "the adapter emits `notifications/tools/list_changed` from the committed published Tool bindings of the Agent snapshot since Story 3.5; `tools/list` stays authoritative for a client that misses the notification",
|
||||
"requirement": "FR-13",
|
||||
"revisions": [
|
||||
"2026-07-28",
|
||||
|
||||
@@ -178,6 +178,8 @@ scrape_configs:
|
||||
- `crank_upstream_requests_total`,
|
||||
`crank_upstream_request_duration_seconds`;
|
||||
- `crank_runtime_inflight`, `crank_runtime_limit_rejections_total`,
|
||||
`crank_runtime_admission_queue_wait_seconds`,
|
||||
`crank_runtime_admission_oldest_waiting_seconds`,
|
||||
`crank_runtime_cache_total`, `crank_idempotency_total`,
|
||||
`crank_confirmation_total`;
|
||||
- `crank_db_pool_connections`;
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6285,"max_rendered_series":19515,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
|
||||
{"schema_version":2,"limits":{"max_families":128,"max_product_labels_per_family":4,"max_logical_series":6381,"max_rendered_series":19611,"max_exemplar_slots":12348,"max_exposition_bytes":8388608},"global_labels":[{"name":"service","class":"process_constant","domain":["admin-api","mcp-server"]},{"name":"version","class":"process_constant","domain":["validated_release_identity"]},{"name":"environment","class":"process_constant","domain":["validated_startup_value"]}],"http_routes":[{"route":"/api/admin/capabilities","processes":["admin_api"]},{"route":"/api/admin/workspaces","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/approve","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}/deny","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/plans/{plan_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/preview","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/plans","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/imports/openapi/{plan_id}/apply","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/export.csv","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/logs/{log_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/events","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/import-context","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/reset-selection","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/onboarding/select-draft","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/analyze-quality","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/import","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs","processes":["admin_api"]},{"route":"/api/admin/workspaces/{workspace_id}/Line truncated
|
||||
@@ -1 +1 @@
|
||||
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
|
||||
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
|
||||
@@ -40,8 +40,30 @@ run_atdd "MCP log level vocabulary" -p crank-community-mcp --lib protocol::loggi
|
||||
run_atdd "MCP safe logging" -p mcp-server --test integration mcp_logging -- --test-threads=1
|
||||
run_atdd "capability inventory projection" -p mcp-server --test integration inventory_projection -- --test-threads=1
|
||||
|
||||
# Story 3.19 (FR-19): a committed Resource/Prompt catalog change is announced to the client that
|
||||
# asked for the signal, in both revisions, and the authoritative list methods stay the recovery path.
|
||||
# The advertisement projection is already covered by the `inventory_projection` entry above.
|
||||
run_atdd "Resource/Prompt change signals" -p mcp-server --test integration mcp_signals -- --test-threads=1
|
||||
# Story 3.19 (FR-19) and Story 3.5 (FR-14): a committed Resource/Prompt/Tool catalog change is
|
||||
# announced to the client that asked for the signal, in both revisions, and the authoritative list
|
||||
# methods (including `tools/list`) stay the recovery path. The advertisement projection is covered by
|
||||
# the `inventory_projection` entry above.
|
||||
run_atdd "Resource/Prompt/Tool change signals" -p mcp-server --test integration mcp_signals -- --test-threads=1
|
||||
run_atdd "change signal wire oracles" -p crank-community-mcp --lib protocol::signals -- --test-threads=1
|
||||
run_atdd "capability advertisement registry" -p crank-community-mcp --lib protocol::registry -- --test-threads=1
|
||||
|
||||
# Story 3.4 (FR-14): progress and cancellation of one synchronous tools/call, on both supported
|
||||
# revisions. The lifecycle bounds (token vocabulary, monotonic bounded queue, pinned frame shape,
|
||||
# cancellation ownership) are unit-level; the end-to-end behaviour (real progress stream, no token,
|
||||
# malformed/oversized/reused token, owner/foreign/late cancellation, ambiguous outbound, client
|
||||
# smoke) runs against the real endpoint, a real published catalog and a real counted upstream.
|
||||
run_atdd "tool call lifecycle bounds" -p crank-community-mcp --lib app::lifecycle -- --test-threads=1
|
||||
run_atdd "tool call progress and cancellation" -p mcp-server --test integration tools_lifecycle -- --test-threads=1
|
||||
|
||||
# Story 4.1 (FR-21, FR-24): a published Tool execution policy and one authoritative admission gate.
|
||||
# The policy/JSON contract and the fair-scheduler arithmetic are unit-level; the authoritative store
|
||||
# (scoped permits, bounded queue, typed overload, durable decisions, version pinning, fail-closed
|
||||
# health) and the runtime wiring (one shared global budget across executors, typed refusals with no
|
||||
# outbound call, adaptive/synchronous fallback) run against a real PostgreSQL schema.
|
||||
run_atdd "execution policy contract" -p crank-core --lib operation::tests::execution_policy -- --test-threads=1
|
||||
run_atdd "admission scheduler bounds" -p crank-core --lib admission:: -- --test-threads=1
|
||||
run_atdd "execution policy validation" -p admin-api --lib operation_validation::tests:: -- --test-threads=1
|
||||
run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1
|
||||
run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1
|
||||
run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"completions":{},"logging":{},"prompts":{"listChanged":true},"resources":{"listChanged":true,"subscribe":false},"tools":{"listChanged":false}}
|
||||
{"completions":{},"logging":{},"prompts":{"listChanged":true},"resources":{"listChanged":true,"subscribe":false},"tools":{"listChanged":true}}
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"revision": "2025-11-25",
|
||||
"inventory_path": "docs/capability-inventory.json",
|
||||
"inventory_sha256": "10ca3538aa1b6b25868f80e19932c7ef918398c4817c52bc1aae52f18c31522b",
|
||||
"inventory_sha256": "98b1f3a7f3c8e9ca1bc15bef8b73b360fbd47a42acaef2e0dffb149e2f46687d",
|
||||
"rule": "remaining-ac-spec.md#5-server-only-projection-rules-1-4",
|
||||
"projection_inputs": [
|
||||
{
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"completions":{},"logging":{},"prompts":{"listChanged":true},"resources":{"listChanged":true,"subscribe":false},"tools":{"listChanged":false}}
|
||||
{"completions":{},"logging":{},"prompts":{"listChanged":true},"resources":{"listChanged":true,"subscribe":false},"tools":{"listChanged":true}}
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"revision": "2026-07-28",
|
||||
"inventory_path": "docs/capability-inventory.json",
|
||||
"inventory_sha256": "10ca3538aa1b6b25868f80e19932c7ef918398c4817c52bc1aae52f18c31522b",
|
||||
"inventory_sha256": "98b1f3a7f3c8e9ca1bc15bef8b73b360fbd47a42acaef2e0dffb149e2f46687d",
|
||||
"rule": "remaining-ac-spec.md#5-server-only-projection-rules-1-4",
|
||||
"projection_inputs": [
|
||||
{
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"oracle": "crank-current-scope-advertised-capabilities",
|
||||
"note": "Independent expected projection of the capability descriptor the CURRENT scope is allowed to advertise: advertisement equals what is implemented. Later capabilities are deferred and must not appear until their owning story implements them. This is a scoped projection, not the official MCP capability list. Story 3.19 implements the Resource/Prompt list-change signals, so both listChanged leaves are advertised; URI-content subscriptions stay unimplemented.",
|
||||
"note": "Independent expected projection of the capability descriptor the CURRENT scope is allowed to advertise: advertisement equals what is implemented. Later capabilities are deferred and must not appear until their owning story implements them. This is a scoped projection, not the official MCP capability list. Story 3.19 implements the Resource/Prompt list-change signals and Story 3.5 the Tools one, so every listChanged leaf is advertised; URI-content subscriptions stay unimplemented.",
|
||||
"expected_keys": ["completions", "logging", "prompts", "resources", "tools"],
|
||||
"expected_descriptor": {
|
||||
"completions": {},
|
||||
@@ -13,7 +13,7 @@
|
||||
"subscribe": false
|
||||
},
|
||||
"tools": {
|
||||
"listChanged": false
|
||||
"listChanged": true
|
||||
}
|
||||
},
|
||||
"must_not_be_advertised": ["experimental", "extensions"],
|
||||
|
||||
@@ -47,14 +47,14 @@ EXPECTED_FROZEN = (
|
||||
b'{"resources":{"listChanged":false,"subscribe":false},"tools":{"listChanged":false}}'
|
||||
)
|
||||
EXPECTED_SHA256 = "e38c795fc28fee74168cb045427df55d882f8f7ace206af5c9a3ca4df5892c9a"
|
||||
# The canonical inventory also implements the logging surface (Story 3.18, FR-19) and the
|
||||
# Resource/Prompt list-change signals (Story 3.19, FR-19), so the committed oracle and the canonical
|
||||
# projection carry `logging` next to `completions`/`prompts`/`resources`/`tools`, and both
|
||||
# `listChanged` leaves are now implemented.
|
||||
# The canonical inventory also implements the logging surface (Story 3.18, FR-19), the
|
||||
# Resource/Prompt list-change signals (Story 3.19, FR-19) and the Tools list-change signal
|
||||
# (Story 3.5, FR-14), so the committed oracle and the canonical projection carry `logging` next to
|
||||
# `completions`/`prompts`/`resources`/`tools`, and every `listChanged` leaf is now implemented.
|
||||
COMMITTED_FROZEN = (
|
||||
b'{"completions":{},"logging":{},"prompts":{"listChanged":true},"resources":{"listChanged":true,"subscribe":false},"tools":{"listChanged":false}}'
|
||||
b'{"completions":{},"logging":{},"prompts":{"listChanged":true},"resources":{"listChanged":true,"subscribe":false},"tools":{"listChanged":true}}'
|
||||
)
|
||||
COMMITTED_SHA256 = "b90b59f32e2e5c43e829b96339b99a52852bc04f48a2ca0ce7e971affa3a878e"
|
||||
COMMITTED_SHA256 = "7ffa56029595fc84b6ee29930d65eada2b4d7cffd7d76f72a945fe795cabfe0f"
|
||||
|
||||
|
||||
class AdvertisedCapabilitiesProjectionTests(unittest.TestCase):
|
||||
|
||||
Loaded 100 of 101 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user