feat(epic4): подтверждать durable Task после атомарного PG-commit (Story 4.2)
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 19m34s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 19m34s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Application entry выдаёт Task ID только после атомарного PostgreSQL commit Task+intent+capacity; replay/conflict, owner scope, bounded envelope, terminal CAS и fail-closed readiness проверены на реальном PG. Также включён принятый план перехода на rmcp (только документ, миграция не запускается).
This commit is contained in:
1 parent
cc612516a2
commit
2219a7217d
37 files changed
+4028
-136
No files matched your search
@@ -1,7 +1,7 @@
|
||||
use std::{io, process::ExitCode, time::Duration};
|
||||
|
||||
use crank_community_mcp::{
|
||||
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts,
|
||||
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts_and_tasks,
|
||||
session::PostgresTransportSessionStore,
|
||||
};
|
||||
use crank_config::{
|
||||
@@ -15,7 +15,7 @@ use crank_observability::{
|
||||
};
|
||||
use crank_registry::{
|
||||
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
|
||||
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry,
|
||||
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, PostgresTaskStore,
|
||||
};
|
||||
use crank_runtime::{
|
||||
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
|
||||
@@ -201,7 +201,21 @@ async fn run(
|
||||
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
|
||||
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
|
||||
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
|
||||
let app = build_app_with_artifacts(
|
||||
// Story 4.2: the durable Task application entry shares the registry pool, the published catalog
|
||||
// refresh interval and the exact admission limits of the runtime; the public wire stays
|
||||
// synchronous/unsupported and no worker is started.
|
||||
let task_store = PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
admission_limits(runtime_limits),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?;
|
||||
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
cache_stores.coordination.clone(),
|
||||
crank_runtime::TaskService::new(std::sync::Arc::new(task_store)),
|
||||
));
|
||||
let app = build_app_with_artifacts_and_tasks(
|
||||
registry,
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
config.runtime.base_url.clone(),
|
||||
@@ -217,6 +231,7 @@ async fn run(
|
||||
std::sync::Arc::new(CommunityMachineCredentialVerifier),
|
||||
runtime_limits.max_concurrent_sessions,
|
||||
artifact_store,
|
||||
tasks,
|
||||
);
|
||||
let listener = TcpListener::bind(config.bind_addr).await?;
|
||||
|
||||
@@ -293,16 +308,20 @@ fn init_observability(
|
||||
)?)
|
||||
}
|
||||
|
||||
fn admission_limits(runtime_limits: RuntimeLimits) -> AdmissionLimits {
|
||||
AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..AdmissionLimits::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn execution_admission(
|
||||
pool: sqlx::PgPool,
|
||||
runtime_limits: RuntimeLimits,
|
||||
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
|
||||
let store = PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..AdmissionLimits::default()
|
||||
},
|
||||
admission_limits(runtime_limits),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?;
|
||||
Ok(std::sync::Arc::new(store))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
mod integration {
|
||||
mod catalog_access;
|
||||
mod common;
|
||||
mod durable_tasks;
|
||||
mod era_isolation;
|
||||
mod execution_certainty;
|
||||
mod execution_stages;
|
||||
|
||||
@@ -0,0 +1,591 @@
|
||||
//! ATDD Story 4.2 (FR-22/FR-23): the durable Task application entry over a real published catalog
|
||||
//! and a real PostgreSQL schema.
|
||||
//!
|
||||
//! These oracles drive the production application entry (`TaskApplication`) that the mcp-server
|
||||
//! composition root constructs: a verified machine credential is checked against the published Tool,
|
||||
//! the Task is accepted durably, a freshly constructed application instance reads the same identity
|
||||
//! and pins, replay/version conflicts never rebind, foreign callers receive no metadata, and a missing
|
||||
//! Task store fails closed and reports readiness as not ready. The public `tools/call` wire stays
|
||||
//! synchronous/unsupported: this suite never calls it.
|
||||
|
||||
use super::common;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
use crank_core::{
|
||||
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState,
|
||||
ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{
|
||||
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
|
||||
};
|
||||
use crank_runtime::{InMemoryCoordinationStateStore, TaskService};
|
||||
use serde_json::json;
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
const TOOL_NAME: &str = "report_gen";
|
||||
const KEY: &str = "retry-key-1";
|
||||
|
||||
fn operation(policy: ToolExecutionPolicy) -> crank_registry::RegistryOperation {
|
||||
let mut operation = common::test_operation("https://api.example.com", TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
operation
|
||||
}
|
||||
|
||||
fn background_policy() -> ToolExecutionPolicy {
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_tool(registry: &PostgresRegistry, policy: ToolExecutionPolicy) {
|
||||
let operation = operation(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &operation.id, 1).await;
|
||||
// A second, explicitly synchronous Tool of the same Agent: the durable entry must refuse it.
|
||||
let synchronous = common::test_operation("https://api.example.com", "report_sync");
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &synchronous, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &synchronous.id, 1).await;
|
||||
let binding = AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
};
|
||||
let synchronous_binding = AgentOperationBinding {
|
||||
operation_id: synchronous.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: synchronous.name.clone(),
|
||||
tool_title: synchronous.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
};
|
||||
common::publish_agent_with_bindings(registry, AGENT_SLUG, vec![binding, synchronous_binding])
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn publish(
|
||||
registry: &PostgresRegistry,
|
||||
operation_id: &crank_core::OperationId,
|
||||
version: u32,
|
||||
) {
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id,
|
||||
version,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
|
||||
let mut next = operation(background_policy());
|
||||
next.version = version;
|
||||
next.status = crank_core::OperationStatus::Draft;
|
||||
next.published_at = None;
|
||||
registry
|
||||
.create_version(CreateVersionRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
snapshot: &next,
|
||||
change_note: Some("next publication"),
|
||||
created_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &next.id, version).await;
|
||||
// The agent catalog must bind the published Tool version for the new publication to be addressed.
|
||||
let agent_id = common::test_agent_id(AGENT_SLUG);
|
||||
let draft = crank_core::AgentVersion {
|
||||
agent_id: agent_id.clone(),
|
||||
version: 2,
|
||||
status: crank_core::AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: Default::default(),
|
||||
created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
|
||||
};
|
||||
registry
|
||||
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: &draft,
|
||||
bindings: &[AgentOperationBinding {
|
||||
operation_id: next.id.clone(),
|
||||
operation_version: version,
|
||||
tool_name: next.name.clone(),
|
||||
tool_title: next.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_agent(crank_registry::PublishAgentRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: 2,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn task_service(pool: PgPool) -> TaskService {
|
||||
let store = PostgresTaskStore::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration");
|
||||
TaskService::new(std::sync::Arc::new(store))
|
||||
}
|
||||
|
||||
fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(0),
|
||||
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
task_service(registry.pool().clone()),
|
||||
)
|
||||
}
|
||||
|
||||
async fn principal(
|
||||
registry: &PostgresRegistry,
|
||||
name: &str,
|
||||
scopes: &[PlatformApiKeyScope],
|
||||
) -> TaskPrincipal {
|
||||
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
|
||||
// The production machine-access path resolves the presented secret against the active platform
|
||||
// key of the addressed Agent and derives the scope identity from that verified record.
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("an active platform key resolves");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: crank_core::VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn unsampled_correlation() -> CorrelationContext {
|
||||
CorrelationContext::new(
|
||||
crank_core::RequestId::generate(),
|
||||
crank_core::TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn counts(pool: &PgPool) -> (i64, i64) {
|
||||
let row = sqlx::query(
|
||||
"select
|
||||
(select count(*)::bigint from tasks) as tasks,
|
||||
(select count(*)::bigint from execution_admission_entries where task_id is not null)
|
||||
as intents",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(row.get("tasks"), row.get("intents"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-accept",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let app = application(®istry);
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("acceptance over the real publication");
|
||||
assert_eq!(acceptance.state, TaskState::Queued);
|
||||
assert!(!acceptance.replayed);
|
||||
assert_eq!(acceptance.operation_version, 1);
|
||||
assert_eq!(acceptance.support.request_id, *correlation.request_id());
|
||||
assert_eq!(acceptance.support.trace_id, *correlation.trace_id());
|
||||
assert!(!acceptance.support.trace_sampled);
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(&acceptance.task_id.to_string())
|
||||
.unwrap()
|
||||
.get_version_num(),
|
||||
7,
|
||||
"the durable Task id is UUIDv7"
|
||||
);
|
||||
// Task, dispatch intent and queue slot committed together.
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// A freshly constructed application + service reads the same durable identity and pins.
|
||||
let reader = application(®istry);
|
||||
let record = reader
|
||||
.status(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(record.state, TaskState::Queued);
|
||||
assert_eq!(record.pins.operation_version, 1);
|
||||
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
|
||||
assert_eq!(record.support.request_id, acceptance.support.request_id);
|
||||
assert_eq!(record.support.traceparent, acceptance.support.traceparent);
|
||||
assert!(!record.support.trace_sampled);
|
||||
let outcome = reader
|
||||
.result(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(outcome.result.is_none() && outcome.error.is_none());
|
||||
|
||||
// Replay of the same scoped key and input returns the accepted Task instead of a new one.
|
||||
let replay = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replay.task_id, acceptance.task_id);
|
||||
assert!(replay.replayed);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_new_publication_never_rebinds_an_accepted_task() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-rebind",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let before = app
|
||||
.status(&caller, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
publish_tool_version(®istry, 2).await;
|
||||
|
||||
// The same retry keeps the accepted Task and its v1 pin; it never becomes a v2 Task.
|
||||
let replay = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replay.task_id, accepted.task_id);
|
||||
assert_eq!(replay.operation_version, 1);
|
||||
let after = app
|
||||
.status(&caller, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(after.pins.operation_version, 1);
|
||||
assert_eq!(after.support.request_id, before.support.request_id);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// An explicit other exact version is a typed conflict, not a rebind.
|
||||
let explicit = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
Some(2),
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a different explicit pin cannot rebind the accepted Task");
|
||||
assert_eq!(explicit, TaskApplicationError::VersionUnavailable);
|
||||
// The original explicit pin still resolves to the accepted Task.
|
||||
let pinned = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
Some(1),
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(pinned.task_id, accepted.task_id);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let owner = principal(
|
||||
®istry,
|
||||
"task-owner",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&owner,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Positive control: the owner reads its Task.
|
||||
assert!(
|
||||
app.status(&owner, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
// A second verified key of the same Agent is another authorization scope: no payload, no metadata.
|
||||
let foreign_scope =
|
||||
principal(®istry, "task-other-scope", &[PlatformApiKeyScope::Read]).await;
|
||||
match app
|
||||
.status(&foreign_scope, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
{
|
||||
Err(TaskApplicationError::NotFound) => {}
|
||||
other => panic!("foreign scope must be not-found without metadata, got {other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
app.result(&foreign_scope, TOOL_NAME, accepted.task_id)
|
||||
.await,
|
||||
Err(TaskApplicationError::NotFound)
|
||||
));
|
||||
|
||||
// A synchronous publication is not accepted as background work at all.
|
||||
let synchronous_app = application(®istry);
|
||||
let refused = synchronous_app
|
||||
.accept_background(
|
||||
&owner,
|
||||
"report_sync",
|
||||
None,
|
||||
"sync-key",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a synchronous policy has no durable acceptance");
|
||||
assert_eq!(refused, TaskApplicationError::NotBackground);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let app = application(®istry);
|
||||
let admin = TaskPrincipal::Admin {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
user_id: "user_admin_1".to_owned(),
|
||||
};
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&admin,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("an authenticated administrator accepts durable work");
|
||||
assert_eq!(accepted.state, TaskState::Queued);
|
||||
// Positive control: the same administrator reads its own Task.
|
||||
assert!(
|
||||
app.status(&admin, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
// An administrator is never a wildcard owner: another admin user and the owning Agent of the
|
||||
// same workspace both see nothing.
|
||||
let other_admin = TaskPrincipal::Admin {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
user_id: "user_admin_2".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
app.status(&other_admin, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
app.result(&other_admin, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
let agent = principal(®istry, "task-admin-cross", &[PlatformApiKeyScope::Read]).await;
|
||||
assert_eq!(
|
||||
app.status(&agent, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
|
||||
// An admin identity addressed to another workspace cannot resolve the publication at all.
|
||||
let foreign_workspace = TaskPrincipal::Admin {
|
||||
workspace_slug: "other-workspace".to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
user_id: "user_admin_1".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
app.accept_background(
|
||||
&foreign_workspace,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"foreign-key",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
|
||||
let database_url = crank_test_support::postgres_schema_url("durable_tasks_outage").await;
|
||||
let pool = PgPool::connect(&database_url).await.unwrap();
|
||||
// No migration: the durable Task state does not exist.
|
||||
let outage_registry = PostgresRegistry::connect(&database_url).await;
|
||||
assert!(
|
||||
outage_registry.is_err(),
|
||||
"an unmigrated registry cannot be used"
|
||||
);
|
||||
|
||||
crank_registry::MigrationAuthority::apply(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-outage",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
assert!(app.health().await.is_ok());
|
||||
|
||||
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
|
||||
sqlx::query("drop table tasks cascade")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(app.health().await.is_err());
|
||||
let error = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect_err("no durable Task store, no acceptance");
|
||||
assert!(matches!(
|
||||
error,
|
||||
TaskApplicationError::Task(crank_core::TaskError::Unavailable { .. })
|
||||
));
|
||||
|
||||
// Readiness of the app that wires the same store reports the durable Task dependency as not ready.
|
||||
let router = common::build_test_app(registry.clone(), Duration::from_millis(0), None);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
});
|
||||
let response = reqwest::get(format!("http://{address}/ready"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE);
|
||||
let body: serde_json::Value = response.json().await.unwrap();
|
||||
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
|
||||
server.abort();
|
||||
}
|
||||
@@ -51,12 +51,14 @@ mod auth_resolution;
|
||||
mod dispatch;
|
||||
mod endpoint;
|
||||
mod legacy_dispatch;
|
||||
mod tasks;
|
||||
|
||||
use self::approvals::{
|
||||
approve_request, deny_request, get_approval_request, list_pending_approvals,
|
||||
};
|
||||
pub(super) use self::auth_resolution::resolve_operation_auth;
|
||||
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
|
||||
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
|
||||
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
|
||||
// through `super`, and the plain lib build does not use them (same pattern as
|
||||
@@ -92,6 +94,9 @@ pub(super) struct AppState {
|
||||
pub(super) signal_streams: Arc<crate::protocol::StreamRegistry>,
|
||||
/// Story 3.4: the bounded table of the `tools/call` requests a legacy client can still cancel.
|
||||
pub(super) in_flight: Arc<self::lifecycle::InFlightCalls>,
|
||||
/// Story 4.2: durable background Tasks over the same registry pool. The public wire stays
|
||||
/// synchronous/unsupported; readiness depends on this mandatory store.
|
||||
tasks: Arc<TaskApplication>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -137,6 +142,7 @@ pub fn build_app(
|
||||
DEFAULT_MAX_CONCURRENT_SESSIONS,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -192,6 +198,7 @@ pub fn build_app_with_background_workers_and_limits(
|
||||
max_concurrent_sessions,
|
||||
true,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -227,6 +234,41 @@ pub fn build_app_with_artifacts(
|
||||
max_concurrent_sessions,
|
||||
true,
|
||||
Some(artifact_store),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
/// Production launcher entry point with the durable Task application entry constructed by the
|
||||
/// process composition root from the same registry pool and admission limits as the runtime.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn build_app_with_artifacts_and_tasks(
|
||||
registry: PostgresRegistry,
|
||||
refresh_interval: Duration,
|
||||
public_base_url: Option<String>,
|
||||
secret_crypto: SecretCrypto,
|
||||
runtime: RuntimeExecutor,
|
||||
api_rate_limiter: RequestRateLimiter,
|
||||
coordination_store: Arc<dyn CoordinationStateStore>,
|
||||
sessions: SharedSessionStore,
|
||||
credential_verifier: SharedMachineCredentialVerifier,
|
||||
max_concurrent_sessions: usize,
|
||||
artifact_store: crank_artifacts::ArtifactStore,
|
||||
tasks: Arc<TaskApplication>,
|
||||
) -> Router {
|
||||
build_app_inner(
|
||||
registry,
|
||||
refresh_interval,
|
||||
public_base_url,
|
||||
secret_crypto,
|
||||
runtime,
|
||||
api_rate_limiter,
|
||||
coordination_store,
|
||||
sessions,
|
||||
credential_verifier,
|
||||
max_concurrent_sessions,
|
||||
true,
|
||||
Some(artifact_store),
|
||||
Some(tasks),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -244,8 +286,23 @@ fn build_app_inner(
|
||||
max_concurrent_sessions: usize,
|
||||
start_background_workers: bool,
|
||||
artifact_store: Option<crank_artifacts::ArtifactStore>,
|
||||
tasks: Option<Arc<TaskApplication>>,
|
||||
) -> Router {
|
||||
let session_metrics = ActiveSessionMetrics::start(Arc::clone(&sessions));
|
||||
let tasks = tasks.unwrap_or_else(|| {
|
||||
let store = crank_registry::PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits::default(),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("default admission limits and scheduler weights are valid");
|
||||
Arc::new(TaskApplication::new(
|
||||
registry.clone(),
|
||||
refresh_interval,
|
||||
Arc::clone(&coordination_store),
|
||||
crank_runtime::TaskService::new(Arc::new(store)),
|
||||
))
|
||||
});
|
||||
let state = Arc::new(AppState {
|
||||
registry: registry.clone(),
|
||||
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
|
||||
@@ -263,6 +320,7 @@ fn build_app_inner(
|
||||
in_flight: Arc::new(lifecycle::InFlightCalls::new(
|
||||
lifecycle::MAX_IN_FLIGHT_CALLS,
|
||||
)),
|
||||
tasks,
|
||||
});
|
||||
if start_background_workers {
|
||||
spawn_approval_recovery(Arc::clone(&state));
|
||||
@@ -315,11 +373,17 @@ async fn readiness(State(state): State<Arc<AppState>>) -> Response {
|
||||
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
|
||||
// reflect it rather than reporting ready on a bare `select 1`.
|
||||
let admission_ready = state.registry.execution_admission_health().await.is_ok();
|
||||
if postgres_ready && admission_ready {
|
||||
// Story 4.2: the durable Task store is mandatory for a durable acceptance acknowledgement.
|
||||
let durable_tasks_ready = state.tasks.health().await.is_ok();
|
||||
if postgres_ready && admission_ready && durable_tasks_ready {
|
||||
return Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": "ready", "execution_admission": "ready" }
|
||||
"checks": {
|
||||
"postgres": "ready",
|
||||
"execution_admission": "ready",
|
||||
"durable_tasks": "ready"
|
||||
}
|
||||
}))
|
||||
.into_response();
|
||||
}
|
||||
@@ -336,6 +400,14 @@ async fn readiness(State(state): State<Arc<AppState>>) -> Response {
|
||||
"not_ready"
|
||||
}),
|
||||
);
|
||||
checks.insert(
|
||||
"durable_tasks".to_owned(),
|
||||
json!(if durable_tasks_ready {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
}),
|
||||
);
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
//! Application entry of durable background Tasks (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! The entry takes an already authenticated principal (a verified machine credential, or an
|
||||
//! administrator identity verified by the calling admin surface), resolves the exact published Tool,
|
||||
//! repeats the caller's authorization, pins the Published Version/policy/deadline and delegates the
|
||||
//! durable acceptance to the runtime Task service. The repository owns the transaction, so the
|
||||
//! returned Task ID exists only after Task + dispatch intent + queue slot committed.
|
||||
//!
|
||||
//! Deliberately not wired: the public MCP `tools/call`/Tasks wire (Story 4.7) and the worker/leases
|
||||
//! (Story 4.3). The existing synchronous/unsupported `tools/call` path is unchanged, and this entry
|
||||
//! never spawns an ephemeral execution future.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord,
|
||||
ToolExecutionMode,
|
||||
};
|
||||
use crank_registry::{PostgresRegistry, PublishedAgentTool};
|
||||
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::access::credential_allows_security_level;
|
||||
use crate::catalog::PublishedToolCatalog;
|
||||
|
||||
/// Authenticated principal of one Task operation. The authorization scope identity is derived here
|
||||
/// from the verified credential/admin identity, never from a caller-supplied request string.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum TaskPrincipal {
|
||||
Agent {
|
||||
workspace_slug: String,
|
||||
agent_slug: String,
|
||||
credential: crank_core::VerifiedMachineCredential,
|
||||
},
|
||||
/// Administrator identity that the admin surface already authenticated. It is never a wildcard:
|
||||
/// owner, workspace and authorization scope all participate in the stored predicate.
|
||||
Admin {
|
||||
workspace_slug: String,
|
||||
agent_slug: String,
|
||||
user_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl TaskPrincipal {
|
||||
fn workspace_slug(&self) -> &str {
|
||||
match self {
|
||||
Self::Agent { workspace_slug, .. } | Self::Admin { workspace_slug, .. } => {
|
||||
workspace_slug
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn agent_slug(&self) -> &str {
|
||||
match self {
|
||||
Self::Agent { agent_slug, .. } | Self::Admin { agent_slug, .. } => agent_slug,
|
||||
}
|
||||
}
|
||||
|
||||
fn owner(&self, tool: &PublishedAgentTool) -> (TaskOwnerKind, String) {
|
||||
match self {
|
||||
Self::Agent { .. } => (TaskOwnerKind::Agent, tool.agent_id.as_str().to_owned()),
|
||||
Self::Admin { user_id, .. } => (TaskOwnerKind::Admin, user_id.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded server-derived authorization scope identity of this principal.
|
||||
fn authorization_scope(&self) -> String {
|
||||
let identity = match self {
|
||||
Self::Agent { credential, .. } => {
|
||||
let mut scopes = credential
|
||||
.scopes
|
||||
.iter()
|
||||
.map(|scope| format!("{scope:?}"))
|
||||
.collect::<Vec<_>>();
|
||||
scopes.sort();
|
||||
format!(
|
||||
"key:{}|mode:{:?}|level:{:?}|scopes:{}",
|
||||
credential
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned())
|
||||
.unwrap_or_else(|| "-".to_owned()),
|
||||
credential.machine_access_mode,
|
||||
credential.max_security_level,
|
||||
scopes.join(",")
|
||||
)
|
||||
}
|
||||
Self::Admin { user_id, .. } => format!("admin:{user_id}"),
|
||||
};
|
||||
identity.chars().take(160).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed refusal of the application entry. `NotFound`/`Denied` never reveal whether a foreign Task
|
||||
/// exists, and no variant carries Task payload or fingerprint.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum TaskApplicationError {
|
||||
#[error("task request was not found")]
|
||||
NotFound,
|
||||
#[error("task request is not authorized")]
|
||||
Denied,
|
||||
#[error("the published Tool policy does not accept durable background work")]
|
||||
NotBackground,
|
||||
#[error("the requested published Tool version is not available")]
|
||||
VersionUnavailable,
|
||||
#[error("this Tool requires a verified approval before acceptance")]
|
||||
ApprovalRequired,
|
||||
#[error(transparent)]
|
||||
Task(#[from] TaskError),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct TaskApplication {
|
||||
catalog: PublishedToolCatalog,
|
||||
service: TaskService,
|
||||
}
|
||||
|
||||
impl TaskApplication {
|
||||
/// Production construction of the application entry over the process registry, the published
|
||||
/// catalog and the durable Task service.
|
||||
pub fn new(
|
||||
registry: PostgresRegistry,
|
||||
refresh_interval: Duration,
|
||||
coordination_store: Arc<dyn crank_core::CoordinationStateStore>,
|
||||
service: TaskService,
|
||||
) -> Self {
|
||||
Self {
|
||||
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
|
||||
service,
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness of the durable Task store this entry depends on.
|
||||
pub async fn health(&self) -> Result<(), TaskError> {
|
||||
self.service.health().await
|
||||
}
|
||||
|
||||
/// Accepts one background request for an already authenticated principal.
|
||||
///
|
||||
/// `expected_version` is the caller's explicit Published Version pin. Without it the current
|
||||
/// publication is pinned for new work and a retry of the same scoped key returns the stored Task
|
||||
/// even after a newer publication. An explicit pin may only retry the Task accepted with that
|
||||
/// exact version: a different pin is a typed conflict, never a rebind.
|
||||
pub async fn accept_background(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
expected_version: Option<u32>,
|
||||
scoped_key: &str,
|
||||
input: &Value,
|
||||
correlation: &CorrelationContext,
|
||||
) -> Result<TaskAcceptance, TaskApplicationError> {
|
||||
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
|
||||
let current_version = tool.operation.version;
|
||||
let Some(policy) = tool.operation.execution_config.execution_policy else {
|
||||
return Err(TaskApplicationError::NotBackground);
|
||||
};
|
||||
if policy.mode == ToolExecutionMode::Synchronous {
|
||||
return Err(TaskApplicationError::NotBackground);
|
||||
}
|
||||
if tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()
|
||||
.is_some_and(|approval| approval.required)
|
||||
{
|
||||
// Without a verified, non-consuming approval reference this entry fails closed instead
|
||||
// of accepting work a worker could never safely execute.
|
||||
return Err(TaskApplicationError::ApprovalRequired);
|
||||
}
|
||||
let draft = |operation_version: u32| {
|
||||
let (owner_kind, owner_id) = principal.owner(&tool);
|
||||
TaskAcceptDraft {
|
||||
workspace_id: tool.workspace_id.as_str().to_owned(),
|
||||
owner_kind,
|
||||
owner_id,
|
||||
authorization_scope: principal.authorization_scope(),
|
||||
operation_id: tool.operation.id.as_str().to_owned(),
|
||||
operation_version,
|
||||
policy_mode: policy.mode,
|
||||
scoped_key: scoped_key.to_owned(),
|
||||
input: input.clone(),
|
||||
correlation: correlation.clone(),
|
||||
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
|
||||
retention_ms: TaskService::default_retention_ms(),
|
||||
approval_reference: None,
|
||||
}
|
||||
};
|
||||
|
||||
let Some(requested) = expected_version else {
|
||||
return Ok(self.service.accept(draft(current_version)).await?);
|
||||
};
|
||||
let draft = draft(requested);
|
||||
match self.service.lookup(&draft).await? {
|
||||
Some(record) => {
|
||||
if record.input_fingerprint != TaskService::input_fingerprint(input)? {
|
||||
return Err(TaskApplicationError::Task(TaskError::FingerprintConflict));
|
||||
}
|
||||
if record.pins.operation_version != requested {
|
||||
return Err(TaskApplicationError::VersionUnavailable);
|
||||
}
|
||||
Ok(TaskAcceptance {
|
||||
task_id: record.task_id,
|
||||
state: record.state,
|
||||
operation_version: record.pins.operation_version,
|
||||
support: record.support,
|
||||
created_at_unix_ms: record.created_at_unix_ms,
|
||||
retain_until_unix_ms: record.pins.retain_until_unix_ms,
|
||||
replayed: true,
|
||||
})
|
||||
}
|
||||
None if requested == current_version => Ok(self.service.accept(draft).await?),
|
||||
// A new Task can only be accepted against the currently published version.
|
||||
None => Err(TaskApplicationError::VersionUnavailable),
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped status read of one Task of the addressed published Tool.
|
||||
pub async fn status(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskRecord, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self.service.status(&access, task_id).await {
|
||||
Ok(record) => Ok(record),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped result projection of one Task of the addressed published Tool.
|
||||
pub async fn result(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskOutcome, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self.service.result(&access, task_id).await {
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn task_access(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
) -> Result<TaskAccess, TaskApplicationError> {
|
||||
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
|
||||
let (owner_kind, owner_id) = principal.owner(&tool);
|
||||
Ok(TaskAccess {
|
||||
workspace_id: tool.workspace_id.as_str().to_owned(),
|
||||
owner_kind,
|
||||
owner_id,
|
||||
authorization_scope: principal.authorization_scope(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves the addressed Tool of the principal's published catalog and repeats its authorization
|
||||
/// check. A foreign workspace, an unknown Tool and an insufficient security level are typed
|
||||
/// refusals without metadata.
|
||||
async fn resolve_authorized_tool(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
) -> Result<PublishedAgentTool, TaskApplicationError> {
|
||||
let catalog = self
|
||||
.catalog
|
||||
.get_catalog(principal.workspace_slug(), principal.agent_slug())
|
||||
.await
|
||||
.map_err(|_| TaskApplicationError::NotFound)?;
|
||||
let tool = catalog
|
||||
.tools
|
||||
.into_iter()
|
||||
.find(|tool| tool.tool_name == tool_name)
|
||||
.ok_or(TaskApplicationError::NotFound)?;
|
||||
if let TaskPrincipal::Admin { workspace_slug, .. } = principal
|
||||
&& tool.workspace_slug != *workspace_slug
|
||||
{
|
||||
return Err(TaskApplicationError::Denied);
|
||||
}
|
||||
if let TaskPrincipal::Agent { credential, .. } = principal
|
||||
&& !credential_allows_security_level(credential, tool.operation.security_level)
|
||||
{
|
||||
return Err(TaskApplicationError::Denied);
|
||||
}
|
||||
Ok(tool)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ mod tool_search;
|
||||
mod transport;
|
||||
|
||||
pub use app::{
|
||||
build_app, build_app_with_artifacts, build_app_with_background_workers,
|
||||
TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts,
|
||||
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
|
||||
build_app_with_background_workers_and_limits,
|
||||
};
|
||||
@@ -18,6 +18,7 @@ pub mod prompt_composition;
|
||||
pub mod protocol;
|
||||
pub mod resource;
|
||||
pub mod secret;
|
||||
pub mod task;
|
||||
pub mod tool_catalog;
|
||||
pub mod tool_quality;
|
||||
pub mod tool_search;
|
||||
@@ -73,6 +74,13 @@ pub mod domain {
|
||||
};
|
||||
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
|
||||
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use crate::task::{
|
||||
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
|
||||
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES,
|
||||
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
|
||||
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds,
|
||||
};
|
||||
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
|
||||
pub use crate::tool_quality::{
|
||||
ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
|
||||
@@ -110,6 +118,7 @@ pub mod ports {
|
||||
PreparedRequest, ProtocolAdapter, ProtocolAdapterError, ResponseCacheScope,
|
||||
RuntimeRequestContext, SharedProtocolAdapter,
|
||||
};
|
||||
pub use crate::task::TaskStore;
|
||||
}
|
||||
|
||||
pub use access::{
|
||||
@@ -192,6 +201,13 @@ pub use product_event::{
|
||||
};
|
||||
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
|
||||
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use task::{
|
||||
TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
|
||||
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
|
||||
TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key,
|
||||
validate_task_envelope,
|
||||
};
|
||||
pub use tool_catalog::{
|
||||
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
|
||||
ToolCatalogBudget, analyze_tool_catalog,
|
||||
|
||||
@@ -0,0 +1,730 @@
|
||||
//! Durable background Task seam (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! A Task is the internal, protocol-independent record of one accepted background request. This
|
||||
//! module owns the domain vocabulary only: the lifecycle states, the accepted identity/pins, the
|
||||
//! bounded persisted envelopes and the `TaskStore` port. The protocol adapter (Story 4.7) and the
|
||||
//! worker (Story 4.3) do not define these semantics; the authoritative store lives in PostgreSQL
|
||||
//! (`crank-registry`), so an accepted Task survives a crash or restart.
|
||||
//!
|
||||
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
|
||||
//! Task was executed. A terminal state is written by a conditional transition only.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::correlation::{RequestId, TraceContext, TraceId};
|
||||
use crate::operation::ToolExecutionMode;
|
||||
|
||||
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
|
||||
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
|
||||
/// Maximum nesting depth of a persisted Task envelope.
|
||||
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
|
||||
/// Maximum number of JSON members/nodes of a persisted Task envelope.
|
||||
pub const TASK_ENVELOPE_MAX_NODES: usize = 256;
|
||||
/// Bound of every scope/owner/operation identity column.
|
||||
pub const TASK_IDENTITY_MAX_BYTES: usize = 160;
|
||||
/// Bound of the caller-supplied idempotency key before it becomes a scoped digest.
|
||||
pub const TASK_SCOPED_KEY_MAX_BYTES: usize = 128;
|
||||
/// Bound of a saved approval reference (an identifier, never a bearer token).
|
||||
pub const TASK_APPROVAL_REFERENCE_MAX_BYTES: usize = 160;
|
||||
/// Default retention of a terminal Task after its execution deadline (internal bounded default).
|
||||
pub const TASK_DEFAULT_RETENTION_MS: i64 = 86_400_000;
|
||||
/// Upper bound on the retention an accepted Task may pin.
|
||||
pub const TASK_MAX_RETENTION_MS: i64 = 30 * 86_400_000;
|
||||
/// Upper bound of one retention/housekeeping batch.
|
||||
pub const TASK_PURGE_BATCH_LIMIT: u32 = 128;
|
||||
|
||||
/// Stable identity of one accepted Task. Generated as UUIDv7 and returned only after the accepting
|
||||
/// transaction committed, so the ID never exists without its durable record.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct TaskId(Uuid);
|
||||
|
||||
impl TaskId {
|
||||
pub fn generate() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
Uuid::parse_str(value)
|
||||
.map(Self)
|
||||
.map_err(|_| TaskError::invalid("task identity is not a UUID"))
|
||||
}
|
||||
|
||||
pub fn as_uuid(self) -> Uuid {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TaskId {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
self.0.fmt(formatter)
|
||||
}
|
||||
}
|
||||
|
||||
/// Closed Task lifecycle vocabulary. `outcome_unknown` is a non-terminal state: a value commit that
|
||||
/// failed to confirm its dispatch is never collapsed into a plain failure.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TaskState {
|
||||
Queued,
|
||||
Working,
|
||||
InputRequired,
|
||||
OutcomeUnknown,
|
||||
Completed,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl TaskState {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Queued => "queued",
|
||||
Self::Working => "working",
|
||||
Self::InputRequired => "input_required",
|
||||
Self::OutcomeUnknown => "outcome_unknown",
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::Cancelled => "cancelled",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"queued" => Ok(Self::Queued),
|
||||
"working" => Ok(Self::Working),
|
||||
"input_required" => Ok(Self::InputRequired),
|
||||
"outcome_unknown" => Ok(Self::OutcomeUnknown),
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"cancelled" => Ok(Self::Cancelled),
|
||||
_ => Err(TaskError::invalid("unknown task state")),
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn is_terminal(self) -> bool {
|
||||
matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
|
||||
}
|
||||
|
||||
/// One conditional transition may only move along this table. A terminal state has no successor,
|
||||
/// so a repeated terminal write can never rewrite an accepted result.
|
||||
pub const fn allows_transition_to(self, next: Self) -> bool {
|
||||
match self {
|
||||
Self::Queued => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
|
||||
Self::Working => matches!(
|
||||
next,
|
||||
Self::InputRequired
|
||||
| Self::OutcomeUnknown
|
||||
| Self::Completed
|
||||
| Self::Failed
|
||||
| Self::Cancelled
|
||||
),
|
||||
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
|
||||
Self::OutcomeUnknown => {
|
||||
matches!(next, Self::Completed | Self::Failed | Self::Cancelled)
|
||||
}
|
||||
Self::Completed | Self::Failed | Self::Cancelled => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Who owns the accepted Task. An administrator is never a wildcard owner of every Task: the owner
|
||||
/// identity and the authorization scope are separate, both-pinned columns.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TaskOwnerKind {
|
||||
Agent,
|
||||
Admin,
|
||||
}
|
||||
|
||||
impl TaskOwnerKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Agent => "agent",
|
||||
Self::Admin => "admin",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"agent" => Ok(Self::Agent),
|
||||
"admin" => Ok(Self::Admin),
|
||||
_ => Err(TaskError::invalid("unknown task owner kind")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Support identities captured at acceptance. They are stored and returned independently of
|
||||
/// sampling, and a replay never rewrites them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskSupportIds {
|
||||
pub request_id: RequestId,
|
||||
pub trace_id: TraceId,
|
||||
pub traceparent: String,
|
||||
pub trace_sampled: bool,
|
||||
}
|
||||
|
||||
impl TaskSupportIds {
|
||||
pub fn new(request_id: RequestId, trace_context: &TraceContext) -> Self {
|
||||
Self {
|
||||
request_id,
|
||||
trace_id: trace_context.trace_id().clone(),
|
||||
traceparent: trace_context.traceparent().to_owned(),
|
||||
trace_sampled: trace_context.is_sampled(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-validates both the trace identity and the parent context: a Task must never persist an
|
||||
/// inconsistent or malformed support pair, and an unsampled context is still a valid parent.
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.trace_id.as_str().is_empty() {
|
||||
return Err(TaskError::invalid("task trace identity is missing"));
|
||||
}
|
||||
let context = TraceContext::parse(&self.traceparent)
|
||||
.map_err(|_| TaskError::invalid("task trace parent is invalid"))?;
|
||||
if context.trace_id() != &self.trace_id || context.is_sampled() != self.trace_sampled {
|
||||
return Err(TaskError::invalid("task trace parent is inconsistent"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Execution semantics pinned at acceptance: a later publication never rewrites them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskPins {
|
||||
pub operation_version: u32,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
pub execution_deadline_unix_ms: i64,
|
||||
/// Retention deadline of the record; never the same thing as the execution deadline.
|
||||
pub retain_until_unix_ms: i64,
|
||||
/// Verified approval evidence reference (identifier only, never a bearer token).
|
||||
pub approval_reference: Option<String>,
|
||||
}
|
||||
|
||||
impl TaskPins {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.operation_version == 0 {
|
||||
return Err(TaskError::invalid(
|
||||
"task operation version must be positive",
|
||||
));
|
||||
}
|
||||
if self.execution_deadline_unix_ms <= 0 {
|
||||
return Err(TaskError::invalid("task execution deadline is invalid"));
|
||||
}
|
||||
if self.retain_until_unix_ms < self.execution_deadline_unix_ms {
|
||||
return Err(TaskError::invalid(
|
||||
"task retention deadline precedes execution",
|
||||
));
|
||||
}
|
||||
if self.retain_until_unix_ms - self.execution_deadline_unix_ms > TASK_MAX_RETENTION_MS {
|
||||
return Err(TaskError::invalid("task retention exceeds its bound"));
|
||||
}
|
||||
if let Some(reference) = self.approval_reference.as_deref()
|
||||
&& (reference.is_empty()
|
||||
|| reference.len() > TASK_APPROVAL_REFERENCE_MAX_BYTES
|
||||
|| reference.bytes().any(|byte| byte.is_ascii_control()))
|
||||
{
|
||||
return Err(TaskError::invalid("task approval reference is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded, classified payload of a Task. Construction validates the byte/depth/node budget and
|
||||
/// redacts secret-bearing fields before anything is hashed or persisted.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskEnvelope {
|
||||
value: Value,
|
||||
}
|
||||
|
||||
impl TaskEnvelope {
|
||||
/// Validates the caller-provided value, then stores a redacted copy of it. The caller hashes the
|
||||
/// original value separately, so redaction never collapses two different requests into one.
|
||||
pub fn try_redacted(input: &Value) -> Result<Self, TaskError> {
|
||||
validate_task_envelope(input)?;
|
||||
Ok(Self {
|
||||
value: redact_task_envelope(input),
|
||||
})
|
||||
}
|
||||
|
||||
/// Accepts a value that was already validated and redacted at a trust boundary.
|
||||
pub fn from_persisted(value: Value) -> Result<Self, TaskError> {
|
||||
validate_task_envelope(&value)?;
|
||||
Ok(Self { value })
|
||||
}
|
||||
|
||||
pub fn value(&self) -> &Value {
|
||||
&self.value
|
||||
}
|
||||
|
||||
pub fn into_value(self) -> Value {
|
||||
self.value
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the size, nesting depth and node count of one Task envelope. Every byte, collection and
|
||||
/// node is counted before the value is cloned, hashed or persisted.
|
||||
pub fn validate_task_envelope(value: &Value) -> Result<(), TaskError> {
|
||||
let encoded = serde_json::to_vec(value)
|
||||
.map_err(|_| TaskError::invalid("task envelope is not serializable"))?;
|
||||
if encoded.len() > TASK_ENVELOPE_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task envelope exceeds its byte budget"));
|
||||
}
|
||||
let mut nodes = 0usize;
|
||||
check_budget(value, 1, &mut nodes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_budget(value: &Value, depth: usize, nodes: &mut usize) -> Result<(), TaskError> {
|
||||
if depth > TASK_ENVELOPE_MAX_DEPTH {
|
||||
return Err(TaskError::invalid("task envelope exceeds its depth budget"));
|
||||
}
|
||||
*nodes += 1;
|
||||
if *nodes > TASK_ENVELOPE_MAX_NODES {
|
||||
return Err(TaskError::invalid("task envelope exceeds its node budget"));
|
||||
}
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
for (key, child) in object {
|
||||
check_budget_name(key)?;
|
||||
check_budget(child, depth + 1, nodes)?;
|
||||
}
|
||||
}
|
||||
Value::Array(values) => {
|
||||
for child in values {
|
||||
check_budget(child, depth + 1, nodes)?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_budget_name(key: &str) -> Result<(), TaskError> {
|
||||
if key.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task envelope key exceeds its budget"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Field-name classification of secret-bearing values. A matching field keeps its position but never
|
||||
/// its plaintext in the persisted envelope.
|
||||
pub fn sensitive_task_key(name: &str) -> bool {
|
||||
let lowered = name.to_ascii_lowercase();
|
||||
[
|
||||
"password",
|
||||
"passwd",
|
||||
"secret",
|
||||
"token",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"authorization",
|
||||
"credential",
|
||||
"private_key",
|
||||
"cookie",
|
||||
"signature",
|
||||
]
|
||||
.iter()
|
||||
.any(|marker| lowered.contains(marker))
|
||||
}
|
||||
|
||||
/// Redacted projection of an already validated envelope.
|
||||
pub fn redact_task_envelope(value: &Value) -> Value {
|
||||
match value {
|
||||
Value::Object(object) => Value::Object(
|
||||
object
|
||||
.iter()
|
||||
.map(|(key, child)| {
|
||||
let value = if sensitive_task_key(key) {
|
||||
Value::String("[redacted]".to_owned())
|
||||
} else {
|
||||
redact_task_envelope(child)
|
||||
};
|
||||
(key.clone(), value)
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
Value::Array(values) => Value::Array(values.iter().map(redact_task_envelope).collect()),
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
|
||||
/// queue capacity commit together, and no acknowledgement exists before that commit.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAcceptRequest {
|
||||
pub task_id: TaskId,
|
||||
pub workspace_id: String,
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
pub operation_id: String,
|
||||
pub pins: TaskPins,
|
||||
/// Digest of the caller key over the full owner/authorization/Tool scope.
|
||||
pub scoped_key: String,
|
||||
/// Digest of the canonical validated executable input.
|
||||
pub input_fingerprint: String,
|
||||
pub input: TaskEnvelope,
|
||||
pub support: TaskSupportIds,
|
||||
}
|
||||
|
||||
impl TaskAcceptRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
for (field, value) in [
|
||||
("workspace", self.workspace_id.as_str()),
|
||||
("owner", self.owner_id.as_str()),
|
||||
("authorization scope", self.authorization_scope.as_str()),
|
||||
("operation", self.operation_id.as_str()),
|
||||
] {
|
||||
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
||||
}
|
||||
}
|
||||
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task scoped key digest is invalid"));
|
||||
}
|
||||
if self.input_fingerprint.len() != 64
|
||||
|| !self
|
||||
.input_fingerprint
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err(TaskError::invalid("task input fingerprint is invalid"));
|
||||
}
|
||||
self.pins.validate()?;
|
||||
self.support.validate()
|
||||
}
|
||||
}
|
||||
|
||||
/// Successful acceptance, returned only after the accepting transaction committed. A replay returns
|
||||
/// the stored identity and support IDs of the original acceptance.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAcceptance {
|
||||
pub task_id: TaskId,
|
||||
pub state: TaskState,
|
||||
/// Exact Published Version pinned by this acceptance; a replay returns the stored pin.
|
||||
pub operation_version: u32,
|
||||
pub support: TaskSupportIds,
|
||||
pub created_at_unix_ms: i64,
|
||||
pub retain_until_unix_ms: i64,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
/// Owner-side access proof of a read or a terminal write. It is built by the application layer from
|
||||
/// the verified principal, never from a caller-supplied marker, and every store query carries the
|
||||
/// whole predicate so a foreign row is never fetched.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAccess {
|
||||
pub workspace_id: String,
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
}
|
||||
|
||||
impl TaskAccess {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
for (field, value) in [
|
||||
("workspace", self.workspace_id.as_str()),
|
||||
("owner", self.owner_id.as_str()),
|
||||
("authorization scope", self.authorization_scope.as_str()),
|
||||
] {
|
||||
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Read model of one Task. Lifecycle fields, pins and support IDs stay readable together; the
|
||||
/// protocol adapter only maps them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskRecord {
|
||||
pub task_id: TaskId,
|
||||
pub state: TaskState,
|
||||
pub workspace_id: String,
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
pub operation_id: String,
|
||||
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
|
||||
pub input_fingerprint: String,
|
||||
pub pins: TaskPins,
|
||||
pub support: TaskSupportIds,
|
||||
pub input: TaskEnvelope,
|
||||
pub progress: Option<TaskEnvelope>,
|
||||
pub result: Option<TaskEnvelope>,
|
||||
pub error: Option<TaskEnvelope>,
|
||||
pub attempts: u32,
|
||||
/// Monotonic lifecycle version used by the terminal CAS.
|
||||
pub version: u32,
|
||||
pub created_at_unix_ms: i64,
|
||||
pub updated_at_unix_ms: i64,
|
||||
}
|
||||
|
||||
/// One conditional terminal transition. Exactly one concurrent writer wins; the losers observe the
|
||||
/// accepted state and never rewrite the result.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskFinishRequest {
|
||||
pub access: TaskAccess,
|
||||
pub task_id: TaskId,
|
||||
pub expected_state: TaskState,
|
||||
pub expected_version: u32,
|
||||
pub target_state: TaskState,
|
||||
pub result: Option<TaskEnvelope>,
|
||||
pub error: Option<TaskEnvelope>,
|
||||
}
|
||||
|
||||
impl TaskFinishRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
self.access.validate()?;
|
||||
if !self.target_state.is_terminal() {
|
||||
return Err(TaskError::invalid("task finish target is not terminal"));
|
||||
}
|
||||
if self.expected_state.is_terminal() {
|
||||
return Err(TaskError::invalid("task finish source is already terminal"));
|
||||
}
|
||||
if !self.expected_state.allows_transition_to(self.target_state) {
|
||||
return Err(TaskError::invalid("task state transition is not allowed"));
|
||||
}
|
||||
if self.target_state == TaskState::Completed && self.result.is_none() {
|
||||
return Err(TaskError::invalid("a completed task requires a result"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded retention outcome.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct TaskPurgeReport {
|
||||
/// Queued Tasks that passed their execution deadline and were failed with their intent removed.
|
||||
pub expired_queued: u64,
|
||||
/// Terminal Tasks that passed their retention deadline.
|
||||
pub expired_terminal: u64,
|
||||
}
|
||||
|
||||
/// Typed Task failure. A denial and a missing Task share one outward shape, so a foreign caller can
|
||||
/// never distinguish "not yours" from "does not exist".
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum TaskError {
|
||||
#[error("task request is invalid: {message}")]
|
||||
Invalid { message: String },
|
||||
#[error("task store is unavailable: {message}")]
|
||||
Unavailable { message: String },
|
||||
#[error("task was not found")]
|
||||
NotFound,
|
||||
#[error("task request conflicts with the accepted request")]
|
||||
FingerprintConflict,
|
||||
#[error("task lifecycle transition was rejected")]
|
||||
StateConflict,
|
||||
#[error("background task capacity is exhausted")]
|
||||
Overloaded { retry_after_ms: u64 },
|
||||
}
|
||||
|
||||
impl TaskError {
|
||||
pub fn invalid(message: impl Into<String>) -> Self {
|
||||
Self::Invalid {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::Unavailable {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the caller may tell this failure apart from an absent Task.
|
||||
pub fn is_not_found(&self) -> bool {
|
||||
matches!(self, Self::NotFound)
|
||||
}
|
||||
}
|
||||
|
||||
/// Authoritative durable Task port.
|
||||
#[async_trait]
|
||||
pub trait TaskStore: Send + Sync {
|
||||
/// Proves the durable Task state is usable (schema and operations reachable).
|
||||
async fn health(&self) -> Result<(), TaskError>;
|
||||
|
||||
/// Atomically accepts a Task: Task record, dispatch intent and one queue slot commit together.
|
||||
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError>;
|
||||
|
||||
/// Reads one Task inside the caller's owner scope.
|
||||
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Scoped lookup of an already accepted Task by its full-scope idempotency digest. Used to keep a
|
||||
/// retry of an explicitly pinned version from becoming new work; `None` means "not accepted yet".
|
||||
async fn find(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
operation_id: &str,
|
||||
scoped_key: &str,
|
||||
) -> Result<Option<TaskRecord>, TaskError>;
|
||||
|
||||
/// Applies exactly one conditional terminal transition and releases its intent atomically.
|
||||
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes
|
||||
/// expired terminal records. Live and non-terminal work is never deleted.
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn request() -> TaskAcceptRequest {
|
||||
TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 3,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_790_000_000_000,
|
||||
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
|
||||
approval_reference: Some("appr_1".to_owned()),
|
||||
},
|
||||
scoped_key: "a".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uuid_v7_task_ids_are_unique_and_round_trip() {
|
||||
let first = TaskId::generate();
|
||||
let second = TaskId::generate();
|
||||
assert_ne!(first, second);
|
||||
assert_eq!(TaskId::parse(&first.to_string()).unwrap(), first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_states_have_no_successor() {
|
||||
for terminal in [
|
||||
TaskState::Completed,
|
||||
TaskState::Failed,
|
||||
TaskState::Cancelled,
|
||||
] {
|
||||
assert!(terminal.is_terminal());
|
||||
for next in [
|
||||
TaskState::Queued,
|
||||
TaskState::Working,
|
||||
TaskState::Completed,
|
||||
TaskState::Failed,
|
||||
] {
|
||||
assert!(!terminal.allows_transition_to(next));
|
||||
}
|
||||
}
|
||||
assert!(TaskState::Queued.allows_transition_to(TaskState::Working));
|
||||
assert!(TaskState::Working.allows_transition_to(TaskState::OutcomeUnknown));
|
||||
assert!(!TaskState::Queued.allows_transition_to(TaskState::Completed));
|
||||
assert!(!TaskState::InputRequired.allows_transition_to(TaskState::Completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn envelope_bounds_reject_bytes_depth_and_nodes() {
|
||||
assert!(TaskEnvelope::try_redacted(&json!({"a": 1})).is_ok());
|
||||
assert!(
|
||||
TaskEnvelope::try_redacted(&json!({"a": "x".repeat(TASK_ENVELOPE_MAX_BYTES)})).is_err()
|
||||
);
|
||||
|
||||
let mut deep = json!(1);
|
||||
for _ in 0..=TASK_ENVELOPE_MAX_DEPTH {
|
||||
deep = json!({"n": deep});
|
||||
}
|
||||
assert!(TaskEnvelope::try_redacted(&deep).is_err());
|
||||
|
||||
let wide = Value::Array((0..=TASK_ENVELOPE_MAX_NODES).map(|_| json!(1)).collect());
|
||||
assert!(TaskEnvelope::try_redacted(&wide).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_bearing_fields_are_redacted_and_keep_their_shape() {
|
||||
let envelope = TaskEnvelope::try_redacted(&json!({
|
||||
"query": "leads",
|
||||
"auth": {"api_key": "sk_live_1", "nested": [{"password": "p"}]}
|
||||
}))
|
||||
.unwrap();
|
||||
let value = envelope.value();
|
||||
assert_eq!(value["query"], json!("leads"));
|
||||
assert_eq!(value["auth"]["api_key"], json!("[redacted]"));
|
||||
assert_eq!(value["auth"]["nested"][0]["password"], json!("[redacted]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsampled_support_ids_are_valid_and_consistent() {
|
||||
let support = TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
);
|
||||
assert!(!support.trace_sampled);
|
||||
assert!(support.traceparent.ends_with("-00"));
|
||||
support.validate().unwrap();
|
||||
|
||||
let mut inconsistent = support.clone();
|
||||
inconsistent.traceparent = inconsistent.traceparent.replace("-00", "-01");
|
||||
assert!(inconsistent.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accept_request_validation_holds_the_identity_and_pin_bounds() {
|
||||
assert!(request().validate().is_ok());
|
||||
|
||||
let mut oversized = request();
|
||||
oversized.workspace_id = "w".repeat(TASK_IDENTITY_MAX_BYTES + 1);
|
||||
assert!(oversized.validate().is_err());
|
||||
|
||||
let mut bad_fingerprint = request();
|
||||
bad_fingerprint.input_fingerprint = "XYZ".to_owned();
|
||||
assert!(bad_fingerprint.validate().is_err());
|
||||
|
||||
let mut bad_retention = request();
|
||||
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
|
||||
assert!(bad_retention.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_requires_a_terminal_reachable_transition() {
|
||||
let access = TaskAccess {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
};
|
||||
let base = TaskFinishRequest {
|
||||
access,
|
||||
task_id: TaskId::generate(),
|
||||
expected_state: TaskState::Working,
|
||||
expected_version: 1,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
error: None,
|
||||
};
|
||||
assert!(base.validate().is_ok());
|
||||
|
||||
let mut non_terminal = base.clone();
|
||||
non_terminal.target_state = TaskState::Working;
|
||||
assert!(non_terminal.validate().is_err());
|
||||
|
||||
let mut terminal_source = base.clone();
|
||||
terminal_source.expected_state = TaskState::Completed;
|
||||
assert!(terminal_source.validate().is_err());
|
||||
|
||||
let mut completed_without_result = base;
|
||||
completed_without_result.result = None;
|
||||
assert!(completed_without_result.validate().is_err());
|
||||
}
|
||||
}
|
||||
@@ -117,6 +117,6 @@ pub use model::{
|
||||
};
|
||||
pub use postgres::{
|
||||
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
|
||||
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, admission_health,
|
||||
admission_snapshot,
|
||||
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, PostgresTaskStore,
|
||||
admission_health, admission_snapshot, tasks_health,
|
||||
};
|
||||
@@ -6,6 +6,7 @@ mod artifact_cleanup_indexes_v13;
|
||||
mod artifact_metadata_v12;
|
||||
mod authority;
|
||||
mod baseline_v1;
|
||||
mod durable_tasks_v25;
|
||||
mod execution_admission_v24;
|
||||
mod execution_outcome_v5;
|
||||
mod import_plan_v14;
|
||||
@@ -37,6 +38,7 @@ mod schema_guard_v20;
|
||||
mod schema_guard_v22;
|
||||
mod schema_guard_v23;
|
||||
mod schema_guard_v24;
|
||||
mod schema_guard_v25;
|
||||
mod schema_guard_v7;
|
||||
mod schema_guard_v8;
|
||||
mod schema_guard_v9;
|
||||
|
||||
@@ -4,6 +4,7 @@ use super::agent_resource_bindings_v18;
|
||||
use super::approval_side_effects_v10;
|
||||
use super::artifact_cleanup_indexes_v13;
|
||||
use super::artifact_metadata_v12;
|
||||
use super::durable_tasks_v25;
|
||||
use super::execution_admission_v24;
|
||||
use super::execution_outcome_v5;
|
||||
use super::import_plan_v14;
|
||||
@@ -40,9 +41,9 @@ use preflight::*;
|
||||
use contract::baseline_source_digest;
|
||||
use contract::validate_descriptors;
|
||||
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
|
||||
const CURRENT_VERSION: i64 = 24;
|
||||
const CURRENT_VERSION: i64 = 25;
|
||||
const IMPLEMENTED_VERSIONS: &[i64] = &[
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
|
||||
];
|
||||
const BASELINE_SOURCE_SHA256: &str =
|
||||
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
|
||||
@@ -337,6 +338,12 @@ impl MigrationAuthority {
|
||||
execution_admission_v24::SOURCE_SHA256,
|
||||
23,
|
||||
),
|
||||
expand_descriptor(
|
||||
25,
|
||||
durable_tasks_v25::NAME,
|
||||
durable_tasks_v25::SOURCE_SHA256,
|
||||
24,
|
||||
),
|
||||
]
|
||||
}
|
||||
pub fn validate_sequence() -> Result<(), MigrationError> {
|
||||
@@ -499,6 +506,9 @@ impl MigrationAuthority {
|
||||
if from < 24 {
|
||||
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
|
||||
}
|
||||
if from < 25 {
|
||||
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
|
||||
}
|
||||
match inspect(&mut transaction).await? {
|
||||
MigrationPreflight::Current {
|
||||
version: CURRENT_VERSION,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v25";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v25.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v25",
|
||||
Some(25),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.canonical_ledger",
|
||||
Some(25),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
-- V25: durable background Tasks (Story 4.2, FR-22, FR-23).
|
||||
--
|
||||
-- Одна строка `tasks` — принятая durable работа. Идентификатор Task (UUIDv7), owner scope,
|
||||
-- authorization scope, запрошенный Tool и точная published-версия, canonical fingerprint входа,
|
||||
-- support IDs (Request/Trace/parent TraceContext независимо от sampling) и bounded envelopes
|
||||
-- закрепляются в той же PostgreSQL transaction, что и dispatch intent
|
||||
-- (`execution_admission_entries.task_id`) и его queue slot. До commit наружу не выдаётся ни ID, ни ack.
|
||||
--
|
||||
-- `scoped_key` — digest caller-ключа по полному (workspace, owner kind/id, authorization scope,
|
||||
-- operation) scope; UNIQUE по этому набору и есть защита идемпотентности. Повтор того же ключа с тем
|
||||
-- же canonical fingerprint возвращает сохранённую Task, другой fingerprint — typed conflict.
|
||||
--
|
||||
-- `retain_until` — срок хранения записи, а не execution deadline: пока запись в retention, replay
|
||||
-- находит тот же Task. Удаление terminal-записей возможно только после retain_until; queued/working/
|
||||
-- input_required/outcome_unknown не удаляются никогда ради лимита.
|
||||
--
|
||||
-- Строка не хранит credentials, bearer/prepared headers и confirmation tokens: секретные поля входа
|
||||
-- редактируются до записи, а approval хранится безопасной ссылкой.
|
||||
|
||||
create table tasks (
|
||||
task_id uuid not null,
|
||||
workspace_id text not null,
|
||||
owner_kind text not null,
|
||||
owner_id text not null,
|
||||
authorization_scope text not null,
|
||||
operation_id text not null,
|
||||
operation_version integer not null,
|
||||
scoped_key text not null,
|
||||
input_fingerprint text not null,
|
||||
request_id text not null,
|
||||
trace_id text not null,
|
||||
traceparent text not null,
|
||||
trace_sampled boolean not null,
|
||||
policy_mode text not null,
|
||||
approval_reference text,
|
||||
state text not null,
|
||||
version integer not null default 1,
|
||||
attempts integer not null default 0,
|
||||
input_envelope jsonb not null,
|
||||
progress_envelope jsonb,
|
||||
result_envelope jsonb,
|
||||
error_envelope jsonb,
|
||||
execution_deadline timestamptz not null,
|
||||
retain_until timestamptz not null,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now(),
|
||||
constraint tasks_pkey primary key (task_id),
|
||||
constraint tasks_workspace_check check (octet_length(workspace_id) between 1 and 160),
|
||||
constraint tasks_owner_kind_check check (owner_kind in ('agent', 'admin')),
|
||||
constraint tasks_owner_check check (octet_length(owner_id) between 1 and 160),
|
||||
constraint tasks_authorization_scope_check check (
|
||||
octet_length(authorization_scope) between 1 and 160
|
||||
),
|
||||
constraint tasks_operation_check check (octet_length(operation_id) between 1 and 160),
|
||||
constraint tasks_operation_version_check check (operation_version >= 1),
|
||||
constraint tasks_scoped_key_check check (octet_length(scoped_key) between 16 and 128),
|
||||
constraint tasks_fingerprint_check check (octet_length(input_fingerprint) = 64),
|
||||
constraint tasks_request_check check (octet_length(request_id) between 1 and 128),
|
||||
constraint tasks_trace_check check (octet_length(trace_id) = 32),
|
||||
constraint tasks_traceparent_check check (octet_length(traceparent) = 55),
|
||||
constraint tasks_policy_mode_check check (
|
||||
policy_mode in ('synchronous', 'background', 'adaptive')
|
||||
),
|
||||
constraint tasks_state_check check (
|
||||
state in ('queued', 'working', 'input_required', 'outcome_unknown',
|
||||
'completed', 'failed', 'cancelled')
|
||||
),
|
||||
constraint tasks_lifecycle_check check (version >= 1 and attempts >= 0),
|
||||
constraint tasks_retention_check check (retain_until >= execution_deadline),
|
||||
constraint tasks_approval_check check (
|
||||
approval_reference is null or octet_length(approval_reference) between 1 and 160
|
||||
)
|
||||
);
|
||||
|
||||
create unique index tasks_scoped_key_uidx
|
||||
on tasks (workspace_id, owner_kind, owner_id, authorization_scope, operation_id, scoped_key);
|
||||
|
||||
create index tasks_retention_idx on tasks (state, retain_until, task_id);
|
||||
|
||||
create index tasks_deadline_idx on tasks (state, execution_deadline, task_id);
|
||||
|
||||
-- Dispatch intent: прежняя reserved-строка admission становится точкой связи Task и capacity.
|
||||
-- FK без cascade: удаление принятой Task не маскируется housekeeping-удалением intent.
|
||||
alter table execution_admission_entries
|
||||
add column task_id uuid,
|
||||
add constraint execution_admission_entries_task_fkey
|
||||
foreign key (task_id) references tasks (task_id);
|
||||
|
||||
create unique index execution_admission_entries_task_uidx
|
||||
on execution_admission_entries (task_id) where task_id is not null;
|
||||
@@ -473,7 +473,15 @@ pub(super) async fn validate_schema_fingerprint(
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v24::validate_v24_execution_admission(connection).await?;
|
||||
super::schema_guard_v24::validate_v24_execution_admission(connection, current_version)
|
||||
.await?;
|
||||
}
|
||||
if current_version < 25 {
|
||||
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -39,7 +39,24 @@ const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("granted_at", "timestamp with time zone", true),
|
||||
];
|
||||
|
||||
const CONSTRAINT_COUNT: &[(&str, usize)] = &[(CONTROLLERS, 4), (ENTRIES, 11)];
|
||||
/// V25 adds the nullable Task dispatch-intent link to the same reserved row. At version 24 the
|
||||
/// column must not exist yet, so the guard validates the shape of the version it describes.
|
||||
fn entry_columns(current_version: i64) -> Vec<(&'static str, &'static str, bool)> {
|
||||
let mut columns = ENTRY_COLUMNS.to_vec();
|
||||
if current_version >= 25 {
|
||||
columns.push(("task_id", "uuid", true));
|
||||
}
|
||||
columns
|
||||
}
|
||||
|
||||
fn constraint_count(table: &str, current_version: i64) -> Option<usize> {
|
||||
match table {
|
||||
CONTROLLERS => Some(4),
|
||||
ENTRIES if current_version >= 25 => Some(12),
|
||||
ENTRIES => Some(11),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"];
|
||||
|
||||
@@ -72,16 +89,22 @@ pub(super) async fn validate_v24_absent(
|
||||
/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24).
|
||||
pub(super) async fn validate_v24_execution_admission(
|
||||
connection: &mut PgConnection,
|
||||
current_version: i64,
|
||||
) -> Result<(), MigrationError> {
|
||||
for relation in OWNED_RELATIONS {
|
||||
if !relation_exists(connection, relation).await? {
|
||||
return Err(v24_error("v24.relations"));
|
||||
}
|
||||
}
|
||||
for (table, columns) in [(CONTROLLERS, CONTROLLER_COLUMNS), (ENTRIES, ENTRY_COLUMNS)] {
|
||||
validate_columns(connection, table, columns).await?;
|
||||
let tables = [
|
||||
(CONTROLLERS, CONTROLLER_COLUMNS.to_vec()),
|
||||
(ENTRIES, entry_columns(current_version)),
|
||||
];
|
||||
for (table, columns) in tables {
|
||||
validate_columns(connection, table, &columns).await?;
|
||||
}
|
||||
for (table, expected) in CONSTRAINT_COUNT {
|
||||
for table in [CONTROLLERS, ENTRIES] {
|
||||
let expected = constraint_count(table, current_version).unwrap_or_default();
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count
|
||||
from pg_catalog.pg_constraint c
|
||||
@@ -95,7 +118,7 @@ pub(super) async fn validate_v24_execution_admission(
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(*expected).unwrap_or_default() {
|
||||
if count != i64::try_from(expected).unwrap_or_default() {
|
||||
return Err(v24_error("v24.constraints"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
//! V25 schema guard: durable background Tasks (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! Схема V25 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
|
||||
//! ожидаемой форме: таблица `tasks` с закрытым словарём state/policy_mode и scoped uniqueness, плюс
|
||||
//! nullable `task_id` на существующей reserved-строке admission с FK и UNIQUE-индексом.
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
const ENTRIES: &str = "execution_admission_entries";
|
||||
const SCOPED_INDEX: &str = "tasks_scoped_key_uidx";
|
||||
const TASK_INDEX: &str = "execution_admission_entries_task_uidx";
|
||||
const TASK_FK: &str = "execution_admission_entries_task_fkey";
|
||||
|
||||
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("task_id", "uuid", false),
|
||||
("workspace_id", "text", false),
|
||||
("owner_kind", "text", false),
|
||||
("owner_id", "text", false),
|
||||
("authorization_scope", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("scoped_key", "text", false),
|
||||
("input_fingerprint", "text", false),
|
||||
("request_id", "text", false),
|
||||
("trace_id", "text", false),
|
||||
("traceparent", "text", false),
|
||||
("trace_sampled", "boolean", false),
|
||||
("policy_mode", "text", false),
|
||||
("approval_reference", "text", true),
|
||||
("state", "text", false),
|
||||
("version", "integer", false),
|
||||
("attempts", "integer", false),
|
||||
("input_envelope", "jsonb", false),
|
||||
("progress_envelope", "jsonb", true),
|
||||
("result_envelope", "jsonb", true),
|
||||
("error_envelope", "jsonb", true),
|
||||
("execution_deadline", "timestamp with time zone", false),
|
||||
("retain_until", "timestamp with time zone", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
];
|
||||
|
||||
fn v25_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(25),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
/// Схема V25 отсутствует целиком (для current_version < 25).
|
||||
pub(super) async fn validate_v25_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
if super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
|
||||
return Ok(false);
|
||||
}
|
||||
for index in [SCOPED_INDEX, TASK_INDEX] {
|
||||
if index_exists(connection, index).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Схема V25 присутствует в ожидаемой форме (для current_version >= 25).
|
||||
pub(super) async fn validate_v25_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
if !relation_exists(connection, TASKS).await? {
|
||||
return Err(v25_error("v25.relations"));
|
||||
}
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
|
||||
return Err(v25_error("v25.columns"));
|
||||
}
|
||||
for (column, data_type, nullable) in TASK_COLUMNS {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v25_error("v25.columns"));
|
||||
}
|
||||
}
|
||||
if !super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
|
||||
return Err(v25_error("v25.intent_column"));
|
||||
}
|
||||
for (constraint, vocabulary) in [
|
||||
(
|
||||
"tasks_state_check",
|
||||
[
|
||||
"queued",
|
||||
"working",
|
||||
"input_required",
|
||||
"outcome_unknown",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
]
|
||||
.as_slice(),
|
||||
),
|
||||
(
|
||||
"tasks_policy_mode_check",
|
||||
["synchronous", "background", "adaptive"].as_slice(),
|
||||
),
|
||||
] {
|
||||
let expression = normalized_expression(connection, TASKS, constraint).await?;
|
||||
if !expression.is_some_and(|expression| {
|
||||
super::schema_guard::enum_constraint_matches(
|
||||
&expression,
|
||||
column_of(constraint),
|
||||
vocabulary,
|
||||
)
|
||||
}) {
|
||||
return Err(v25_error("v25.vocabulary"));
|
||||
}
|
||||
}
|
||||
for (index, table) in [(SCOPED_INDEX, TASKS), (TASK_INDEX, ENTRIES)] {
|
||||
if index_unique(connection, table, index).await? != Some(true) {
|
||||
return Err(v25_error("v25.indexes"));
|
||||
}
|
||||
}
|
||||
if !constraint_exists(connection, ENTRIES, TASK_FK).await? {
|
||||
return Err(v25_error("v25.intent_fk"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn column_of(constraint: &str) -> &'static str {
|
||||
if constraint.contains("policy_mode") {
|
||||
"policy_mode"
|
||||
} else {
|
||||
"state"
|
||||
}
|
||||
}
|
||||
|
||||
async fn constraint_exists(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
) as present",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn normalized_expression(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<Option<String>, MigrationError> {
|
||||
let row = query(
|
||||
"select pg_get_expr(c.conbin, c.conrelid) as expression
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
and c.contype = 'c'",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
|
||||
}
|
||||
|
||||
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_class idx
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1
|
||||
) as present",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn index_unique(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
index: &str,
|
||||
) -> Result<Option<bool>, MigrationError> {
|
||||
let row = query(
|
||||
"select i.indisunique, i.indisvalid
|
||||
from pg_catalog.pg_index i
|
||||
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
|
||||
join pg_catalog.pg_class t on t.oid = i.indrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1 and t.relname = $2",
|
||||
)
|
||||
.bind(index)
|
||||
.bind(table)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| {
|
||||
let unique = row.try_get::<bool, _>("indisunique").ok()?;
|
||||
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
|
||||
Some(unique && valid)
|
||||
}))
|
||||
}
|
||||
@@ -13,7 +13,8 @@ use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
|
||||
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
|
||||
AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, select_next_grant,
|
||||
AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, TaskError,
|
||||
select_next_grant,
|
||||
};
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
@@ -194,7 +195,8 @@ impl ExecutionAdmission for PostgresExecutionAdmission {
|
||||
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError> {
|
||||
let result = sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id = $1 and owner_token = $2 and state = 'executing'",
|
||||
where ticket_id = $1 and owner_token = $2 and state = 'executing'
|
||||
and task_id is null",
|
||||
)
|
||||
.bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX))
|
||||
.bind(&permit.owner_token)
|
||||
@@ -207,7 +209,8 @@ impl ExecutionAdmission for PostgresExecutionAdmission {
|
||||
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError> {
|
||||
let result = sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')",
|
||||
where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')
|
||||
and task_id is null",
|
||||
)
|
||||
.bind(i64::try_from(ticket_id).unwrap_or(i64::MAX))
|
||||
.bind(owner_token)
|
||||
@@ -301,9 +304,19 @@ impl PostgresExecutionAdmission {
|
||||
Ok(Attempt::Overloaded)
|
||||
}
|
||||
Err(AttemptError::Failure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(AdmissionStoreError::unavailable(error.to_string()))
|
||||
}
|
||||
Err(AttemptError::AdmissionFailure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(error)
|
||||
}
|
||||
Err(AttemptError::UniqueViolation) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(AdmissionStoreError::unavailable(
|
||||
"execution admission hit an unexpected unique conflict",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,82 +327,32 @@ impl PostgresExecutionAdmission {
|
||||
ticket: &mut Option<u64>,
|
||||
) -> Result<Attempt, AttemptError> {
|
||||
let workspace_id = context.request.scope.workspace_id.as_str();
|
||||
sqlx::query(
|
||||
"insert into execution_admission_controllers (workspace_id)
|
||||
values ($1), ($2)
|
||||
on conflict (workspace_id) do nothing",
|
||||
)
|
||||
.bind(DEPLOYMENT_CONTROLLER)
|
||||
.bind(workspace_id)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let cursor = lock_admission_in_tx(transaction, workspace_id).await?;
|
||||
|
||||
// Fixed lock order: deployment controller first, then the workspace controller.
|
||||
lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?;
|
||||
let cursor = lock_controller(transaction, workspace_id).await?;
|
||||
|
||||
// Expired queue decisions and waiting tickets leave the bounded queue; held permits never do.
|
||||
sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id in (
|
||||
select ticket_id from execution_admission_entries
|
||||
where state in ('waiting', 'reserved') and expires_at <= now()
|
||||
order by ticket_id
|
||||
limit $1
|
||||
)",
|
||||
)
|
||||
.bind(EXPIRED_CLEANUP_BATCH)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
// Expired queue decisions and waiting tickets leave the bounded queue; held permits and
|
||||
// Task-linked dispatch intents never do (Story 4.2 owns the durable Task lifetime).
|
||||
cleanup_expired_unlinked(transaction).await?;
|
||||
|
||||
if ticket.is_none() {
|
||||
let queued: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries
|
||||
where state in ('waiting', 'reserved')",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
if u64::try_from(queued).unwrap_or(u64::MAX)
|
||||
>= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX)
|
||||
{
|
||||
let queued = bounded_queue_len(transaction).await?;
|
||||
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
|
||||
return Err(AttemptError::Overloaded);
|
||||
}
|
||||
let state = match context.request.mode {
|
||||
AdmissionMode::Synchronous => "waiting",
|
||||
AdmissionMode::Background => "reserved",
|
||||
};
|
||||
let row = sqlx::query(
|
||||
"insert into execution_admission_entries (
|
||||
workspace_id, agent_key, operation_id, operation_version, mode, policy_mode,
|
||||
state, owner_token, fingerprint, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
to_timestamp($10::double precision / 1000.0))
|
||||
returning ticket_id",
|
||||
let ticket_id = insert_admission_entry(
|
||||
transaction,
|
||||
context.request,
|
||||
context.owner_token,
|
||||
state,
|
||||
None,
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(&context.request.scope.agent_key)
|
||||
.bind(&context.request.scope.operation_id)
|
||||
.bind(i32::try_from(context.request.scope.operation_version).unwrap_or(i32::MAX))
|
||||
.bind(context.request.mode.as_str())
|
||||
.bind(policy_mode_str(context.request))
|
||||
.bind(state)
|
||||
.bind(context.owner_token)
|
||||
.bind(&context.request.fingerprint)
|
||||
.bind(context.request.expires_at_unix_ms)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
|
||||
*ticket = Some(u64::try_from(ticket_id).unwrap_or_default());
|
||||
.await?;
|
||||
*ticket = Some(ticket_id);
|
||||
if context.request.mode == AdmissionMode::Background {
|
||||
return Ok(Attempt::Enqueued(
|
||||
u64::try_from(ticket_id).unwrap_or_default(),
|
||||
));
|
||||
return Ok(Attempt::Enqueued(ticket_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -520,22 +483,121 @@ impl PostgresExecutionAdmission {
|
||||
}
|
||||
}
|
||||
|
||||
enum AttemptError {
|
||||
/// Takes both controller locks in the fixed `'*'` -> workspace order. Shared by the synchronous
|
||||
/// admission attempt and the durable Task repository, so both serialize on the same controller rows
|
||||
/// and one task/queue invariant holds. Callers own the transaction; nothing here commits or sleeps.
|
||||
pub(super) async fn lock_admission_in_tx(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
) -> Result<AdmissionCursor, AttemptError> {
|
||||
sqlx::query(
|
||||
"insert into execution_admission_controllers (workspace_id)
|
||||
values ($1), ($2)
|
||||
on conflict (workspace_id) do nothing",
|
||||
)
|
||||
.bind(DEPLOYMENT_CONTROLLER)
|
||||
.bind(workspace_id)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?;
|
||||
lock_controller(transaction, workspace_id).await
|
||||
}
|
||||
|
||||
/// Bounded cleanup of expired waiting/reserved rows that are not a durable Task dispatch intent.
|
||||
pub(super) async fn cleanup_expired_unlinked(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> Result<(), AttemptError> {
|
||||
sqlx::query(
|
||||
"delete from execution_admission_entries
|
||||
where ticket_id in (
|
||||
select ticket_id from execution_admission_entries
|
||||
where state in ('waiting', 'reserved') and expires_at <= now()
|
||||
and task_id is null
|
||||
order by ticket_id
|
||||
limit $1
|
||||
)",
|
||||
)
|
||||
.bind(EXPIRED_CLEANUP_BATCH)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Authoritative bounded queue length: waiting and reserved rows share one budget.
|
||||
pub(super) async fn bounded_queue_len(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> Result<u64, AttemptError> {
|
||||
let queued: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries
|
||||
where state in ('waiting', 'reserved')",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
Ok(u64::try_from(queued).unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
/// Inserts one bounded admission row. A Task-linked row is the durable dispatch intent of the Task
|
||||
/// accepted in the same transaction, so its capacity slot can never be silently reclaimed.
|
||||
pub(super) async fn insert_admission_entry(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
request: &AdmissionRequest,
|
||||
owner_token: &str,
|
||||
state: &str,
|
||||
task_id: Option<Uuid>,
|
||||
) -> Result<u64, AttemptError> {
|
||||
let row = sqlx::query(
|
||||
"insert into execution_admission_entries (
|
||||
workspace_id, agent_key, operation_id, operation_version, mode, policy_mode,
|
||||
state, owner_token, fingerprint, expires_at, task_id
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
to_timestamp($10::double precision / 1000.0), $11)
|
||||
returning ticket_id",
|
||||
)
|
||||
.bind(&request.scope.workspace_id)
|
||||
.bind(&request.scope.agent_key)
|
||||
.bind(&request.scope.operation_id)
|
||||
.bind(i32::try_from(request.scope.operation_version).unwrap_or(i32::MAX))
|
||||
.bind(request.mode.as_str())
|
||||
.bind(policy_mode_str(request))
|
||||
.bind(state)
|
||||
.bind(owner_token)
|
||||
.bind(&request.fingerprint)
|
||||
.bind(request.expires_at_unix_ms)
|
||||
.bind(task_id)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
|
||||
Ok(u64::try_from(ticket_id).unwrap_or_default())
|
||||
}
|
||||
|
||||
pub(super) enum AttemptError {
|
||||
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
|
||||
Contended,
|
||||
/// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait.
|
||||
Overloaded,
|
||||
Failure(AdmissionStoreError),
|
||||
/// A foreign unique constraint aborted the transaction; the caller rolls back and rereads.
|
||||
UniqueViolation,
|
||||
/// Durable-Task side failure (the Task repository maps it to its own typed error).
|
||||
Failure(TaskError),
|
||||
/// Admission side failure.
|
||||
AdmissionFailure(AdmissionStoreError),
|
||||
}
|
||||
|
||||
impl AttemptError {
|
||||
fn from_sql(error: sqlx::Error) -> Self {
|
||||
pub(super) fn from_sql(error: sqlx::Error) -> Self {
|
||||
if let Some(database) = error.as_database_error()
|
||||
&& matches!(database.code().as_deref(), Some("55P03" | "57014"))
|
||||
{
|
||||
return Self::Contended;
|
||||
}
|
||||
Self::Failure(PostgresExecutionAdmission::unavailable(error))
|
||||
Self::AdmissionFailure(PostgresExecutionAdmission::unavailable(error))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,11 @@ impl PostgresRegistry {
|
||||
super::admission::admission_health(&self.pool).await
|
||||
}
|
||||
|
||||
/// Readiness probe of the durable Task state (Story 4.2).
|
||||
pub async fn durable_tasks_health(&self) -> Result<(), crank_core::TaskError> {
|
||||
super::tasks::tasks_health(&self.pool).await
|
||||
}
|
||||
|
||||
/// Authoritative admission queue readings for low-cardinality metrics.
|
||||
pub async fn execution_admission_snapshot(
|
||||
&self,
|
||||
|
||||
@@ -21,6 +21,7 @@ mod product_event;
|
||||
mod prompt;
|
||||
mod resource;
|
||||
mod secret;
|
||||
mod tasks;
|
||||
mod upstream;
|
||||
mod workspace;
|
||||
mod yaml_import;
|
||||
@@ -42,6 +43,8 @@ pub use admission::{
|
||||
};
|
||||
pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError};
|
||||
pub use resource::{AuthorizedResourceRead, BoundResourceAdmission};
|
||||
pub use tasks::PostgresTaskStore;
|
||||
pub use tasks::tasks_health;
|
||||
|
||||
use crate::{
|
||||
error::RegistryError,
|
||||
|
||||
@@ -0,0 +1,586 @@
|
||||
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
|
||||
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
|
||||
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
|
||||
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
|
||||
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
|
||||
//!
|
||||
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
|
||||
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
|
||||
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
|
||||
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::admission::{
|
||||
AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked,
|
||||
insert_admission_entry, lock_admission_in_tx,
|
||||
};
|
||||
|
||||
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
|
||||
authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode,
|
||||
approval_reference,
|
||||
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
|
||||
result_envelope, error_envelope, attempts, version,
|
||||
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
|
||||
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
|
||||
(extract(epoch from created_at) * 1000)::bigint as created_ms,
|
||||
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PostgresTaskStore {
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: crank_core::AdmissionSchedulerConfig,
|
||||
) -> Result<Self, TaskError> {
|
||||
limits
|
||||
.validate()
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
scheduler
|
||||
.validate()
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
Ok(Self { pool, limits })
|
||||
}
|
||||
|
||||
fn unavailable(error: impl std::fmt::Display) -> TaskError {
|
||||
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
|
||||
}
|
||||
|
||||
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
|
||||
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local lock_timeout = '3000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local statement_timeout = '5000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
Ok(transaction)
|
||||
}
|
||||
|
||||
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
|
||||
/// query, so a foreign row is never fetched.
|
||||
async fn stored_acceptance(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(&request.scoped_key)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let Some(row) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
|
||||
Ok(Some((
|
||||
acceptance_from_record(&record),
|
||||
record.input_fingerprint,
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
|
||||
TaskAcceptance {
|
||||
task_id: record.task_id,
|
||||
state: record.state,
|
||||
operation_version: record.pins.operation_version,
|
||||
support: record.support.clone(),
|
||||
created_at_unix_ms: record.created_at_unix_ms,
|
||||
retain_until_unix_ms: record.pins.retain_until_unix_ms,
|
||||
replayed: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
|
||||
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
|
||||
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
|
||||
sqlx::query("select count(*)::bigint from tasks where false")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresTaskStore::unavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One transaction's decision of the acceptance path.
|
||||
enum Accepted {
|
||||
Created(TaskAcceptance),
|
||||
Replayed(TaskAcceptance),
|
||||
Conflict,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskStore for PostgresTaskStore {
|
||||
async fn health(&self) -> Result<(), TaskError> {
|
||||
sqlx::query("select count(*)::bigint from tasks where false")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
super::admission::admission_health(&self.pool)
|
||||
.await
|
||||
.map_err(|error| TaskError::unavailable(error.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
let outcome = self.accept_locked(&mut transaction, &request).await;
|
||||
match outcome {
|
||||
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
|
||||
// The acknowledgement of a created Task is only possible after a real COMMIT.
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(TaskAcceptance {
|
||||
replayed: acceptance.replayed,
|
||||
..acceptance
|
||||
})
|
||||
}
|
||||
Ok(Accepted::Conflict) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::FingerprintConflict)
|
||||
}
|
||||
Err(AttemptError::Overloaded) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::Overloaded {
|
||||
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
})
|
||||
}
|
||||
Err(AttemptError::Contended) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::unavailable(
|
||||
"durable task acceptance could not take the admission locks",
|
||||
))
|
||||
}
|
||||
Err(AttemptError::UniqueViolation) => {
|
||||
// Roll back the aborted transaction, then reread the scoped key once outside it.
|
||||
transaction.rollback().await.ok();
|
||||
self.reread_after_unique(&request).await
|
||||
}
|
||||
Err(AttemptError::Failure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(error)
|
||||
}
|
||||
Err(AttemptError::AdmissionFailure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::unavailable(error.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5"
|
||||
)))
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
record_from_row(&row).map_err(Self::unavailable)
|
||||
}
|
||||
|
||||
async fn find(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
operation_id: &str,
|
||||
scoped_key: &str,
|
||||
) -> Result<Option<TaskRecord>, TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(operation_id)
|
||||
.bind(scoped_key)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
row.map(|row| record_from_row(&row)).transpose()
|
||||
}
|
||||
|
||||
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
let access = &request.access;
|
||||
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = $1,
|
||||
result_envelope = $2,
|
||||
error_envelope = $3,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
||||
and owner_id = $7 and authorization_scope = $8
|
||||
and state = $9 and version = $10
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(request.target_state.as_str())
|
||||
.bind(request.result.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.error.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(request.expected_state.as_str())
|
||||
.bind(i64::from(request.expected_version))
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = update else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(self.classify_missed_finish(&request).await);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(request.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
|
||||
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
|
||||
let mut transaction = self.start().await?;
|
||||
let expired = sqlx::query(
|
||||
"with expired as (
|
||||
update tasks
|
||||
set state = 'failed',
|
||||
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id in (
|
||||
select task_id from tasks
|
||||
where state = 'queued' and execution_deadline <= now()
|
||||
order by task_id
|
||||
limit $1
|
||||
)
|
||||
returning task_id
|
||||
), released as (
|
||||
delete from execution_admission_entries
|
||||
where task_id in (select task_id from expired)
|
||||
returning task_id
|
||||
)
|
||||
select (select count(*) from expired)::bigint as expired_queued",
|
||||
)
|
||||
.bind(batch)
|
||||
.fetch_one(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let expired_queued: i64 = expired
|
||||
.try_get("expired_queued")
|
||||
.map_err(Self::unavailable)?;
|
||||
let purged = sqlx::query(
|
||||
"delete from tasks
|
||||
where task_id in (
|
||||
select task_id from tasks
|
||||
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
|
||||
order by task_id
|
||||
limit $1
|
||||
)",
|
||||
)
|
||||
.bind(batch)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(TaskPurgeReport {
|
||||
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
|
||||
expired_terminal: purged.rows_affected(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
async fn accept_locked(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<Accepted, AttemptError> {
|
||||
// Replay first: an already accepted Task is returned without capacity mutation, so it works
|
||||
// with a full queue and after the original execution deadline while the record is retained.
|
||||
if let Some((acceptance, fingerprint)) =
|
||||
self.stored_acceptance(transaction, request).await?
|
||||
{
|
||||
return Ok(if fingerprint == request.input_fingerprint {
|
||||
Accepted::Replayed(acceptance)
|
||||
} else {
|
||||
Accepted::Conflict
|
||||
});
|
||||
}
|
||||
|
||||
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
|
||||
lock_admission_in_tx(transaction, &request.workspace_id).await?;
|
||||
|
||||
// The competitor may have committed while this transaction waited for the controller lock.
|
||||
if let Some((acceptance, fingerprint)) =
|
||||
self.stored_acceptance(transaction, request).await?
|
||||
{
|
||||
return Ok(if fingerprint == request.input_fingerprint {
|
||||
Accepted::Replayed(acceptance)
|
||||
} else {
|
||||
Accepted::Conflict
|
||||
});
|
||||
}
|
||||
|
||||
cleanup_expired_unlinked(transaction).await?;
|
||||
let queued = bounded_queue_len(transaction).await?;
|
||||
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
|
||||
return Err(AttemptError::Overloaded);
|
||||
}
|
||||
|
||||
let created_ms: i64 = sqlx::query(
|
||||
"insert into tasks (
|
||||
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
|
||||
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
|
||||
trace_sampled, policy_mode, approval_reference, state, input_envelope,
|
||||
execution_deadline, retain_until
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued',
|
||||
$16, to_timestamp($17::double precision / 1000.0),
|
||||
to_timestamp($18::double precision / 1000.0)
|
||||
)
|
||||
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
|
||||
.bind(&request.scoped_key)
|
||||
.bind(&request.input_fingerprint)
|
||||
.bind(request.support.request_id.as_str())
|
||||
.bind(request.support.trace_id.as_str())
|
||||
.bind(&request.support.traceparent)
|
||||
.bind(request.support.trace_sampled)
|
||||
.bind(policy_mode_str(request.pins.policy_mode))
|
||||
.bind(request.pins.approval_reference.as_deref())
|
||||
.bind(request.input.value().clone())
|
||||
.bind(request.pins.execution_deadline_unix_ms)
|
||||
.bind(request.pins.retain_until_unix_ms)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(task_sql_error)?
|
||||
.try_get("created_ms")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
let admission_request = AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: request.workspace_id.clone(),
|
||||
agent_key: request.owner_id.clone(),
|
||||
operation_id: request.operation_id.clone(),
|
||||
operation_version: request.pins.operation_version,
|
||||
},
|
||||
mode: AdmissionMode::Background,
|
||||
policy_mode: request.pins.policy_mode,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
// Never reached: the intent row is inserted by this transaction, not by `admit`.
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: request.input_fingerprint.clone(),
|
||||
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
|
||||
};
|
||||
let owner_token = format!("task_{}", request.task_id);
|
||||
insert_admission_entry(
|
||||
transaction,
|
||||
&admission_request,
|
||||
&owner_token,
|
||||
"reserved",
|
||||
Some(request.task_id.as_uuid()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Accepted::Created(TaskAcceptance {
|
||||
task_id: request.task_id,
|
||||
state: TaskState::Queued,
|
||||
operation_version: request.pins.operation_version,
|
||||
support: request.support.clone(),
|
||||
created_at_unix_ms: created_ms,
|
||||
retain_until_unix_ms: request.pins.retain_until_unix_ms,
|
||||
replayed: false,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
|
||||
async fn reread_after_unique(
|
||||
&self,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<TaskAcceptance, TaskError> {
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(&request.scoped_key)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::unavailable(
|
||||
"durable task acceptance ended with an unresolved unique conflict",
|
||||
));
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
if record.input_fingerprint != request.input_fingerprint {
|
||||
return Err(TaskError::FingerprintConflict);
|
||||
}
|
||||
Ok(acceptance_from_record(&record))
|
||||
}
|
||||
|
||||
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
|
||||
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
|
||||
async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
|
||||
let access = &request.access;
|
||||
match sqlx::query(
|
||||
"select state, version from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => TaskError::StateConflict,
|
||||
Ok(None) => TaskError::NotFound,
|
||||
Err(error) => Self::unavailable(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
|
||||
match mode {
|
||||
ToolExecutionMode::Synchronous => "synchronous",
|
||||
ToolExecutionMode::Background => "background",
|
||||
ToolExecutionMode::Adaptive => "adaptive",
|
||||
}
|
||||
}
|
||||
|
||||
fn task_sql_error(error: sqlx::Error) -> AttemptError {
|
||||
if error
|
||||
.as_database_error()
|
||||
.and_then(|database| database.code())
|
||||
.as_deref()
|
||||
== Some("23505")
|
||||
{
|
||||
return AttemptError::UniqueViolation;
|
||||
}
|
||||
AttemptError::from_sql(error)
|
||||
}
|
||||
|
||||
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
|
||||
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
|
||||
let state: String = row.try_get("state").map_err(unavailable_row)?;
|
||||
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
|
||||
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
|
||||
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
|
||||
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
|
||||
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
|
||||
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
|
||||
value.map(TaskEnvelope::from_persisted).transpose()
|
||||
};
|
||||
Ok(TaskRecord {
|
||||
task_id: TaskId::parse(&task_id.to_string())?,
|
||||
state: TaskState::parse(&state)?,
|
||||
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
|
||||
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
|
||||
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
|
||||
authorization_scope: row
|
||||
.try_get("authorization_scope")
|
||||
.map_err(unavailable_row)?,
|
||||
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
|
||||
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
|
||||
pins: TaskPins {
|
||||
operation_version: u32::try_from(
|
||||
row.try_get::<i32, _>("operation_version")
|
||||
.map_err(unavailable_row)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
policy_mode: match policy_mode.as_str() {
|
||||
"background" => ToolExecutionMode::Background,
|
||||
"adaptive" => ToolExecutionMode::Adaptive,
|
||||
_ => ToolExecutionMode::Synchronous,
|
||||
},
|
||||
execution_deadline_unix_ms: row
|
||||
.try_get("execution_deadline_ms")
|
||||
.map_err(unavailable_row)?,
|
||||
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
|
||||
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
|
||||
},
|
||||
support: TaskSupportIds {
|
||||
request_id: crank_core::RequestId::parse(&request_id)
|
||||
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
|
||||
trace_id: crank_core::TraceId::parse(&trace_id)
|
||||
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
|
||||
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
|
||||
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
|
||||
},
|
||||
input: envelope("input_envelope")?
|
||||
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
|
||||
progress: envelope("progress_envelope")?,
|
||||
result: envelope("result_envelope")?,
|
||||
error: envelope("error_envelope")?,
|
||||
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
|
||||
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn unavailable_row(error: sqlx::Error) -> TaskError {
|
||||
PostgresTaskStore::unavailable(error)
|
||||
}
|
||||
@@ -17,5 +17,6 @@ mod integration {
|
||||
mod prompt_drafts;
|
||||
mod prompt_publication;
|
||||
mod resource_drafts;
|
||||
mod tasks;
|
||||
mod workspace_access;
|
||||
}
|
||||
@@ -70,6 +70,8 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
|
||||
(21, "prompts-v21"),
|
||||
(22, "prompt-publications-v22"),
|
||||
(23, "prompt-dependencies-v23"),
|
||||
(24, "execution-admission-v24"),
|
||||
(25, "durable-tasks-v25"),
|
||||
];
|
||||
assert_eq!(rows.len(), expected.len());
|
||||
for (row, (version, name)) in rows.iter().zip(expected) {
|
||||
@@ -180,7 +182,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(first.pool()).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
@@ -228,7 +230,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
|
||||
.expect("the V18-local guard must be restorable");
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"restoring the exact V18 contract must return the current verdict"
|
||||
);
|
||||
}
|
||||
@@ -354,7 +356,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -467,13 +469,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 2,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
let trace_column: bool = sqlx::query_scalar(
|
||||
"select exists (
|
||||
@@ -516,7 +518,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 3,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -576,7 +578,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 4,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -830,6 +832,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 11,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
|
||||
for (drift, restore) in [
|
||||
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
|
||||
+8
-8
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"a rejected primary-key nullability change must leave the schema current"
|
||||
);
|
||||
|
||||
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
|
||||
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"the baseline before the drift matrix must be the current V16 schema"
|
||||
);
|
||||
let mut observations: Vec<String> = Vec::new();
|
||||
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
|
||||
.await
|
||||
.expect("restoring the exact definition must apply");
|
||||
let restored = MigrationAuthority::preflight(&pool).await;
|
||||
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 23 }));
|
||||
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 }));
|
||||
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
|
||||
// мнением того же guard-а, который принимает неправильные определения (F4).
|
||||
let restored_exact = catalog_definition_snapshot(&pool).await == before;
|
||||
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"baseline before inserting the shadow schema"
|
||||
);
|
||||
|
||||
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
// Тень не меняет вердикт guard-а по текущей схеме.
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"a same-named object in another schema must not change the verdict for the current schema"
|
||||
);
|
||||
|
||||
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 },
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
"the exact restoration must return the current V16 schema"
|
||||
);
|
||||
|
||||
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
let tables = sqlx::query(
|
||||
"select table_name
|
||||
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 13,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
|
||||
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||
assert_eq!(operation_count, 1);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
|
||||
assert!(supports_elicitation);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
|
||||
.fetch_one(&pool)
|
||||
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 18,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::apply(&pool).await.unwrap(),
|
||||
MigrationApplyResult::Applied { from: 18, to: 23 }
|
||||
MigrationApplyResult::Applied { from: 18, to: 25 }
|
||||
);
|
||||
let after: Vec<(i64, String, String)> =
|
||||
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
|
||||
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
|
||||
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 23 }
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -84,18 +84,18 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_v19_future_version").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
// The same `version > CURRENT_VERSION` branch an old V23 binary takes when it
|
||||
// meets a V24 database; a synthetic V24 row exercises it without a second binary.
|
||||
// The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer
|
||||
// database; a synthetic version above the current one exercises it without a second binary.
|
||||
sqlx::query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values (24, 'future-v24', repeat('a', 64), 'expand', 'n-minus-one-readable')",
|
||||
values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||
assert_eq!(error.code(), "future_version");
|
||||
assert_eq!(error.version(), Some(24));
|
||||
assert_eq!(error.version(), Some(26));
|
||||
assert_eq!(error.recovery(), "install_matching_application");
|
||||
assert!(
|
||||
MigrationAuthority::require_current(&pool).await.is_err(),
|
||||
@@ -155,7 +155,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 18,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 11,
|
||||
target: 23,
|
||||
target: 25,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ pub(crate) async fn remove_v4_schema(pool: &sqlx::PgPool) {
|
||||
.unwrap();
|
||||
}
|
||||
pub(crate) async fn remove_v5_schema(pool: &sqlx::PgPool) {
|
||||
remove_v24_schema(pool).await;
|
||||
remove_v6_schema(pool).await;
|
||||
sqlx::raw_sql(
|
||||
"drop index if exists invocation_logs_workspace_operation_version_idx;
|
||||
@@ -154,6 +155,7 @@ pub(crate) async fn remove_v11_schema(pool: &sqlx::PgPool) {
|
||||
.unwrap();
|
||||
}
|
||||
pub(crate) async fn remove_v14_schema(pool: &sqlx::PgPool) {
|
||||
remove_v24_schema(pool).await;
|
||||
remove_v15_schema(pool).await;
|
||||
sqlx::raw_sql(
|
||||
"drop table if exists import_plan_applications;
|
||||
@@ -224,6 +226,7 @@ pub(crate) async fn remove_v18_schema(pool: &sqlx::PgPool) {
|
||||
.unwrap();
|
||||
}
|
||||
pub(crate) async fn remove_v19_schema(pool: &sqlx::PgPool) {
|
||||
remove_v24_schema(pool).await;
|
||||
remove_v20_schema(pool).await;
|
||||
sqlx::raw_sql(
|
||||
"drop table if exists resource_security_revocations;
|
||||
@@ -307,6 +310,7 @@ pub(crate) async fn remove_v20_schema(pool: &sqlx::PgPool) {
|
||||
.unwrap();
|
||||
}
|
||||
pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
|
||||
remove_v24_schema(pool).await;
|
||||
remove_v14_schema(pool).await;
|
||||
sqlx::raw_sql(
|
||||
"drop index if exists import_jobs_openapi_source_idx;
|
||||
@@ -323,3 +327,27 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
|
||||
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
|
||||
sqlx::raw_sql(
|
||||
"drop table if exists tasks cascade;
|
||||
delete from __crank_migrations where version = 25;",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Story 4.1: снять V24 (authoritative execution admission) перед возвратом к < 24.
|
||||
pub(crate) async fn remove_v24_schema(pool: &sqlx::PgPool) {
|
||||
remove_v25_schema(pool).await;
|
||||
sqlx::raw_sql(
|
||||
"drop table if exists execution_admission_entries;
|
||||
drop table if exists execution_admission_controllers;
|
||||
delete from __crank_migrations where version = 24;",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
@@ -0,0 +1,682 @@
|
||||
//! ATDD Story 4.2 (FR-22/FR-23): durable Task acceptance on a real PostgreSQL schema.
|
||||
//!
|
||||
//! These oracles drive the real store: Task + dispatch intent + queue slot in one transaction, replay
|
||||
//! and typed conflict before any capacity mutation, bounded shared capacity, the Task-linked intent
|
||||
//! guard of the legacy admission paths, one-winner terminal CAS with an immutable result, bounded
|
||||
//! retention, owner/authorization-scope isolation and fail-closed health.
|
||||
|
||||
use super::common;
|
||||
|
||||
use crank_core::{
|
||||
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence,
|
||||
ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore};
|
||||
use serde_json::json;
|
||||
use sqlx::{PgPool, Row};
|
||||
|
||||
const WORKSPACE: &str = "ws_default";
|
||||
const AGENT: &str = "agent_a";
|
||||
|
||||
fn limits(queue: usize) -> AdmissionLimits {
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: queue,
|
||||
}
|
||||
}
|
||||
|
||||
fn store_with(pool: PgPool, queue: usize) -> PostgresTaskStore {
|
||||
PostgresTaskStore::new(pool, limits(queue), Default::default()).expect("valid store config")
|
||||
}
|
||||
|
||||
fn unsampled_support() -> TaskSupportIds {
|
||||
TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00").unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> TaskAcceptRequest {
|
||||
let deadline = current_unix_ms() + deadline_in_ms;
|
||||
TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: WORKSPACE.to_owned(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: agent.to_owned(),
|
||||
authorization_scope: format!("credential:pk_{agent}"),
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 3,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: deadline,
|
||||
retain_until_unix_ms: deadline + 3_600_000,
|
||||
approval_reference: Some("appr_1".to_owned()),
|
||||
},
|
||||
scoped_key: key.to_owned(),
|
||||
input_fingerprint: fingerprint.to_owned(),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(),
|
||||
support: unsampled_support(),
|
||||
}
|
||||
}
|
||||
|
||||
fn access(agent: &str) -> TaskAccess {
|
||||
TaskAccess {
|
||||
workspace_id: WORKSPACE.to_owned(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: agent.to_owned(),
|
||||
authorization_scope: format!("credential:pk_{agent}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn key(seed: &str) -> String {
|
||||
format!("{seed:0<64}").chars().take(64).collect()
|
||||
}
|
||||
|
||||
fn digest(seed: &str) -> String {
|
||||
format!("{seed:0<64}").chars().take(64).collect()
|
||||
}
|
||||
|
||||
async fn row_counts(pool: &PgPool) -> (i64, i64) {
|
||||
let row = sqlx::query(
|
||||
"select
|
||||
(select count(*)::bigint from tasks) as tasks,
|
||||
(select count(*)::bigint from execution_admission_entries where task_id is not null)
|
||||
as intents",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(row.get("tasks"), row.get("intents"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acceptance_commits_task_intent_and_slot_together() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let request = request(&key("k1"), &digest("f1"), AGENT, 30_000);
|
||||
let task_id = request.task_id;
|
||||
let support = request.support.clone();
|
||||
|
||||
// Another connection holds both controller rows: the acceptance cannot commit yet.
|
||||
let mut blocker = registry.pool().begin().await.unwrap();
|
||||
sqlx::query(
|
||||
"insert into execution_admission_controllers (workspace_id) values ('*'), ($1)
|
||||
on conflict (workspace_id) do nothing",
|
||||
)
|
||||
.bind(WORKSPACE)
|
||||
.execute(&mut *blocker)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("update execution_admission_controllers set updated_at = updated_at")
|
||||
.execute(&mut *blocker)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let pool = registry.pool().clone();
|
||||
let acceptance = tokio::spawn(async move { store.accept(request).await });
|
||||
tokio::time::sleep(std::time::Duration::from_millis(150)).await;
|
||||
|
||||
// No partial Task, no intent and no accepted response exist before COMMIT.
|
||||
assert!(!acceptance.is_finished());
|
||||
assert_eq!(row_counts(&pool).await, (0, 0));
|
||||
|
||||
blocker.commit().await.unwrap();
|
||||
let acceptance = acceptance.await.unwrap().expect("acceptance commits");
|
||||
assert_eq!(acceptance.state, TaskState::Queued);
|
||||
assert!(!acceptance.replayed);
|
||||
assert_eq!(acceptance.task_id, task_id);
|
||||
assert_eq!(acceptance.support, support);
|
||||
assert_eq!(row_counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// A fresh connection and store read the committed pins/identities back.
|
||||
let reader = store_with(database.concurrent_registry(2).await.pool().clone(), 8);
|
||||
let record = reader.read(&access(AGENT), task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Queued);
|
||||
assert_eq!(record.pins.operation_version, 3);
|
||||
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
|
||||
assert_eq!(record.pins.approval_reference.as_deref(), Some("appr_1"));
|
||||
assert_eq!(record.support.request_id, support.request_id);
|
||||
assert_eq!(record.support.trace_id, support.trace_id);
|
||||
assert_eq!(record.support.traceparent, support.traceparent);
|
||||
assert!(!record.support.trace_sampled);
|
||||
// Secret-bearing input is classified before persistence.
|
||||
assert_eq!(record.input.value()["api_key"], json!("[redacted]"));
|
||||
assert_eq!(record.input.value()["report"], json!("q3"));
|
||||
assert!(record.result.is_none());
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replay_returns_the_same_task_and_one_slot() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let first = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.expect("first acceptance");
|
||||
let before = store.read(&access(AGENT), first.task_id).await.unwrap();
|
||||
|
||||
let replay = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.expect("replay returns the accepted Task");
|
||||
assert_eq!(replay.task_id, first.task_id);
|
||||
assert!(replay.replayed);
|
||||
assert_eq!(replay.support.request_id, first.support.request_id);
|
||||
assert_eq!(row_counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// Replay never rewrites the accepted row or its audit evidence.
|
||||
let after = store.read(&access(AGENT), first.task_id).await.unwrap();
|
||||
assert_eq!(after.version, before.version);
|
||||
assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms);
|
||||
assert_eq!(after.created_at_unix_ms, before.created_at_unix_ms);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replay_stays_available_with_a_full_queue_and_past_execution_deadline() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 1);
|
||||
|
||||
// A short deadline: the Task is still retained after its execution deadline elapsed.
|
||||
let first = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
|
||||
.await
|
||||
.expect("first acceptance");
|
||||
sqlx::query(
|
||||
"update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1",
|
||||
)
|
||||
.bind(first.task_id.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
|
||||
let replayed = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
|
||||
.await
|
||||
.expect("replay works with a full queue and an elapsed execution deadline");
|
||||
assert_eq!(replayed.task_id, first.task_id);
|
||||
assert!(replayed.replayed);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_different_fingerprint_on_the_same_key_is_a_typed_conflict() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let first = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.expect("first acceptance");
|
||||
let before = store.read(&access(AGENT), first.task_id).await.unwrap();
|
||||
|
||||
let conflict = store
|
||||
.accept(request(&key("k1"), &digest("f2"), AGENT, 30_000))
|
||||
.await
|
||||
.expect_err("different input is a conflict");
|
||||
assert_eq!(conflict, TaskError::FingerprintConflict);
|
||||
assert_eq!(row_counts(registry.pool()).await, (1, 1));
|
||||
|
||||
let after = store.read(&access(AGENT), first.task_id).await.unwrap();
|
||||
assert_eq!(after.version, before.version);
|
||||
assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_capacity_is_shared_and_overload_creates_nothing() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 1);
|
||||
let admission =
|
||||
PostgresExecutionAdmission::new(registry.pool().clone(), limits(1), Default::default())
|
||||
.unwrap();
|
||||
|
||||
// A durable enqueue decision and a durable Task share the same bounded queue budget.
|
||||
let waiting = admission
|
||||
.admit(AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: WORKSPACE.to_owned(),
|
||||
agent_key: AGENT.to_owned(),
|
||||
operation_id: "op_sync".to_owned(),
|
||||
operation_version: 1,
|
||||
},
|
||||
mode: AdmissionMode::Background,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_sync".to_owned(),
|
||||
expires_at_unix_ms: current_unix_ms() + 30_000,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(waiting, AdmissionOutcome::Enqueued(_)));
|
||||
|
||||
let overloaded = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.expect_err("queue is full");
|
||||
assert!(matches!(overloaded, TaskError::Overloaded { .. }));
|
||||
assert_eq!(row_counts(registry.pool()).await, (0, 0));
|
||||
assert_eq!(
|
||||
admission.snapshot().await.unwrap().waiting,
|
||||
1,
|
||||
"the durable refusal changed no capacity"
|
||||
);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_cleanup_and_cancel_never_remove_a_task_linked_intent() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
let admission =
|
||||
PostgresExecutionAdmission::new(registry.pool().clone(), limits(8), Default::default())
|
||||
.unwrap();
|
||||
|
||||
let accepted = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 1))
|
||||
.await
|
||||
.expect("acceptance");
|
||||
// The linked reserved row looks expired; the legacy TTL cleanup must ignore it.
|
||||
sqlx::query("update execution_admission_entries set expires_at = now() - interval '1 hour'")
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let ticket: i64 =
|
||||
sqlx::query("select ticket_id from execution_admission_entries where task_id = $1")
|
||||
.bind(accepted.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.get("ticket_id");
|
||||
let owner_token: String =
|
||||
sqlx::query("select owner_token from execution_admission_entries where ticket_id = $1")
|
||||
.bind(ticket)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.get("owner_token");
|
||||
|
||||
// A legacy attempt runs the bounded expiry cleanup; a foreign/own legacy cancel is refused.
|
||||
let _ = admission
|
||||
.admit(AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: WORKSPACE.to_owned(),
|
||||
agent_key: "agent_b".to_owned(),
|
||||
operation_id: "op_sync".to_owned(),
|
||||
operation_version: 1,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_sync".to_owned(),
|
||||
expires_at_unix_ms: current_unix_ms() + 50,
|
||||
})
|
||||
.await;
|
||||
assert!(!admission.cancel(ticket as u64, &owner_token).await.unwrap());
|
||||
assert_eq!(row_counts(registry.pool()).await, (1, 1));
|
||||
assert_eq!(
|
||||
store
|
||||
.read(&access(AGENT), accepted.task_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
TaskState::Queued
|
||||
);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_transition_has_one_winner_and_an_immutable_result() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let accepted = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let queued = store.read(&access(AGENT), accepted.task_id).await.unwrap();
|
||||
assert_eq!(queued.version, 1);
|
||||
|
||||
// Moving a queued Task straight to completed is not an allowed transition.
|
||||
let skipped = TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: accepted.task_id,
|
||||
expected_state: TaskState::Queued,
|
||||
expected_version: queued.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
error: None,
|
||||
};
|
||||
assert_eq!(
|
||||
store
|
||||
.finish(skipped)
|
||||
.await
|
||||
.expect_err("transition rejected"),
|
||||
TaskError::Invalid {
|
||||
message: "task state transition is not allowed".to_owned()
|
||||
}
|
||||
);
|
||||
|
||||
let started = TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: accepted.task_id,
|
||||
expected_state: TaskState::Queued,
|
||||
expected_version: queued.version,
|
||||
target_state: TaskState::Working,
|
||||
result: None,
|
||||
error: None,
|
||||
};
|
||||
assert_eq!(
|
||||
store
|
||||
.finish(started)
|
||||
.await
|
||||
.expect_err("working is not terminal"),
|
||||
TaskError::Invalid {
|
||||
message: "task finish target is not terminal".to_owned()
|
||||
}
|
||||
);
|
||||
sqlx::query("update tasks set state = 'working' where task_id = $1")
|
||||
.bind(accepted.task_id.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let working = store.read(&access(AGENT), accepted.task_id).await.unwrap();
|
||||
|
||||
let mut finishers = Vec::new();
|
||||
for marker in ["first", "second"] {
|
||||
let store = store.clone();
|
||||
let mut finish = TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: accepted.task_id,
|
||||
expected_state: TaskState::Working,
|
||||
expected_version: working.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"marker": marker})).unwrap()),
|
||||
error: None,
|
||||
};
|
||||
finish.target_state = TaskState::Completed;
|
||||
finishers.push(tokio::spawn(async move { store.finish(finish).await }));
|
||||
}
|
||||
let first = finishers.remove(0).await.unwrap();
|
||||
let second = finishers.remove(0).await.unwrap();
|
||||
let winners = [&first, &second]
|
||||
.iter()
|
||||
.filter(|result| result.is_ok())
|
||||
.count();
|
||||
assert_eq!(winners, 1, "exactly one concurrent CAS wins");
|
||||
let loser = if first.is_err() { &first } else { &second };
|
||||
assert_eq!(
|
||||
loser.as_ref().err().unwrap(),
|
||||
&TaskError::StateConflict,
|
||||
"the loser observes a rejected transition, not a rewrite"
|
||||
);
|
||||
|
||||
let terminal = store.read(&access(AGENT), accepted.task_id).await.unwrap();
|
||||
assert_eq!(terminal.state, TaskState::Completed);
|
||||
assert_eq!(terminal.version, working.version + 1);
|
||||
let marker = terminal.result.as_ref().unwrap().value()["marker"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
assert!(marker == "first" || marker == "second");
|
||||
|
||||
// A repeated terminal write and a terminal->terminal move are rejected.
|
||||
let repeated = TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: accepted.task_id,
|
||||
expected_state: TaskState::Completed,
|
||||
expected_version: terminal.version,
|
||||
target_state: TaskState::Failed,
|
||||
result: None,
|
||||
error: Some(TaskEnvelope::try_redacted(&json!({"reason": "x"})).unwrap()),
|
||||
};
|
||||
assert!(repeated.validate().is_err());
|
||||
|
||||
// Terminal Task and its dispatch intent left together.
|
||||
assert_eq!(row_counts(registry.pool()).await, (1, 0));
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_is_bounded_and_never_deletes_live_work() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let queued = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let expired_queued = store
|
||||
.accept(request(&key("k2"), &digest("f2"), AGENT, 1))
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1",
|
||||
)
|
||||
.bind(expired_queued.task_id.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
// A retained terminal Task whose retention has not expired yet and one that has.
|
||||
let terminal = store
|
||||
.accept(request(&key("k3"), &digest("f3"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"update tasks set state = 'completed', result_envelope = '{\"ok\":true}'::jsonb,
|
||||
retain_until = now() + interval '1 hour' where task_id = $1",
|
||||
)
|
||||
.bind(terminal.task_id.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(terminal.task_id.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let expired_terminal = store
|
||||
.accept(request(&key("k4"), &digest("f4"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"update tasks set state = 'cancelled', execution_deadline = now() - interval '2 hours',
|
||||
retain_until = now() - interval '1 hour'
|
||||
where task_id = $1",
|
||||
)
|
||||
.bind(expired_terminal.task_id.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(expired_terminal.task_id.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let report = store.purge_expired(64).await.unwrap();
|
||||
assert_eq!(report.expired_queued, 1);
|
||||
assert_eq!(report.expired_terminal, 1);
|
||||
|
||||
// Live queued work is never deleted or failed; the expired queued Task released its slot.
|
||||
assert_eq!(
|
||||
store
|
||||
.read(&access(AGENT), queued.task_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
TaskState::Queued
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.read(&access(AGENT), terminal.task_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
TaskState::Completed
|
||||
);
|
||||
assert!(matches!(
|
||||
store
|
||||
.read(&access(AGENT), expired_queued.task_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.state,
|
||||
TaskState::Failed
|
||||
));
|
||||
assert!(
|
||||
store
|
||||
.read(&access(AGENT), expired_terminal.task_id)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let remaining: i64 = sqlx::query("select count(*)::bigint as count from execution_admission_entries where task_id is not null")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.get("count");
|
||||
assert_eq!(remaining, 1, "only the live queued Task keeps its intent");
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreign_scope_reads_are_not_found_and_never_leak_payload() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let accepted = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Positive control: the owner reads its Task.
|
||||
assert!(store.read(&access(AGENT), accepted.task_id).await.is_ok());
|
||||
|
||||
// Foreign owner, foreign authorization scope and foreign workspace are all not-found.
|
||||
let mut foreign_owner = access(AGENT);
|
||||
foreign_owner.owner_id = "agent_b".to_owned();
|
||||
assert_eq!(
|
||||
store
|
||||
.read(&foreign_owner, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskError::NotFound
|
||||
);
|
||||
let mut foreign_scope = access(AGENT);
|
||||
foreign_scope.authorization_scope = "credential:pk_other".to_owned();
|
||||
assert_eq!(
|
||||
store
|
||||
.read(&foreign_scope, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskError::NotFound
|
||||
);
|
||||
let mut foreign_workspace = access(AGENT);
|
||||
foreign_workspace.workspace_id = "ws_other".to_owned();
|
||||
assert_eq!(
|
||||
store
|
||||
.read(&foreign_workspace, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskError::NotFound
|
||||
);
|
||||
|
||||
// The same caller key under a foreign owner creates its own Task, never a visible collision.
|
||||
let other = store
|
||||
.accept(request(&key("k1"), &digest("f1"), "agent_b", 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_ne!(other.task_id, accepted.task_id);
|
||||
assert_eq!(row_counts(registry.pool()).await, (2, 2));
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_bounded_envelope_is_rejected_before_persistence() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// Fresh schema: the first connection applies the migration, later pools reuse the same schema.
|
||||
let _migrated = database.registry().await;
|
||||
let registry = database.concurrent_registry(4).await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
assert!(
|
||||
TaskEnvelope::try_redacted(&json!({"blob": "x".repeat(64 * 1024)})).is_err(),
|
||||
"an oversized envelope is rejected before any accept"
|
||||
);
|
||||
|
||||
let mut invalid = request(&key("k1"), &digest("f1"), AGENT, 30_000);
|
||||
invalid.scoped_key = "short".to_owned();
|
||||
assert!(matches!(invalid.validate(), Err(TaskError::Invalid { .. })));
|
||||
assert_eq!(
|
||||
store.accept(invalid).await.unwrap_err(),
|
||||
TaskError::Invalid {
|
||||
message: "task scoped key digest is invalid".to_owned()
|
||||
}
|
||||
);
|
||||
assert_eq!(row_counts(registry.pool()).await, (0, 0));
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_failure_fails_closed_without_records() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
// No migration on this schema: the durable Task state does not exist yet.
|
||||
let unmigrated = database.raw_pool().await;
|
||||
let store = store_with(unmigrated, 8);
|
||||
|
||||
assert!(store.health().await.is_err());
|
||||
let error = store
|
||||
.accept(request(&key("k1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.expect_err("acceptance fails closed");
|
||||
assert!(matches!(error, TaskError::Unavailable { .. }));
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
@@ -314,7 +314,7 @@ fn request_fingerprint(input: &Value) -> Result<String, RuntimeError> {
|
||||
Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded)))
|
||||
}
|
||||
|
||||
fn canonical_json(value: &Value) -> Value {
|
||||
pub(crate) fn canonical_json(value: &Value) -> Value {
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
let mut entries = object.iter().collect::<Vec<_>>();
|
||||
|
||||
@@ -15,6 +15,7 @@ mod rate_limit;
|
||||
mod request_context;
|
||||
mod request_preparation;
|
||||
mod response_cache;
|
||||
mod tasks;
|
||||
pub fn validate_resource_template_parameter(
|
||||
schema: &crank_schema::Schema,
|
||||
value: &serde_json::Value,
|
||||
@@ -114,3 +115,4 @@ pub use rate_limit::{
|
||||
};
|
||||
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
|
||||
pub use secret_crypto::SecretCrypto;
|
||||
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
|
||||
@@ -0,0 +1,266 @@
|
||||
//! Runtime application service of durable Tasks (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! The service is the only place that turns an approved background request into the durable domain
|
||||
//! request: bounded validated input, canonical input fingerprint, full-scope idempotency digest,
|
||||
//! pinned Published Version/deadline/retention and the support identities of the request. It holds an
|
||||
//! owned request, never `ResolvedAuth` or raw secrets, and it never dispatches: the durable store
|
||||
//! commits Task + dispatch intent + queue slot in one transaction and only then produces an
|
||||
//! acknowledgement.
|
||||
//!
|
||||
//! Reads go through the same owner-scoped access proof, so the durable row is only fetched for the
|
||||
//! caller that owns it. There is no in-memory fallback: a store failure is a typed refusal.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use crank_core::{
|
||||
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS,
|
||||
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds,
|
||||
ToolExecutionMode, current_unix_ms, validate_task_envelope,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// One owned background request, already authorized by the application layer.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct TaskAcceptDraft {
|
||||
pub workspace_id: String,
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
/// Server-derived authorization scope identity of the verified principal.
|
||||
pub authorization_scope: String,
|
||||
pub operation_id: String,
|
||||
pub operation_version: u32,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
/// Caller key, bounded before it becomes a full-scope digest.
|
||||
pub scoped_key: String,
|
||||
/// Canonical executable input; it is validated, fingerprinted, then persisted redacted.
|
||||
pub input: serde_json::Value,
|
||||
pub correlation: CorrelationContext,
|
||||
/// Absolute execution deadline of the accepted work.
|
||||
pub deadline: Instant,
|
||||
/// Retention after the execution deadline.
|
||||
pub retention_ms: i64,
|
||||
pub approval_reference: Option<String>,
|
||||
}
|
||||
|
||||
/// Projection of the durable status/result read. `result`/`error` are only present once a terminal
|
||||
/// transition committed; a non-terminal Task reports its state and inputs without inventing output.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskOutcome {
|
||||
pub task_id: TaskId,
|
||||
pub state: crank_core::TaskState,
|
||||
pub result: Option<serde_json::Value>,
|
||||
pub error: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl TaskOutcome {
|
||||
fn from_record(record: TaskRecord) -> Self {
|
||||
Self {
|
||||
task_id: record.task_id,
|
||||
state: record.state,
|
||||
result: record.result.map(|envelope| envelope.value().clone()),
|
||||
error: record.error.map(|envelope| envelope.value().clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct TaskService {
|
||||
store: Arc<dyn TaskStore>,
|
||||
}
|
||||
|
||||
impl TaskService {
|
||||
pub fn new(store: Arc<dyn TaskStore>) -> Self {
|
||||
Self { store }
|
||||
}
|
||||
|
||||
/// Readiness of the mandatory durable Task state.
|
||||
pub async fn health(&self) -> Result<(), TaskError> {
|
||||
self.store.health().await
|
||||
}
|
||||
|
||||
/// Durable acceptance: the returned identity exists only after the store committed it.
|
||||
pub async fn accept(&self, draft: TaskAcceptDraft) -> Result<TaskAcceptance, TaskError> {
|
||||
validate_task_envelope(&draft.input)?;
|
||||
let (scoped_key, input_fingerprint) = (
|
||||
scoped_key_digest(&draft)?,
|
||||
canonical_input_fingerprint(&draft.input)?,
|
||||
);
|
||||
let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline);
|
||||
let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: draft.workspace_id,
|
||||
owner_kind: draft.owner_kind,
|
||||
owner_id: draft.owner_id,
|
||||
authorization_scope: draft.authorization_scope,
|
||||
operation_id: draft.operation_id,
|
||||
pins: TaskPins {
|
||||
operation_version: draft.operation_version,
|
||||
policy_mode: draft.policy_mode,
|
||||
execution_deadline_unix_ms,
|
||||
retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms),
|
||||
approval_reference: draft.approval_reference,
|
||||
},
|
||||
scoped_key,
|
||||
input_fingerprint,
|
||||
input: TaskEnvelope::try_redacted(&draft.input)?,
|
||||
support: TaskSupportIds::new(
|
||||
draft.correlation.request_id().clone(),
|
||||
draft.correlation.trace_context(),
|
||||
),
|
||||
};
|
||||
self.store.accept(request).await
|
||||
}
|
||||
|
||||
/// Owner-scoped status read. A foreign or absent Task is one typed not-found.
|
||||
pub async fn status(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
self.store.read(access, task_id).await
|
||||
}
|
||||
|
||||
/// Owner-scoped result read, projected from the same authorized row.
|
||||
pub async fn result(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskOutcome, TaskError> {
|
||||
Ok(TaskOutcome::from_record(
|
||||
self.store.read(access, task_id).await?,
|
||||
))
|
||||
}
|
||||
|
||||
/// Default retention pinned by the application layer when it does not choose one explicitly.
|
||||
pub fn default_retention_ms() -> i64 {
|
||||
TASK_DEFAULT_RETENTION_MS
|
||||
}
|
||||
|
||||
/// Scoped lookup of an already accepted Task by the same full-scope identity, used to keep a
|
||||
/// retry of an explicitly pinned version from being accepted as new work.
|
||||
pub async fn lookup(&self, draft: &TaskAcceptDraft) -> Result<Option<TaskRecord>, TaskError> {
|
||||
let scoped_key = scoped_key_digest(draft)?;
|
||||
let access = TaskAccess {
|
||||
workspace_id: draft.workspace_id.clone(),
|
||||
owner_kind: draft.owner_kind,
|
||||
owner_id: draft.owner_id.clone(),
|
||||
authorization_scope: draft.authorization_scope.clone(),
|
||||
};
|
||||
self.store
|
||||
.find(&access, &draft.operation_id, &scoped_key)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Canonical digest of a validated input, exposed so the application layer can compare a stored
|
||||
/// Task against the request it is retrying without duplicating the canonicalization.
|
||||
pub fn input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
|
||||
canonical_input_fingerprint(input)
|
||||
}
|
||||
}
|
||||
|
||||
fn deadline_unix_ms(deadline: Instant) -> i64 {
|
||||
let remaining = u64::try_from(
|
||||
deadline
|
||||
.saturating_duration_since(Instant::now())
|
||||
.as_millis(),
|
||||
)
|
||||
.unwrap_or_default();
|
||||
current_unix_ms().saturating_add(i64::try_from(remaining.max(1)).unwrap_or(i64::MAX))
|
||||
}
|
||||
|
||||
/// Digest of the caller key over the full Task identity. The scope is never abbreviated to a
|
||||
/// connection, protocol revision or raw credential, so two callers cannot collide on one key.
|
||||
fn scoped_key_digest(draft: &TaskAcceptDraft) -> Result<String, TaskError> {
|
||||
let key = draft.scoped_key.as_str();
|
||||
if key.is_empty()
|
||||
|| key.len() > TASK_SCOPED_KEY_MAX_BYTES
|
||||
|| key.bytes().any(|byte| byte.is_ascii_control())
|
||||
{
|
||||
return Err(TaskError::invalid("task idempotency key is invalid"));
|
||||
}
|
||||
let material = format!(
|
||||
"{}\n{}\n{}\n{}\n{}\n{}",
|
||||
draft.workspace_id,
|
||||
draft.owner_kind.as_str(),
|
||||
draft.owner_id,
|
||||
draft.authorization_scope,
|
||||
draft.operation_id,
|
||||
key
|
||||
);
|
||||
Ok(hex_digest(material.as_bytes()))
|
||||
}
|
||||
|
||||
/// Canonical digest of the validated executable input. Object key order never changes the digest,
|
||||
/// while any material business or secret value does.
|
||||
fn canonical_input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
|
||||
let canonical = crate::idempotency::canonical_json(input);
|
||||
let encoded = serde_json::to_vec(&canonical)
|
||||
.map_err(|_| TaskError::invalid("task input is not canonicalizable"))?;
|
||||
Ok(hex_digest(&encoded))
|
||||
}
|
||||
|
||||
fn hex_digest(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn draft(key: &str, input: serde_json::Value) -> TaskAcceptDraft {
|
||||
TaskAcceptDraft {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
operation_id: "op_report".to_owned(),
|
||||
operation_version: 2,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
scoped_key: key.to_owned(),
|
||||
input,
|
||||
correlation: CorrelationContext::generate(),
|
||||
deadline: Instant::now() + std::time::Duration::from_secs(30),
|
||||
retention_ms: TASK_DEFAULT_RETENTION_MS,
|
||||
approval_reference: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_fingerprint_ignores_key_order_and_tracks_values() {
|
||||
let first = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 2}})).unwrap();
|
||||
let reordered = canonical_input_fingerprint(&json!({"b": {"c": 2}, "a": 1})).unwrap();
|
||||
assert_eq!(first, reordered);
|
||||
assert_eq!(first.len(), 64);
|
||||
|
||||
let changed = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 3}})).unwrap();
|
||||
assert_ne!(first, changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_key_digest_covers_the_whole_identity() {
|
||||
let base = scoped_key_digest(&draft("key_1", json!({}))).unwrap();
|
||||
let mut foreign = draft("key_1", json!({}));
|
||||
foreign.owner_id = "agent_b".to_owned();
|
||||
assert_ne!(base, scoped_key_digest(&foreign).unwrap());
|
||||
|
||||
let mut other_scope = draft("key_1", json!({}));
|
||||
other_scope.authorization_scope = "credential:pk_2".to_owned();
|
||||
assert_ne!(base, scoped_key_digest(&other_scope).unwrap());
|
||||
|
||||
assert!(scoped_key_digest(&draft("", json!({}))).is_err());
|
||||
assert!(
|
||||
scoped_key_digest(&draft(
|
||||
&"k".repeat(TASK_SCOPED_KEY_MAX_BYTES + 1),
|
||||
json!({})
|
||||
))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
# Переход Crank на официальный MCP SDK rmcp
|
||||
|
||||
Статус: планирование одобрено владельцем 27.10.2026. Полная миграция и production-переключение этим планом не запускаются. Владелец планирования и очереди — hermes_tl; реализация — hermes_dev; независимая проверка — hermes_qa; Git/CI/релизы — hermes_devops. ARC подключается к конкретному техническому затруднению, не как обязательный новый этап.
|
||||
|
||||
## Результат
|
||||
|
||||
Стандартный протокольный слой MCP обслуживает официальный SDK. Crank сохраняет авторизацию, права Agent/workspace, опубликованные версии, REST mapping, admission и исполнение команд. Обе действующие версии протокола и функции Tools/Resources/Prompts остаются доступны. Снижается объём собственного протокольного кода; ускорение и производительность не объявляются измеренными заранее.
|
||||
|
||||
Исходная точка: принятый main после 4.1, известный merge cc612516a29d48f2831ba6bdf58c1dc777aa6abf. Перед исполнением получить актуальный main и убедиться, что в него вошла проверенная поставка. Плановый кандидат SDK — опубликованный rmcp 3.7.0; окончательный pin и lock проверяются на первом этапе, автоматического обновления на другую версию нет.
|
||||
|
||||
## 1. Проверить тонкий адаптер на реальном MCP-пути
|
||||
|
||||
Исполнитель: DEV, независимая QA. Риск: высокий — ошибка может нарушить доступ к инструментам.
|
||||
|
||||
Сделать изолированный исполняемый прототип SDK на существующей границе типизированных команд, сохранив авторизацию и динамический каталог. Публичный production endpoint пока обслуживает прежний путь.
|
||||
|
||||
Проверяемый результат:
|
||||
|
||||
- Динамические tools/list и tools/call работают через SDK с действующими сервисами Crank, включая запрет чужого Agent/неверных credentials и сохранение ошибок инструмента.
|
||||
- Через настоящий HTTP-путь проверены обе revisions: legacy initialize/session и modern discover/stateless, JSON/SSE, headers, progress и cancellation.
|
||||
- Отдельно проверены пригодность SessionManager для существующего хранения сессий, переподключение и владение отменой; наличие SDK-типа само по себе не считается доказательством.
|
||||
- Установлены существенные gaps для Resources/Prompts/notifications/subscriptions и текущих execution modes. Наличие интерфейса не выдаётся за полную совместимость.
|
||||
- На небольшом smoke-наборе проверено, что adapter не обходит действующие ограничения; замеченный риск памяти/очередей записан. Нагрузочная квалификация относится к завершённой функции, не превращается в условие этого прототипа. Существующие ceilings не меняются.
|
||||
- Результат — работающий код и краткое решение TL: готов перенос либо конкретная несовместимость с минимальным вариантом решения. Если потребуется ослабить контракт, заменить хранилище или создать большой дополнительный framework — вопрос владельцу до продолжения.
|
||||
|
||||
Это один достаточный опыт, а не серия исследований SDK. Использовать существующие wire fixtures и точечные проверки, сохранять непроверенные пункты явно.
|
||||
|
||||
## 2. Перенести действующие MCP-функции на SDK
|
||||
|
||||
Зависит от положительного результата этапа 1 и решения TL о достаточности совместимости. Исполнитель: DEV, независимая QA. Риск: высокий — затрагивается общий клиентский интерфейс.
|
||||
|
||||
Подключить SDK к действующему пути и перенести стандартные envelopes, dispatch, lifecycle и transport. Порядок переноса устанавливается по результату первого опыта, без параллельных writers одних MCP-файлов.
|
||||
|
||||
Проверяемый результат:
|
||||
|
||||
- Через SDK проходят действующие Tools, Resources, Prompts, completion, уведомления и подписки в применимых revisions, а также progress/cancel и существующая logging policy.
|
||||
- Сохранены авторизация/отзыв доступа, agent scope, advertised capabilities, admission, deadlines и семантика ошибок. SDK defaults не расширяют объявленные capabilities или список разрешённых revisions.
|
||||
- Сессии, reconnect и ограничения памяти/очередей сохраняют действующий контракт; in-memory хранилище не заменяет существующее долговременное хранение без решения.
|
||||
- Перенесённые участки собственного протокольного кода удаляются по мере принятия. Временный способ безопасного сравнения/отката не превращается во второй постоянный MCP runtime.
|
||||
- Существующие affected ATDD подтверждают старое поведение; тесты меняются только там, где проверяли приватное устройство старого адаптера, а не нужный клиенту контракт.
|
||||
|
||||
Новые бизнес-возможности, переработка REST runtime, базы, модели или UI в эту работу не входят.
|
||||
|
||||
## 3. Завершить замену и поставить результат
|
||||
|
||||
Зависит от принятого этапа 2. Исполнители: DEV/QA, затем DevOps. Риск: высокий — финальный переход общего MCP-пути. Внешняя проверка относительно реализации: независимая QA на существующем клиентском/wire-наборе обеих revisions.
|
||||
|
||||
Проверяемый результат:
|
||||
|
||||
- Удалены ставшие ненужными собственные parser/dispatch/transport участки, временные переключатели и неиспользуемые зависимости. Поддержка legacy protocol при этом сохраняется средствами SDK.
|
||||
- Один необходимый финальный feature-прогон подтверждает реальный клиентский путь обеих revisions, изоляцию доступа, streaming/cancel/session и применимые ограничения ресурсов. Полный набор не повторяется отдельно каждой ролью; обязательные CI-гейты сохраняются.
|
||||
- DevOps выполнил commit/push/PR, protected merge и честный release с проверенными артефактами. Документация описывает активный SDK, поддерживаемые версии и реальные ограничения.
|
||||
- План отката указывает предыдущую принятую версию и проверяет совместимость состояния. Production-переключение выполняется только отдельным разрешённым manual deployment по существующему runbook; эта карточка не даёт разрешения на выкатку.
|
||||
|
||||
## Порядок относительно текущей разработки
|
||||
|
||||
1. Закончить уже выполняемую проверку поставки 4.1, не открывать её разработку заново.
|
||||
2. Планировать SDK-адаптер перед дальнейшим расширением заменяемого transport/protocol слоя. Этап 1 не ждёт завершения будущих 4.2–4.7 или всей нагрузочной кампании.
|
||||
3. Доменные части будущих задач могут идти параллельно при отсутствии общего writer и явных интерфейсных зависимостях. Новую протокольную реализацию и миграцию одних файлов параллельно не вести.
|
||||
4. TL сопоставляет существующие 4.2–4.7 с SDK, чтобы не реализовать вручную уже предоставленный механизм. Действующие продуктовые AC не сокращаются. Неизвестная совместимость — проверка этапа 1, а не повод считать будущие функции готовыми.
|
||||
|
||||
## Постановка и контроль
|
||||
|
||||
Достаточно трёх последовательных исполнений по этапам выше, с same-card QA и действующей приёмкой TL; не создавать отдельный новый бизнес-эпик, PRD, новый тестовый framework или цепочку архитектурных согласований.
|
||||
|
||||
На текущем шаге TL фиксирует эти этапы в существующем backlog/Gitea, зависимости и приоритет относительно текущей работы. Будущие исполнительские карточки оставить в triage/backlog; не запускать полную миграцию автоматически из задачи планирования. Сроки не выдумывать до первого опыта. Подтверждённая непригодность SDK эскалируется владельцу с фактами и вариантами, а не молчаливым возвращением к самописному MCP.
|
||||
|
||||
## Основания
|
||||
|
||||
Канонические пути ниже находятся на Hermes `192.168.1.127` (SSH-пользователь `codex`); относительные пути считаются от `/home/codex/code/crank`. Это доступный TL источник R, а не текущая локальная папка оператора. Локальная копия аудита у оператора: `/home/bsodfather/code/hermes/deploy/mcp-sdk-audit-20261027/REPORT.md`.
|
||||
|
||||
- Решение владельца 27.10.2026: запланировать переход на rmcp.
|
||||
- Аудит: `/home/codex/.hermes/codex-operator/handoffs/crank-rmcp-audit-20261027.md`.
|
||||
- Действующие AC: `_bmad-output/initiative-crank/epic-mcp-capabilities/story-mcp-revision-capabilities.md`, в том числе оценка SDK AT-3.1-14; остальные MCP AC остаются действующими.
|
||||
- Официальный SDK: https://github.com/modelcontextprotocol/rust-sdk ; документация ServerHandler: https://docs.rs/rmcp/latest/rmcp/handler/server/trait.ServerHandler.html .
|
||||
- Общий процесс: `/home/codex/.hermes/team/development-delivery.md`, `validation-policy.md`, `delivery-efficiency.md`.
|
||||
|
||||
Предыдущее default-правило «сохранять собственный transport» не является техническим доказательством отказа от SDK. Цель нового плана — проверить и принять SDK-путь; продуктовые и security-контракты сохраняются.
|
||||
@@ -35,3 +35,19 @@ version, stage, stable error code, retryability и certainty. Для legacy v4
|
||||
Пути Admin API, MCP methods и успешные response fields не изменены. Поля ошибки
|
||||
добавлены обратно совместимо. Outbound DNS/redirect/retry/SSRF policy принадлежит
|
||||
Story 1.9 и этой границей не считается завершённой.
|
||||
|
||||
## Durable background Task (Story 4.2)
|
||||
|
||||
Принятие фоновой работы идёт через `TaskApplication → crank-runtime::TaskService →
|
||||
TaskStore`. PostgreSQL-реализация выполняет одну транзакцию: Task, dispatch intent
|
||||
(`execution_admission_entries.task_id`) и queue slot коммитятся вместе, и только
|
||||
после commit выдаётся ack с Task ID/поддержкой Request/Trace/parent TraceContext.
|
||||
Повтор того же полного scoped key возвращает сохранённую Task, другой canonical
|
||||
input fingerprint — типизированный конфликт; явная другая published-версия при том
|
||||
же ключе не перезакрепляет уже принятую работу. Owner-скоуп (workspace, Agent/admin,
|
||||
authorization scope) входит в каждый read; чужой Task неотличим от отсутствующего.
|
||||
|
||||
Границы этой story: публичный MCP wire Tasks/capabilities/activation остаётся
|
||||
Story 4.7, worker/leases/recovery — Story 4.3. Существующий `tools/call` сохраняет
|
||||
synchronous/unsupported поведение; durable acceptance не запускает ephemeral
|
||||
execution и не рекламирует клиенту background capability.
|
||||
@@ -1 +1 @@
|
||||
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
|
||||
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
|
||||
@@ -67,3 +67,14 @@ run_atdd "execution policy validation" -p admin-api --lib operation_validation::
|
||||
run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1
|
||||
run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1
|
||||
run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1
|
||||
|
||||
# Story 4.2 (FR-22, FR-23): durable background Tasks. The lifecycle/envelope bounds are unit-level; the
|
||||
# authoritative store (one-transaction acceptance, replay/typed conflict, shared bounded capacity,
|
||||
# Task-linked intent guard, terminal CAS, retention, scope isolation, fail-closed health) and the
|
||||
# application entry (verified principal, exact publication pin, foreign-scope denial, readiness) run
|
||||
# against a real PostgreSQL schema. The migration suite proves a fresh and an already-migrated v25.
|
||||
run_atdd "durable task domain bounds" -p crank-core --lib task:: -- --test-threads=1
|
||||
run_atdd "runtime task identity" -p crank-runtime --lib tasks:: -- --test-threads=1
|
||||
run_atdd "durable task store" -p crank-registry --test integration tasks:: -- --test-threads=1
|
||||
run_atdd "durable task application entry" -p mcp-server --test integration durable_tasks:: -- --test-threads=1
|
||||
run_atdd "durable task migration" -p crank-registry --test integration migrations:: -- --test-threads=1
|
||||
Reference in new issue
Block a user