From 2219a7217d60e149cec4583835c92e3644ff443b Mon Sep 17 00:00:00 2001 From: bsodfather Date: Tue, 27 Oct 2026 00:09:36 +0000 Subject: [PATCH] =?UTF-8?q?feat(epic4):=20=D0=BF=D0=BE=D0=B4=D1=82=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=B6=D0=B4=D0=B0=D1=82=D1=8C=20durable=20Task=20?= =?UTF-8?q?=D0=BF=D0=BE=D1=81=D0=BB=D0=B5=20=D0=B0=D1=82=D0=BE=D0=BC=D0=B0?= =?UTF-8?q?=D1=80=D0=BD=D0=BE=D0=B3=D0=BE=20PG-commit=20(Story=204.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Application entry выдаёт Task ID только после атомарного PostgreSQL commit Task+intent+capacity; replay/conflict, owner scope, bounded envelope, terminal CAS и fail-closed readiness проверены на реальном PG. Также включён принятый план перехода на rmcp (только документ, миграция не запускается). --- apps/mcp-server/src/main.rs | 33 +- apps/mcp-server/tests/integration.rs | 1 + .../tests/integration/durable_tasks.rs | 591 ++++++++++++++ crates/crank-community-mcp/src/app.rs | 76 +- crates/crank-community-mcp/src/app/tasks.rs | 295 +++++++ crates/crank-community-mcp/src/lib.rs | 3 +- crates/crank-core/src/lib.rs | 16 + crates/crank-core/src/task.rs | 730 ++++++++++++++++++ crates/crank-registry/src/lib.rs | 4 +- crates/crank-registry/src/migrations.rs | 2 + .../src/migrations/authority.rs | 14 +- .../src/migrations/durable_tasks_v25.rs | 45 ++ .../src/migrations/durable_tasks_v25.sql | 90 +++ .../src/migrations/schema_guard.rs | 10 +- .../src/migrations/schema_guard_v24.rs | 33 +- .../src/migrations/schema_guard_v25.rs | 247 ++++++ .../crank-registry/src/postgres/admission.rs | 206 +++-- .../crank-registry/src/postgres/connection.rs | 5 + crates/crank-registry/src/postgres/mod.rs | 3 + crates/crank-registry/src/postgres/tasks.rs | 586 ++++++++++++++ crates/crank-registry/tests/integration.rs | 1 + .../tests/integration/migrations.rs | 18 +- .../migrations/artifact_metadata.rs | 12 +- .../artifact_metadata/exhaustive_v12.rs | 16 +- .../integration/migrations/import_plan.rs | 4 +- .../integration/migrations/legacy_adoption.rs | 16 +- .../resource_security_revocations.rs | 18 +- .../tests/integration/migrations/rollback.rs | 2 +- .../integration/migrations/schema_removal.rs | 28 + .../crank-registry/tests/integration/tasks.rs | 682 ++++++++++++++++ crates/crank-runtime/src/idempotency.rs | 2 +- crates/crank-runtime/src/lib.rs | 2 + crates/crank-runtime/src/tasks.rs | 266 +++++++ docs/development/rmcp-migration-plan.md | 78 ++ docs/execution-boundary.md | 16 + docs/schemas/migration-sequence.json | 2 +- scripts/ci-story-checks.sh | 11 + 37 files changed, 4028 insertions(+), 136 deletions(-) create mode 100644 apps/mcp-server/tests/integration/durable_tasks.rs create mode 100644 crates/crank-community-mcp/src/app/tasks.rs create mode 100644 crates/crank-core/src/task.rs create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v25.rs create mode 100644 crates/crank-registry/src/migrations/durable_tasks_v25.sql create mode 100644 crates/crank-registry/src/migrations/schema_guard_v25.rs create mode 100644 crates/crank-registry/src/postgres/tasks.rs create mode 100644 crates/crank-registry/tests/integration/tasks.rs create mode 100644 crates/crank-runtime/src/tasks.rs create mode 100644 docs/development/rmcp-migration-plan.md diff --git a/apps/mcp-server/src/main.rs b/apps/mcp-server/src/main.rs index 805bcd9..8a0e72a 100644 --- a/apps/mcp-server/src/main.rs +++ b/apps/mcp-server/src/main.rs @@ -1,7 +1,7 @@ use std::{io, process::ExitCode, time::Duration}; use crank_community_mcp::{ - auth::CommunityMachineCredentialVerifier, build_app_with_artifacts, + auth::CommunityMachineCredentialVerifier, build_app_with_artifacts_and_tasks, session::PostgresTransportSessionStore, }; use crank_config::{ @@ -15,7 +15,7 @@ use crank_observability::{ }; use crank_registry::{ AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, - PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, + PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, PostgresTaskStore, }; use crank_runtime::{ RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores, @@ -201,7 +201,21 @@ async fn run( // admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root // fail-closed per read instead of preventing startup; nothing is ever written from this process. let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone()); - let app = build_app_with_artifacts( + // Story 4.2: the durable Task application entry shares the registry pool, the published catalog + // refresh interval and the exact admission limits of the runtime; the public wire stays + // synchronous/unsupported and no worker is started. + let task_store = PostgresTaskStore::new( + registry.pool().clone(), + admission_limits(runtime_limits), + crank_core::AdmissionSchedulerConfig::default(), + )?; + let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new( + registry.clone(), + Duration::from_millis(config.refresh_ms), + cache_stores.coordination.clone(), + crank_runtime::TaskService::new(std::sync::Arc::new(task_store)), + )); + let app = build_app_with_artifacts_and_tasks( registry, Duration::from_millis(config.refresh_ms), config.runtime.base_url.clone(), @@ -217,6 +231,7 @@ async fn run( std::sync::Arc::new(CommunityMachineCredentialVerifier), runtime_limits.max_concurrent_sessions, artifact_store, + tasks, ); let listener = TcpListener::bind(config.bind_addr).await?; @@ -293,16 +308,20 @@ fn init_observability( )?) } +fn admission_limits(runtime_limits: RuntimeLimits) -> AdmissionLimits { + AdmissionLimits { + global_limit: runtime_limits.max_concurrent_unary, + ..AdmissionLimits::default() + } +} + fn execution_admission( pool: sqlx::PgPool, runtime_limits: RuntimeLimits, ) -> Result, Box> { let store = PostgresExecutionAdmission::new( pool, - AdmissionLimits { - global_limit: runtime_limits.max_concurrent_unary, - ..AdmissionLimits::default() - }, + admission_limits(runtime_limits), crank_core::AdmissionSchedulerConfig::default(), )?; Ok(std::sync::Arc::new(store)) diff --git a/apps/mcp-server/tests/integration.rs b/apps/mcp-server/tests/integration.rs index f7f357e..e727e61 100644 --- a/apps/mcp-server/tests/integration.rs +++ b/apps/mcp-server/tests/integration.rs @@ -1,6 +1,7 @@ mod integration { mod catalog_access; mod common; + mod durable_tasks; mod era_isolation; mod execution_certainty; mod execution_stages; diff --git a/apps/mcp-server/tests/integration/durable_tasks.rs b/apps/mcp-server/tests/integration/durable_tasks.rs new file mode 100644 index 0000000..9b4c867 --- /dev/null +++ b/apps/mcp-server/tests/integration/durable_tasks.rs @@ -0,0 +1,591 @@ +//! ATDD Story 4.2 (FR-22/FR-23): the durable Task application entry over a real published catalog +//! and a real PostgreSQL schema. +//! +//! These oracles drive the production application entry (`TaskApplication`) that the mcp-server +//! composition root constructs: a verified machine credential is checked against the published Tool, +//! the Task is accepted durably, a freshly constructed application instance reads the same identity +//! and pins, replay/version conflicts never rebind, foreign callers receive no metadata, and a missing +//! Task store fails closed and reports readiness as not ready. The public `tools/call` wire stays +//! synchronous/unsupported: this suite never calls it. + +use super::common; + +use std::time::Duration; + +use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal}; +use crank_core::{ + AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState, + ToolExecutionMode, ToolExecutionPolicy, +}; +use crank_registry::{ + AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest, +}; +use crank_runtime::{InMemoryCoordinationStateStore, TaskService}; +use serde_json::json; +use sqlx::{PgPool, Row}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; + +const AGENT_SLUG: &str = "reports"; +const TOOL_NAME: &str = "report_gen"; +const KEY: &str = "retry-key-1"; + +fn operation(policy: ToolExecutionPolicy) -> crank_registry::RegistryOperation { + let mut operation = common::test_operation("https://api.example.com", TOOL_NAME); + operation.execution_config.execution_policy = Some(policy); + operation +} + +fn background_policy() -> ToolExecutionPolicy { + ToolExecutionPolicy { + mode: ToolExecutionMode::Background, + deadline_ms: 60_000, + no_tasks_fallback: NoTasksFallback::Unsupported, + } +} + +async fn seed_tool(registry: &PostgresRegistry, policy: ToolExecutionPolicy) { + let operation = operation(policy); + registry + .create_operation(&common::test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish(registry, &operation.id, 1).await; + // A second, explicitly synchronous Tool of the same Agent: the durable entry must refuse it. + let synchronous = common::test_operation("https://api.example.com", "report_sync"); + registry + .create_operation(&common::test_workspace_id(), &synchronous, Some("alice")) + .await + .unwrap(); + publish(registry, &synchronous.id, 1).await; + let binding = AgentOperationBinding { + operation_id: operation.id.clone(), + operation_version: 1, + tool_name: operation.name.clone(), + tool_title: operation.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }; + let synchronous_binding = AgentOperationBinding { + operation_id: synchronous.id.clone(), + operation_version: 1, + tool_name: synchronous.name.clone(), + tool_title: synchronous.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }; + common::publish_agent_with_bindings(registry, AGENT_SLUG, vec![binding, synchronous_binding]) + .await; +} + +async fn publish( + registry: &PostgresRegistry, + operation_id: &crank_core::OperationId, + version: u32, +) { + registry + .publish_operation(PublishRequest { + workspace_id: &common::test_workspace_id(), + operation_id, + version, + published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(), + published_by: Some("alice"), + }) + .await + .unwrap(); +} + +async fn publish_tool_version(registry: &PostgresRegistry, version: u32) { + let mut next = operation(background_policy()); + next.version = version; + next.status = crank_core::OperationStatus::Draft; + next.published_at = None; + registry + .create_version(CreateVersionRequest { + workspace_id: &common::test_workspace_id(), + snapshot: &next, + change_note: Some("next publication"), + created_by: Some("alice"), + }) + .await + .unwrap(); + publish(registry, &next.id, version).await; + // The agent catalog must bind the published Tool version for the new publication to be addressed. + let agent_id = common::test_agent_id(AGENT_SLUG); + let draft = crank_core::AgentVersion { + agent_id: agent_id.clone(), + version: 2, + status: crank_core::AgentStatus::Draft, + instructions: json!({}), + tool_selection_policy: Default::default(), + created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(), + }; + registry + .create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest { + workspace_id: &common::test_workspace_id(), + agent_id: &agent_id, + version: &draft, + bindings: &[AgentOperationBinding { + operation_id: next.id.clone(), + operation_version: version, + tool_name: next.name.clone(), + tool_title: next.tool_description.title.clone(), + tool_description_override: None, + enabled: true, + }], + updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(), + }) + .await + .unwrap(); + registry + .publish_agent(crank_registry::PublishAgentRequest { + workspace_id: &common::test_workspace_id(), + agent_id: &agent_id, + version: 2, + published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(), + published_by: Some("alice"), + expected_state: None, + }) + .await + .unwrap(); +} + +fn task_service(pool: PgPool) -> TaskService { + let store = PostgresTaskStore::new( + pool, + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: 8, + }, + Default::default(), + ) + .expect("valid task store configuration"); + TaskService::new(std::sync::Arc::new(store)) +} + +fn application(registry: &PostgresRegistry) -> TaskApplication { + TaskApplication::new( + registry.clone(), + Duration::from_millis(0), + std::sync::Arc::new(InMemoryCoordinationStateStore::default()), + task_service(registry.pool().clone()), + ) +} + +async fn principal( + registry: &PostgresRegistry, + name: &str, + scopes: &[PlatformApiKeyScope], +) -> TaskPrincipal { + let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await; + // The production machine-access path resolves the presented secret against the active platform + // key of the addressed Agent and derives the scope identity from that verified record. + let secret_hash = common::hash_access_secret(&token); + let api_key = registry + .get_platform_api_key_by_secret_for_agent_slug( + common::test_workspace_slug(), + AGENT_SLUG, + &secret_hash, + ) + .await + .unwrap() + .expect("an active platform key resolves"); + registry + .touch_platform_api_key( + &api_key.api_key.workspace_id, + &api_key.api_key.id, + &OffsetDateTime::now_utc(), + ) + .await + .unwrap(); + TaskPrincipal::Agent { + workspace_slug: common::test_workspace_slug().to_owned(), + agent_slug: AGENT_SLUG.to_owned(), + credential: crank_core::VerifiedMachineCredential { + machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey, + max_security_level: crank_core::OperationSecurityLevel::Standard, + scopes: api_key.api_key.scopes.clone(), + platform_api_key_id: Some(api_key.api_key.id.clone()), + }, + } +} + +fn unsampled_correlation() -> CorrelationContext { + CorrelationContext::new( + crank_core::RequestId::generate(), + crank_core::TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00") + .unwrap(), + ) +} + +async fn counts(pool: &PgPool) -> (i64, i64) { + let row = sqlx::query( + "select + (select count(*)::bigint from tasks) as tasks, + (select count(*)::bigint from execution_admission_entries where task_id is not null) + as intents", + ) + .fetch_one(pool) + .await + .unwrap(); + (row.get("tasks"), row.get("intents")) +} + +#[tokio::test] +async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() { + let registry = common::test_registry().await; + seed_tool(®istry, background_policy()).await; + let caller = principal( + ®istry, + "task-accept", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let correlation = unsampled_correlation(); + let app = application(®istry); + + let acceptance = app + .accept_background( + &caller, + TOOL_NAME, + None, + KEY, + &json!({"report": "q3"}), + &correlation, + ) + .await + .expect("acceptance over the real publication"); + assert_eq!(acceptance.state, TaskState::Queued); + assert!(!acceptance.replayed); + assert_eq!(acceptance.operation_version, 1); + assert_eq!(acceptance.support.request_id, *correlation.request_id()); + assert_eq!(acceptance.support.trace_id, *correlation.trace_id()); + assert!(!acceptance.support.trace_sampled); + assert_eq!( + uuid::Uuid::parse_str(&acceptance.task_id.to_string()) + .unwrap() + .get_version_num(), + 7, + "the durable Task id is UUIDv7" + ); + // Task, dispatch intent and queue slot committed together. + assert_eq!(counts(registry.pool()).await, (1, 1)); + + // A freshly constructed application + service reads the same durable identity and pins. + let reader = application(®istry); + let record = reader + .status(&caller, TOOL_NAME, acceptance.task_id) + .await + .unwrap(); + assert_eq!(record.state, TaskState::Queued); + assert_eq!(record.pins.operation_version, 1); + assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background); + assert_eq!(record.support.request_id, acceptance.support.request_id); + assert_eq!(record.support.traceparent, acceptance.support.traceparent); + assert!(!record.support.trace_sampled); + let outcome = reader + .result(&caller, TOOL_NAME, acceptance.task_id) + .await + .unwrap(); + assert!(outcome.result.is_none() && outcome.error.is_none()); + + // Replay of the same scoped key and input returns the accepted Task instead of a new one. + let replay = app + .accept_background( + &caller, + TOOL_NAME, + None, + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .unwrap(); + assert_eq!(replay.task_id, acceptance.task_id); + assert!(replay.replayed); + assert_eq!(counts(registry.pool()).await, (1, 1)); +} + +#[tokio::test] +async fn a_new_publication_never_rebinds_an_accepted_task() { + let registry = common::test_registry().await; + seed_tool(®istry, background_policy()).await; + let caller = principal( + ®istry, + "task-rebind", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let app = application(®istry); + let accepted = app + .accept_background( + &caller, + TOOL_NAME, + None, + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .unwrap(); + let before = app + .status(&caller, TOOL_NAME, accepted.task_id) + .await + .unwrap(); + + publish_tool_version(®istry, 2).await; + + // The same retry keeps the accepted Task and its v1 pin; it never becomes a v2 Task. + let replay = app + .accept_background( + &caller, + TOOL_NAME, + None, + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .unwrap(); + assert_eq!(replay.task_id, accepted.task_id); + assert_eq!(replay.operation_version, 1); + let after = app + .status(&caller, TOOL_NAME, accepted.task_id) + .await + .unwrap(); + assert_eq!(after.pins.operation_version, 1); + assert_eq!(after.support.request_id, before.support.request_id); + assert_eq!(counts(registry.pool()).await, (1, 1)); + + // An explicit other exact version is a typed conflict, not a rebind. + let explicit = app + .accept_background( + &caller, + TOOL_NAME, + Some(2), + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .expect_err("a different explicit pin cannot rebind the accepted Task"); + assert_eq!(explicit, TaskApplicationError::VersionUnavailable); + // The original explicit pin still resolves to the accepted Task. + let pinned = app + .accept_background( + &caller, + TOOL_NAME, + Some(1), + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .unwrap(); + assert_eq!(pinned.task_id, accepted.task_id); + assert_eq!(counts(registry.pool()).await, (1, 1)); +} + +#[tokio::test] +async fn foreign_and_insufficient_callers_receive_no_task_metadata() { + let registry = common::test_registry().await; + seed_tool(®istry, background_policy()).await; + let owner = principal( + ®istry, + "task-owner", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let app = application(®istry); + let accepted = app + .accept_background( + &owner, + TOOL_NAME, + None, + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .unwrap(); + // Positive control: the owner reads its Task. + assert!( + app.status(&owner, TOOL_NAME, accepted.task_id) + .await + .is_ok() + ); + + // A second verified key of the same Agent is another authorization scope: no payload, no metadata. + let foreign_scope = + principal(®istry, "task-other-scope", &[PlatformApiKeyScope::Read]).await; + match app + .status(&foreign_scope, TOOL_NAME, accepted.task_id) + .await + { + Err(TaskApplicationError::NotFound) => {} + other => panic!("foreign scope must be not-found without metadata, got {other:?}"), + } + assert!(matches!( + app.result(&foreign_scope, TOOL_NAME, accepted.task_id) + .await, + Err(TaskApplicationError::NotFound) + )); + + // A synchronous publication is not accepted as background work at all. + let synchronous_app = application(®istry); + let refused = synchronous_app + .accept_background( + &owner, + "report_sync", + None, + "sync-key", + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .expect_err("a synchronous policy has no durable acceptance"); + assert_eq!(refused, TaskApplicationError::NotBackground); + assert_eq!(counts(registry.pool()).await, (1, 1)); +} + +#[tokio::test] +async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() { + let registry = common::test_registry().await; + seed_tool(®istry, background_policy()).await; + let app = application(®istry); + let admin = TaskPrincipal::Admin { + workspace_slug: common::test_workspace_slug().to_owned(), + agent_slug: AGENT_SLUG.to_owned(), + user_id: "user_admin_1".to_owned(), + }; + let accepted = app + .accept_background( + &admin, + TOOL_NAME, + None, + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .expect("an authenticated administrator accepts durable work"); + assert_eq!(accepted.state, TaskState::Queued); + // Positive control: the same administrator reads its own Task. + assert!( + app.status(&admin, TOOL_NAME, accepted.task_id) + .await + .is_ok() + ); + + // An administrator is never a wildcard owner: another admin user and the owning Agent of the + // same workspace both see nothing. + let other_admin = TaskPrincipal::Admin { + workspace_slug: common::test_workspace_slug().to_owned(), + agent_slug: AGENT_SLUG.to_owned(), + user_id: "user_admin_2".to_owned(), + }; + assert_eq!( + app.status(&other_admin, TOOL_NAME, accepted.task_id) + .await + .unwrap_err(), + TaskApplicationError::NotFound + ); + assert_eq!( + app.result(&other_admin, TOOL_NAME, accepted.task_id) + .await + .unwrap_err(), + TaskApplicationError::NotFound + ); + let agent = principal(®istry, "task-admin-cross", &[PlatformApiKeyScope::Read]).await; + assert_eq!( + app.status(&agent, TOOL_NAME, accepted.task_id) + .await + .unwrap_err(), + TaskApplicationError::NotFound + ); + + // An admin identity addressed to another workspace cannot resolve the publication at all. + let foreign_workspace = TaskPrincipal::Admin { + workspace_slug: "other-workspace".to_owned(), + agent_slug: AGENT_SLUG.to_owned(), + user_id: "user_admin_1".to_owned(), + }; + assert_eq!( + app.accept_background( + &foreign_workspace, + TOOL_NAME, + None, + "foreign-key", + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .unwrap_err(), + TaskApplicationError::NotFound + ); + assert_eq!(counts(registry.pool()).await, (1, 1)); +} + +#[tokio::test] +async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() { + let database_url = crank_test_support::postgres_schema_url("durable_tasks_outage").await; + let pool = PgPool::connect(&database_url).await.unwrap(); + // No migration: the durable Task state does not exist. + let outage_registry = PostgresRegistry::connect(&database_url).await; + assert!( + outage_registry.is_err(), + "an unmigrated registry cannot be used" + ); + + crank_registry::MigrationAuthority::apply(&pool) + .await + .unwrap(); + let registry = PostgresRegistry::connect(&database_url).await.unwrap(); + seed_tool(®istry, background_policy()).await; + let caller = principal( + ®istry, + "task-outage", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let app = application(®istry); + assert!(app.health().await.is_ok()); + + // Fail closed: without the Task schema the entry accepts nothing and reports dependency failure. + sqlx::query("drop table tasks cascade") + .execute(&pool) + .await + .unwrap(); + assert!(app.health().await.is_err()); + let error = app + .accept_background( + &caller, + TOOL_NAME, + None, + KEY, + &json!({"report": "q3"}), + &unsampled_correlation(), + ) + .await + .expect_err("no durable Task store, no acceptance"); + assert!(matches!( + error, + TaskApplicationError::Task(crank_core::TaskError::Unavailable { .. }) + )); + + // Readiness of the app that wires the same store reports the durable Task dependency as not ready. + let router = common::build_test_app(registry.clone(), Duration::from_millis(0), None); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let response = reqwest::get(format!("http://{address}/ready")) + .await + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE); + let body: serde_json::Value = response.json().await.unwrap(); + assert_eq!(body["checks"]["durable_tasks"], json!("not_ready")); + server.abort(); +} diff --git a/crates/crank-community-mcp/src/app.rs b/crates/crank-community-mcp/src/app.rs index a76511e..2be7fad 100644 --- a/crates/crank-community-mcp/src/app.rs +++ b/crates/crank-community-mcp/src/app.rs @@ -51,12 +51,14 @@ mod auth_resolution; mod dispatch; mod endpoint; mod legacy_dispatch; +mod tasks; use self::approvals::{ approve_request, deny_request, get_approval_request, list_pending_approvals, }; pub(super) use self::auth_resolution::resolve_operation_auth; use self::endpoint::{mcp_delete, mcp_get, mcp_post}; +pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal}; // Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two // through `super`, and the plain lib build does not use them (same pattern as @@ -92,6 +94,9 @@ pub(super) struct AppState { pub(super) signal_streams: Arc, /// Story 3.4: the bounded table of the `tools/call` requests a legacy client can still cancel. pub(super) in_flight: Arc, + /// Story 4.2: durable background Tasks over the same registry pool. The public wire stays + /// synchronous/unsupported; readiness depends on this mandatory store. + tasks: Arc, } #[derive(Clone)] @@ -137,6 +142,7 @@ pub fn build_app( DEFAULT_MAX_CONCURRENT_SESSIONS, false, None, + None, ) } @@ -192,6 +198,7 @@ pub fn build_app_with_background_workers_and_limits( max_concurrent_sessions, true, None, + None, ) } @@ -227,6 +234,41 @@ pub fn build_app_with_artifacts( max_concurrent_sessions, true, Some(artifact_store), + None, + ) +} + +/// Production launcher entry point with the durable Task application entry constructed by the +/// process composition root from the same registry pool and admission limits as the runtime. +#[allow(clippy::too_many_arguments)] +pub fn build_app_with_artifacts_and_tasks( + registry: PostgresRegistry, + refresh_interval: Duration, + public_base_url: Option, + secret_crypto: SecretCrypto, + runtime: RuntimeExecutor, + api_rate_limiter: RequestRateLimiter, + coordination_store: Arc, + sessions: SharedSessionStore, + credential_verifier: SharedMachineCredentialVerifier, + max_concurrent_sessions: usize, + artifact_store: crank_artifacts::ArtifactStore, + tasks: Arc, +) -> Router { + build_app_inner( + registry, + refresh_interval, + public_base_url, + secret_crypto, + runtime, + api_rate_limiter, + coordination_store, + sessions, + credential_verifier, + max_concurrent_sessions, + true, + Some(artifact_store), + Some(tasks), ) } @@ -244,8 +286,23 @@ fn build_app_inner( max_concurrent_sessions: usize, start_background_workers: bool, artifact_store: Option, + tasks: Option>, ) -> Router { let session_metrics = ActiveSessionMetrics::start(Arc::clone(&sessions)); + let tasks = tasks.unwrap_or_else(|| { + let store = crank_registry::PostgresTaskStore::new( + registry.pool().clone(), + crank_registry::AdmissionLimits::default(), + crank_core::AdmissionSchedulerConfig::default(), + ) + .expect("default admission limits and scheduler weights are valid"); + Arc::new(TaskApplication::new( + registry.clone(), + refresh_interval, + Arc::clone(&coordination_store), + crank_runtime::TaskService::new(Arc::new(store)), + )) + }); let state = Arc::new(AppState { registry: registry.clone(), catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store), @@ -263,6 +320,7 @@ fn build_app_inner( in_flight: Arc::new(lifecycle::InFlightCalls::new( lifecycle::MAX_IN_FLIGHT_CALLS, )), + tasks, }); if start_background_workers { spawn_approval_recovery(Arc::clone(&state)); @@ -315,11 +373,17 @@ async fn readiness(State(state): State>) -> Response { // Story 4.1: admission state is a mandatory dependency of every execution, so readiness must // reflect it rather than reporting ready on a bare `select 1`. let admission_ready = state.registry.execution_admission_health().await.is_ok(); - if postgres_ready && admission_ready { + // Story 4.2: the durable Task store is mandatory for a durable acceptance acknowledgement. + let durable_tasks_ready = state.tasks.health().await.is_ok(); + if postgres_ready && admission_ready && durable_tasks_ready { return Json(json!({ "service": "mcp-server", "status": "ready", - "checks": { "postgres": "ready", "execution_admission": "ready" } + "checks": { + "postgres": "ready", + "execution_admission": "ready", + "durable_tasks": "ready" + } })) .into_response(); } @@ -336,6 +400,14 @@ async fn readiness(State(state): State>) -> Response { "not_ready" }), ); + checks.insert( + "durable_tasks".to_owned(), + json!(if durable_tasks_ready { + "ready" + } else { + "not_ready" + }), + ); ( StatusCode::SERVICE_UNAVAILABLE, Json(json!({ diff --git a/crates/crank-community-mcp/src/app/tasks.rs b/crates/crank-community-mcp/src/app/tasks.rs new file mode 100644 index 0000000..c25f85d --- /dev/null +++ b/crates/crank-community-mcp/src/app/tasks.rs @@ -0,0 +1,295 @@ +//! Application entry of durable background Tasks (Story 4.2, FR-22/FR-23). +//! +//! The entry takes an already authenticated principal (a verified machine credential, or an +//! administrator identity verified by the calling admin surface), resolves the exact published Tool, +//! repeats the caller's authorization, pins the Published Version/policy/deadline and delegates the +//! durable acceptance to the runtime Task service. The repository owns the transaction, so the +//! returned Task ID exists only after Task + dispatch intent + queue slot committed. +//! +//! Deliberately not wired: the public MCP `tools/call`/Tasks wire (Story 4.7) and the worker/leases +//! (Story 4.3). The existing synchronous/unsupported `tools/call` path is unchanged, and this entry +//! never spawns an ephemeral execution future. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crank_core::{ + CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord, + ToolExecutionMode, +}; +use crank_registry::{PostgresRegistry, PublishedAgentTool}; +use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService}; +use serde_json::Value; + +use crate::access::credential_allows_security_level; +use crate::catalog::PublishedToolCatalog; + +/// Authenticated principal of one Task operation. The authorization scope identity is derived here +/// from the verified credential/admin identity, never from a caller-supplied request string. +#[derive(Clone, Debug)] +pub enum TaskPrincipal { + Agent { + workspace_slug: String, + agent_slug: String, + credential: crank_core::VerifiedMachineCredential, + }, + /// Administrator identity that the admin surface already authenticated. It is never a wildcard: + /// owner, workspace and authorization scope all participate in the stored predicate. + Admin { + workspace_slug: String, + agent_slug: String, + user_id: String, + }, +} + +impl TaskPrincipal { + fn workspace_slug(&self) -> &str { + match self { + Self::Agent { workspace_slug, .. } | Self::Admin { workspace_slug, .. } => { + workspace_slug + } + } + } + + fn agent_slug(&self) -> &str { + match self { + Self::Agent { agent_slug, .. } | Self::Admin { agent_slug, .. } => agent_slug, + } + } + + fn owner(&self, tool: &PublishedAgentTool) -> (TaskOwnerKind, String) { + match self { + Self::Agent { .. } => (TaskOwnerKind::Agent, tool.agent_id.as_str().to_owned()), + Self::Admin { user_id, .. } => (TaskOwnerKind::Admin, user_id.clone()), + } + } + + /// Bounded server-derived authorization scope identity of this principal. + fn authorization_scope(&self) -> String { + let identity = match self { + Self::Agent { credential, .. } => { + let mut scopes = credential + .scopes + .iter() + .map(|scope| format!("{scope:?}")) + .collect::>(); + scopes.sort(); + format!( + "key:{}|mode:{:?}|level:{:?}|scopes:{}", + credential + .platform_api_key_id + .as_ref() + .map(|id| id.as_str().to_owned()) + .unwrap_or_else(|| "-".to_owned()), + credential.machine_access_mode, + credential.max_security_level, + scopes.join(",") + ) + } + Self::Admin { user_id, .. } => format!("admin:{user_id}"), + }; + identity.chars().take(160).collect() + } +} + +/// Typed refusal of the application entry. `NotFound`/`Denied` never reveal whether a foreign Task +/// exists, and no variant carries Task payload or fingerprint. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum TaskApplicationError { + #[error("task request was not found")] + NotFound, + #[error("task request is not authorized")] + Denied, + #[error("the published Tool policy does not accept durable background work")] + NotBackground, + #[error("the requested published Tool version is not available")] + VersionUnavailable, + #[error("this Tool requires a verified approval before acceptance")] + ApprovalRequired, + #[error(transparent)] + Task(#[from] TaskError), +} + +#[derive(Clone)] +pub struct TaskApplication { + catalog: PublishedToolCatalog, + service: TaskService, +} + +impl TaskApplication { + /// Production construction of the application entry over the process registry, the published + /// catalog and the durable Task service. + pub fn new( + registry: PostgresRegistry, + refresh_interval: Duration, + coordination_store: Arc, + service: TaskService, + ) -> Self { + Self { + catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store), + service, + } + } + + /// Readiness of the durable Task store this entry depends on. + pub async fn health(&self) -> Result<(), TaskError> { + self.service.health().await + } + + /// Accepts one background request for an already authenticated principal. + /// + /// `expected_version` is the caller's explicit Published Version pin. Without it the current + /// publication is pinned for new work and a retry of the same scoped key returns the stored Task + /// even after a newer publication. An explicit pin may only retry the Task accepted with that + /// exact version: a different pin is a typed conflict, never a rebind. + pub async fn accept_background( + &self, + principal: &TaskPrincipal, + tool_name: &str, + expected_version: Option, + scoped_key: &str, + input: &Value, + correlation: &CorrelationContext, + ) -> Result { + let tool = self.resolve_authorized_tool(principal, tool_name).await?; + let current_version = tool.operation.version; + let Some(policy) = tool.operation.execution_config.execution_policy else { + return Err(TaskApplicationError::NotBackground); + }; + if policy.mode == ToolExecutionMode::Synchronous { + return Err(TaskApplicationError::NotBackground); + } + if tool + .operation + .execution_config + .approval_policy + .as_ref() + .is_some_and(|approval| approval.required) + { + // Without a verified, non-consuming approval reference this entry fails closed instead + // of accepting work a worker could never safely execute. + return Err(TaskApplicationError::ApprovalRequired); + } + let draft = |operation_version: u32| { + let (owner_kind, owner_id) = principal.owner(&tool); + TaskAcceptDraft { + workspace_id: tool.workspace_id.as_str().to_owned(), + owner_kind, + owner_id, + authorization_scope: principal.authorization_scope(), + operation_id: tool.operation.id.as_str().to_owned(), + operation_version, + policy_mode: policy.mode, + scoped_key: scoped_key.to_owned(), + input: input.clone(), + correlation: correlation.clone(), + deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)), + retention_ms: TaskService::default_retention_ms(), + approval_reference: None, + } + }; + + let Some(requested) = expected_version else { + return Ok(self.service.accept(draft(current_version)).await?); + }; + let draft = draft(requested); + match self.service.lookup(&draft).await? { + Some(record) => { + if record.input_fingerprint != TaskService::input_fingerprint(input)? { + return Err(TaskApplicationError::Task(TaskError::FingerprintConflict)); + } + if record.pins.operation_version != requested { + return Err(TaskApplicationError::VersionUnavailable); + } + Ok(TaskAcceptance { + task_id: record.task_id, + state: record.state, + operation_version: record.pins.operation_version, + support: record.support, + created_at_unix_ms: record.created_at_unix_ms, + retain_until_unix_ms: record.pins.retain_until_unix_ms, + replayed: true, + }) + } + None if requested == current_version => Ok(self.service.accept(draft).await?), + // A new Task can only be accepted against the currently published version. + None => Err(TaskApplicationError::VersionUnavailable), + } + } + + /// Owner-scoped status read of one Task of the addressed published Tool. + pub async fn status( + &self, + principal: &TaskPrincipal, + tool_name: &str, + task_id: TaskId, + ) -> Result { + let access = self.task_access(principal, tool_name).await?; + match self.service.status(&access, task_id).await { + Ok(record) => Ok(record), + Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound), + Err(error) => Err(TaskApplicationError::Task(error)), + } + } + + /// Owner-scoped result projection of one Task of the addressed published Tool. + pub async fn result( + &self, + principal: &TaskPrincipal, + tool_name: &str, + task_id: TaskId, + ) -> Result { + let access = self.task_access(principal, tool_name).await?; + match self.service.result(&access, task_id).await { + Ok(outcome) => Ok(outcome), + Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound), + Err(error) => Err(TaskApplicationError::Task(error)), + } + } + + async fn task_access( + &self, + principal: &TaskPrincipal, + tool_name: &str, + ) -> Result { + let tool = self.resolve_authorized_tool(principal, tool_name).await?; + let (owner_kind, owner_id) = principal.owner(&tool); + Ok(TaskAccess { + workspace_id: tool.workspace_id.as_str().to_owned(), + owner_kind, + owner_id, + authorization_scope: principal.authorization_scope(), + }) + } + + /// Resolves the addressed Tool of the principal's published catalog and repeats its authorization + /// check. A foreign workspace, an unknown Tool and an insufficient security level are typed + /// refusals without metadata. + async fn resolve_authorized_tool( + &self, + principal: &TaskPrincipal, + tool_name: &str, + ) -> Result { + let catalog = self + .catalog + .get_catalog(principal.workspace_slug(), principal.agent_slug()) + .await + .map_err(|_| TaskApplicationError::NotFound)?; + let tool = catalog + .tools + .into_iter() + .find(|tool| tool.tool_name == tool_name) + .ok_or(TaskApplicationError::NotFound)?; + if let TaskPrincipal::Admin { workspace_slug, .. } = principal + && tool.workspace_slug != *workspace_slug + { + return Err(TaskApplicationError::Denied); + } + if let TaskPrincipal::Agent { credential, .. } = principal + && !credential_allows_security_level(credential, tool.operation.security_level) + { + return Err(TaskApplicationError::Denied); + } + Ok(tool) + } +} diff --git a/crates/crank-community-mcp/src/lib.rs b/crates/crank-community-mcp/src/lib.rs index bdd6124..e2c4aca 100644 --- a/crates/crank-community-mcp/src/lib.rs +++ b/crates/crank-community-mcp/src/lib.rs @@ -16,6 +16,7 @@ mod tool_search; mod transport; pub use app::{ - build_app, build_app_with_artifacts, build_app_with_background_workers, + TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts, + build_app_with_artifacts_and_tasks, build_app_with_background_workers, build_app_with_background_workers_and_limits, }; diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index 5f8a2b1..54ecc1d 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -18,6 +18,7 @@ pub mod prompt_composition; pub mod protocol; pub mod resource; pub mod secret; +pub mod task; pub mod tool_catalog; pub mod tool_quality; pub mod tool_search; @@ -73,6 +74,13 @@ pub mod domain { }; pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion}; + pub use crate::task::{ + TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, + TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES, + TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, + TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, + TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds, + }; pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget}; pub use crate::tool_quality::{ ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet, @@ -110,6 +118,7 @@ pub mod ports { PreparedRequest, ProtocolAdapter, ProtocolAdapterError, ResponseCacheScope, RuntimeRequestContext, SharedProtocolAdapter, }; + pub use crate::task::TaskStore; } pub use access::{ @@ -192,6 +201,13 @@ pub use product_event::{ }; pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion}; +pub use task::{ + TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, + TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, + TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, + TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key, + validate_task_envelope, +}; pub use tool_catalog::{ RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError, ToolCatalogBudget, analyze_tool_catalog, diff --git a/crates/crank-core/src/task.rs b/crates/crank-core/src/task.rs new file mode 100644 index 0000000..997da77 --- /dev/null +++ b/crates/crank-core/src/task.rs @@ -0,0 +1,730 @@ +//! Durable background Task seam (Story 4.2, FR-22/FR-23). +//! +//! A Task is the internal, protocol-independent record of one accepted background request. This +//! module owns the domain vocabulary only: the lifecycle states, the accepted identity/pins, the +//! bounded persisted envelopes and the `TaskStore` port. The protocol adapter (Story 4.7) and the +//! worker (Story 4.3) do not define these semantics; the authoritative store lives in PostgreSQL +//! (`crank-registry`), so an accepted Task survives a crash or restart. +//! +//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a +//! Task was executed. A terminal state is written by a conditional transition only. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +use crate::correlation::{RequestId, TraceContext, TraceId}; +use crate::operation::ToolExecutionMode; + +/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata). +pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768; +/// Maximum nesting depth of a persisted Task envelope. +pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8; +/// Maximum number of JSON members/nodes of a persisted Task envelope. +pub const TASK_ENVELOPE_MAX_NODES: usize = 256; +/// Bound of every scope/owner/operation identity column. +pub const TASK_IDENTITY_MAX_BYTES: usize = 160; +/// Bound of the caller-supplied idempotency key before it becomes a scoped digest. +pub const TASK_SCOPED_KEY_MAX_BYTES: usize = 128; +/// Bound of a saved approval reference (an identifier, never a bearer token). +pub const TASK_APPROVAL_REFERENCE_MAX_BYTES: usize = 160; +/// Default retention of a terminal Task after its execution deadline (internal bounded default). +pub const TASK_DEFAULT_RETENTION_MS: i64 = 86_400_000; +/// Upper bound on the retention an accepted Task may pin. +pub const TASK_MAX_RETENTION_MS: i64 = 30 * 86_400_000; +/// Upper bound of one retention/housekeeping batch. +pub const TASK_PURGE_BATCH_LIMIT: u32 = 128; + +/// Stable identity of one accepted Task. Generated as UUIDv7 and returned only after the accepting +/// transaction committed, so the ID never exists without its durable record. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct TaskId(Uuid); + +impl TaskId { + pub fn generate() -> Self { + Self(Uuid::now_v7()) + } + + pub fn parse(value: &str) -> Result { + Uuid::parse_str(value) + .map(Self) + .map_err(|_| TaskError::invalid("task identity is not a UUID")) + } + + pub fn as_uuid(self) -> Uuid { + self.0 + } +} + +impl std::fmt::Display for TaskId { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(formatter) + } +} + +/// Closed Task lifecycle vocabulary. `outcome_unknown` is a non-terminal state: a value commit that +/// failed to confirm its dispatch is never collapsed into a plain failure. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskState { + Queued, + Working, + InputRequired, + OutcomeUnknown, + Completed, + Failed, + Cancelled, +} + +impl TaskState { + pub const fn as_str(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Working => "working", + Self::InputRequired => "input_required", + Self::OutcomeUnknown => "outcome_unknown", + Self::Completed => "completed", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + } + } + + pub fn parse(value: &str) -> Result { + match value { + "queued" => Ok(Self::Queued), + "working" => Ok(Self::Working), + "input_required" => Ok(Self::InputRequired), + "outcome_unknown" => Ok(Self::OutcomeUnknown), + "completed" => Ok(Self::Completed), + "failed" => Ok(Self::Failed), + "cancelled" => Ok(Self::Cancelled), + _ => Err(TaskError::invalid("unknown task state")), + } + } + + pub const fn is_terminal(self) -> bool { + matches!(self, Self::Completed | Self::Failed | Self::Cancelled) + } + + /// One conditional transition may only move along this table. A terminal state has no successor, + /// so a repeated terminal write can never rewrite an accepted result. + pub const fn allows_transition_to(self, next: Self) -> bool { + match self { + Self::Queued => matches!(next, Self::Working | Self::Failed | Self::Cancelled), + Self::Working => matches!( + next, + Self::InputRequired + | Self::OutcomeUnknown + | Self::Completed + | Self::Failed + | Self::Cancelled + ), + Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled), + Self::OutcomeUnknown => { + matches!(next, Self::Completed | Self::Failed | Self::Cancelled) + } + Self::Completed | Self::Failed | Self::Cancelled => false, + } + } +} + +/// Who owns the accepted Task. An administrator is never a wildcard owner of every Task: the owner +/// identity and the authorization scope are separate, both-pinned columns. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TaskOwnerKind { + Agent, + Admin, +} + +impl TaskOwnerKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Agent => "agent", + Self::Admin => "admin", + } + } + + pub fn parse(value: &str) -> Result { + match value { + "agent" => Ok(Self::Agent), + "admin" => Ok(Self::Admin), + _ => Err(TaskError::invalid("unknown task owner kind")), + } + } +} + +/// Support identities captured at acceptance. They are stored and returned independently of +/// sampling, and a replay never rewrites them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskSupportIds { + pub request_id: RequestId, + pub trace_id: TraceId, + pub traceparent: String, + pub trace_sampled: bool, +} + +impl TaskSupportIds { + pub fn new(request_id: RequestId, trace_context: &TraceContext) -> Self { + Self { + request_id, + trace_id: trace_context.trace_id().clone(), + traceparent: trace_context.traceparent().to_owned(), + trace_sampled: trace_context.is_sampled(), + } + } + + /// Re-validates both the trace identity and the parent context: a Task must never persist an + /// inconsistent or malformed support pair, and an unsampled context is still a valid parent. + pub fn validate(&self) -> Result<(), TaskError> { + if self.trace_id.as_str().is_empty() { + return Err(TaskError::invalid("task trace identity is missing")); + } + let context = TraceContext::parse(&self.traceparent) + .map_err(|_| TaskError::invalid("task trace parent is invalid"))?; + if context.trace_id() != &self.trace_id || context.is_sampled() != self.trace_sampled { + return Err(TaskError::invalid("task trace parent is inconsistent")); + } + Ok(()) + } +} + +/// Execution semantics pinned at acceptance: a later publication never rewrites them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskPins { + pub operation_version: u32, + pub policy_mode: ToolExecutionMode, + pub execution_deadline_unix_ms: i64, + /// Retention deadline of the record; never the same thing as the execution deadline. + pub retain_until_unix_ms: i64, + /// Verified approval evidence reference (identifier only, never a bearer token). + pub approval_reference: Option, +} + +impl TaskPins { + pub fn validate(&self) -> Result<(), TaskError> { + if self.operation_version == 0 { + return Err(TaskError::invalid( + "task operation version must be positive", + )); + } + if self.execution_deadline_unix_ms <= 0 { + return Err(TaskError::invalid("task execution deadline is invalid")); + } + if self.retain_until_unix_ms < self.execution_deadline_unix_ms { + return Err(TaskError::invalid( + "task retention deadline precedes execution", + )); + } + if self.retain_until_unix_ms - self.execution_deadline_unix_ms > TASK_MAX_RETENTION_MS { + return Err(TaskError::invalid("task retention exceeds its bound")); + } + if let Some(reference) = self.approval_reference.as_deref() + && (reference.is_empty() + || reference.len() > TASK_APPROVAL_REFERENCE_MAX_BYTES + || reference.bytes().any(|byte| byte.is_ascii_control())) + { + return Err(TaskError::invalid("task approval reference is invalid")); + } + Ok(()) + } +} + +/// Bounded, classified payload of a Task. Construction validates the byte/depth/node budget and +/// redacts secret-bearing fields before anything is hashed or persisted. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskEnvelope { + value: Value, +} + +impl TaskEnvelope { + /// Validates the caller-provided value, then stores a redacted copy of it. The caller hashes the + /// original value separately, so redaction never collapses two different requests into one. + pub fn try_redacted(input: &Value) -> Result { + validate_task_envelope(input)?; + Ok(Self { + value: redact_task_envelope(input), + }) + } + + /// Accepts a value that was already validated and redacted at a trust boundary. + pub fn from_persisted(value: Value) -> Result { + validate_task_envelope(&value)?; + Ok(Self { value }) + } + + pub fn value(&self) -> &Value { + &self.value + } + + pub fn into_value(self) -> Value { + self.value + } +} + +/// Validates the size, nesting depth and node count of one Task envelope. Every byte, collection and +/// node is counted before the value is cloned, hashed or persisted. +pub fn validate_task_envelope(value: &Value) -> Result<(), TaskError> { + let encoded = serde_json::to_vec(value) + .map_err(|_| TaskError::invalid("task envelope is not serializable"))?; + if encoded.len() > TASK_ENVELOPE_MAX_BYTES { + return Err(TaskError::invalid("task envelope exceeds its byte budget")); + } + let mut nodes = 0usize; + check_budget(value, 1, &mut nodes)?; + Ok(()) +} + +fn check_budget(value: &Value, depth: usize, nodes: &mut usize) -> Result<(), TaskError> { + if depth > TASK_ENVELOPE_MAX_DEPTH { + return Err(TaskError::invalid("task envelope exceeds its depth budget")); + } + *nodes += 1; + if *nodes > TASK_ENVELOPE_MAX_NODES { + return Err(TaskError::invalid("task envelope exceeds its node budget")); + } + match value { + Value::Object(object) => { + for (key, child) in object { + check_budget_name(key)?; + check_budget(child, depth + 1, nodes)?; + } + } + Value::Array(values) => { + for child in values { + check_budget(child, depth + 1, nodes)?; + } + } + _ => {} + } + Ok(()) +} + +fn check_budget_name(key: &str) -> Result<(), TaskError> { + if key.len() > TASK_IDENTITY_MAX_BYTES { + return Err(TaskError::invalid("task envelope key exceeds its budget")); + } + Ok(()) +} + +/// Field-name classification of secret-bearing values. A matching field keeps its position but never +/// its plaintext in the persisted envelope. +pub fn sensitive_task_key(name: &str) -> bool { + let lowered = name.to_ascii_lowercase(); + [ + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "credential", + "private_key", + "cookie", + "signature", + ] + .iter() + .any(|marker| lowered.contains(marker)) +} + +/// Redacted projection of an already validated envelope. +pub fn redact_task_envelope(value: &Value) -> Value { + match value { + Value::Object(object) => Value::Object( + object + .iter() + .map(|(key, child)| { + let value = if sensitive_task_key(key) { + Value::String("[redacted]".to_owned()) + } else { + redact_task_envelope(child) + }; + (key.clone(), value) + }) + .collect(), + ), + Value::Array(values) => Value::Array(values.iter().map(redact_task_envelope).collect()), + other => other.clone(), + } +} + +/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and +/// queue capacity commit together, and no acknowledgement exists before that commit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskAcceptRequest { + pub task_id: TaskId, + pub workspace_id: String, + pub owner_kind: TaskOwnerKind, + pub owner_id: String, + pub authorization_scope: String, + pub operation_id: String, + pub pins: TaskPins, + /// Digest of the caller key over the full owner/authorization/Tool scope. + pub scoped_key: String, + /// Digest of the canonical validated executable input. + pub input_fingerprint: String, + pub input: TaskEnvelope, + pub support: TaskSupportIds, +} + +impl TaskAcceptRequest { + pub fn validate(&self) -> Result<(), TaskError> { + for (field, value) in [ + ("workspace", self.workspace_id.as_str()), + ("owner", self.owner_id.as_str()), + ("authorization scope", self.authorization_scope.as_str()), + ("operation", self.operation_id.as_str()), + ] { + if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES { + return Err(TaskError::invalid(format!("task {field} is invalid"))); + } + } + if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES { + return Err(TaskError::invalid("task scoped key digest is invalid")); + } + if self.input_fingerprint.len() != 64 + || !self + .input_fingerprint + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err(TaskError::invalid("task input fingerprint is invalid")); + } + self.pins.validate()?; + self.support.validate() + } +} + +/// Successful acceptance, returned only after the accepting transaction committed. A replay returns +/// the stored identity and support IDs of the original acceptance. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskAcceptance { + pub task_id: TaskId, + pub state: TaskState, + /// Exact Published Version pinned by this acceptance; a replay returns the stored pin. + pub operation_version: u32, + pub support: TaskSupportIds, + pub created_at_unix_ms: i64, + pub retain_until_unix_ms: i64, + pub replayed: bool, +} + +/// Owner-side access proof of a read or a terminal write. It is built by the application layer from +/// the verified principal, never from a caller-supplied marker, and every store query carries the +/// whole predicate so a foreign row is never fetched. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskAccess { + pub workspace_id: String, + pub owner_kind: TaskOwnerKind, + pub owner_id: String, + pub authorization_scope: String, +} + +impl TaskAccess { + pub fn validate(&self) -> Result<(), TaskError> { + for (field, value) in [ + ("workspace", self.workspace_id.as_str()), + ("owner", self.owner_id.as_str()), + ("authorization scope", self.authorization_scope.as_str()), + ] { + if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES { + return Err(TaskError::invalid(format!("task {field} is invalid"))); + } + } + Ok(()) + } +} + +/// Read model of one Task. Lifecycle fields, pins and support IDs stay readable together; the +/// protocol adapter only maps them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskRecord { + pub task_id: TaskId, + pub state: TaskState, + pub workspace_id: String, + pub owner_kind: TaskOwnerKind, + pub owner_id: String, + pub authorization_scope: String, + pub operation_id: String, + /// Canonical digest of the accepted executable input (compare-and-conflict identity). + pub input_fingerprint: String, + pub pins: TaskPins, + pub support: TaskSupportIds, + pub input: TaskEnvelope, + pub progress: Option, + pub result: Option, + pub error: Option, + pub attempts: u32, + /// Monotonic lifecycle version used by the terminal CAS. + pub version: u32, + pub created_at_unix_ms: i64, + pub updated_at_unix_ms: i64, +} + +/// One conditional terminal transition. Exactly one concurrent writer wins; the losers observe the +/// accepted state and never rewrite the result. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskFinishRequest { + pub access: TaskAccess, + pub task_id: TaskId, + pub expected_state: TaskState, + pub expected_version: u32, + pub target_state: TaskState, + pub result: Option, + pub error: Option, +} + +impl TaskFinishRequest { + pub fn validate(&self) -> Result<(), TaskError> { + self.access.validate()?; + if !self.target_state.is_terminal() { + return Err(TaskError::invalid("task finish target is not terminal")); + } + if self.expected_state.is_terminal() { + return Err(TaskError::invalid("task finish source is already terminal")); + } + if !self.expected_state.allows_transition_to(self.target_state) { + return Err(TaskError::invalid("task state transition is not allowed")); + } + if self.target_state == TaskState::Completed && self.result.is_none() { + return Err(TaskError::invalid("a completed task requires a result")); + } + Ok(()) + } +} + +/// Bounded retention outcome. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TaskPurgeReport { + /// Queued Tasks that passed their execution deadline and were failed with their intent removed. + pub expired_queued: u64, + /// Terminal Tasks that passed their retention deadline. + pub expired_terminal: u64, +} + +/// Typed Task failure. A denial and a missing Task share one outward shape, so a foreign caller can +/// never distinguish "not yours" from "does not exist". +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum TaskError { + #[error("task request is invalid: {message}")] + Invalid { message: String }, + #[error("task store is unavailable: {message}")] + Unavailable { message: String }, + #[error("task was not found")] + NotFound, + #[error("task request conflicts with the accepted request")] + FingerprintConflict, + #[error("task lifecycle transition was rejected")] + StateConflict, + #[error("background task capacity is exhausted")] + Overloaded { retry_after_ms: u64 }, +} + +impl TaskError { + pub fn invalid(message: impl Into) -> Self { + Self::Invalid { + message: message.into(), + } + } + + pub fn unavailable(message: impl Into) -> Self { + Self::Unavailable { + message: message.into(), + } + } + + /// Whether the caller may tell this failure apart from an absent Task. + pub fn is_not_found(&self) -> bool { + matches!(self, Self::NotFound) + } +} + +/// Authoritative durable Task port. +#[async_trait] +pub trait TaskStore: Send + Sync { + /// Proves the durable Task state is usable (schema and operations reachable). + async fn health(&self) -> Result<(), TaskError>; + + /// Atomically accepts a Task: Task record, dispatch intent and one queue slot commit together. + async fn accept(&self, request: TaskAcceptRequest) -> Result; + + /// Reads one Task inside the caller's owner scope. + async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result; + + /// Scoped lookup of an already accepted Task by its full-scope idempotency digest. Used to keep a + /// retry of an explicitly pinned version from becoming new work; `None` means "not accepted yet". + async fn find( + &self, + access: &TaskAccess, + operation_id: &str, + scoped_key: &str, + ) -> Result, TaskError>; + + /// Applies exactly one conditional terminal transition and releases its intent atomically. + async fn finish(&self, request: TaskFinishRequest) -> Result; + + /// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes + /// expired terminal records. Live and non-terminal work is never deleted. + async fn purge_expired(&self, batch: u32) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn request() -> TaskAcceptRequest { + TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: "ws_default".to_owned(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_1".to_owned(), + operation_id: "op_report".to_owned(), + pins: TaskPins { + operation_version: 3, + policy_mode: ToolExecutionMode::Background, + execution_deadline_unix_ms: 1_790_000_000_000, + retain_until_unix_ms: 1_790_000_000_000 + 1_000, + approval_reference: Some("appr_1".to_owned()), + }, + scoped_key: "a".repeat(64), + input_fingerprint: "b".repeat(64), + input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(), + support: TaskSupportIds::new( + RequestId::generate(), + &TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00") + .unwrap(), + ), + } + } + + #[test] + fn uuid_v7_task_ids_are_unique_and_round_trip() { + let first = TaskId::generate(); + let second = TaskId::generate(); + assert_ne!(first, second); + assert_eq!(TaskId::parse(&first.to_string()).unwrap(), first); + } + + #[test] + fn terminal_states_have_no_successor() { + for terminal in [ + TaskState::Completed, + TaskState::Failed, + TaskState::Cancelled, + ] { + assert!(terminal.is_terminal()); + for next in [ + TaskState::Queued, + TaskState::Working, + TaskState::Completed, + TaskState::Failed, + ] { + assert!(!terminal.allows_transition_to(next)); + } + } + assert!(TaskState::Queued.allows_transition_to(TaskState::Working)); + assert!(TaskState::Working.allows_transition_to(TaskState::OutcomeUnknown)); + assert!(!TaskState::Queued.allows_transition_to(TaskState::Completed)); + assert!(!TaskState::InputRequired.allows_transition_to(TaskState::Completed)); + } + + #[test] + fn envelope_bounds_reject_bytes_depth_and_nodes() { + assert!(TaskEnvelope::try_redacted(&json!({"a": 1})).is_ok()); + assert!( + TaskEnvelope::try_redacted(&json!({"a": "x".repeat(TASK_ENVELOPE_MAX_BYTES)})).is_err() + ); + + let mut deep = json!(1); + for _ in 0..=TASK_ENVELOPE_MAX_DEPTH { + deep = json!({"n": deep}); + } + assert!(TaskEnvelope::try_redacted(&deep).is_err()); + + let wide = Value::Array((0..=TASK_ENVELOPE_MAX_NODES).map(|_| json!(1)).collect()); + assert!(TaskEnvelope::try_redacted(&wide).is_err()); + } + + #[test] + fn secret_bearing_fields_are_redacted_and_keep_their_shape() { + let envelope = TaskEnvelope::try_redacted(&json!({ + "query": "leads", + "auth": {"api_key": "sk_live_1", "nested": [{"password": "p"}]} + })) + .unwrap(); + let value = envelope.value(); + assert_eq!(value["query"], json!("leads")); + assert_eq!(value["auth"]["api_key"], json!("[redacted]")); + assert_eq!(value["auth"]["nested"][0]["password"], json!("[redacted]")); + } + + #[test] + fn unsampled_support_ids_are_valid_and_consistent() { + let support = TaskSupportIds::new( + RequestId::generate(), + &TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00") + .unwrap(), + ); + assert!(!support.trace_sampled); + assert!(support.traceparent.ends_with("-00")); + support.validate().unwrap(); + + let mut inconsistent = support.clone(); + inconsistent.traceparent = inconsistent.traceparent.replace("-00", "-01"); + assert!(inconsistent.validate().is_err()); + } + + #[test] + fn accept_request_validation_holds_the_identity_and_pin_bounds() { + assert!(request().validate().is_ok()); + + let mut oversized = request(); + oversized.workspace_id = "w".repeat(TASK_IDENTITY_MAX_BYTES + 1); + assert!(oversized.validate().is_err()); + + let mut bad_fingerprint = request(); + bad_fingerprint.input_fingerprint = "XYZ".to_owned(); + assert!(bad_fingerprint.validate().is_err()); + + let mut bad_retention = request(); + bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1; + assert!(bad_retention.validate().is_err()); + } + + #[test] + fn finish_requires_a_terminal_reachable_transition() { + let access = TaskAccess { + workspace_id: "ws_default".to_owned(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_1".to_owned(), + }; + let base = TaskFinishRequest { + access, + task_id: TaskId::generate(), + expected_state: TaskState::Working, + expected_version: 1, + target_state: TaskState::Completed, + result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()), + error: None, + }; + assert!(base.validate().is_ok()); + + let mut non_terminal = base.clone(); + non_terminal.target_state = TaskState::Working; + assert!(non_terminal.validate().is_err()); + + let mut terminal_source = base.clone(); + terminal_source.expected_state = TaskState::Completed; + assert!(terminal_source.validate().is_err()); + + let mut completed_without_result = base; + completed_without_result.result = None; + assert!(completed_without_result.validate().is_err()); + } +} diff --git a/crates/crank-registry/src/lib.rs b/crates/crank-registry/src/lib.rs index 7647058..e073be2 100644 --- a/crates/crank-registry/src/lib.rs +++ b/crates/crank-registry/src/lib.rs @@ -117,6 +117,6 @@ pub use model::{ }; pub use postgres::{ AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission, - PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, admission_health, - admission_snapshot, + PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, PostgresTaskStore, + admission_health, admission_snapshot, tasks_health, }; diff --git a/crates/crank-registry/src/migrations.rs b/crates/crank-registry/src/migrations.rs index c25d66d..bb8e989 100644 --- a/crates/crank-registry/src/migrations.rs +++ b/crates/crank-registry/src/migrations.rs @@ -6,6 +6,7 @@ mod artifact_cleanup_indexes_v13; mod artifact_metadata_v12; mod authority; mod baseline_v1; +mod durable_tasks_v25; mod execution_admission_v24; mod execution_outcome_v5; mod import_plan_v14; @@ -37,6 +38,7 @@ mod schema_guard_v20; mod schema_guard_v22; mod schema_guard_v23; mod schema_guard_v24; +mod schema_guard_v25; mod schema_guard_v7; mod schema_guard_v8; mod schema_guard_v9; diff --git a/crates/crank-registry/src/migrations/authority.rs b/crates/crank-registry/src/migrations/authority.rs index 4bde7fa..eeb5354 100644 --- a/crates/crank-registry/src/migrations/authority.rs +++ b/crates/crank-registry/src/migrations/authority.rs @@ -4,6 +4,7 @@ use super::agent_resource_bindings_v18; use super::approval_side_effects_v10; use super::artifact_cleanup_indexes_v13; use super::artifact_metadata_v12; +use super::durable_tasks_v25; use super::execution_admission_v24; use super::execution_outcome_v5; use super::import_plan_v14; @@ -40,9 +41,9 @@ use preflight::*; use contract::baseline_source_digest; use contract::validate_descriptors; const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947; -const CURRENT_VERSION: i64 = 24; +const CURRENT_VERSION: i64 = 25; const IMPLEMENTED_VERSIONS: &[i64] = &[ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, ]; const BASELINE_SOURCE_SHA256: &str = "eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675"; @@ -337,6 +338,12 @@ impl MigrationAuthority { execution_admission_v24::SOURCE_SHA256, 23, ), + expand_descriptor( + 25, + durable_tasks_v25::NAME, + durable_tasks_v25::SOURCE_SHA256, + 24, + ), ] } pub fn validate_sequence() -> Result<(), MigrationError> { @@ -499,6 +506,9 @@ impl MigrationAuthority { if from < 24 { execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?; } + if from < 25 { + durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?; + } match inspect(&mut transaction).await? { MigrationPreflight::Current { version: CURRENT_VERSION, diff --git a/crates/crank-registry/src/migrations/durable_tasks_v25.rs b/crates/crank-registry/src/migrations/durable_tasks_v25.rs new file mode 100644 index 0000000..ccb18cb --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v25.rs @@ -0,0 +1,45 @@ +use sqlx::{Postgres, Transaction, query}; + +use super::authority::{MigrationDescriptor, MigrationError}; + +pub(super) const NAME: &str = "durable-tasks-v25"; +pub(super) const SOURCE: &str = include_str!("durable_tasks_v25.sql"); +pub(super) const SOURCE_SHA256: &str = + "ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd"; + +pub(super) async fn apply( + transaction: &mut Transaction<'_, Postgres>, + descriptor: &MigrationDescriptor, +) -> Result<(), MigrationError> { + sqlx::raw_sql(SOURCE) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.durable_tasks_v25", + Some(25), + "restore_known_good_backup", + ) + })?; + query( + "insert into __crank_migrations (version, name, checksum, phase, compatibility) + values ($1, $2, $3, $4, $5)", + ) + .bind(descriptor.version) + .bind(descriptor.name) + .bind(&descriptor.checksum) + .bind(descriptor.phase) + .bind(descriptor.compatibility) + .execute(&mut **transaction) + .await + .map_err(|_| { + MigrationError::new( + "apply_failed", + "apply.canonical_ledger", + Some(25), + "restore_known_good_backup", + ) + })?; + Ok(()) +} diff --git a/crates/crank-registry/src/migrations/durable_tasks_v25.sql b/crates/crank-registry/src/migrations/durable_tasks_v25.sql new file mode 100644 index 0000000..5d21abb --- /dev/null +++ b/crates/crank-registry/src/migrations/durable_tasks_v25.sql @@ -0,0 +1,90 @@ +-- V25: durable background Tasks (Story 4.2, FR-22, FR-23). +-- +-- Одна строка `tasks` — принятая durable работа. Идентификатор Task (UUIDv7), owner scope, +-- authorization scope, запрошенный Tool и точная published-версия, canonical fingerprint входа, +-- support IDs (Request/Trace/parent TraceContext независимо от sampling) и bounded envelopes +-- закрепляются в той же PostgreSQL transaction, что и dispatch intent +-- (`execution_admission_entries.task_id`) и его queue slot. До commit наружу не выдаётся ни ID, ни ack. +-- +-- `scoped_key` — digest caller-ключа по полному (workspace, owner kind/id, authorization scope, +-- operation) scope; UNIQUE по этому набору и есть защита идемпотентности. Повтор того же ключа с тем +-- же canonical fingerprint возвращает сохранённую Task, другой fingerprint — typed conflict. +-- +-- `retain_until` — срок хранения записи, а не execution deadline: пока запись в retention, replay +-- находит тот же Task. Удаление terminal-записей возможно только после retain_until; queued/working/ +-- input_required/outcome_unknown не удаляются никогда ради лимита. +-- +-- Строка не хранит credentials, bearer/prepared headers и confirmation tokens: секретные поля входа +-- редактируются до записи, а approval хранится безопасной ссылкой. + +create table tasks ( + task_id uuid not null, + workspace_id text not null, + owner_kind text not null, + owner_id text not null, + authorization_scope text not null, + operation_id text not null, + operation_version integer not null, + scoped_key text not null, + input_fingerprint text not null, + request_id text not null, + trace_id text not null, + traceparent text not null, + trace_sampled boolean not null, + policy_mode text not null, + approval_reference text, + state text not null, + version integer not null default 1, + attempts integer not null default 0, + input_envelope jsonb not null, + progress_envelope jsonb, + result_envelope jsonb, + error_envelope jsonb, + execution_deadline timestamptz not null, + retain_until timestamptz not null, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + constraint tasks_pkey primary key (task_id), + constraint tasks_workspace_check check (octet_length(workspace_id) between 1 and 160), + constraint tasks_owner_kind_check check (owner_kind in ('agent', 'admin')), + constraint tasks_owner_check check (octet_length(owner_id) between 1 and 160), + constraint tasks_authorization_scope_check check ( + octet_length(authorization_scope) between 1 and 160 + ), + constraint tasks_operation_check check (octet_length(operation_id) between 1 and 160), + constraint tasks_operation_version_check check (operation_version >= 1), + constraint tasks_scoped_key_check check (octet_length(scoped_key) between 16 and 128), + constraint tasks_fingerprint_check check (octet_length(input_fingerprint) = 64), + constraint tasks_request_check check (octet_length(request_id) between 1 and 128), + constraint tasks_trace_check check (octet_length(trace_id) = 32), + constraint tasks_traceparent_check check (octet_length(traceparent) = 55), + constraint tasks_policy_mode_check check ( + policy_mode in ('synchronous', 'background', 'adaptive') + ), + constraint tasks_state_check check ( + state in ('queued', 'working', 'input_required', 'outcome_unknown', + 'completed', 'failed', 'cancelled') + ), + constraint tasks_lifecycle_check check (version >= 1 and attempts >= 0), + constraint tasks_retention_check check (retain_until >= execution_deadline), + constraint tasks_approval_check check ( + approval_reference is null or octet_length(approval_reference) between 1 and 160 + ) +); + +create unique index tasks_scoped_key_uidx + on tasks (workspace_id, owner_kind, owner_id, authorization_scope, operation_id, scoped_key); + +create index tasks_retention_idx on tasks (state, retain_until, task_id); + +create index tasks_deadline_idx on tasks (state, execution_deadline, task_id); + +-- Dispatch intent: прежняя reserved-строка admission становится точкой связи Task и capacity. +-- FK без cascade: удаление принятой Task не маскируется housekeeping-удалением intent. +alter table execution_admission_entries + add column task_id uuid, + add constraint execution_admission_entries_task_fkey + foreign key (task_id) references tasks (task_id); + +create unique index execution_admission_entries_task_uidx + on execution_admission_entries (task_id) where task_id is not null; diff --git a/crates/crank-registry/src/migrations/schema_guard.rs b/crates/crank-registry/src/migrations/schema_guard.rs index d668351..7eaa520 100644 --- a/crates/crank-registry/src/migrations/schema_guard.rs +++ b/crates/crank-registry/src/migrations/schema_guard.rs @@ -473,7 +473,15 @@ pub(super) async fn validate_schema_fingerprint( return Err(schema_error(current_version)); } } else { - super::schema_guard_v24::validate_v24_execution_admission(connection).await?; + super::schema_guard_v24::validate_v24_execution_admission(connection, current_version) + .await?; + } + if current_version < 25 { + if !super::schema_guard_v25::validate_v25_absent(connection).await? { + return Err(schema_error(current_version)); + } + } else { + super::schema_guard_v25::validate_v25_durable_tasks(connection).await?; } Ok(()) } diff --git a/crates/crank-registry/src/migrations/schema_guard_v24.rs b/crates/crank-registry/src/migrations/schema_guard_v24.rs index aad6bf1..6492fd8 100644 --- a/crates/crank-registry/src/migrations/schema_guard_v24.rs +++ b/crates/crank-registry/src/migrations/schema_guard_v24.rs @@ -39,7 +39,24 @@ const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[ ("granted_at", "timestamp with time zone", true), ]; -const CONSTRAINT_COUNT: &[(&str, usize)] = &[(CONTROLLERS, 4), (ENTRIES, 11)]; +/// V25 adds the nullable Task dispatch-intent link to the same reserved row. At version 24 the +/// column must not exist yet, so the guard validates the shape of the version it describes. +fn entry_columns(current_version: i64) -> Vec<(&'static str, &'static str, bool)> { + let mut columns = ENTRY_COLUMNS.to_vec(); + if current_version >= 25 { + columns.push(("task_id", "uuid", true)); + } + columns +} + +fn constraint_count(table: &str, current_version: i64) -> Option { + match table { + CONTROLLERS => Some(4), + ENTRIES if current_version >= 25 => Some(12), + ENTRIES => Some(11), + _ => None, + } +} const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"]; @@ -72,16 +89,22 @@ pub(super) async fn validate_v24_absent( /// Схема V24 присутствует в ожидаемой форме (для current_version >= 24). pub(super) async fn validate_v24_execution_admission( connection: &mut PgConnection, + current_version: i64, ) -> Result<(), MigrationError> { for relation in OWNED_RELATIONS { if !relation_exists(connection, relation).await? { return Err(v24_error("v24.relations")); } } - for (table, columns) in [(CONTROLLERS, CONTROLLER_COLUMNS), (ENTRIES, ENTRY_COLUMNS)] { - validate_columns(connection, table, columns).await?; + let tables = [ + (CONTROLLERS, CONTROLLER_COLUMNS.to_vec()), + (ENTRIES, entry_columns(current_version)), + ]; + for (table, columns) in tables { + validate_columns(connection, table, &columns).await?; } - for (table, expected) in CONSTRAINT_COUNT { + for table in [CONTROLLERS, ENTRIES] { + let expected = constraint_count(table, current_version).unwrap_or_default(); let count: i64 = query( "select count(*)::bigint as count from pg_catalog.pg_constraint c @@ -95,7 +118,7 @@ pub(super) async fn validate_v24_execution_admission( .map_err(|_| MigrationError::storage("preflight.schema"))? .try_get("count") .map_err(|_| MigrationError::storage("preflight.schema"))?; - if count != i64::try_from(*expected).unwrap_or_default() { + if count != i64::try_from(expected).unwrap_or_default() { return Err(v24_error("v24.constraints")); } } diff --git a/crates/crank-registry/src/migrations/schema_guard_v25.rs b/crates/crank-registry/src/migrations/schema_guard_v25.rs new file mode 100644 index 0000000..2acf929 --- /dev/null +++ b/crates/crank-registry/src/migrations/schema_guard_v25.rs @@ -0,0 +1,247 @@ +//! V25 schema guard: durable background Tasks (Story 4.2, FR-22/FR-23). +//! +//! Схема V25 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в +//! ожидаемой форме: таблица `tasks` с закрытым словарём state/policy_mode и scoped uniqueness, плюс +//! nullable `task_id` на существующей reserved-строке admission с FK и UNIQUE-индексом. + +use sqlx::{PgConnection, Row, query}; + +use super::authority::MigrationError; +use super::schema_guard::relation_exists; + +const TASKS: &str = "tasks"; +const ENTRIES: &str = "execution_admission_entries"; +const SCOPED_INDEX: &str = "tasks_scoped_key_uidx"; +const TASK_INDEX: &str = "execution_admission_entries_task_uidx"; +const TASK_FK: &str = "execution_admission_entries_task_fkey"; + +const TASK_COLUMNS: &[(&str, &str, bool)] = &[ + ("task_id", "uuid", false), + ("workspace_id", "text", false), + ("owner_kind", "text", false), + ("owner_id", "text", false), + ("authorization_scope", "text", false), + ("operation_id", "text", false), + ("operation_version", "integer", false), + ("scoped_key", "text", false), + ("input_fingerprint", "text", false), + ("request_id", "text", false), + ("trace_id", "text", false), + ("traceparent", "text", false), + ("trace_sampled", "boolean", false), + ("policy_mode", "text", false), + ("approval_reference", "text", true), + ("state", "text", false), + ("version", "integer", false), + ("attempts", "integer", false), + ("input_envelope", "jsonb", false), + ("progress_envelope", "jsonb", true), + ("result_envelope", "jsonb", true), + ("error_envelope", "jsonb", true), + ("execution_deadline", "timestamp with time zone", false), + ("retain_until", "timestamp with time zone", false), + ("created_at", "timestamp with time zone", false), + ("updated_at", "timestamp with time zone", false), +]; + +fn v25_error(stage: &'static str) -> MigrationError { + MigrationError::new( + "partial_sequence", + stage, + Some(25), + "restore_known_good_backup", + ) +} + +/// Схема V25 отсутствует целиком (для current_version < 25). +pub(super) async fn validate_v25_absent( + connection: &mut PgConnection, +) -> Result { + if relation_exists(connection, TASKS).await? { + return Ok(false); + } + if super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? { + return Ok(false); + } + for index in [SCOPED_INDEX, TASK_INDEX] { + if index_exists(connection, index).await? { + return Ok(false); + } + } + Ok(true) +} + +/// Схема V25 присутствует в ожидаемой форме (для current_version >= 25). +pub(super) async fn validate_v25_durable_tasks( + connection: &mut PgConnection, +) -> Result<(), MigrationError> { + if !relation_exists(connection, TASKS).await? { + return Err(v25_error("v25.relations")); + } + let count: i64 = query( + "select count(*)::bigint as count from information_schema.columns + where table_schema = current_schema() and table_name = $1", + ) + .bind(TASKS) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get("count") + .map_err(|_| MigrationError::storage("preflight.schema"))?; + if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() { + return Err(v25_error("v25.columns")); + } + for (column, data_type, nullable) in TASK_COLUMNS { + let row = query( + "select data_type, is_nullable from information_schema.columns + where table_schema = current_schema() and table_name = $1 and column_name = $2", + ) + .bind(TASKS) + .bind(column) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + let valid = row.is_some_and(|row| { + row.try_get::("data_type").ok().as_deref() == Some(*data_type) + && row.try_get::("is_nullable").ok().as_deref() + == Some(if *nullable { "YES" } else { "NO" }) + }); + if !valid { + return Err(v25_error("v25.columns")); + } + } + if !super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? { + return Err(v25_error("v25.intent_column")); + } + for (constraint, vocabulary) in [ + ( + "tasks_state_check", + [ + "queued", + "working", + "input_required", + "outcome_unknown", + "completed", + "failed", + "cancelled", + ] + .as_slice(), + ), + ( + "tasks_policy_mode_check", + ["synchronous", "background", "adaptive"].as_slice(), + ), + ] { + let expression = normalized_expression(connection, TASKS, constraint).await?; + if !expression.is_some_and(|expression| { + super::schema_guard::enum_constraint_matches( + &expression, + column_of(constraint), + vocabulary, + ) + }) { + return Err(v25_error("v25.vocabulary")); + } + } + for (index, table) in [(SCOPED_INDEX, TASKS), (TASK_INDEX, ENTRIES)] { + if index_unique(connection, table, index).await? != Some(true) { + return Err(v25_error("v25.indexes")); + } + } + if !constraint_exists(connection, ENTRIES, TASK_FK).await? { + return Err(v25_error("v25.intent_fk")); + } + Ok(()) +} + +fn column_of(constraint: &str) -> &'static str { + if constraint.contains("policy_mode") { + "policy_mode" + } else { + "state" + } +} + +async fn constraint_exists( + connection: &mut PgConnection, + table: &str, + constraint: &str, +) -> Result { + query( + "select exists ( + select 1 from pg_catalog.pg_constraint c + join pg_catalog.pg_class t on t.oid = c.conrelid + join pg_catalog.pg_namespace n on n.oid = t.relnamespace + where n.nspname = current_schema() and t.relname = $1 and c.conname = $2 + ) as present", + ) + .bind(table) + .bind(constraint) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get::("present") + .map_err(|_| MigrationError::storage("preflight.schema")) +} + +async fn normalized_expression( + connection: &mut PgConnection, + table: &str, + constraint: &str, +) -> Result, MigrationError> { + let row = query( + "select pg_get_expr(c.conbin, c.conrelid) as expression + from pg_catalog.pg_constraint c + join pg_catalog.pg_class t on t.oid = c.conrelid + join pg_catalog.pg_namespace n on n.oid = t.relnamespace + where n.nspname = current_schema() and t.relname = $1 and c.conname = $2 + and c.contype = 'c'", + ) + .bind(table) + .bind(constraint) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + Ok(row.and_then(|row| row.try_get::("expression").ok())) +} + +async fn index_exists(connection: &mut PgConnection, index: &str) -> Result { + query( + "select exists ( + select 1 from pg_catalog.pg_class idx + join pg_catalog.pg_namespace n on n.oid = idx.relnamespace + where n.nspname = current_schema() and idx.relname = $1 + ) as present", + ) + .bind(index) + .fetch_one(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))? + .try_get::("present") + .map_err(|_| MigrationError::storage("preflight.schema")) +} + +async fn index_unique( + connection: &mut PgConnection, + table: &str, + index: &str, +) -> Result, MigrationError> { + let row = query( + "select i.indisunique, i.indisvalid + from pg_catalog.pg_index i + join pg_catalog.pg_class idx on idx.oid = i.indexrelid + join pg_catalog.pg_class t on t.oid = i.indrelid + join pg_catalog.pg_namespace n on n.oid = idx.relnamespace + where n.nspname = current_schema() and idx.relname = $1 and t.relname = $2", + ) + .bind(index) + .bind(table) + .fetch_optional(&mut *connection) + .await + .map_err(|_| MigrationError::storage("preflight.schema"))?; + Ok(row.and_then(|row| { + let unique = row.try_get::("indisunique").ok()?; + let valid = row.try_get::("indisvalid").ok()?; + Some(unique && valid) + })) +} diff --git a/crates/crank-registry/src/postgres/admission.rs b/crates/crank-registry/src/postgres/admission.rs index e9a5f59..5807e64 100644 --- a/crates/crank-registry/src/postgres/admission.rs +++ b/crates/crank-registry/src/postgres/admission.rs @@ -13,7 +13,8 @@ use async_trait::async_trait; use crank_core::{ AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit, AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig, - AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, select_next_grant, + AdmissionStoreError, ApprovalEvidence, ExecutionAdmission, NoTasksFallback, TaskError, + select_next_grant, }; use sqlx::{PgPool, Postgres, Row, Transaction}; use uuid::Uuid; @@ -194,7 +195,8 @@ impl ExecutionAdmission for PostgresExecutionAdmission { async fn release(&self, permit: &AdmissionPermit) -> Result { let result = sqlx::query( "delete from execution_admission_entries - where ticket_id = $1 and owner_token = $2 and state = 'executing'", + where ticket_id = $1 and owner_token = $2 and state = 'executing' + and task_id is null", ) .bind(i64::try_from(permit.ticket_id).unwrap_or(i64::MAX)) .bind(&permit.owner_token) @@ -207,7 +209,8 @@ impl ExecutionAdmission for PostgresExecutionAdmission { async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result { let result = sqlx::query( "delete from execution_admission_entries - where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved')", + where ticket_id = $1 and owner_token = $2 and state in ('waiting', 'reserved') + and task_id is null", ) .bind(i64::try_from(ticket_id).unwrap_or(i64::MAX)) .bind(owner_token) @@ -301,9 +304,19 @@ impl PostgresExecutionAdmission { Ok(Attempt::Overloaded) } Err(AttemptError::Failure(error)) => { + transaction.rollback().await.ok(); + Err(AdmissionStoreError::unavailable(error.to_string())) + } + Err(AttemptError::AdmissionFailure(error)) => { transaction.rollback().await.ok(); Err(error) } + Err(AttemptError::UniqueViolation) => { + transaction.rollback().await.ok(); + Err(AdmissionStoreError::unavailable( + "execution admission hit an unexpected unique conflict", + )) + } } } @@ -314,82 +327,32 @@ impl PostgresExecutionAdmission { ticket: &mut Option, ) -> Result { let workspace_id = context.request.scope.workspace_id.as_str(); - sqlx::query( - "insert into execution_admission_controllers (workspace_id) - values ($1), ($2) - on conflict (workspace_id) do nothing", - ) - .bind(DEPLOYMENT_CONTROLLER) - .bind(workspace_id) - .execute(&mut **transaction) - .await - .map_err(AttemptError::from_sql)?; + let cursor = lock_admission_in_tx(transaction, workspace_id).await?; - // Fixed lock order: deployment controller first, then the workspace controller. - lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?; - let cursor = lock_controller(transaction, workspace_id).await?; - - // Expired queue decisions and waiting tickets leave the bounded queue; held permits never do. - sqlx::query( - "delete from execution_admission_entries - where ticket_id in ( - select ticket_id from execution_admission_entries - where state in ('waiting', 'reserved') and expires_at <= now() - order by ticket_id - limit $1 - )", - ) - .bind(EXPIRED_CLEANUP_BATCH) - .execute(&mut **transaction) - .await - .map_err(AttemptError::from_sql)?; + // Expired queue decisions and waiting tickets leave the bounded queue; held permits and + // Task-linked dispatch intents never do (Story 4.2 owns the durable Task lifetime). + cleanup_expired_unlinked(transaction).await?; if ticket.is_none() { - let queued: i64 = sqlx::query( - "select count(*)::bigint as count from execution_admission_entries - where state in ('waiting', 'reserved')", - ) - .fetch_one(&mut **transaction) - .await - .map_err(AttemptError::from_sql)? - .try_get("count") - .map_err(AttemptError::from_sql)?; - if u64::try_from(queued).unwrap_or(u64::MAX) - >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) - { + let queued = bounded_queue_len(transaction).await?; + if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) { return Err(AttemptError::Overloaded); } let state = match context.request.mode { AdmissionMode::Synchronous => "waiting", AdmissionMode::Background => "reserved", }; - let row = sqlx::query( - "insert into execution_admission_entries ( - workspace_id, agent_key, operation_id, operation_version, mode, policy_mode, - state, owner_token, fingerprint, expires_at - ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, - to_timestamp($10::double precision / 1000.0)) - returning ticket_id", + let ticket_id = insert_admission_entry( + transaction, + context.request, + context.owner_token, + state, + None, ) - .bind(workspace_id) - .bind(&context.request.scope.agent_key) - .bind(&context.request.scope.operation_id) - .bind(i32::try_from(context.request.scope.operation_version).unwrap_or(i32::MAX)) - .bind(context.request.mode.as_str()) - .bind(policy_mode_str(context.request)) - .bind(state) - .bind(context.owner_token) - .bind(&context.request.fingerprint) - .bind(context.request.expires_at_unix_ms) - .fetch_one(&mut **transaction) - .await - .map_err(AttemptError::from_sql)?; - let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?; - *ticket = Some(u64::try_from(ticket_id).unwrap_or_default()); + .await?; + *ticket = Some(ticket_id); if context.request.mode == AdmissionMode::Background { - return Ok(Attempt::Enqueued( - u64::try_from(ticket_id).unwrap_or_default(), - )); + return Ok(Attempt::Enqueued(ticket_id)); } } @@ -520,22 +483,121 @@ impl PostgresExecutionAdmission { } } -enum AttemptError { +/// Takes both controller locks in the fixed `'*'` -> workspace order. Shared by the synchronous +/// admission attempt and the durable Task repository, so both serialize on the same controller rows +/// and one task/queue invariant holds. Callers own the transaction; nothing here commits or sleeps. +pub(super) async fn lock_admission_in_tx( + transaction: &mut Transaction<'_, Postgres>, + workspace_id: &str, +) -> Result { + sqlx::query( + "insert into execution_admission_controllers (workspace_id) + values ($1), ($2) + on conflict (workspace_id) do nothing", + ) + .bind(DEPLOYMENT_CONTROLLER) + .bind(workspace_id) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + + lock_controller(transaction, DEPLOYMENT_CONTROLLER).await?; + lock_controller(transaction, workspace_id).await +} + +/// Bounded cleanup of expired waiting/reserved rows that are not a durable Task dispatch intent. +pub(super) async fn cleanup_expired_unlinked( + transaction: &mut Transaction<'_, Postgres>, +) -> Result<(), AttemptError> { + sqlx::query( + "delete from execution_admission_entries + where ticket_id in ( + select ticket_id from execution_admission_entries + where state in ('waiting', 'reserved') and expires_at <= now() + and task_id is null + order by ticket_id + limit $1 + )", + ) + .bind(EXPIRED_CLEANUP_BATCH) + .execute(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + Ok(()) +} + +/// Authoritative bounded queue length: waiting and reserved rows share one budget. +pub(super) async fn bounded_queue_len( + transaction: &mut Transaction<'_, Postgres>, +) -> Result { + let queued: i64 = sqlx::query( + "select count(*)::bigint as count from execution_admission_entries + where state in ('waiting', 'reserved')", + ) + .fetch_one(&mut **transaction) + .await + .map_err(AttemptError::from_sql)? + .try_get("count") + .map_err(AttemptError::from_sql)?; + Ok(u64::try_from(queued).unwrap_or(u64::MAX)) +} + +/// Inserts one bounded admission row. A Task-linked row is the durable dispatch intent of the Task +/// accepted in the same transaction, so its capacity slot can never be silently reclaimed. +pub(super) async fn insert_admission_entry( + transaction: &mut Transaction<'_, Postgres>, + request: &AdmissionRequest, + owner_token: &str, + state: &str, + task_id: Option, +) -> Result { + let row = sqlx::query( + "insert into execution_admission_entries ( + workspace_id, agent_key, operation_id, operation_version, mode, policy_mode, + state, owner_token, fingerprint, expires_at, task_id + ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, + to_timestamp($10::double precision / 1000.0), $11) + returning ticket_id", + ) + .bind(&request.scope.workspace_id) + .bind(&request.scope.agent_key) + .bind(&request.scope.operation_id) + .bind(i32::try_from(request.scope.operation_version).unwrap_or(i32::MAX)) + .bind(request.mode.as_str()) + .bind(policy_mode_str(request)) + .bind(state) + .bind(owner_token) + .bind(&request.fingerprint) + .bind(request.expires_at_unix_ms) + .bind(task_id) + .fetch_one(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?; + Ok(u64::try_from(ticket_id).unwrap_or_default()) +} + +pub(super) enum AttemptError { /// Lock or statement timeout: the transaction made no decision and the waiter retries. Contended, /// Bounded queue capacity is exhausted: a typed overload outcome, never an unbounded wait. Overloaded, - Failure(AdmissionStoreError), + /// A foreign unique constraint aborted the transaction; the caller rolls back and rereads. + UniqueViolation, + /// Durable-Task side failure (the Task repository maps it to its own typed error). + Failure(TaskError), + /// Admission side failure. + AdmissionFailure(AdmissionStoreError), } impl AttemptError { - fn from_sql(error: sqlx::Error) -> Self { + pub(super) fn from_sql(error: sqlx::Error) -> Self { if let Some(database) = error.as_database_error() && matches!(database.code().as_deref(), Some("55P03" | "57014")) { return Self::Contended; } - Self::Failure(PostgresExecutionAdmission::unavailable(error)) + Self::AdmissionFailure(PostgresExecutionAdmission::unavailable(error)) } } diff --git a/crates/crank-registry/src/postgres/connection.rs b/crates/crank-registry/src/postgres/connection.rs index 675bc3b..b90b965 100644 --- a/crates/crank-registry/src/postgres/connection.rs +++ b/crates/crank-registry/src/postgres/connection.rs @@ -52,6 +52,11 @@ impl PostgresRegistry { super::admission::admission_health(&self.pool).await } + /// Readiness probe of the durable Task state (Story 4.2). + pub async fn durable_tasks_health(&self) -> Result<(), crank_core::TaskError> { + super::tasks::tasks_health(&self.pool).await + } + /// Authoritative admission queue readings for low-cardinality metrics. pub async fn execution_admission_snapshot( &self, diff --git a/crates/crank-registry/src/postgres/mod.rs b/crates/crank-registry/src/postgres/mod.rs index c53b767..78ae840 100644 --- a/crates/crank-registry/src/postgres/mod.rs +++ b/crates/crank-registry/src/postgres/mod.rs @@ -21,6 +21,7 @@ mod product_event; mod prompt; mod resource; mod secret; +mod tasks; mod upstream; mod workspace; mod yaml_import; @@ -42,6 +43,8 @@ pub use admission::{ }; pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError}; pub use resource::{AuthorizedResourceRead, BoundResourceAdmission}; +pub use tasks::PostgresTaskStore; +pub use tasks::tasks_health; use crate::{ error::RegistryError, diff --git a/crates/crank-registry/src/postgres/tasks.rs b/crates/crank-registry/src/postgres/tasks.rs new file mode 100644 index 0000000..473c512 --- /dev/null +++ b/crates/crank-registry/src/postgres/tasks.rs @@ -0,0 +1,586 @@ +//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23). +//! +//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent +//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no +//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware +//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`, +//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction. +//! +//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous +//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no +//! compensating delete and no synchronous fallback can happen after an uncertain acceptance. + +use async_trait::async_trait; +use crank_core::{ + AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback, + TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, + TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore, + TaskSupportIds, ToolExecutionMode, +}; +use serde_json::Value; +use sqlx::{PgPool, Postgres, Row, Transaction}; +use uuid::Uuid; + +use super::admission::{ + AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked, + insert_admission_entry, lock_admission_in_tx, +}; + +const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id, + authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode, + approval_reference, + request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope, + result_envelope, error_envelope, attempts, version, + (extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms, + (extract(epoch from retain_until) * 1000)::bigint as retain_until_ms, + (extract(epoch from created_at) * 1000)::bigint as created_ms, + (extract(epoch from updated_at) * 1000)::bigint as updated_ms"; + +#[derive(Clone)] +pub struct PostgresTaskStore { + pool: PgPool, + limits: AdmissionLimits, +} + +impl PostgresTaskStore { + pub fn new( + pool: PgPool, + limits: AdmissionLimits, + scheduler: crank_core::AdmissionSchedulerConfig, + ) -> Result { + limits + .validate() + .map_err(|error| TaskError::invalid(error.to_string()))?; + scheduler + .validate() + .map_err(|error| TaskError::invalid(error.to_string()))?; + Ok(Self { pool, limits }) + } + + fn unavailable(error: impl std::fmt::Display) -> TaskError { + TaskError::unavailable(format!("durable task store is unavailable: {error}")) + } + + async fn start(&self) -> Result, TaskError> { + let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?; + sqlx::query("set local lock_timeout = '3000ms'") + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + sqlx::query("set local statement_timeout = '5000ms'") + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + Ok(transaction) + } + + /// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the + /// query, so a foreign row is never fetched. + async fn stored_acceptance( + &self, + transaction: &mut Transaction<'_, Postgres>, + request: &TaskAcceptRequest, + ) -> Result, AttemptError> { + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks + where workspace_id = $1 and owner_kind = $2 and owner_id = $3 + and authorization_scope = $4 and operation_id = $5 and scoped_key = $6" + ))) + .bind(&request.workspace_id) + .bind(request.owner_kind.as_str()) + .bind(&request.owner_id) + .bind(&request.authorization_scope) + .bind(&request.operation_id) + .bind(&request.scoped_key) + .fetch_optional(&mut **transaction) + .await + .map_err(AttemptError::from_sql)?; + let Some(row) = row else { + return Ok(None); + }; + let record = record_from_row(&row).map_err(AttemptError::Failure)?; + Ok(Some(( + acceptance_from_record(&record), + record.input_fingerprint, + ))) + } +} + +fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance { + TaskAcceptance { + task_id: record.task_id, + state: record.state, + operation_version: record.pins.operation_version, + support: record.support.clone(), + created_at_unix_ms: record.created_at_unix_ms, + retain_until_unix_ms: record.pins.retain_until_unix_ms, + replayed: true, + } +} + +/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or +/// unreadable schema is a dependency failure, never a degraded-but-ready state. +pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> { + sqlx::query("select count(*)::bigint from tasks where false") + .fetch_one(pool) + .await + .map_err(PostgresTaskStore::unavailable)?; + Ok(()) +} + +/// One transaction's decision of the acceptance path. +enum Accepted { + Created(TaskAcceptance), + Replayed(TaskAcceptance), + Conflict, +} + +#[async_trait] +impl TaskStore for PostgresTaskStore { + async fn health(&self) -> Result<(), TaskError> { + sqlx::query("select count(*)::bigint from tasks where false") + .fetch_one(&self.pool) + .await + .map_err(Self::unavailable)?; + super::admission::admission_health(&self.pool) + .await + .map_err(|error| TaskError::unavailable(error.to_string()))?; + Ok(()) + } + + async fn accept(&self, request: TaskAcceptRequest) -> Result { + request.validate()?; + let mut transaction = self.start().await?; + let outcome = self.accept_locked(&mut transaction, &request).await; + match outcome { + Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => { + // The acknowledgement of a created Task is only possible after a real COMMIT. + transaction.commit().await.map_err(Self::unavailable)?; + Ok(TaskAcceptance { + replayed: acceptance.replayed, + ..acceptance + }) + } + Ok(Accepted::Conflict) => { + transaction.rollback().await.ok(); + Err(TaskError::FingerprintConflict) + } + Err(AttemptError::Overloaded) => { + transaction.rollback().await.ok(); + Err(TaskError::Overloaded { + retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS, + }) + } + Err(AttemptError::Contended) => { + transaction.rollback().await.ok(); + Err(TaskError::unavailable( + "durable task acceptance could not take the admission locks", + )) + } + Err(AttemptError::UniqueViolation) => { + // Roll back the aborted transaction, then reread the scoped key once outside it. + transaction.rollback().await.ok(); + self.reread_after_unique(&request).await + } + Err(AttemptError::Failure(error)) => { + transaction.rollback().await.ok(); + Err(error) + } + Err(AttemptError::AdmissionFailure(error)) => { + transaction.rollback().await.ok(); + Err(TaskError::unavailable(error.to_string())) + } + } + } + + async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result { + access.validate()?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks + where task_id = $1 and workspace_id = $2 and owner_kind = $3 + and owner_id = $4 and authorization_scope = $5" + ))) + .bind(task_id.as_uuid()) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .fetch_optional(&self.pool) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + return Err(TaskError::NotFound); + }; + record_from_row(&row).map_err(Self::unavailable) + } + + async fn find( + &self, + access: &TaskAccess, + operation_id: &str, + scoped_key: &str, + ) -> Result, TaskError> { + access.validate()?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks + where workspace_id = $1 and owner_kind = $2 and owner_id = $3 + and authorization_scope = $4 and operation_id = $5 and scoped_key = $6" + ))) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .bind(operation_id) + .bind(scoped_key) + .fetch_optional(&self.pool) + .await + .map_err(Self::unavailable)?; + row.map(|row| record_from_row(&row)).transpose() + } + + async fn finish(&self, request: TaskFinishRequest) -> Result { + request.validate()?; + let mut transaction = self.start().await?; + let access = &request.access; + let update = sqlx::query(sqlx::AssertSqlSafe(format!( + "update tasks + set state = $1, + result_envelope = $2, + error_envelope = $3, + version = version + 1, + updated_at = now() + where task_id = $4 and workspace_id = $5 and owner_kind = $6 + and owner_id = $7 and authorization_scope = $8 + and state = $9 and version = $10 + returning {TASK_PROJECTION}" + ))) + .bind(request.target_state.as_str()) + .bind(request.result.as_ref().map(|value| value.value().clone())) + .bind(request.error.as_ref().map(|value| value.value().clone())) + .bind(request.task_id.as_uuid()) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .bind(request.expected_state.as_str()) + .bind(i64::from(request.expected_version)) + .fetch_optional(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let Some(row) = update else { + transaction.rollback().await.ok(); + return Err(self.classify_missed_finish(&request).await); + }; + let record = record_from_row(&row).map_err(Self::unavailable)?; + // Terminal Task and its dispatch intent leave together: a half-done release is rolled back. + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(request.task_id.as_uuid()) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(record) + } + + async fn purge_expired(&self, batch: u32) -> Result { + let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT)); + let mut transaction = self.start().await?; + let expired = sqlx::query( + "with expired as ( + update tasks + set state = 'failed', + error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'), + version = version + 1, + updated_at = now() + where task_id in ( + select task_id from tasks + where state = 'queued' and execution_deadline <= now() + order by task_id + limit $1 + ) + returning task_id + ), released as ( + delete from execution_admission_entries + where task_id in (select task_id from expired) + returning task_id + ) + select (select count(*) from expired)::bigint as expired_queued", + ) + .bind(batch) + .fetch_one(&mut *transaction) + .await + .map_err(Self::unavailable)?; + let expired_queued: i64 = expired + .try_get("expired_queued") + .map_err(Self::unavailable)?; + let purged = sqlx::query( + "delete from tasks + where task_id in ( + select task_id from tasks + where state in ('completed', 'failed', 'cancelled') and retain_until <= now() + order by task_id + limit $1 + )", + ) + .bind(batch) + .execute(&mut *transaction) + .await + .map_err(Self::unavailable)?; + transaction.commit().await.map_err(Self::unavailable)?; + Ok(TaskPurgeReport { + expired_queued: u64::try_from(expired_queued).unwrap_or_default(), + expired_terminal: purged.rows_affected(), + }) + } +} + +impl PostgresTaskStore { + async fn accept_locked( + &self, + transaction: &mut Transaction<'_, Postgres>, + request: &TaskAcceptRequest, + ) -> Result { + // Replay first: an already accepted Task is returned without capacity mutation, so it works + // with a full queue and after the original execution deadline while the record is retained. + if let Some((acceptance, fingerprint)) = + self.stored_acceptance(transaction, request).await? + { + return Ok(if fingerprint == request.input_fingerprint { + Accepted::Replayed(acceptance) + } else { + Accepted::Conflict + }); + } + + // Serialize with synchronous admission and any concurrent acceptance of the same scope. + lock_admission_in_tx(transaction, &request.workspace_id).await?; + + // The competitor may have committed while this transaction waited for the controller lock. + if let Some((acceptance, fingerprint)) = + self.stored_acceptance(transaction, request).await? + { + return Ok(if fingerprint == request.input_fingerprint { + Accepted::Replayed(acceptance) + } else { + Accepted::Conflict + }); + } + + cleanup_expired_unlinked(transaction).await?; + let queued = bounded_queue_len(transaction).await?; + if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) { + return Err(AttemptError::Overloaded); + } + + let created_ms: i64 = sqlx::query( + "insert into tasks ( + task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id, + operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent, + trace_sampled, policy_mode, approval_reference, state, input_envelope, + execution_deadline, retain_until + ) values ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', + $16, to_timestamp($17::double precision / 1000.0), + to_timestamp($18::double precision / 1000.0) + ) + returning (extract(epoch from created_at) * 1000)::bigint as created_ms", + ) + .bind(request.task_id.as_uuid()) + .bind(&request.workspace_id) + .bind(request.owner_kind.as_str()) + .bind(&request.owner_id) + .bind(&request.authorization_scope) + .bind(&request.operation_id) + .bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX)) + .bind(&request.scoped_key) + .bind(&request.input_fingerprint) + .bind(request.support.request_id.as_str()) + .bind(request.support.trace_id.as_str()) + .bind(&request.support.traceparent) + .bind(request.support.trace_sampled) + .bind(policy_mode_str(request.pins.policy_mode)) + .bind(request.pins.approval_reference.as_deref()) + .bind(request.input.value().clone()) + .bind(request.pins.execution_deadline_unix_ms) + .bind(request.pins.retain_until_unix_ms) + .fetch_one(&mut **transaction) + .await + .map_err(task_sql_error)? + .try_get("created_ms") + .map_err(AttemptError::from_sql)?; + + let admission_request = AdmissionRequest { + scope: AdmissionScope { + workspace_id: request.workspace_id.clone(), + agent_key: request.owner_id.clone(), + operation_id: request.operation_id.clone(), + operation_version: request.pins.operation_version, + }, + mode: AdmissionMode::Background, + policy_mode: request.pins.policy_mode, + no_tasks_fallback: NoTasksFallback::Unsupported, + // Never reached: the intent row is inserted by this transaction, not by `admit`. + approval: ApprovalEvidence::NotRequired, + fingerprint: request.input_fingerprint.clone(), + expires_at_unix_ms: request.pins.execution_deadline_unix_ms, + }; + let owner_token = format!("task_{}", request.task_id); + insert_admission_entry( + transaction, + &admission_request, + &owner_token, + "reserved", + Some(request.task_id.as_uuid()), + ) + .await?; + + Ok(Accepted::Created(TaskAcceptance { + task_id: request.task_id, + state: TaskState::Queued, + operation_version: request.pins.operation_version, + support: request.support.clone(), + created_at_unix_ms: created_ms, + retain_until_unix_ms: request.pins.retain_until_unix_ms, + replayed: false, + })) + } + + /// Bounded scoped reread after a foreign unique conflict aborted the transaction. + async fn reread_after_unique( + &self, + request: &TaskAcceptRequest, + ) -> Result { + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "select {TASK_PROJECTION} from tasks + where workspace_id = $1 and owner_kind = $2 and owner_id = $3 + and authorization_scope = $4 and operation_id = $5 and scoped_key = $6" + ))) + .bind(&request.workspace_id) + .bind(request.owner_kind.as_str()) + .bind(&request.owner_id) + .bind(&request.authorization_scope) + .bind(&request.operation_id) + .bind(&request.scoped_key) + .fetch_optional(&self.pool) + .await + .map_err(Self::unavailable)?; + let Some(row) = row else { + return Err(TaskError::unavailable( + "durable task acceptance ended with an unresolved unique conflict", + )); + }; + let record = record_from_row(&row)?; + if record.input_fingerprint != request.input_fingerprint { + return Err(TaskError::FingerprintConflict); + } + Ok(acceptance_from_record(&record)) + } + + /// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are + /// reported without payload, and a foreign Task is indistinguishable from an absent one. + async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError { + let access = &request.access; + match sqlx::query( + "select state, version from tasks + where task_id = $1 and workspace_id = $2 and owner_kind = $3 + and owner_id = $4 and authorization_scope = $5", + ) + .bind(request.task_id.as_uuid()) + .bind(&access.workspace_id) + .bind(access.owner_kind.as_str()) + .bind(&access.owner_id) + .bind(&access.authorization_scope) + .fetch_optional(&self.pool) + .await + { + Ok(Some(_)) => TaskError::StateConflict, + Ok(None) => TaskError::NotFound, + Err(error) => Self::unavailable(error), + } + } +} + +fn policy_mode_str(mode: ToolExecutionMode) -> &'static str { + match mode { + ToolExecutionMode::Synchronous => "synchronous", + ToolExecutionMode::Background => "background", + ToolExecutionMode::Adaptive => "adaptive", + } +} + +fn task_sql_error(error: sqlx::Error) -> AttemptError { + if error + .as_database_error() + .and_then(|database| database.code()) + .as_deref() + == Some("23505") + { + return AttemptError::UniqueViolation; + } + AttemptError::from_sql(error) +} + +fn record_from_row(row: &sqlx::postgres::PgRow) -> Result { + let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?; + let state: String = row.try_get("state").map_err(unavailable_row)?; + let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?; + let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?; + let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?; + let request_id: String = row.try_get("request_id").map_err(unavailable_row)?; + let envelope = |column: &str| -> Result, TaskError> { + let value: Option = row.try_get(column).map_err(unavailable_row)?; + value.map(TaskEnvelope::from_persisted).transpose() + }; + Ok(TaskRecord { + task_id: TaskId::parse(&task_id.to_string())?, + state: TaskState::parse(&state)?, + workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?, + owner_kind: TaskOwnerKind::parse(&owner_kind)?, + owner_id: row.try_get("owner_id").map_err(unavailable_row)?, + authorization_scope: row + .try_get("authorization_scope") + .map_err(unavailable_row)?, + operation_id: row.try_get("operation_id").map_err(unavailable_row)?, + input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?, + pins: TaskPins { + operation_version: u32::try_from( + row.try_get::("operation_version") + .map_err(unavailable_row)?, + ) + .unwrap_or_default(), + policy_mode: match policy_mode.as_str() { + "background" => ToolExecutionMode::Background, + "adaptive" => ToolExecutionMode::Adaptive, + _ => ToolExecutionMode::Synchronous, + }, + execution_deadline_unix_ms: row + .try_get("execution_deadline_ms") + .map_err(unavailable_row)?, + retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?, + approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?, + }, + support: TaskSupportIds { + request_id: crank_core::RequestId::parse(&request_id) + .map_err(|_| TaskError::unavailable("stored request id is invalid"))?, + trace_id: crank_core::TraceId::parse(&trace_id) + .map_err(|_| TaskError::unavailable("stored trace id is invalid"))?, + traceparent: row.try_get("traceparent").map_err(unavailable_row)?, + trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?, + }, + input: envelope("input_envelope")? + .ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?, + progress: envelope("progress_envelope")?, + result: envelope("result_envelope")?, + error: envelope("error_envelope")?, + attempts: u32::try_from(row.try_get::("attempts").map_err(unavailable_row)?) + .unwrap_or_default(), + version: u32::try_from(row.try_get::("version").map_err(unavailable_row)?) + .unwrap_or_default(), + created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?, + updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?, + }) +} + +fn unavailable_row(error: sqlx::Error) -> TaskError { + PostgresTaskStore::unavailable(error) +} diff --git a/crates/crank-registry/tests/integration.rs b/crates/crank-registry/tests/integration.rs index 1de7ee3..511820a 100644 --- a/crates/crank-registry/tests/integration.rs +++ b/crates/crank-registry/tests/integration.rs @@ -17,5 +17,6 @@ mod integration { mod prompt_drafts; mod prompt_publication; mod resource_drafts; + mod tasks; mod workspace_access; } diff --git a/crates/crank-registry/tests/integration/migrations.rs b/crates/crank-registry/tests/integration/migrations.rs index a5ed79f..720e85b 100644 --- a/crates/crank-registry/tests/integration/migrations.rs +++ b/crates/crank-registry/tests/integration/migrations.rs @@ -70,6 +70,8 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() { (21, "prompts-v21"), (22, "prompt-publications-v22"), (23, "prompt-dependencies-v23"), + (24, "execution-admission-v24"), + (25, "durable-tasks-v25"), ]; assert_eq!(rows.len(), expected.len()); for (row, (version, name)) in rows.iter().zip(expected) { @@ -180,7 +182,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() { ); assert_eq!( MigrationAuthority::preflight(first.pool()).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } #[tokio::test] @@ -228,7 +230,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() { .expect("the V18-local guard must be restorable"); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "restoring the exact V18 contract must return the current verdict" ); } @@ -354,7 +356,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -467,13 +469,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 2, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); let trace_column: bool = sqlx::query_scalar( "select exists ( @@ -516,7 +518,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 3, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -576,7 +578,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 4, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -830,6 +832,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } diff --git a/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs b/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs index ceaed97..d2a102d 100644 --- a/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs +++ b/crates/crank-registry/tests/integration/migrations/artifact_metadata.rs @@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 11, - target: 23, + target: 25, } ); @@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "restore: {restore}" ); } @@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "restore: {restore}" ); } @@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) { .unwrap(); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "restore: {restore}" ); } @@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); for (drift, restore) in [ @@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() { sqlx::raw_sql(restore).execute(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "restore: {restore}" ); } diff --git a/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs b/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs index a0d5b1e..430d931 100644 --- a/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs +++ b/crates/crank-registry/tests/integration/migrations/artifact_metadata/exhaustive_v12.rs @@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column ); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "a rejected primary-key nullability change must leave the schema current" ); @@ -473,7 +473,7 @@ async fn assert_definition_only_drift( assert_eq!(super::catalog_constraint_definitions(pool).await, before); assert_eq!( MigrationAuthority::preflight(pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } @@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "the baseline before the drift matrix must be the current V16 schema" ); let mut observations: Vec = Vec::new(); @@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift .await .expect("restoring the exact definition must apply"); let restored = MigrationAuthority::preflight(&pool).await; - let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 23 })); + let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 })); // Точное восстановление доказывается сравнением снятого снимка с baseline, а не // мнением того же guard-а, который принимает неправильные определения (F4). let restored_exact = catalog_definition_snapshot(&pool).await == before; @@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "baseline before inserting the shadow schema" ); @@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() // Тень не меняет вердикт guard-а по текущей схеме. assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "a same-named object in another schema must not change the verdict for the current schema" ); @@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() ); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 }, + MigrationPreflight::Current { version: 25 }, "the exact restoration must return the current V16 schema" ); @@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema() .unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); assert!( diff --git a/crates/crank-registry/tests/integration/migrations/import_plan.rs b/crates/crank-registry/tests/integration/migrations/import_plan.rs index 4fad481..2dcf092 100644 --- a/crates/crank-registry/tests/integration/migrations/import_plan.rs +++ b/crates/crank-registry/tests/integration/migrations/import_plan.rs @@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); let tables = sqlx::query( "select table_name @@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 13, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); diff --git a/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs b/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs index 5b24994..2bc519c 100644 --- a/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs +++ b/crates/crank-registry/tests/integration/migrations/legacy_adoption.rs @@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() { assert_eq!(operation_count, 1); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } @@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los ); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } @@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); @@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() { assert!(supports_elicitation); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } @@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 0, - target: 23, + target: 25, } ); MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } diff --git a/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs b/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs index 64cb118..7db1994 100644 --- a/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs +++ b/crates/crank-registry/tests/integration/migrations/resource_security_revocations.rs @@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() { MigrationAuthority::apply(&pool).await.unwrap(); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations") .fetch_one(&pool) @@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 18, - target: 23, + target: 25, } ); assert_eq!( MigrationAuthority::apply(&pool).await.unwrap(), - MigrationApplyResult::Applied { from: 18, to: 23 } + MigrationApplyResult::Applied { from: 18, to: 25 } ); let after: Vec<(i64, String, String)> = sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version") @@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() { assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten"); assert_eq!( MigrationAuthority::preflight(&pool).await.unwrap(), - MigrationPreflight::Current { version: 23 } + MigrationPreflight::Current { version: 25 } ); } @@ -84,18 +84,18 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() { let database_url = crank_test_support::postgres_schema_url("test_v19_future_version").await; let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); MigrationAuthority::apply(&pool).await.unwrap(); - // The same `version > CURRENT_VERSION` branch an old V23 binary takes when it - // meets a V24 database; a synthetic V24 row exercises it without a second binary. + // The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer + // database; a synthetic version above the current one exercises it without a second binary. sqlx::query( "insert into __crank_migrations (version, name, checksum, phase, compatibility) - values (24, 'future-v24', repeat('a', 64), 'expand', 'n-minus-one-readable')", + values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')", ) .execute(&pool) .await .unwrap(); let error = MigrationAuthority::preflight(&pool).await.unwrap_err(); assert_eq!(error.code(), "future_version"); - assert_eq!(error.version(), Some(24)); + assert_eq!(error.version(), Some(26)); assert_eq!(error.recovery(), "install_matching_application"); assert!( MigrationAuthority::require_current(&pool).await.is_err(), @@ -155,7 +155,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 18, - target: 23, + target: 25, } ); } diff --git a/crates/crank-registry/tests/integration/migrations/rollback.rs b/crates/crank-registry/tests/integration/migrations/rollback.rs index af65c71..add4333 100644 --- a/crates/crank-registry/tests/integration/migrations/rollback.rs +++ b/crates/crank-registry/tests/integration/migrations/rollback.rs @@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() { MigrationAuthority::preflight(&pool).await.unwrap(), MigrationPreflight::MigrationRequired { current: 11, - target: 23, + target: 25, } ); } diff --git a/crates/crank-registry/tests/integration/migrations/schema_removal.rs b/crates/crank-registry/tests/integration/migrations/schema_removal.rs index b26d056..1bfa76a 100644 --- a/crates/crank-registry/tests/integration/migrations/schema_removal.rs +++ b/crates/crank-registry/tests/integration/migrations/schema_removal.rs @@ -38,6 +38,7 @@ pub(crate) async fn remove_v4_schema(pool: &sqlx::PgPool) { .unwrap(); } pub(crate) async fn remove_v5_schema(pool: &sqlx::PgPool) { + remove_v24_schema(pool).await; remove_v6_schema(pool).await; sqlx::raw_sql( "drop index if exists invocation_logs_workspace_operation_version_idx; @@ -154,6 +155,7 @@ pub(crate) async fn remove_v11_schema(pool: &sqlx::PgPool) { .unwrap(); } pub(crate) async fn remove_v14_schema(pool: &sqlx::PgPool) { + remove_v24_schema(pool).await; remove_v15_schema(pool).await; sqlx::raw_sql( "drop table if exists import_plan_applications; @@ -224,6 +226,7 @@ pub(crate) async fn remove_v18_schema(pool: &sqlx::PgPool) { .unwrap(); } pub(crate) async fn remove_v19_schema(pool: &sqlx::PgPool) { + remove_v24_schema(pool).await; remove_v20_schema(pool).await; sqlx::raw_sql( "drop table if exists resource_security_revocations; @@ -307,6 +310,7 @@ pub(crate) async fn remove_v20_schema(pool: &sqlx::PgPool) { .unwrap(); } pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) { + remove_v24_schema(pool).await; remove_v14_schema(pool).await; sqlx::raw_sql( "drop index if exists import_jobs_openapi_source_idx; @@ -323,3 +327,27 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) { .await .unwrap(); } + +/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25. +pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) { + sqlx::raw_sql( + "drop table if exists tasks cascade; + delete from __crank_migrations where version = 25;", + ) + .execute(pool) + .await + .unwrap(); +} + +/// Story 4.1: снять V24 (authoritative execution admission) перед возвратом к < 24. +pub(crate) async fn remove_v24_schema(pool: &sqlx::PgPool) { + remove_v25_schema(pool).await; + sqlx::raw_sql( + "drop table if exists execution_admission_entries; + drop table if exists execution_admission_controllers; + delete from __crank_migrations where version = 24;", + ) + .execute(pool) + .await + .unwrap(); +} diff --git a/crates/crank-registry/tests/integration/tasks.rs b/crates/crank-registry/tests/integration/tasks.rs new file mode 100644 index 0000000..f169fff --- /dev/null +++ b/crates/crank-registry/tests/integration/tasks.rs @@ -0,0 +1,682 @@ +//! ATDD Story 4.2 (FR-22/FR-23): durable Task acceptance on a real PostgreSQL schema. +//! +//! These oracles drive the real store: Task + dispatch intent + queue slot in one transaction, replay +//! and typed conflict before any capacity mutation, bounded shared capacity, the Task-linked intent +//! guard of the legacy admission paths, one-winner terminal CAS with an immutable result, bounded +//! retention, owner/authorization-scope isolation and fail-closed health. + +use super::common; + +use crank_core::{ + AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence, + ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope, + TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore, + TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms, +}; +use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore}; +use serde_json::json; +use sqlx::{PgPool, Row}; + +const WORKSPACE: &str = "ws_default"; +const AGENT: &str = "agent_a"; + +fn limits(queue: usize) -> AdmissionLimits { + AdmissionLimits { + global_limit: 4, + agent_limit: 4, + tool_limit: 4, + queue_limit: queue, + } +} + +fn store_with(pool: PgPool, queue: usize) -> PostgresTaskStore { + PostgresTaskStore::new(pool, limits(queue), Default::default()).expect("valid store config") +} + +fn unsampled_support() -> TaskSupportIds { + TaskSupportIds::new( + RequestId::generate(), + &TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00").unwrap(), + ) +} + +fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> TaskAcceptRequest { + let deadline = current_unix_ms() + deadline_in_ms; + TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: WORKSPACE.to_owned(), + owner_kind: TaskOwnerKind::Agent, + owner_id: agent.to_owned(), + authorization_scope: format!("credential:pk_{agent}"), + operation_id: "op_report".to_owned(), + pins: TaskPins { + operation_version: 3, + policy_mode: ToolExecutionMode::Background, + execution_deadline_unix_ms: deadline, + retain_until_unix_ms: deadline + 3_600_000, + approval_reference: Some("appr_1".to_owned()), + }, + scoped_key: key.to_owned(), + input_fingerprint: fingerprint.to_owned(), + input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(), + support: unsampled_support(), + } +} + +fn access(agent: &str) -> TaskAccess { + TaskAccess { + workspace_id: WORKSPACE.to_owned(), + owner_kind: TaskOwnerKind::Agent, + owner_id: agent.to_owned(), + authorization_scope: format!("credential:pk_{agent}"), + } +} + +fn key(seed: &str) -> String { + format!("{seed:0<64}").chars().take(64).collect() +} + +fn digest(seed: &str) -> String { + format!("{seed:0<64}").chars().take(64).collect() +} + +async fn row_counts(pool: &PgPool) -> (i64, i64) { + let row = sqlx::query( + "select + (select count(*)::bigint from tasks) as tasks, + (select count(*)::bigint from execution_admission_entries where task_id is not null) + as intents", + ) + .fetch_one(pool) + .await + .unwrap(); + (row.get("tasks"), row.get("intents")) +} + +#[tokio::test] +async fn acceptance_commits_task_intent_and_slot_together() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + + let request = request(&key("k1"), &digest("f1"), AGENT, 30_000); + let task_id = request.task_id; + let support = request.support.clone(); + + // Another connection holds both controller rows: the acceptance cannot commit yet. + let mut blocker = registry.pool().begin().await.unwrap(); + sqlx::query( + "insert into execution_admission_controllers (workspace_id) values ('*'), ($1) + on conflict (workspace_id) do nothing", + ) + .bind(WORKSPACE) + .execute(&mut *blocker) + .await + .unwrap(); + sqlx::query("update execution_admission_controllers set updated_at = updated_at") + .execute(&mut *blocker) + .await + .unwrap(); + + let pool = registry.pool().clone(); + let acceptance = tokio::spawn(async move { store.accept(request).await }); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + + // No partial Task, no intent and no accepted response exist before COMMIT. + assert!(!acceptance.is_finished()); + assert_eq!(row_counts(&pool).await, (0, 0)); + + blocker.commit().await.unwrap(); + let acceptance = acceptance.await.unwrap().expect("acceptance commits"); + assert_eq!(acceptance.state, TaskState::Queued); + assert!(!acceptance.replayed); + assert_eq!(acceptance.task_id, task_id); + assert_eq!(acceptance.support, support); + assert_eq!(row_counts(registry.pool()).await, (1, 1)); + + // A fresh connection and store read the committed pins/identities back. + let reader = store_with(database.concurrent_registry(2).await.pool().clone(), 8); + let record = reader.read(&access(AGENT), task_id).await.unwrap(); + assert_eq!(record.state, TaskState::Queued); + assert_eq!(record.pins.operation_version, 3); + assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background); + assert_eq!(record.pins.approval_reference.as_deref(), Some("appr_1")); + assert_eq!(record.support.request_id, support.request_id); + assert_eq!(record.support.trace_id, support.trace_id); + assert_eq!(record.support.traceparent, support.traceparent); + assert!(!record.support.trace_sampled); + // Secret-bearing input is classified before persistence. + assert_eq!(record.input.value()["api_key"], json!("[redacted]")); + assert_eq!(record.input.value()["report"], json!("q3")); + assert!(record.result.is_none()); + + database.cleanup().await; +} + +#[tokio::test] +async fn replay_returns_the_same_task_and_one_slot() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + + let first = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .expect("first acceptance"); + let before = store.read(&access(AGENT), first.task_id).await.unwrap(); + + let replay = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .expect("replay returns the accepted Task"); + assert_eq!(replay.task_id, first.task_id); + assert!(replay.replayed); + assert_eq!(replay.support.request_id, first.support.request_id); + assert_eq!(row_counts(registry.pool()).await, (1, 1)); + + // Replay never rewrites the accepted row or its audit evidence. + let after = store.read(&access(AGENT), first.task_id).await.unwrap(); + assert_eq!(after.version, before.version); + assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms); + assert_eq!(after.created_at_unix_ms, before.created_at_unix_ms); + + database.cleanup().await; +} + +#[tokio::test] +async fn replay_stays_available_with_a_full_queue_and_past_execution_deadline() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 1); + + // A short deadline: the Task is still retained after its execution deadline elapsed. + let first = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 1)) + .await + .expect("first acceptance"); + sqlx::query( + "update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1", + ) + .bind(first.task_id.as_uuid()) + .execute(registry.pool()) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + + let replayed = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 1)) + .await + .expect("replay works with a full queue and an elapsed execution deadline"); + assert_eq!(replayed.task_id, first.task_id); + assert!(replayed.replayed); + + database.cleanup().await; +} + +#[tokio::test] +async fn a_different_fingerprint_on_the_same_key_is_a_typed_conflict() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + + let first = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .expect("first acceptance"); + let before = store.read(&access(AGENT), first.task_id).await.unwrap(); + + let conflict = store + .accept(request(&key("k1"), &digest("f2"), AGENT, 30_000)) + .await + .expect_err("different input is a conflict"); + assert_eq!(conflict, TaskError::FingerprintConflict); + assert_eq!(row_counts(registry.pool()).await, (1, 1)); + + let after = store.read(&access(AGENT), first.task_id).await.unwrap(); + assert_eq!(after.version, before.version); + assert_eq!(after.updated_at_unix_ms, before.updated_at_unix_ms); + + database.cleanup().await; +} + +#[tokio::test] +async fn bounded_capacity_is_shared_and_overload_creates_nothing() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 1); + let admission = + PostgresExecutionAdmission::new(registry.pool().clone(), limits(1), Default::default()) + .unwrap(); + + // A durable enqueue decision and a durable Task share the same bounded queue budget. + let waiting = admission + .admit(AdmissionRequest { + scope: AdmissionScope { + workspace_id: WORKSPACE.to_owned(), + agent_key: AGENT.to_owned(), + operation_id: "op_sync".to_owned(), + operation_version: 1, + }, + mode: AdmissionMode::Background, + policy_mode: ToolExecutionMode::Background, + no_tasks_fallback: NoTasksFallback::Synchronous, + approval: ApprovalEvidence::NotRequired, + fingerprint: "fp_sync".to_owned(), + expires_at_unix_ms: current_unix_ms() + 30_000, + }) + .await + .unwrap(); + assert!(matches!(waiting, AdmissionOutcome::Enqueued(_))); + + let overloaded = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .expect_err("queue is full"); + assert!(matches!(overloaded, TaskError::Overloaded { .. })); + assert_eq!(row_counts(registry.pool()).await, (0, 0)); + assert_eq!( + admission.snapshot().await.unwrap().waiting, + 1, + "the durable refusal changed no capacity" + ); + + database.cleanup().await; +} + +#[tokio::test] +async fn legacy_cleanup_and_cancel_never_remove_a_task_linked_intent() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + let admission = + PostgresExecutionAdmission::new(registry.pool().clone(), limits(8), Default::default()) + .unwrap(); + + let accepted = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 1)) + .await + .expect("acceptance"); + // The linked reserved row looks expired; the legacy TTL cleanup must ignore it. + sqlx::query("update execution_admission_entries set expires_at = now() - interval '1 hour'") + .execute(registry.pool()) + .await + .unwrap(); + + let ticket: i64 = + sqlx::query("select ticket_id from execution_admission_entries where task_id = $1") + .bind(accepted.task_id.as_uuid()) + .fetch_one(registry.pool()) + .await + .unwrap() + .get("ticket_id"); + let owner_token: String = + sqlx::query("select owner_token from execution_admission_entries where ticket_id = $1") + .bind(ticket) + .fetch_one(registry.pool()) + .await + .unwrap() + .get("owner_token"); + + // A legacy attempt runs the bounded expiry cleanup; a foreign/own legacy cancel is refused. + let _ = admission + .admit(AdmissionRequest { + scope: AdmissionScope { + workspace_id: WORKSPACE.to_owned(), + agent_key: "agent_b".to_owned(), + operation_id: "op_sync".to_owned(), + operation_version: 1, + }, + mode: AdmissionMode::Synchronous, + policy_mode: ToolExecutionMode::Synchronous, + no_tasks_fallback: NoTasksFallback::Synchronous, + approval: ApprovalEvidence::NotRequired, + fingerprint: "fp_sync".to_owned(), + expires_at_unix_ms: current_unix_ms() + 50, + }) + .await; + assert!(!admission.cancel(ticket as u64, &owner_token).await.unwrap()); + assert_eq!(row_counts(registry.pool()).await, (1, 1)); + assert_eq!( + store + .read(&access(AGENT), accepted.task_id) + .await + .unwrap() + .state, + TaskState::Queued + ); + + database.cleanup().await; +} + +#[tokio::test] +async fn terminal_transition_has_one_winner_and_an_immutable_result() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + + let accepted = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .unwrap(); + let queued = store.read(&access(AGENT), accepted.task_id).await.unwrap(); + assert_eq!(queued.version, 1); + + // Moving a queued Task straight to completed is not an allowed transition. + let skipped = TaskFinishRequest { + access: access(AGENT), + task_id: accepted.task_id, + expected_state: TaskState::Queued, + expected_version: queued.version, + target_state: TaskState::Completed, + result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()), + error: None, + }; + assert_eq!( + store + .finish(skipped) + .await + .expect_err("transition rejected"), + TaskError::Invalid { + message: "task state transition is not allowed".to_owned() + } + ); + + let started = TaskFinishRequest { + access: access(AGENT), + task_id: accepted.task_id, + expected_state: TaskState::Queued, + expected_version: queued.version, + target_state: TaskState::Working, + result: None, + error: None, + }; + assert_eq!( + store + .finish(started) + .await + .expect_err("working is not terminal"), + TaskError::Invalid { + message: "task finish target is not terminal".to_owned() + } + ); + sqlx::query("update tasks set state = 'working' where task_id = $1") + .bind(accepted.task_id.as_uuid()) + .execute(registry.pool()) + .await + .unwrap(); + let working = store.read(&access(AGENT), accepted.task_id).await.unwrap(); + + let mut finishers = Vec::new(); + for marker in ["first", "second"] { + let store = store.clone(); + let mut finish = TaskFinishRequest { + access: access(AGENT), + task_id: accepted.task_id, + expected_state: TaskState::Working, + expected_version: working.version, + target_state: TaskState::Completed, + result: Some(TaskEnvelope::try_redacted(&json!({"marker": marker})).unwrap()), + error: None, + }; + finish.target_state = TaskState::Completed; + finishers.push(tokio::spawn(async move { store.finish(finish).await })); + } + let first = finishers.remove(0).await.unwrap(); + let second = finishers.remove(0).await.unwrap(); + let winners = [&first, &second] + .iter() + .filter(|result| result.is_ok()) + .count(); + assert_eq!(winners, 1, "exactly one concurrent CAS wins"); + let loser = if first.is_err() { &first } else { &second }; + assert_eq!( + loser.as_ref().err().unwrap(), + &TaskError::StateConflict, + "the loser observes a rejected transition, not a rewrite" + ); + + let terminal = store.read(&access(AGENT), accepted.task_id).await.unwrap(); + assert_eq!(terminal.state, TaskState::Completed); + assert_eq!(terminal.version, working.version + 1); + let marker = terminal.result.as_ref().unwrap().value()["marker"] + .as_str() + .unwrap() + .to_owned(); + assert!(marker == "first" || marker == "second"); + + // A repeated terminal write and a terminal->terminal move are rejected. + let repeated = TaskFinishRequest { + access: access(AGENT), + task_id: accepted.task_id, + expected_state: TaskState::Completed, + expected_version: terminal.version, + target_state: TaskState::Failed, + result: None, + error: Some(TaskEnvelope::try_redacted(&json!({"reason": "x"})).unwrap()), + }; + assert!(repeated.validate().is_err()); + + // Terminal Task and its dispatch intent left together. + assert_eq!(row_counts(registry.pool()).await, (1, 0)); + + database.cleanup().await; +} + +#[tokio::test] +async fn retention_is_bounded_and_never_deletes_live_work() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + + let queued = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .unwrap(); + let expired_queued = store + .accept(request(&key("k2"), &digest("f2"), AGENT, 1)) + .await + .unwrap(); + sqlx::query( + "update tasks set execution_deadline = now() - interval '1 hour' where task_id = $1", + ) + .bind(expired_queued.task_id.as_uuid()) + .execute(registry.pool()) + .await + .unwrap(); + // A retained terminal Task whose retention has not expired yet and one that has. + let terminal = store + .accept(request(&key("k3"), &digest("f3"), AGENT, 30_000)) + .await + .unwrap(); + sqlx::query( + "update tasks set state = 'completed', result_envelope = '{\"ok\":true}'::jsonb, + retain_until = now() + interval '1 hour' where task_id = $1", + ) + .bind(terminal.task_id.as_uuid()) + .execute(registry.pool()) + .await + .unwrap(); + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(terminal.task_id.as_uuid()) + .execute(registry.pool()) + .await + .unwrap(); + let expired_terminal = store + .accept(request(&key("k4"), &digest("f4"), AGENT, 30_000)) + .await + .unwrap(); + sqlx::query( + "update tasks set state = 'cancelled', execution_deadline = now() - interval '2 hours', + retain_until = now() - interval '1 hour' + where task_id = $1", + ) + .bind(expired_terminal.task_id.as_uuid()) + .execute(registry.pool()) + .await + .unwrap(); + sqlx::query("delete from execution_admission_entries where task_id = $1") + .bind(expired_terminal.task_id.as_uuid()) + .execute(registry.pool()) + .await + .unwrap(); + + let report = store.purge_expired(64).await.unwrap(); + assert_eq!(report.expired_queued, 1); + assert_eq!(report.expired_terminal, 1); + + // Live queued work is never deleted or failed; the expired queued Task released its slot. + assert_eq!( + store + .read(&access(AGENT), queued.task_id) + .await + .unwrap() + .state, + TaskState::Queued + ); + assert_eq!( + store + .read(&access(AGENT), terminal.task_id) + .await + .unwrap() + .state, + TaskState::Completed + ); + assert!(matches!( + store + .read(&access(AGENT), expired_queued.task_id) + .await + .unwrap() + .state, + TaskState::Failed + )); + assert!( + store + .read(&access(AGENT), expired_terminal.task_id) + .await + .is_err() + ); + let remaining: i64 = sqlx::query("select count(*)::bigint as count from execution_admission_entries where task_id is not null") + .fetch_one(registry.pool()) + .await + .unwrap() + .get("count"); + assert_eq!(remaining, 1, "only the live queued Task keeps its intent"); + + database.cleanup().await; +} + +#[tokio::test] +async fn foreign_scope_reads_are_not_found_and_never_leak_payload() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + + let accepted = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .unwrap(); + + // Positive control: the owner reads its Task. + assert!(store.read(&access(AGENT), accepted.task_id).await.is_ok()); + + // Foreign owner, foreign authorization scope and foreign workspace are all not-found. + let mut foreign_owner = access(AGENT); + foreign_owner.owner_id = "agent_b".to_owned(); + assert_eq!( + store + .read(&foreign_owner, accepted.task_id) + .await + .unwrap_err(), + TaskError::NotFound + ); + let mut foreign_scope = access(AGENT); + foreign_scope.authorization_scope = "credential:pk_other".to_owned(); + assert_eq!( + store + .read(&foreign_scope, accepted.task_id) + .await + .unwrap_err(), + TaskError::NotFound + ); + let mut foreign_workspace = access(AGENT); + foreign_workspace.workspace_id = "ws_other".to_owned(); + assert_eq!( + store + .read(&foreign_workspace, accepted.task_id) + .await + .unwrap_err(), + TaskError::NotFound + ); + + // The same caller key under a foreign owner creates its own Task, never a visible collision. + let other = store + .accept(request(&key("k1"), &digest("f1"), "agent_b", 30_000)) + .await + .unwrap(); + assert_ne!(other.task_id, accepted.task_id); + assert_eq!(row_counts(registry.pool()).await, (2, 2)); + + database.cleanup().await; +} + +#[tokio::test] +async fn invalid_bounded_envelope_is_rejected_before_persistence() { + let database = common::TestDatabase::new().await; + // Fresh schema: the first connection applies the migration, later pools reuse the same schema. + let _migrated = database.registry().await; + let registry = database.concurrent_registry(4).await; + let store = store_with(registry.pool().clone(), 8); + + assert!( + TaskEnvelope::try_redacted(&json!({"blob": "x".repeat(64 * 1024)})).is_err(), + "an oversized envelope is rejected before any accept" + ); + + let mut invalid = request(&key("k1"), &digest("f1"), AGENT, 30_000); + invalid.scoped_key = "short".to_owned(); + assert!(matches!(invalid.validate(), Err(TaskError::Invalid { .. }))); + assert_eq!( + store.accept(invalid).await.unwrap_err(), + TaskError::Invalid { + message: "task scoped key digest is invalid".to_owned() + } + ); + assert_eq!(row_counts(registry.pool()).await, (0, 0)); + + database.cleanup().await; +} + +#[tokio::test] +async fn store_failure_fails_closed_without_records() { + let database = common::TestDatabase::new().await; + // No migration on this schema: the durable Task state does not exist yet. + let unmigrated = database.raw_pool().await; + let store = store_with(unmigrated, 8); + + assert!(store.health().await.is_err()); + let error = store + .accept(request(&key("k1"), &digest("f1"), AGENT, 30_000)) + .await + .expect_err("acceptance fails closed"); + assert!(matches!(error, TaskError::Unavailable { .. })); + + database.cleanup().await; +} diff --git a/crates/crank-runtime/src/idempotency.rs b/crates/crank-runtime/src/idempotency.rs index c3434c1..d9f9118 100644 --- a/crates/crank-runtime/src/idempotency.rs +++ b/crates/crank-runtime/src/idempotency.rs @@ -314,7 +314,7 @@ fn request_fingerprint(input: &Value) -> Result { Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded))) } -fn canonical_json(value: &Value) -> Value { +pub(crate) fn canonical_json(value: &Value) -> Value { match value { Value::Object(object) => { let mut entries = object.iter().collect::>(); diff --git a/crates/crank-runtime/src/lib.rs b/crates/crank-runtime/src/lib.rs index 12a0e0a..41114ba 100644 --- a/crates/crank-runtime/src/lib.rs +++ b/crates/crank-runtime/src/lib.rs @@ -15,6 +15,7 @@ mod rate_limit; mod request_context; mod request_preparation; mod response_cache; +mod tasks; pub fn validate_resource_template_parameter( schema: &crank_schema::Schema, value: &serde_json::Value, @@ -114,3 +115,4 @@ pub use rate_limit::{ }; pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext}; pub use secret_crypto::SecretCrypto; +pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService}; diff --git a/crates/crank-runtime/src/tasks.rs b/crates/crank-runtime/src/tasks.rs new file mode 100644 index 0000000..1acafa3 --- /dev/null +++ b/crates/crank-runtime/src/tasks.rs @@ -0,0 +1,266 @@ +//! Runtime application service of durable Tasks (Story 4.2, FR-22/FR-23). +//! +//! The service is the only place that turns an approved background request into the durable domain +//! request: bounded validated input, canonical input fingerprint, full-scope idempotency digest, +//! pinned Published Version/deadline/retention and the support identities of the request. It holds an +//! owned request, never `ResolvedAuth` or raw secrets, and it never dispatches: the durable store +//! commits Task + dispatch intent + queue slot in one transaction and only then produces an +//! acknowledgement. +//! +//! Reads go through the same owner-scoped access proof, so the durable row is only fetched for the +//! caller that owns it. There is no in-memory fallback: a store failure is a typed refusal. + +use std::sync::Arc; +use std::time::Instant; + +use crank_core::{ + CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, + TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, + TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds, + ToolExecutionMode, current_unix_ms, validate_task_envelope, +}; +use sha2::{Digest, Sha256}; + +/// One owned background request, already authorized by the application layer. +#[derive(Clone, Debug)] +pub struct TaskAcceptDraft { + pub workspace_id: String, + pub owner_kind: TaskOwnerKind, + pub owner_id: String, + /// Server-derived authorization scope identity of the verified principal. + pub authorization_scope: String, + pub operation_id: String, + pub operation_version: u32, + pub policy_mode: ToolExecutionMode, + /// Caller key, bounded before it becomes a full-scope digest. + pub scoped_key: String, + /// Canonical executable input; it is validated, fingerprinted, then persisted redacted. + pub input: serde_json::Value, + pub correlation: CorrelationContext, + /// Absolute execution deadline of the accepted work. + pub deadline: Instant, + /// Retention after the execution deadline. + pub retention_ms: i64, + pub approval_reference: Option, +} + +/// Projection of the durable status/result read. `result`/`error` are only present once a terminal +/// transition committed; a non-terminal Task reports its state and inputs without inventing output. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaskOutcome { + pub task_id: TaskId, + pub state: crank_core::TaskState, + pub result: Option, + pub error: Option, +} + +impl TaskOutcome { + fn from_record(record: TaskRecord) -> Self { + Self { + task_id: record.task_id, + state: record.state, + result: record.result.map(|envelope| envelope.value().clone()), + error: record.error.map(|envelope| envelope.value().clone()), + } + } +} + +#[derive(Clone)] +pub struct TaskService { + store: Arc, +} + +impl TaskService { + pub fn new(store: Arc) -> Self { + Self { store } + } + + /// Readiness of the mandatory durable Task state. + pub async fn health(&self) -> Result<(), TaskError> { + self.store.health().await + } + + /// Durable acceptance: the returned identity exists only after the store committed it. + pub async fn accept(&self, draft: TaskAcceptDraft) -> Result { + validate_task_envelope(&draft.input)?; + let (scoped_key, input_fingerprint) = ( + scoped_key_digest(&draft)?, + canonical_input_fingerprint(&draft.input)?, + ); + let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline); + let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS); + let request = TaskAcceptRequest { + task_id: TaskId::generate(), + workspace_id: draft.workspace_id, + owner_kind: draft.owner_kind, + owner_id: draft.owner_id, + authorization_scope: draft.authorization_scope, + operation_id: draft.operation_id, + pins: TaskPins { + operation_version: draft.operation_version, + policy_mode: draft.policy_mode, + execution_deadline_unix_ms, + retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms), + approval_reference: draft.approval_reference, + }, + scoped_key, + input_fingerprint, + input: TaskEnvelope::try_redacted(&draft.input)?, + support: TaskSupportIds::new( + draft.correlation.request_id().clone(), + draft.correlation.trace_context(), + ), + }; + self.store.accept(request).await + } + + /// Owner-scoped status read. A foreign or absent Task is one typed not-found. + pub async fn status( + &self, + access: &TaskAccess, + task_id: TaskId, + ) -> Result { + self.store.read(access, task_id).await + } + + /// Owner-scoped result read, projected from the same authorized row. + pub async fn result( + &self, + access: &TaskAccess, + task_id: TaskId, + ) -> Result { + Ok(TaskOutcome::from_record( + self.store.read(access, task_id).await?, + )) + } + + /// Default retention pinned by the application layer when it does not choose one explicitly. + pub fn default_retention_ms() -> i64 { + TASK_DEFAULT_RETENTION_MS + } + + /// Scoped lookup of an already accepted Task by the same full-scope identity, used to keep a + /// retry of an explicitly pinned version from being accepted as new work. + pub async fn lookup(&self, draft: &TaskAcceptDraft) -> Result, TaskError> { + let scoped_key = scoped_key_digest(draft)?; + let access = TaskAccess { + workspace_id: draft.workspace_id.clone(), + owner_kind: draft.owner_kind, + owner_id: draft.owner_id.clone(), + authorization_scope: draft.authorization_scope.clone(), + }; + self.store + .find(&access, &draft.operation_id, &scoped_key) + .await + } + + /// Canonical digest of a validated input, exposed so the application layer can compare a stored + /// Task against the request it is retrying without duplicating the canonicalization. + pub fn input_fingerprint(input: &serde_json::Value) -> Result { + canonical_input_fingerprint(input) + } +} + +fn deadline_unix_ms(deadline: Instant) -> i64 { + let remaining = u64::try_from( + deadline + .saturating_duration_since(Instant::now()) + .as_millis(), + ) + .unwrap_or_default(); + current_unix_ms().saturating_add(i64::try_from(remaining.max(1)).unwrap_or(i64::MAX)) +} + +/// Digest of the caller key over the full Task identity. The scope is never abbreviated to a +/// connection, protocol revision or raw credential, so two callers cannot collide on one key. +fn scoped_key_digest(draft: &TaskAcceptDraft) -> Result { + let key = draft.scoped_key.as_str(); + if key.is_empty() + || key.len() > TASK_SCOPED_KEY_MAX_BYTES + || key.bytes().any(|byte| byte.is_ascii_control()) + { + return Err(TaskError::invalid("task idempotency key is invalid")); + } + let material = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + draft.workspace_id, + draft.owner_kind.as_str(), + draft.owner_id, + draft.authorization_scope, + draft.operation_id, + key + ); + Ok(hex_digest(material.as_bytes())) +} + +/// Canonical digest of the validated executable input. Object key order never changes the digest, +/// while any material business or secret value does. +fn canonical_input_fingerprint(input: &serde_json::Value) -> Result { + let canonical = crate::idempotency::canonical_json(input); + let encoded = serde_json::to_vec(&canonical) + .map_err(|_| TaskError::invalid("task input is not canonicalizable"))?; + Ok(hex_digest(&encoded)) +} + +fn hex_digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn draft(key: &str, input: serde_json::Value) -> TaskAcceptDraft { + TaskAcceptDraft { + workspace_id: "ws_default".to_owned(), + owner_kind: TaskOwnerKind::Agent, + owner_id: "agent_a".to_owned(), + authorization_scope: "credential:pk_1".to_owned(), + operation_id: "op_report".to_owned(), + operation_version: 2, + policy_mode: ToolExecutionMode::Background, + scoped_key: key.to_owned(), + input, + correlation: CorrelationContext::generate(), + deadline: Instant::now() + std::time::Duration::from_secs(30), + retention_ms: TASK_DEFAULT_RETENTION_MS, + approval_reference: None, + } + } + + #[test] + fn canonical_fingerprint_ignores_key_order_and_tracks_values() { + let first = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 2}})).unwrap(); + let reordered = canonical_input_fingerprint(&json!({"b": {"c": 2}, "a": 1})).unwrap(); + assert_eq!(first, reordered); + assert_eq!(first.len(), 64); + + let changed = canonical_input_fingerprint(&json!({"a": 1, "b": {"c": 3}})).unwrap(); + assert_ne!(first, changed); + } + + #[test] + fn scoped_key_digest_covers_the_whole_identity() { + let base = scoped_key_digest(&draft("key_1", json!({}))).unwrap(); + let mut foreign = draft("key_1", json!({})); + foreign.owner_id = "agent_b".to_owned(); + assert_ne!(base, scoped_key_digest(&foreign).unwrap()); + + let mut other_scope = draft("key_1", json!({})); + other_scope.authorization_scope = "credential:pk_2".to_owned(); + assert_ne!(base, scoped_key_digest(&other_scope).unwrap()); + + assert!(scoped_key_digest(&draft("", json!({}))).is_err()); + assert!( + scoped_key_digest(&draft( + &"k".repeat(TASK_SCOPED_KEY_MAX_BYTES + 1), + json!({}) + )) + .is_err() + ); + } +} diff --git a/docs/development/rmcp-migration-plan.md b/docs/development/rmcp-migration-plan.md new file mode 100644 index 0000000..6ba06c9 --- /dev/null +++ b/docs/development/rmcp-migration-plan.md @@ -0,0 +1,78 @@ +# Переход Crank на официальный MCP SDK rmcp + +Статус: планирование одобрено владельцем 27.10.2026. Полная миграция и production-переключение этим планом не запускаются. Владелец планирования и очереди — hermes_tl; реализация — hermes_dev; независимая проверка — hermes_qa; Git/CI/релизы — hermes_devops. ARC подключается к конкретному техническому затруднению, не как обязательный новый этап. + +## Результат + +Стандартный протокольный слой MCP обслуживает официальный SDK. Crank сохраняет авторизацию, права Agent/workspace, опубликованные версии, REST mapping, admission и исполнение команд. Обе действующие версии протокола и функции Tools/Resources/Prompts остаются доступны. Снижается объём собственного протокольного кода; ускорение и производительность не объявляются измеренными заранее. + +Исходная точка: принятый main после 4.1, известный merge cc612516a29d48f2831ba6bdf58c1dc777aa6abf. Перед исполнением получить актуальный main и убедиться, что в него вошла проверенная поставка. Плановый кандидат SDK — опубликованный rmcp 3.7.0; окончательный pin и lock проверяются на первом этапе, автоматического обновления на другую версию нет. + +## 1. Проверить тонкий адаптер на реальном MCP-пути + +Исполнитель: DEV, независимая QA. Риск: высокий — ошибка может нарушить доступ к инструментам. + +Сделать изолированный исполняемый прототип SDK на существующей границе типизированных команд, сохранив авторизацию и динамический каталог. Публичный production endpoint пока обслуживает прежний путь. + +Проверяемый результат: + +- Динамические tools/list и tools/call работают через SDK с действующими сервисами Crank, включая запрет чужого Agent/неверных credentials и сохранение ошибок инструмента. +- Через настоящий HTTP-путь проверены обе revisions: legacy initialize/session и modern discover/stateless, JSON/SSE, headers, progress и cancellation. +- Отдельно проверены пригодность SessionManager для существующего хранения сессий, переподключение и владение отменой; наличие SDK-типа само по себе не считается доказательством. +- Установлены существенные gaps для Resources/Prompts/notifications/subscriptions и текущих execution modes. Наличие интерфейса не выдаётся за полную совместимость. +- На небольшом smoke-наборе проверено, что adapter не обходит действующие ограничения; замеченный риск памяти/очередей записан. Нагрузочная квалификация относится к завершённой функции, не превращается в условие этого прототипа. Существующие ceilings не меняются. +- Результат — работающий код и краткое решение TL: готов перенос либо конкретная несовместимость с минимальным вариантом решения. Если потребуется ослабить контракт, заменить хранилище или создать большой дополнительный framework — вопрос владельцу до продолжения. + +Это один достаточный опыт, а не серия исследований SDK. Использовать существующие wire fixtures и точечные проверки, сохранять непроверенные пункты явно. + +## 2. Перенести действующие MCP-функции на SDK + +Зависит от положительного результата этапа 1 и решения TL о достаточности совместимости. Исполнитель: DEV, независимая QA. Риск: высокий — затрагивается общий клиентский интерфейс. + +Подключить SDK к действующему пути и перенести стандартные envelopes, dispatch, lifecycle и transport. Порядок переноса устанавливается по результату первого опыта, без параллельных writers одних MCP-файлов. + +Проверяемый результат: + +- Через SDK проходят действующие Tools, Resources, Prompts, completion, уведомления и подписки в применимых revisions, а также progress/cancel и существующая logging policy. +- Сохранены авторизация/отзыв доступа, agent scope, advertised capabilities, admission, deadlines и семантика ошибок. SDK defaults не расширяют объявленные capabilities или список разрешённых revisions. +- Сессии, reconnect и ограничения памяти/очередей сохраняют действующий контракт; in-memory хранилище не заменяет существующее долговременное хранение без решения. +- Перенесённые участки собственного протокольного кода удаляются по мере принятия. Временный способ безопасного сравнения/отката не превращается во второй постоянный MCP runtime. +- Существующие affected ATDD подтверждают старое поведение; тесты меняются только там, где проверяли приватное устройство старого адаптера, а не нужный клиенту контракт. + +Новые бизнес-возможности, переработка REST runtime, базы, модели или UI в эту работу не входят. + +## 3. Завершить замену и поставить результат + +Зависит от принятого этапа 2. Исполнители: DEV/QA, затем DevOps. Риск: высокий — финальный переход общего MCP-пути. Внешняя проверка относительно реализации: независимая QA на существующем клиентском/wire-наборе обеих revisions. + +Проверяемый результат: + +- Удалены ставшие ненужными собственные parser/dispatch/transport участки, временные переключатели и неиспользуемые зависимости. Поддержка legacy protocol при этом сохраняется средствами SDK. +- Один необходимый финальный feature-прогон подтверждает реальный клиентский путь обеих revisions, изоляцию доступа, streaming/cancel/session и применимые ограничения ресурсов. Полный набор не повторяется отдельно каждой ролью; обязательные CI-гейты сохраняются. +- DevOps выполнил commit/push/PR, protected merge и честный release с проверенными артефактами. Документация описывает активный SDK, поддерживаемые версии и реальные ограничения. +- План отката указывает предыдущую принятую версию и проверяет совместимость состояния. Production-переключение выполняется только отдельным разрешённым manual deployment по существующему runbook; эта карточка не даёт разрешения на выкатку. + +## Порядок относительно текущей разработки + +1. Закончить уже выполняемую проверку поставки 4.1, не открывать её разработку заново. +2. Планировать SDK-адаптер перед дальнейшим расширением заменяемого transport/protocol слоя. Этап 1 не ждёт завершения будущих 4.2–4.7 или всей нагрузочной кампании. +3. Доменные части будущих задач могут идти параллельно при отсутствии общего writer и явных интерфейсных зависимостях. Новую протокольную реализацию и миграцию одних файлов параллельно не вести. +4. TL сопоставляет существующие 4.2–4.7 с SDK, чтобы не реализовать вручную уже предоставленный механизм. Действующие продуктовые AC не сокращаются. Неизвестная совместимость — проверка этапа 1, а не повод считать будущие функции готовыми. + +## Постановка и контроль + +Достаточно трёх последовательных исполнений по этапам выше, с same-card QA и действующей приёмкой TL; не создавать отдельный новый бизнес-эпик, PRD, новый тестовый framework или цепочку архитектурных согласований. + +На текущем шаге TL фиксирует эти этапы в существующем backlog/Gitea, зависимости и приоритет относительно текущей работы. Будущие исполнительские карточки оставить в triage/backlog; не запускать полную миграцию автоматически из задачи планирования. Сроки не выдумывать до первого опыта. Подтверждённая непригодность SDK эскалируется владельцу с фактами и вариантами, а не молчаливым возвращением к самописному MCP. + +## Основания + +Канонические пути ниже находятся на Hermes `192.168.1.127` (SSH-пользователь `codex`); относительные пути считаются от `/home/codex/code/crank`. Это доступный TL источник R, а не текущая локальная папка оператора. Локальная копия аудита у оператора: `/home/bsodfather/code/hermes/deploy/mcp-sdk-audit-20261027/REPORT.md`. + +- Решение владельца 27.10.2026: запланировать переход на rmcp. +- Аудит: `/home/codex/.hermes/codex-operator/handoffs/crank-rmcp-audit-20261027.md`. +- Действующие AC: `_bmad-output/initiative-crank/epic-mcp-capabilities/story-mcp-revision-capabilities.md`, в том числе оценка SDK AT-3.1-14; остальные MCP AC остаются действующими. +- Официальный SDK: https://github.com/modelcontextprotocol/rust-sdk ; документация ServerHandler: https://docs.rs/rmcp/latest/rmcp/handler/server/trait.ServerHandler.html . +- Общий процесс: `/home/codex/.hermes/team/development-delivery.md`, `validation-policy.md`, `delivery-efficiency.md`. + +Предыдущее default-правило «сохранять собственный transport» не является техническим доказательством отказа от SDK. Цель нового плана — проверить и принять SDK-путь; продуктовые и security-контракты сохраняются. diff --git a/docs/execution-boundary.md b/docs/execution-boundary.md index d4c6e2b..d7f4697 100644 --- a/docs/execution-boundary.md +++ b/docs/execution-boundary.md @@ -35,3 +35,19 @@ version, stage, stable error code, retryability и certainty. Для legacy v4 Пути Admin API, MCP methods и успешные response fields не изменены. Поля ошибки добавлены обратно совместимо. Outbound DNS/redirect/retry/SSRF policy принадлежит Story 1.9 и этой границей не считается завершённой. + +## Durable background Task (Story 4.2) + +Принятие фоновой работы идёт через `TaskApplication → crank-runtime::TaskService → +TaskStore`. PostgreSQL-реализация выполняет одну транзакцию: Task, dispatch intent +(`execution_admission_entries.task_id`) и queue slot коммитятся вместе, и только +после commit выдаётся ack с Task ID/поддержкой Request/Trace/parent TraceContext. +Повтор того же полного scoped key возвращает сохранённую Task, другой canonical +input fingerprint — типизированный конфликт; явная другая published-версия при том +же ключе не перезакрепляет уже принятую работу. Owner-скоуп (workspace, Agent/admin, +authorization scope) входит в каждый read; чужой Task неотличим от отсутствующего. + +Границы этой story: публичный MCP wire Tasks/capabilities/activation остаётся +Story 4.7, worker/leases/recovery — Story 4.3. Существующий `tools/call` сохраняет +synchronous/unsupported поведение; durable acceptance не запускает ephemeral +execution и не рекламирует клиенту background capability. diff --git a/docs/schemas/migration-sequence.json b/docs/schemas/migration-sequence.json index ec7743a..0facea5 100644 --- a/docs/schemas/migration-sequence.json +++ b/docs/schemas/migration-sequence.json @@ -1 +1 @@ -{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23},{"backfill":{"kind":"none"},"checksum":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-admission-v24","owner":"crank-registry","phase":"expand","readable_schema_max":24,"readable_schema_min":23,"source_digest":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","transactional":true,"version":24}]} +{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-cleanup-indexes-v13","owner":"crank-registry","phase":"expand","readable_schema_max":13,"readable_schema_min":12,"source_digest":"abe822c967e1b2cc3966ede988e05eb2d9b2a062b539c45aaa8cad677395ab05","transactional":true,"version":13},{"backfill":{"kind":"none"},"checksum":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"import-plan-v14","owner":"crank-registry","phase":"expand","readable_schema_max":14,"readable_schema_min":13,"source_digest":"176c5b3cd12ddaf78dd0d312626a0c387f9c240a765194b34e88f7e0d3b4ca2a","transactional":true,"version":14},{"backfill":{"kind":"none"},"checksum":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-import-selection-v15","owner":"crank-registry","phase":"expand","readable_schema_max":15,"readable_schema_min":14,"source_digest":"0dbe85c649ca227f15ac141e7bf004948d3240c3495489b30e2197980ecd0c5e","transactional":true,"version":15},{"backfill":{"kind":"none"},"checksum":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-drafts-v16","owner":"crank-registry","phase":"expand","readable_schema_max":16,"readable_schema_min":15,"source_digest":"0ed07dabd37f97e212923650b80278898c4e9519cee0cda4170f6b2b2a3d92d7","transactional":true,"version":16},{"backfill":{"kind":"none"},"checksum":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-publications-v17","owner":"crank-registry","phase":"expand","readable_schema_max":17,"readable_schema_min":16,"source_digest":"eaefc08d25ba5ab92775dd9263ced14d927e28b558444f83ea789f1f0023c092","transactional":true,"version":17},{"backfill":{"kind":"none"},"checksum":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-resource-bindings-v18","owner":"crank-registry","phase":"expand","readable_schema_max":18,"readable_schema_min":17,"source_digest":"cb9b82183e04eb72c1f1a9ec853fcfc618f2b1c2a73edd051245890292beeb8f","transactional":true,"version":18},{"backfill":{"kind":"none"},"checksum":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-security-revocations-v19","owner":"crank-registry","phase":"expand","readable_schema_max":19,"readable_schema_min":18,"source_digest":"e66493a4014af5cbf1ed7532ab7140ce773dc30b0995dea242b791f51dad5bc0","transactional":true,"version":19},{"backfill":{"kind":"none"},"checksum":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"resource-template-sources-v20","owner":"crank-registry","phase":"expand","readable_schema_max":20,"readable_schema_min":19,"source_digest":"1537f7267f4bb5317242f3c20d613dd9a5b4e5b01cec5c578949f431e91f7502","transactional":true,"version":20},{"backfill":{"kind":"none"},"checksum":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompts-v21","owner":"crank-registry","phase":"expand","readable_schema_max":21,"readable_schema_min":20,"source_digest":"c62717f8eb87b249b7771c74517ce99baeaf2793bfd5fca5d15e8412112bc53e","transactional":true,"version":21},{"backfill":{"kind":"none"},"checksum":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-publications-v22","owner":"crank-registry","phase":"expand","readable_schema_max":22,"readable_schema_min":21,"source_digest":"b330b248f164fc6f3635739668f5ea94dc48f0e84d18f0d3dfceb92ff6c445e7","transactional":true,"version":22},{"backfill":{"kind":"none"},"checksum":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"prompt-dependencies-v23","owner":"crank-registry","phase":"expand","readable_schema_max":23,"readable_schema_min":22,"source_digest":"15799d41e73021386bf1bae5952cbd34df93317cbce1a97fa8f99b3bd15cac64","transactional":true,"version":23},{"backfill":{"kind":"none"},"checksum":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-admission-v24","owner":"crank-registry","phase":"expand","readable_schema_max":24,"readable_schema_min":23,"source_digest":"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681","transactional":true,"version":24},{"backfill":{"kind":"none"},"checksum":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"durable-tasks-v25","owner":"crank-registry","phase":"expand","readable_schema_max":25,"readable_schema_min":24,"source_digest":"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd","transactional":true,"version":25}]} diff --git a/scripts/ci-story-checks.sh b/scripts/ci-story-checks.sh index a8ad8f2..6b276a7 100755 --- a/scripts/ci-story-checks.sh +++ b/scripts/ci-story-checks.sh @@ -67,3 +67,14 @@ run_atdd "execution policy validation" -p admin-api --lib operation_validation:: run_atdd "execution admission store" -p crank-registry --test integration admission:: -- --test-threads=1 run_atdd "runtime admission budget" -p mcp-server --test admission_budget -- --test-threads=1 run_atdd "admission readiness" -p admin-api --lib routes::tests::readiness_checks -- --test-threads=1 + +# Story 4.2 (FR-22, FR-23): durable background Tasks. The lifecycle/envelope bounds are unit-level; the +# authoritative store (one-transaction acceptance, replay/typed conflict, shared bounded capacity, +# Task-linked intent guard, terminal CAS, retention, scope isolation, fail-closed health) and the +# application entry (verified principal, exact publication pin, foreign-scope denial, readiness) run +# against a real PostgreSQL schema. The migration suite proves a fresh and an already-migrated v25. +run_atdd "durable task domain bounds" -p crank-core --lib task:: -- --test-threads=1 +run_atdd "runtime task identity" -p crank-runtime --lib tasks:: -- --test-threads=1 +run_atdd "durable task store" -p crank-registry --test integration tasks:: -- --test-threads=1 +run_atdd "durable task application entry" -p mcp-server --test integration durable_tasks:: -- --test-threads=1 +run_atdd "durable task migration" -p crank-registry --test integration migrations:: -- --test-threads=1 -- 2.54.0