//! ATDD Story 4.2 (FR-22/FR-23): the durable Task application entry over a real published catalog //! and a real PostgreSQL schema. //! //! These oracles drive the production application entry (`TaskApplication`) that the mcp-server //! composition root constructs: a verified machine credential is checked against the published Tool, //! the Task is accepted durably, a freshly constructed application instance reads the same identity //! and pins, replay/version conflicts never rebind, foreign callers receive no metadata, and a missing //! Task store fails closed and reports readiness as not ready. The public `tools/call` wire stays //! synchronous/unsupported: this suite never calls it. use super::common; use std::time::Duration; use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal}; use crank_core::{ AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId, TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore, TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy, }; use crank_registry::{ AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest, }; use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService}; use serde_json::json; use sqlx::{PgPool, Row}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; const AGENT_SLUG: &str = "reports"; const TOOL_NAME: &str = "report_gen"; const KEY: &str = "retry-key-1"; fn operation(policy: ToolExecutionPolicy) -> crank_registry::RegistryOperation { let mut operation = common::test_operation("https://api.example.com", TOOL_NAME); operation.execution_config.execution_policy = Some(policy); operation } fn background_policy() -> ToolExecutionPolicy { ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, } } async fn seed_tool(registry: &PostgresRegistry, policy: ToolExecutionPolicy) { let operation = operation(policy); registry .create_operation(&common::test_workspace_id(), &operation, Some("alice")) .await .unwrap(); publish(registry, &operation.id, 1).await; // A second, explicitly synchronous Tool of the same Agent: the durable entry must refuse it. let synchronous = common::test_operation("https://api.example.com", "report_sync"); registry .create_operation(&common::test_workspace_id(), &synchronous, Some("alice")) .await .unwrap(); publish(registry, &synchronous.id, 1).await; let binding = AgentOperationBinding { operation_id: operation.id.clone(), operation_version: 1, tool_name: operation.name.clone(), tool_title: operation.tool_description.title.clone(), tool_description_override: None, enabled: true, }; let synchronous_binding = AgentOperationBinding { operation_id: synchronous.id.clone(), operation_version: 1, tool_name: synchronous.name.clone(), tool_title: synchronous.tool_description.title.clone(), tool_description_override: None, enabled: true, }; common::publish_agent_with_bindings(registry, AGENT_SLUG, vec![binding, synchronous_binding]) .await; } async fn publish( registry: &PostgresRegistry, operation_id: &crank_core::OperationId, version: u32, ) { registry .publish_operation(PublishRequest { workspace_id: &common::test_workspace_id(), operation_id, version, published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(), published_by: Some("alice"), }) .await .unwrap(); } async fn publish_tool_version(registry: &PostgresRegistry, version: u32) { let mut next = operation(background_policy()); next.version = version; next.status = crank_core::OperationStatus::Draft; next.published_at = None; registry .create_version(CreateVersionRequest { workspace_id: &common::test_workspace_id(), snapshot: &next, change_note: Some("next publication"), created_by: Some("alice"), }) .await .unwrap(); publish(registry, &next.id, version).await; // The agent catalog must bind the published Tool version for the new publication to be addressed. let agent_id = common::test_agent_id(AGENT_SLUG); let draft = crank_core::AgentVersion { agent_id: agent_id.clone(), version: 2, status: crank_core::AgentStatus::Draft, instructions: json!({}), tool_selection_policy: Default::default(), created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(), }; registry .create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest { workspace_id: &common::test_workspace_id(), agent_id: &agent_id, version: &draft, bindings: &[AgentOperationBinding { operation_id: next.id.clone(), operation_version: version, tool_name: next.name.clone(), tool_title: next.tool_description.title.clone(), tool_description_override: None, enabled: true, }], updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(), }) .await .unwrap(); registry .publish_agent(crank_registry::PublishAgentRequest { workspace_id: &common::test_workspace_id(), agent_id: &agent_id, version: 2, published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(), published_by: Some("alice"), expected_state: None, }) .await .unwrap(); } const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000"; fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService { let store = PostgresTaskStore::new( pool, AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"); TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto)) } /// The application entry under the same encrypted-input authority the production process uses. async fn application(registry: &PostgresRegistry) -> TaskApplication { let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key"); let observed_at = OffsetDateTime::now_utc(); registry .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { epoch: crypto.master_key_epoch(), fingerprint: crypto.master_key_fingerprint(), cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, observed_at: &observed_at, }) .await .expect("test master-key identity registered"); TaskApplication::new( registry.clone(), Duration::from_millis(0), std::sync::Arc::new(InMemoryCoordinationStateStore::default()), task_service(registry.pool().clone(), crypto), ) } async fn principal( registry: &PostgresRegistry, name: &str, scopes: &[PlatformApiKeyScope], ) -> TaskPrincipal { let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await; // The production machine-access path resolves the presented secret against the active platform // key of the addressed Agent and derives the scope identity from that verified record. let secret_hash = common::hash_access_secret(&token); let api_key = registry .get_platform_api_key_by_secret_for_agent_slug( common::test_workspace_slug(), AGENT_SLUG, &secret_hash, ) .await .unwrap() .expect("an active platform key resolves"); registry .touch_platform_api_key( &api_key.api_key.workspace_id, &api_key.api_key.id, &OffsetDateTime::now_utc(), ) .await .unwrap(); TaskPrincipal::Agent { workspace_slug: common::test_workspace_slug().to_owned(), agent_slug: AGENT_SLUG.to_owned(), credential: crank_core::VerifiedMachineCredential { machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey, max_security_level: crank_core::OperationSecurityLevel::Standard, scopes: api_key.api_key.scopes.clone(), platform_api_key_id: Some(api_key.api_key.id.clone()), }, } } fn unsampled_correlation() -> CorrelationContext { CorrelationContext::new( crank_core::RequestId::generate(), crank_core::TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00") .unwrap(), ) } async fn counts(pool: &PgPool) -> (i64, i64) { let row = sqlx::query( "select (select count(*)::bigint from tasks) as tasks, (select count(*)::bigint from execution_admission_entries where task_id is not null) as intents", ) .fetch_one(pool) .await .unwrap(); (row.get("tasks"), row.get("intents")) } #[tokio::test] async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() { let registry = common::test_registry().await; seed_tool(®istry, background_policy()).await; let caller = principal( ®istry, "task-accept", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let correlation = unsampled_correlation(); let app = application(®istry).await; let acceptance = app .accept_background( &caller, TOOL_NAME, None, KEY, &json!({"report": "q3"}), &correlation, ) .await .expect("acceptance over the real publication"); assert_eq!(acceptance.state, TaskState::Queued); assert!(!acceptance.replayed); assert_eq!(acceptance.operation_version, 1); assert_eq!(acceptance.support.request_id, *correlation.request_id()); assert_eq!(acceptance.support.trace_id, *correlation.trace_id()); assert!(!acceptance.support.trace_sampled); assert_eq!( uuid::Uuid::parse_str(&acceptance.task_id.to_string()) .unwrap() .get_version_num(), 7, "the durable Task id is UUIDv7" ); // Task, dispatch intent and queue slot committed together. assert_eq!(counts(registry.pool()).await, (1, 1)); // A freshly constructed application + service reads the same durable identity and pins. let reader = application(®istry).await; let record = reader .status(&caller, TOOL_NAME, acceptance.task_id) .await .unwrap(); assert_eq!(record.state, TaskState::Queued); assert_eq!(record.pins.operation_version, 1); assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background); assert_eq!(record.support.request_id, acceptance.support.request_id); assert_eq!(record.support.traceparent, acceptance.support.traceparent); assert!(!record.support.trace_sampled); let outcome = reader .result(&caller, TOOL_NAME, acceptance.task_id) .await .unwrap(); assert!(outcome.result.is_none() && outcome.error.is_none()); // Replay of the same scoped key and input returns the accepted Task instead of a new one. let replay = app .accept_background( &caller, TOOL_NAME, None, KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .unwrap(); assert_eq!(replay.task_id, acceptance.task_id); assert!(replay.replayed); assert_eq!(counts(registry.pool()).await, (1, 1)); } #[tokio::test] async fn a_new_publication_never_rebinds_an_accepted_task() { let registry = common::test_registry().await; seed_tool(®istry, background_policy()).await; let caller = principal( ®istry, "task-rebind", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let app = application(®istry).await; let accepted = app .accept_background( &caller, TOOL_NAME, None, KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .unwrap(); let before = app .status(&caller, TOOL_NAME, accepted.task_id) .await .unwrap(); publish_tool_version(®istry, 2).await; // The same retry keeps the accepted Task and its v1 pin; it never becomes a v2 Task. let replay = app .accept_background( &caller, TOOL_NAME, None, KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .unwrap(); assert_eq!(replay.task_id, accepted.task_id); assert_eq!(replay.operation_version, 1); let after = app .status(&caller, TOOL_NAME, accepted.task_id) .await .unwrap(); assert_eq!(after.pins.operation_version, 1); assert_eq!(after.support.request_id, before.support.request_id); assert_eq!(counts(registry.pool()).await, (1, 1)); // An explicit other exact version is a typed conflict, not a rebind. let explicit = app .accept_background( &caller, TOOL_NAME, Some(2), KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect_err("a different explicit pin cannot rebind the accepted Task"); assert_eq!(explicit, TaskApplicationError::VersionUnavailable); // The original explicit pin still resolves to the accepted Task. let pinned = app .accept_background( &caller, TOOL_NAME, Some(1), KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .unwrap(); assert_eq!(pinned.task_id, accepted.task_id); assert_eq!(counts(registry.pool()).await, (1, 1)); } #[tokio::test] async fn foreign_and_insufficient_callers_receive_no_task_metadata() { let registry = common::test_registry().await; seed_tool(®istry, background_policy()).await; let owner = principal( ®istry, "task-owner", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let app = application(®istry).await; let accepted = app .accept_background( &owner, TOOL_NAME, None, KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .unwrap(); // Positive control: the owner reads its Task. assert!( app.status(&owner, TOOL_NAME, accepted.task_id) .await .is_ok() ); // A second verified key of the same Agent is another authorization scope: no payload, no metadata. let foreign_scope = principal(®istry, "task-other-scope", &[PlatformApiKeyScope::Read]).await; match app .status(&foreign_scope, TOOL_NAME, accepted.task_id) .await { Err(TaskApplicationError::NotFound) => {} other => panic!("foreign scope must be not-found without metadata, got {other:?}"), } assert!(matches!( app.result(&foreign_scope, TOOL_NAME, accepted.task_id) .await, Err(TaskApplicationError::NotFound) )); // A synchronous publication is not accepted as background work at all. let synchronous_app = application(®istry).await; let refused = synchronous_app .accept_background( &owner, "report_sync", None, "sync-key", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect_err("a synchronous policy has no durable acceptance"); assert_eq!(refused, TaskApplicationError::NotBackground); assert_eq!(counts(registry.pool()).await, (1, 1)); } #[tokio::test] async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() { let registry = common::test_registry().await; seed_tool(®istry, background_policy()).await; let app = application(®istry).await; let admin = TaskPrincipal::Admin { workspace_slug: common::test_workspace_slug().to_owned(), agent_slug: AGENT_SLUG.to_owned(), user_id: "user_admin_1".to_owned(), }; let accepted = app .accept_background( &admin, TOOL_NAME, None, KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("an authenticated administrator accepts durable work"); assert_eq!(accepted.state, TaskState::Queued); // Positive control: the same administrator reads its own Task. assert!( app.status(&admin, TOOL_NAME, accepted.task_id) .await .is_ok() ); // An administrator is never a wildcard owner: another admin user and the owning Agent of the // same workspace both see nothing. let other_admin = TaskPrincipal::Admin { workspace_slug: common::test_workspace_slug().to_owned(), agent_slug: AGENT_SLUG.to_owned(), user_id: "user_admin_2".to_owned(), }; assert_eq!( app.status(&other_admin, TOOL_NAME, accepted.task_id) .await .unwrap_err(), TaskApplicationError::NotFound ); assert_eq!( app.result(&other_admin, TOOL_NAME, accepted.task_id) .await .unwrap_err(), TaskApplicationError::NotFound ); let agent = principal(®istry, "task-admin-cross", &[PlatformApiKeyScope::Read]).await; assert_eq!( app.status(&agent, TOOL_NAME, accepted.task_id) .await .unwrap_err(), TaskApplicationError::NotFound ); // An admin identity addressed to another workspace cannot resolve the publication at all. let foreign_workspace = TaskPrincipal::Admin { workspace_slug: "other-workspace".to_owned(), agent_slug: AGENT_SLUG.to_owned(), user_id: "user_admin_1".to_owned(), }; assert_eq!( app.accept_background( &foreign_workspace, TOOL_NAME, None, "foreign-key", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .unwrap_err(), TaskApplicationError::NotFound ); assert_eq!(counts(registry.pool()).await, (1, 1)); } #[tokio::test] async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() { let database_url = crank_test_support::postgres_schema_url("durable_tasks_outage").await; let pool = PgPool::connect(&database_url).await.unwrap(); // No migration: the durable Task state does not exist. let outage_registry = PostgresRegistry::connect(&database_url).await; assert!( outage_registry.is_err(), "an unmigrated registry cannot be used" ); crank_registry::MigrationAuthority::apply(&pool) .await .unwrap(); let registry = PostgresRegistry::connect(&database_url).await.unwrap(); seed_tool(®istry, background_policy()).await; let caller = principal( ®istry, "task-outage", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let app = application(®istry).await; assert!(app.health().await.is_ok()); // Fail closed: without the Task schema the entry accepts nothing and reports dependency failure. sqlx::query("drop table tasks cascade") .execute(&pool) .await .unwrap(); assert!(app.health().await.is_err()); let error = app .accept_background( &caller, TOOL_NAME, None, KEY, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect_err("no durable Task store, no acceptance"); assert!(matches!( error, TaskApplicationError::Task(crank_core::TaskError::Unavailable { .. }) )); // Readiness of the app that wires the same store reports the durable Task dependency as not ready. let router = common::build_test_app(registry.clone(), Duration::from_millis(0), None); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap(); }); let response = reqwest::get(format!("http://{address}/ready")) .await .unwrap(); assert_eq!(response.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE); let body: serde_json::Value = response.json().await.unwrap(); assert_eq!(body["checks"]["durable_tasks"], json!("not_ready")); server.abort(); } /// Commits and resolves exactly one durable attempt of the current lease through the real store. /// /// Story 4.4 moved the dispatch checkpoint and the attempt resolution into the runtime seam, so a /// worker-level test double persists exactly one attempt instead of inventing a second retry loop. pub(crate) async fn commit_attempt( context: &crank_runtime::TaskAttemptContext, outcome: crank_core::AttemptOutcome, ) -> crank_runtime::TaskAttemptOutcome { use crank_core::{AttemptResolution, BeginAttempt, TaskState}; let lease = context.lease(); let begin = BeginAttempt { task_id: lease.record.task_id, owner: lease.owner.clone(), fence: lease.fence, expected_version: lease.record.version, attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()), input_fingerprint: lease.record.input_fingerprint.clone(), upstream_key_digest: None, retry_allowed: lease.record.pins.retry.max_attempts > 1, }; match context.store().checkpoint_dispatch(begin).await { Ok(attempt) => match context.store().resolve_attempt(&attempt, outcome).await { Ok(AttemptResolution::Finished(task)) if task.state == TaskState::Completed => { crank_runtime::TaskAttemptOutcome::Completed } Ok(AttemptResolution::Finished(_)) => crank_runtime::TaskAttemptOutcome::Failed { code: "test_failure".to_owned(), }, Ok(AttemptResolution::Unknown(_)) => crank_runtime::TaskAttemptOutcome::OutcomeUnknown, Ok(AttemptResolution::RetryAt { .. }) | Err(_) => { crank_runtime::TaskAttemptOutcome::Stopped } }, Err(_) => crank_runtime::TaskAttemptOutcome::Stopped, } } /// Bounded success outcome used by the worker-level test doubles. pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::AttemptOutcome { crank_core::AttemptOutcome::Completed( crank_core::TaskEnvelope::try_redacted(&value).expect("test result is bounded"), ) } mod attempt_guards; mod attempt_trace; mod cancel_chain; mod input_continuation; mod input_continuation_guards; mod shutdown_window; mod task_worker;