//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1). //! //! End-to-end: a real durable Task worker dispatches a retrying GET through a real local //! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is //! persisted in `tasks.current_attempt_id` on both attempts. use super::super::common; use axum::Router; use axum::http::StatusCode; use axum::routing::get; use crank_community_mcp::TaskWorkerHandler; use crank_core::{ AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy, }; use crank_runtime::{ OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy, }; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use time::OffsetDateTime; use tracing::{Id, Subscriber, field::Visit}; use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan}; const AGENT_SLUG: &str = "reports"; const TOOL_NAME: &str = "attempt_trace"; const KEY: &str = "attempt-trace-key-1"; async fn publish( registry: &crank_registry::PostgresRegistry, operation_id: &crank_core::OperationId, version: u32, ) { registry .publish_operation(crank_registry::PublishRequest { workspace_id: &common::test_workspace_id(), operation_id, version, published_at: &OffsetDateTime::parse( "2026-03-26T10:00:00Z", &time::format_description::well_known::Rfc3339, ) .unwrap(), published_by: Some("alice"), }) .await .unwrap(); } async fn application_with_store( registry: &crank_registry::PostgresRegistry, ) -> ( crank_community_mcp::TaskApplication, std::sync::Arc, SecretCrypto, ) { let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(); let observed_at = OffsetDateTime::now_utc(); registry .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { epoch: crypto.master_key_epoch(), fingerprint: crypto.master_key_fingerprint(), cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, observed_at: &observed_at, }) .await .expect("test master-key identity registered"); let store = std::sync::Arc::new( crank_registry::PostgresTaskStore::new( registry.pool().clone(), crank_registry::AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"), ); let tasks = crank_community_mcp::TaskApplication::new( registry.clone(), std::time::Duration::ZERO, std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()), crank_runtime::TaskService::with_secret_crypto( store.clone(), std::sync::Arc::new(crypto.clone()), ), ); (tasks, store, crypto) } async fn principal( registry: &crank_registry::PostgresRegistry, name: &str, scopes: &[crank_core::PlatformApiKeyScope], ) -> crank_community_mcp::TaskPrincipal { let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await; let secret_hash = common::hash_access_secret(&token); let api_key = registry .get_platform_api_key_by_secret_for_agent_slug( common::test_workspace_slug(), AGENT_SLUG, &secret_hash, ) .await .unwrap() .expect("an active platform key resolves"); registry .touch_platform_api_key( &api_key.api_key.workspace_id, &api_key.api_key.id, &OffsetDateTime::now_utc(), ) .await .unwrap(); crank_community_mcp::TaskPrincipal::Agent { workspace_slug: common::test_workspace_slug().to_owned(), agent_slug: AGENT_SLUG.to_owned(), credential: crank_core::VerifiedMachineCredential { machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey, max_security_level: crank_core::OperationSecurityLevel::Standard, scopes: api_key.api_key.scopes.clone(), platform_api_key_id: Some(api_key.api_key.id.clone()), }, } } #[tokio::test] async fn adapter_attempt_span_carries_durable_attempt_id() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", get({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { let n = calls.fetch_add(1, Ordering::SeqCst); if n == 0 { return ( StatusCode::INTERNAL_SERVER_ERROR, axum::Json(serde_json::json!({"error":"boom"})), ); } ( StatusCode::OK, axum::Json(serde_json::json!({"id":"lead-1"})), ) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let upstream = format!("http://{address}"); let mut operation = common::test_operation(&upstream, TOOL_NAME); let Target::Rest(target) = &mut operation.target; target.method = crank_core::HttpMethod::Get; operation.execution_config.execution_policy = Some(ToolExecutionPolicy { mode: crank_core::ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: crank_core::NoTasksFallback::Unsupported, }); operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 }); registry .create_operation(&common::test_workspace_id(), &operation, Some("alice")) .await .unwrap(); publish(®istry, &operation.id, 1).await; common::publish_agent_with_bindings( ®istry, AGENT_SLUG, vec![AgentOperationBinding { operation_id: operation.id.clone(), operation_version: 1, tool_name: operation.name.clone(), tool_title: operation.tool_description.title.clone(), tool_description_override: None, enabled: true, }], ) .await; let caller = principal( ®istry, "attempt-trace", &[ crank_core::PlatformApiKeyScope::Read, crank_core::PlatformApiKeyScope::Write, ], ) .await; let (tasks, store, crypto) = application_with_store(®istry).await; let acceptance = tasks .accept_background( &caller, TOOL_NAME, None, KEY, &serde_json::json!({"email": "user@example.com"}), &CorrelationContext::generate(), ) .await .expect("durable acceptance"); let runtime = community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build(); let handler = Arc::new(TaskWorkerHandler::from_application( &tasks, registry.clone(), runtime, crypto, )); let config = TaskWorkerConfig { owner: "runtime-worker-attempt-trace".to_owned(), batch_limit: 1, lease_ms: 10_000, poll_interval: std::time::Duration::from_millis(50), max_inflight: 1, shutdown_timeout: std::time::Duration::from_secs(5), ..TaskWorkerConfig::default() }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let capture = TraceCapture::default(); let subscriber = tracing_subscriber::registry().with(capture.clone()); let _guard = tracing::subscriber::set_default(subscriber); let worker_task = tokio::spawn(async move { worker.run().await }); let mut state = None; for _ in 0..200 { tokio::time::sleep(std::time::Duration::from_millis(50)).await; let current: Option = sqlx::query_scalar("select state from tasks where task_id = $1") .bind(acceptance.task_id.as_uuid()) .fetch_optional(registry.pool()) .await .unwrap(); state = current; if state.as_deref() == Some("completed") { break; } } stop_flag.store(true, Ordering::SeqCst); tokio::time::timeout(std::time::Duration::from_secs(10), worker_task) .await .expect("worker stops inside the budget") .expect("worker exits cleanly") .expect("worker reports no error"); server.abort(); let durable_attempt_id: Option = sqlx::query_scalar::<_, Option>( "select current_attempt_id from tasks where task_id = $1", ) .bind(acceptance.task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!(state.as_deref(), Some("completed")); // Story 4.4: every outbound send is its own durable attempt, so the two retries of one lease carry // two distinct attempt identities, both recorded in the immutable attempt history. let attempt_rows: Vec<(String, i32)> = sqlx::query_as( "select attempt_id, ordinal from task_attempts where task_id = $1 order by ordinal", ) .bind(acceptance.task_id.as_uuid()) .fetch_all(registry.pool()) .await .unwrap(); assert_eq!(attempt_rows.len(), 2, "two durable attempts were committed"); assert_eq!(attempt_rows[0].1, 1); assert_eq!(attempt_rows[1].1, 2); assert_eq!( Some(attempt_rows[1].0.as_str()), durable_attempt_id.as_deref(), "the Task points at its latest durable attempt" ); let spans = capture.snapshot(); let attempt_spans: Vec<_> = spans .iter() .filter(|s| s.name == "adapter_attempt") .collect(); assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans"); for span in &attempt_spans { let aid = span.fields.get("attempt_id").cloned().unwrap_or_default(); assert!(!aid.is_empty(), "attempt_id must not be empty"); assert!( !aid.contains("traceparent"), "attempt_id must not contain traceparent" ); assert!( attempt_rows.iter().any(|(id, _)| id == &aid), "every attempt span names an attempt that was durably committed" ); } let ids: Vec = attempt_spans .iter() .map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default()) .collect(); assert_ne!(ids[0], ids[1], "each retry has its own durable attempt id"); assert_eq!( spans .iter() .filter(|span| span.name == "durable_task_attempt") .count(), 1, "one durable attempt span per dispatch" ); let runtime_span = spans .iter() .find(|span| span.name == "runtime.execute") .expect("runtime.execute span"); assert_eq!( runtime_span.parent_name, Some("durable_task_attempt"), "the execution continues the durable attempt trace" ); for span in &attempt_spans { assert_eq!( span.parent_name, Some("runtime.execute"), "each retry is a child attempt span under the durable execution" ); } let attempts: Vec = attempt_spans .iter() .map(|s| s.fields.get("attempt").cloned().unwrap_or_default()) .collect(); assert!(attempts.contains(&"1".to_string())); assert!(attempts.contains(&"2".to_string())); for span in &attempt_spans { assert_eq!( span.fields.get("max_attempts").cloned().unwrap_or_default(), "2" ); } } // TraceCapture minimal port from crates/crank-runtime/tests/stages.rs #[derive(Clone, Default)] struct TraceCapture { spans: Arc>>, } impl TraceCapture { fn snapshot(&self) -> Vec { self.spans.lock().expect("span lock").clone() } } #[derive(Clone, Debug)] struct CapturedSpan { name: &'static str, parent_name: Option<&'static str>, fields: std::collections::BTreeMap, } impl Layer for TraceCapture where S: Subscriber + for<'lookup> LookupSpan<'lookup>, { fn on_new_span( &self, attributes: &tracing::span::Attributes<'_>, id: &Id, context: tracing_subscriber::layer::Context<'_, S>, ) { let parent = attributes .parent() .and_then(|parent| context.span(parent)) .or_else(|| { attributes .is_contextual() .then(|| context.lookup_current()) .flatten() }); let mut visitor = FieldVisitor::default(); attributes.record(&mut visitor); let mut spans = self.spans.lock().expect("span lock"); let index = spans.len(); spans.push(CapturedSpan { name: attributes.metadata().name(), parent_name: parent.map(|span| span.metadata().name()), fields: visitor.fields, }); context .span(id) .expect("span exists") .extensions_mut() .insert(index); } fn on_record( &self, id: &Id, values: &tracing::span::Record<'_>, context: tracing_subscriber::layer::Context<'_, S>, ) { let mut visitor = FieldVisitor::default(); values.record(&mut visitor); let span = context.span(id).expect("span exists"); let index = *span.extensions().get::().expect("capture index"); self.spans.lock().expect("span lock")[index] .fields .extend(visitor.fields); } } #[derive(Default)] struct FieldVisitor { fields: std::collections::BTreeMap, } impl Visit for FieldVisitor { fn record_str(&mut self, field: &tracing::field::Field, value: &str) { self.fields .insert(field.name().to_owned(), value.to_owned()); } fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { self.fields .insert(field.name().to_owned(), format!("{value:?}")); } }