feat: harden community production foundation through story 1.5

apps/admin-api/tests/integration/operations_agents.rs

@@ -10,7 +10,7 @@ use std::{
 };
 
 use async_trait::async_trait;
-use axum::{Json, Router, routing::post};
+use axum::{Json, Router, extract::State, http::HeaderMap, routing::post};
 use crank_core::{
     ExecutionConfig, HttpMethod, MembershipRole, OperationSecurityLevel, Protocol,
     ResponseCachePolicy, RestTarget, SecretKind, Target, ToolDescription, WorkspaceId,
@@ -97,8 +97,8 @@ impl IdentityProvider for RejectingIdentityProvider {
 async fn creates_publishes_and_tests_rest_operation() {
     let registry = test_registry().await;
     let storage_root = test_storage_root("lifecycle");
-    let upstream_base_url = spawn_upstream_server().await;
-    let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
+    let (upstream_base_url, observed_upstream_headers) = spawn_correlation_upstream_server().await;
+    let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await;
     let client = authorized_client(&base_url).await;
 
     let created = client
@@ -132,18 +132,25 @@ async fn creates_publishes_and_tests_rest_operation() {
         .json::<Value>()
         .await
         .unwrap();
-    let test_run = client
+    let test_run_response = client
         .post(format!("{base_url}/operations/{operation_id}/test-runs"))
+        .header("x-request-id", "req_admin_test_run")
+        .header(
+            "traceparent",
+            "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+        )
         .json(&json!({
             "version": 1,
             "input": { "email": "user@example.com" }
         }))
         .send()
         .await
-        .unwrap()
-        .json::<Value>()
-        .await
         .unwrap();
+    assert_eq!(
+        test_run_response.headers()["x-trace-id"].to_str().unwrap(),
+        "0af7651916cd43dd8448eb211c80319c"
+    );
+    let test_run = test_run_response.json::<Value>().await.unwrap();
 
     assert_eq!(listed["items"][0]["name"], "crm_create_lead");
     assert_eq!(
@@ -158,6 +165,65 @@ async fn creates_publishes_and_tests_rest_operation() {
         "user@example.com"
     );
     assert_eq!(test_run["response_preview"]["id"], "lead_123");
+    let logs = registry
+        .list_invocation_logs(crank_registry::ListInvocationLogsQuery {
+            workspace_id: &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
+            level: None,
+            search_text: None,
+            source: Some(crank_core::InvocationSource::AdminTestRun),
+            operation_id: Some(&crank_core::OperationId::new(&operation_id)),
+            agent_id: None,
+            created_after: None,
+            limit: 10,
+        })
+        .await
+        .unwrap();
+    assert_eq!(logs.len(), 1);
+    assert_eq!(
+        logs[0].log.request_id.as_deref(),
+        Some("req_admin_test_run")
+    );
+    assert_eq!(
+        logs[0].log.trace_id.as_deref(),
+        Some("0af7651916cd43dd8448eb211c80319c")
+    );
+    let upstream_headers = observed_upstream_headers.lock().await;
+    let upstream_headers = upstream_headers.as_ref().unwrap();
+    assert_eq!(
+        upstream_headers["x-request-id"].to_str().unwrap(),
+        "req_admin_test_run"
+    );
+    assert_eq!(
+        &upstream_headers["traceparent"].to_str().unwrap()[3..35],
+        "0af7651916cd43dd8448eb211c80319c"
+    );
+}
+
+async fn spawn_correlation_upstream_server() -> (String, Arc<tokio::sync::Mutex<Option<HeaderMap>>>)
+{
+    let observed = Arc::new(tokio::sync::Mutex::new(None));
+    let app = Router::new()
+        .route("/crm/leads", post(capture_correlation_and_create_lead))
+        .with_state(Arc::clone(&observed));
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+    (format!("http://{address}"), observed)
+}
+
+async fn capture_correlation_and_create_lead(
+    State(observed): State<Arc<tokio::sync::Mutex<Option<HeaderMap>>>>,
+    headers: HeaderMap,
+    Json(payload): Json<Value>,
+) -> Json<Value> {
+    *observed.lock().await = Some(headers);
+    Json(json!({
+        "id": "lead_123",
+        "status": "created",
+        "input": payload,
+    }))
 }
 
 #[tokio::test(flavor = "multi_thread")]