исправить: закрыть ревью сквозной корреляции

apps/admin-api/src/request_context.rs

@@ -1,6 +1,6 @@
 use axum::{
     extract::Request,
-    http::{HeaderMap, HeaderName, HeaderValue},
+    http::{HeaderName, HeaderValue},
     middleware::Next,
     response::Response,
 };
@@ -16,7 +16,7 @@ pub struct RequestContext {
 
 pub async fn apply_request_context(mut request: Request, next: Next) -> Response {
     let context = RequestContext {
-        request_id: resolve_request_id(request.headers()),
+        request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
     };
     let method = request.method().clone();
     let path = request.uri().path().to_owned();
@@ -46,15 +46,6 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
     .await
 }
 
-fn resolve_request_id(headers: &HeaderMap) -> String {
-    RequestId::resolve(
-        headers
-            .get(&REQUEST_ID_HEADER)
-            .and_then(|value| value.to_str().ok()),
-    )
-    .into_string()
-}
-
 #[cfg(test)]
 mod tests {
     #[test]

apps/admin-api/src/service.rs

@@ -5,10 +5,10 @@ use std::sync::Arc;
 use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
 use crank_core::{
     AuditSink, AuthProfile, CapabilityProfile, CommunityCapabilityProfile, EditionCapabilities,
-    ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId, NoopAuditSink,
-    OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine, ProductEdition, Protocol,
-    ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind, ToolQualitySchemaNode,
-    UsagePeriod, WorkspaceId,
+    ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId,
+    InvocationSource, NoopAuditSink, OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine,
+    ProductEdition, Protocol, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind,
+    ToolQualitySchemaNode, UsagePeriod, WorkspaceId,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_registry::{
@@ -479,9 +479,7 @@ impl AdminService {
 
         let history_span = crank_trace::Stage::HistoryWrite.span();
         let (outcome, db_span) = async {
-            let db_span = crank_trace::Stage::DbQuery
-                .db_span(crank_trace::DbOperation::InvocationHistoryWrite)
-                .expect("database stage");
+            let db_span = crank_trace::DbOperation::InvocationHistoryWrite.span();
             let outcome = self
                 .registry
                 .create_invocation_log(CreateInvocationLogRequest { log: &log })
@@ -509,7 +507,7 @@ impl AdminService {
             outcome,
             request.request_id,
             request.status,
-            "admin_test_run",
+            request.source,
         );
         outcome
     }
@@ -519,7 +517,7 @@ fn observe_invocation_history_outcome(
     outcome: InvocationHistoryWriteOutcome,
     request_id: Option<&str>,
     status: crank_core::InvocationStatus,
-    source: &'static str,
+    source: InvocationSource,
 ) {
     let Some(loss) = outcome.loss() else {
         return;
@@ -530,13 +528,20 @@ fn observe_invocation_history_outcome(
     tracing::warn!(
         name: "admin.invocation_history.lost",
         request_id = request_id.unwrap_or_default(),
-        source,
+        source = invocation_source_label(source),
         invocation_status = invocation_status_label(status),
         error_category = loss.category.as_str(),
         "invocation history was not recorded"
     );
 }
 
+fn invocation_source_label(source: InvocationSource) -> &'static str {
+    match source {
+        InvocationSource::AdminTestRun => "admin_test_run",
+        InvocationSource::AgentToolCall => "agent_tool_call",
+    }
+}
+
 fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
     match status {
         crank_core::InvocationStatus::Ok => "ok",
@@ -606,9 +611,9 @@ fn new_prefixed_id(prefix: &str) -> String {
     format!("{prefix}_{}", Uuid::now_v7().simple())
 }
 
-fn generate_access_secret(prefix: &str) -> String {
+fn generate_access_secret(marker: &str) -> String {
     let random = URL_SAFE_NO_PAD.encode(Uuid::now_v7().as_bytes());
-    format!("{prefix}_{random}")
+    format!("{marker}{random}")
 }
 
 fn hash_access_secret(secret: &str) -> String {
@@ -874,7 +879,7 @@ mod tests {
         sync::{Arc, Mutex},
     };
 
-    use crank_core::InvocationStatus;
+    use crank_core::{InvocationSource, InvocationStatus};
     use crank_observability::{
         ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
         operational_incident_total,
@@ -931,7 +936,7 @@ mod tests {
             }),
             Some("req_admin_dc08"),
             InvocationStatus::Error,
-            "admin_test_run",
+            InvocationSource::AgentToolCall,
         );
 
         let output = writer.output();
@@ -942,7 +947,7 @@ mod tests {
             .find(|event: &Value| event["event"] == "admin.invocation_history.lost")
             .unwrap();
         assert_eq!(event["request_id"], "req_admin_dc08");
-        assert_eq!(event["fields"]["source"], "admin_test_run");
+        assert_eq!(event["fields"]["source"], "agent_tool_call");
         assert_eq!(event["fields"]["invocation_status"], "error");
         assert_eq!(event["fields"]["error_category"], "invalid_record");
         assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);

apps/admin-api/src/service/api_keys.rs

@@ -65,10 +65,7 @@ impl AdminService {
             ),
             None => None,
         };
-        let secret = generate_access_secret(match payload.key_kind {
-            PlatformApiKeyKind::McpClient => "crk",
-            PlatformApiKeyKind::Approval => "crk_appr",
-        });
+        let secret = generate_access_secret(payload.key_kind.secret_marker());
         let api_key = PlatformApiKeyRecord {
             api_key: PlatformApiKey {
                 id: PlatformApiKeyId::new(new_prefixed_id("pk")),

apps/admin-api/tests/dc08.rs

@@ -2,11 +2,21 @@ use std::sync::{
     Arc,
     atomic::{AtomicUsize, Ordering},
 };
+use std::{io, sync::Mutex};
 
 use axum::{Json, Router, extract::State, routing::post};
+use crank_core::{OperationId, WorkspaceId};
+use serde_json::{Value, json};
+use serial_test::serial;
 use tokio::{net::TcpListener, sync::Notify};
+use tracing_subscriber::fmt::MakeWriter;
 
-use super::*;
+#[path = "integration/common.rs"]
+mod common;
+
+use common::*;
+
+const DEFAULT_WORKSPACE_ID: &str = "ws_default";
 
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
@@ -15,6 +25,17 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
     let registry_for_failure = registry.clone();
     let storage_root = test_storage_root("observability_history_loss");
     let upstream = spawn_blocking_upstream_server().await;
+    let log_writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        crank_observability::ObservabilityConfig::new(
+            crank_observability::ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            "info",
+            crank_observability::RedactionLimits::default(),
+        ),
+        log_writer.clone(),
+    )
+    .unwrap();
+    tracing::subscriber::set_global_default(subscriber).unwrap();
     let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
     let client = authorized_client(&base_url).await;
 
@@ -85,6 +106,16 @@ async fn preserves_external_success_when_invocation_history_is_lost() {
         .await
         .unwrap();
     assert!(logs["items"].as_array().unwrap().is_empty());
+
+    let output = log_writer.output();
+    assert!(!output.contains("dc08-canary-secret"));
+    let incident = output
+        .lines()
+        .filter_map(|line| serde_json::from_str::<Value>(line).ok())
+        .find(|event| event["event"] == "admin.invocation_history.lost")
+        .expect("DC-08 incident");
+    assert_eq!(incident["request_id"], "req_dc08_admin");
+    assert_eq!(incident["fields"]["source"], "admin_test_run");
 }
 
 struct BlockingUpstream {
@@ -139,3 +170,39 @@ async fn blocking_create_lead(
         "email": payload["email"]
     }))
 }
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

apps/admin-api/tests/integration/community_access_usage.rs

@@ -41,8 +41,6 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
 const TEST_SESSION_SECRET: &str = "test-session-secret";
 const TEST_MASTER_KEY: &str = "test-master-key";
 
-mod history_loss;
-
 struct TestServer {
     base_url: String,
     shutdown: Option<tokio::sync::oneshot::Sender<()>>,

apps/admin-api/tests/integration/request_context.rs

@@ -7,7 +7,7 @@ use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
 use axum::{
     Router,
     body::Body,
-    http::{HeaderMap, Request, StatusCode},
+    http::{HeaderMap, HeaderValue, Request, StatusCode},
     routing::get,
 };
 use crank_observability::{
@@ -16,12 +16,14 @@ use crank_observability::{
 use opentelemetry::{global, trace::TracerProvider as _};
 use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
 use tower::ServiceExt;
-use tracing::instrument::WithSubscriber;
 use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
 use uuid::Version;
 
+static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
 #[tokio::test(flavor = "current_thread")]
 async fn logs_request_completion_and_rejects_untrusted_values() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
     let writer = SharedLogWriter::default();
     let subscriber = crank_observability::build_subscriber(
         ObservabilityConfig::new(
@@ -33,6 +35,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
     )
     .unwrap();
     let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
     let app = probe_app();
 
     let response = app
@@ -44,7 +47,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
                 .body(Body::empty())
                 .unwrap(),
         )
-        .with_subscriber(dispatch.clone())
         .await
         .unwrap();
 
@@ -73,7 +75,6 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
                 .body(Body::empty())
                 .unwrap(),
         )
-        .with_subscriber(dispatch)
         .await
         .unwrap();
     let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
@@ -88,12 +89,14 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
 
 #[tokio::test(flavor = "current_thread")]
 async fn covers_valid_invalid_and_absent_traceparent() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
     global::set_text_map_propagator(TraceContextPropagator::new());
     let provider = SdkTracerProvider::builder().build();
     let tracer = provider.tracer("admin-request-context-test");
     let subscriber =
         tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
     let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
     let app = trace_probe_app();
 
     let valid = observed_trace_id(
@@ -109,7 +112,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
                     .body(Body::empty())
                     .unwrap(),
             )
-            .with_subscriber(dispatch.clone())
             .await
             .unwrap(),
     );
@@ -122,7 +124,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
                     .body(Body::empty())
                     .unwrap(),
             )
-            .with_subscriber(dispatch.clone())
             .await
             .unwrap(),
     );
@@ -133,7 +134,6 @@ async fn covers_valid_invalid_and_absent_traceparent() {
                 .body(Body::empty())
                 .unwrap(),
         )
-        .with_subscriber(dispatch)
         .await
         .unwrap(),
     );
@@ -145,6 +145,35 @@ async fn covers_valid_invalid_and_absent_traceparent() {
     provider.shutdown().unwrap();
 }
 
+#[tokio::test(flavor = "current_thread")]
+async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    let dispatch = tracing::Dispatch::new(tracing_subscriber::registry());
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let mut request = Request::builder()
+        .uri("/probe")
+        .body(Body::empty())
+        .unwrap();
+    request.headers_mut().append(
+        REQUEST_ID_HEADER,
+        HeaderValue::from_static("first-request-id"),
+    );
+    request.headers_mut().append(
+        REQUEST_ID_HEADER,
+        HeaderValue::from_static("second-request-id"),
+    );
+
+    let response = probe_app().oneshot(request).await.unwrap();
+    let generated = response.headers()[REQUEST_ID_HEADER].to_str().unwrap();
+
+    assert_ne!(generated, "first-request-id");
+    assert_ne!(generated, "second-request-id");
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
+}
+
 fn probe_app() -> Router {
     Router::new()
         .route("/probe", get(|| async { "ok" }))