наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

apps/admin-api/Cargo.toml

@@ -3,6 +3,7 @@ name = "admin-api"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [[bin]]
@@ -18,9 +19,12 @@ crank-community-auth = { path = "../../crates/crank-community-auth" }
 crank-core = { path = "../../crates/crank-core" }
 crank-import = { path = "../../crates/crank-import" }
 crank-mapping = { path = "../../crates/crank-mapping" }
+crank-observability = { path = "../../crates/crank-observability" }
 crank-registry = { path = "../../crates/crank-registry" }
 crank-runtime = { path = "../../crates/crank-runtime" }
 crank-schema = { path = "../../crates/crank-schema" }
+crank-trace = { path = "../../crates/crank-trace" }
+metrics.workspace = true
 rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -37,5 +41,9 @@ uuid.workspace = true
 [dev-dependencies]
 async-trait = "0.1"
 crank-test-support = { path = "../../crates/crank-test-support" }
+opentelemetry.workspace = true
+opentelemetry_sdk.workspace = true
 reqwest.workspace = true
 serial_test = "3"
+tower.workspace = true
+tracing-opentelemetry.workspace = true

apps/admin-api/src/app.rs

@@ -166,6 +166,7 @@ pub fn build_app(state: AppState) -> Router {
 
     Router::new()
         .route("/health", get(crate::routes::health))
+        .route("/ready", get(crate::routes::readiness))
         .nest(
             "/api/auth",
             Router::new()
@@ -178,6 +179,9 @@ pub fn build_app(state: AppState) -> Router {
             apply_api_rate_limit,
         ))
         .layer(middleware::from_fn(apply_request_context))
+        .layer(middleware::from_fn(
+            crank_observability::record_http_request,
+        ))
         .with_state(state)
 }
 

apps/admin-api/src/dto.rs

@@ -269,7 +269,12 @@ pub struct CreatedPlatformApiKeyResponse {
 }
 
 #[derive(Clone, Debug, Serialize)]
-pub struct WorkspaceExportResponse {
+pub struct WorkspaceCatalogSnapshotResponse {
+    pub kind: String,
+    pub format_version: String,
+    pub restorable: bool,
+    pub included: Vec<String>,
+    pub excluded: Vec<String>,
     pub workspace: WorkspaceRecord,
     pub operations: Vec<OperationSummaryView>,
     pub agents: Vec<AgentSummaryView>,

apps/admin-api/src/error.rs

@@ -137,16 +137,24 @@ impl ApiError {
 impl IntoResponse for ApiError {
     fn into_response(self) -> Response {
         match &self {
-            Self::Internal { message, .. } => {
-                error!(error_code = self.code(), error_message = %message)
+            Self::Internal { .. } => {
+                error!(
+                    name: "admin.response.internal_error",
+                    error_code = self.code(),
+                    "internal API error response"
+                )
             }
-            Self::Unauthorized { message, .. }
-            | Self::Forbidden { message, .. }
-            | Self::Validation { message, .. }
-            | Self::NotFound { message, .. }
-            | Self::Conflict { message, .. }
-            | Self::RateLimited { message, .. } => {
-                warn!(error_code = self.code(), error_message = %message)
+            Self::Unauthorized { .. }
+            | Self::Forbidden { .. }
+            | Self::Validation { .. }
+            | Self::NotFound { .. }
+            | Self::Conflict { .. }
+            | Self::RateLimited { .. } => {
+                warn!(
+                    name: "admin.response.rejected",
+                    error_code = self.code(),
+                    "API request rejected"
+                )
             }
         }
 
@@ -351,6 +359,10 @@ impl From<RegistryError> for ApiError {
                 format!("import job {job_id} was not found"),
                 json!({ "job_id": job_id }),
             ),
+            RegistryError::ImportJobAlreadyApplied { job_id } => Self::conflict_with_context(
+                format!("import job {job_id} was already applied with different parameters"),
+                json!({ "job_id": job_id }),
+            ),
             RegistryError::Storage(_) | RegistryError::Serialization(_) => {
                 Self::internal(value.to_string())
             }
@@ -407,6 +419,10 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "runtime_confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "runtime_confirmation_error",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "runtime_confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "runtime_idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "runtime_idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "runtime_idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "runtime_idempotency_outcome_unknown",
         RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error",
         RuntimeError::MissingAuthProfile { .. } => "runtime_auth_profile_error",
         RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
@@ -434,7 +450,11 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
             "safety_class": safety_class,
         })),
         RuntimeError::InvalidConfirmationToken { operation_id }
-        | RuntimeError::ConfirmationStoreUnavailable { operation_id } => Some(json!({
+        | RuntimeError::ConfirmationStoreUnavailable { operation_id }
+        | RuntimeError::IdempotencyStoreUnavailable { operation_id }
+        | RuntimeError::IdempotencyInProgress { operation_id }
+        | RuntimeError::IdempotencyConflict { operation_id }
+        | RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({
             "operation_id": operation_id,
         })),
         RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({

apps/admin-api/src/main.rs

@@ -1,4 +1,4 @@
-use std::{env, net::SocketAddr, path::PathBuf};
+use std::{env, net::SocketAddr, path::PathBuf, time::Duration};
 
 use admin_api::{
     app::build_app,
@@ -7,22 +7,50 @@ use admin_api::{
     state::AppState,
 };
 use crank_community_auth::PasswordIdentityProvider;
+use crank_observability::{
+    CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
+    capture_critical_error,
+};
 use crank_registry::{PostgresPoolConfig, PostgresRegistry};
 use crank_runtime::{
     RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
     RuntimeLimits, SecretCrypto,
 };
-use sqlx::postgres::PgConnectOptions;
+use sqlx::{PgPool, postgres::PgConnectOptions};
 use tokio::net::TcpListener;
-use tracing::info;
+use tracing::{info, warn};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    tracing_subscriber::fmt()
-        .with_env_filter(
-            env::var("CRANK_LOG_LEVEL").unwrap_or_else(|_| "admin_api=info,tower_http=info".into()),
-        )
-        .init();
+    let observability = crank_observability::init(ObservabilityConfig::from_env(
+        "admin-api",
+        env!("CARGO_PKG_VERSION"),
+        "admin_api=info,tower_http=info",
+    )?)?;
+    let mut startup_completed = false;
+    let result = run(&observability, &mut startup_completed).await;
+    if result.is_err() {
+        capture_critical_error(if startup_completed {
+            CriticalErrorCategory::Internal
+        } else {
+            CriticalErrorCategory::Startup
+        });
+    }
+    result
+}
+
+async fn run(
+    observability: &ObservabilityLifecycle,
+    startup_completed: &mut bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let metrics_config =
+        MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?;
+    let metrics_enabled = metrics_config.enabled();
+    let metrics_server = if metrics_config.enabled() {
+        Some(observability.metrics_surface(metrics_config).bind().await?)
+    } else {
+        None
+    };
 
     let storage_root = PathBuf::from(
         env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()),
@@ -36,6 +64,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         pool_config,
     )
     .await?;
+    if metrics_enabled {
+        spawn_postgres_pool_metrics(registry.pool().clone());
+    }
     let auth_settings = AuthSettings {
         session_secret: env::var("CRANK_SESSION_SECRET")?,
         password_pepper: env::var("CRANK_PASSWORD_PEPPER")?,
@@ -74,10 +105,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     .with_outbound_http_policy(outbound_http_policy)
     .with_identity_provider(std::sync::Arc::new(identity_provider))
     .build();
+    let invocation_log_retention_days =
+        positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
     service.bootstrap_admin_user().await?;
     if env_flag("CRANK_DEMO_SEED") {
         service.seed_demo_assets().await?;
     }
+    spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days);
     let state = AppState {
         service,
         api_rate_limiter: if cache_config.backend.is_external() {
@@ -92,6 +126,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
 
     info!(
+        name: "admin.postgres_pool.configured",
         runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
         admin_rate_limit_rps = api_rate_limit.requests_per_second,
         admin_rate_limit_burst = api_rate_limit.burst,
@@ -101,15 +136,76 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         acquire_timeout_ms = pool_config.acquire_timeout_ms,
         idle_timeout_ms = pool_config.idle_timeout_ms,
         max_lifetime_ms = pool_config.max_lifetime_ms,
+        invocation_log_retention_days,
         "postgres pool configured"
     );
-    info!("admin-api listening on {}", socket_addr);
+    info!(
+        name: "admin.server.listening",
+        bind_address = %socket_addr,
+        "admin-api listening"
+    );
+    *startup_completed = true;
 
-    axum::serve(listener, make_service).await?;
+    if let Some(metrics_server) = metrics_server {
+        tokio::select! {
+            result = axum::serve(listener, make_service) => result?,
+            result = metrics_server.serve() => result?,
+        }
+    } else {
+        axum::serve(listener, make_service).await?;
+    }
 
     Ok(())
 }
 
+fn positive_i64_from_env(
+    name: &'static str,
+    default: i64,
+) -> Result<i64, Box<dyn std::error::Error>> {
+    let value = match env::var(name) {
+        Ok(raw) => raw.parse::<i64>()?,
+        Err(env::VarError::NotPresent) => default,
+        Err(error) => return Err(error.into()),
+    };
+    if value <= 0 {
+        return Err(format!("{name} must be greater than zero").into());
+    }
+    Ok(value)
+}
+
+fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(60 * 60));
+        loop {
+            interval.tick().await;
+            let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days);
+            match service.cleanup_invocation_logs_before(cutoff).await {
+                Ok(removed) if removed > 0 => info!(
+                    name: "admin.invocation_log_cleanup.completed",
+                    removed,
+                    "expired invocation logs removed"
+                ),
+                Ok(_) => {}
+                Err(_) => warn!(
+                    name: "admin.invocation_log_cleanup.failed",
+                    error_category = "registry_cleanup",
+                    "failed to remove expired invocation logs"
+                ),
+            }
+        }
+    });
+}
+
+fn spawn_postgres_pool_metrics(pool: PgPool) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(5));
+        loop {
+            interval.tick().await;
+            crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
+        }
+    });
+}
+
 fn env_flag(name: &str) -> bool {
     matches!(
         env::var(name)

apps/admin-api/src/rate_limit.rs

@@ -6,7 +6,7 @@ use axum::{
     middleware::Next,
     response::Response,
 };
-use crank_runtime::RateLimitRejection;
+use crank_runtime::{RateLimitCheckError, RateLimitRejection};
 
 use crate::{error::ApiError, state::AppState};
 
@@ -25,11 +25,16 @@ pub async fn apply_api_rate_limit(
         peer_ip,
         state.trust_forwarded_headers,
     );
-    if let Err(rejection) = state.api_rate_limiter.check(&key).await {
-        return Err(ApiError::rate_limited_with_context(
-            "request rate limit exceeded",
-            rejection_context(rejection),
-        ));
+    if let Err(error) = state.api_rate_limiter.check(&key).await {
+        return match error {
+            RateLimitCheckError::Rejected(rejection) => Err(ApiError::rate_limited_with_context(
+                "request rate limit exceeded",
+                rejection_context(rejection),
+            )),
+            RateLimitCheckError::StoreUnavailable => {
+                Err(ApiError::internal("rate limit service unavailable"))
+            }
+        };
     }
 
     Ok(next.run(request).await)

apps/admin-api/src/request_context.rs

@@ -4,11 +4,10 @@ use axum::{
     middleware::Next,
     response::Response,
 };
-use tracing::info;
-use uuid::Uuid;
+use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
+use tracing::{Instrument, info, info_span};
 
 pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
-const MAX_REQUEST_ID_LEN: usize = 128;
 
 #[derive(Clone, Debug)]
 pub struct RequestContext {
@@ -21,145 +20,53 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
     };
     let method = request.method().clone();
     let path = request.uri().path().to_owned();
+    let span = info_span!(
+        target: "crank::trace",
+        "http.request",
+        request_id = %context.request_id,
+    );
+    set_remote_trace_parent(&span, request.headers());
     request.extensions_mut().insert(context.clone());
 
-    let mut response = next.run(request).await;
-    info!(
-        request_id = %context.request_id,
-        method = %method,
-        path,
-        status = response.status().as_u16(),
-        "admin request completed"
-    );
-    if let Ok(value) = HeaderValue::from_str(&context.request_id) {
-        response.headers_mut().insert(REQUEST_ID_HEADER, value);
-    }
-    response
+    with_request_correlation(context.request_id.clone(), async move {
+        let mut response = next.run(request).instrument(span).await;
+        info!(
+            name: "admin.request.completed",
+            request_id = %context.request_id,
+            method = %method,
+            path,
+            status = response.status().as_u16(),
+            "admin request completed"
+        );
+        if let Ok(value) = HeaderValue::from_str(&context.request_id) {
+            response.headers_mut().insert(REQUEST_ID_HEADER, value);
+        }
+        response
+    })
+    .await
 }
 
 fn resolve_request_id(headers: &HeaderMap) -> String {
-    headers
-        .get(&REQUEST_ID_HEADER)
-        .and_then(|value| value.to_str().ok())
-        .map(str::trim)
-        .filter(|value| is_valid_request_id(value))
-        .map(ToOwned::to_owned)
-        .unwrap_or_else(|| Uuid::now_v7().to_string())
-}
-
-fn is_valid_request_id(value: &str) -> bool {
-    !value.is_empty()
-        && value.len() <= MAX_REQUEST_ID_LEN
-        && value
-            .bytes()
-            .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
+    RequestId::resolve(
+        headers
+            .get(&REQUEST_ID_HEADER)
+            .and_then(|value| value.to_str().ok()),
+    )
+    .into_string()
 }
 
 #[cfg(test)]
 mod tests {
-    use std::io;
-    use std::sync::{Arc, Mutex};
-
-    use axum::{Router, routing::get};
-    use reqwest::Client;
-    use tokio::net::TcpListener;
-    use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
-
-    use super::{REQUEST_ID_HEADER, apply_request_context, is_valid_request_id};
-
     #[test]
     fn accepts_visible_ascii_request_ids() {
-        assert!(is_valid_request_id("req_test_123"));
-        assert!(is_valid_request_id("trace-123/abc"));
+        assert!(crank_observability::RequestId::is_valid("req_test_123"));
+        assert!(crank_observability::RequestId::is_valid("trace-123/abc"));
     }
 
     #[test]
     fn rejects_empty_or_control_request_ids() {
-        assert!(!is_valid_request_id(""));
-        assert!(!is_valid_request_id("bad value"));
-        assert!(!is_valid_request_id("bad\nvalue"));
-    }
-
-    #[derive(Clone, Default)]
-    struct SharedLogWriter {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    impl SharedLogWriter {
-        fn output(&self) -> String {
-            String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
-        }
-    }
-
-    impl<'a> MakeWriter<'a> for SharedLogWriter {
-        type Writer = SharedLogGuard;
-
-        fn make_writer(&'a self) -> Self::Writer {
-            SharedLogGuard {
-                buffer: Arc::clone(&self.buffer),
-            }
-        }
-    }
-
-    struct SharedLogGuard {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    impl io::Write for SharedLogGuard {
-        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
-            self.buffer.lock().unwrap().extend_from_slice(bytes);
-            Ok(bytes.len())
-        }
-
-        fn flush(&mut self) -> io::Result<()> {
-            Ok(())
-        }
-    }
-
-    #[tokio::test]
-    async fn logs_request_completion_with_request_id() {
-        let writer = SharedLogWriter::default();
-        let subscriber = tracing_subscriber::registry().with(
-            tracing_subscriber::fmt::layer()
-                .with_writer(writer.clone())
-                .without_time()
-                .with_ansi(false)
-                .with_target(false)
-                .compact()
-                .with_filter(LevelFilter::INFO),
-        );
-        let dispatch = tracing::Dispatch::new(subscriber);
-        let app = Router::new()
-            .route("/probe", get(|| async { "ok" }))
-            .layer(axum::middleware::from_fn(apply_request_context));
-        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let address = listener.local_addr().unwrap();
-
-        let _guard = tracing::dispatcher::set_default(&dispatch);
-        tokio::spawn(async move {
-            axum::serve(listener, app).await.unwrap();
-        });
-
-        let response = Client::new()
-            .get(format!("http://{address}/probe"))
-            .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
-            .send()
-            .await
-            .unwrap();
-
-        assert_eq!(response.status(), reqwest::StatusCode::OK);
-        assert_eq!(
-            response.headers()[REQUEST_ID_HEADER.as_str()]
-                .to_str()
-                .unwrap(),
-            "req_admin_trace_123"
-        );
-
-        let logs = writer.output();
-        assert!(logs.contains("admin request completed"));
-        assert!(logs.contains("req_admin_trace_123"));
-        assert!(logs.contains("GET"));
-        assert!(logs.contains("/probe"));
-        assert!(logs.contains("status=200"));
+        assert!(!crank_observability::RequestId::is_valid(""));
+        assert!(!crank_observability::RequestId::is_valid("bad value"));
+        assert!(!crank_observability::RequestId::is_valid("bad\nvalue"));
     }
 }

apps/admin-api/src/routes.rs

@@ -10,12 +10,36 @@ pub mod secrets;
 pub mod upstreams;
 pub mod workspaces;
 
-use axum::Json;
+use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
 use serde_json::json;
 
+use crate::state::AppState;
+
 pub async fn health() -> Json<serde_json::Value> {
     Json(json!({
         "service": "admin-api",
         "status": "ok"
     }))
 }
+
+pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
+    match state.service.readiness().await {
+        Ok(()) => (
+            StatusCode::OK,
+            Json(json!({
+                "service": "admin-api",
+                "status": "ready",
+                "checks": { "postgres": "ready" }
+            })),
+        ),
+        Err(error) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "service": "admin-api",
+                "status": "not_ready",
+                "checks": { "postgres": "not_ready" },
+                "error": error.to_string()
+            })),
+        ),
+    }
+}

apps/admin-api/src/routes/access.rs

@@ -18,7 +18,7 @@ pub async fn export_workspace(
 ) -> Result<Json<Value>, ApiError> {
     let exported = state
         .service
-        .export_workspace(&path.workspace_id.as_str().into())
+        .export_workspace_catalog_snapshot(&path.workspace_id.as_str().into())
         .await?;
     Ok(Json(json!(exported)))
 }

apps/admin-api/src/routes/auth.rs

@@ -103,7 +103,7 @@ pub async fn change_password(
 ) -> Result<StatusCode, ApiError> {
     state
         .service
-        .change_password(&session.user.id, payload)
+        .change_password(&session.user.id, &session.session_id, payload)
         .await?;
     Ok(StatusCode::NO_CONTENT)
 }

apps/admin-api/src/service.rs

@@ -12,16 +12,18 @@ use crank_core::{
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_registry::{
-    AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary,
-    OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
+    AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef,
+    OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
 };
 use crank_runtime::{
     OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
 };
 use crank_schema::{Schema, SchemaKind};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use serde_json::{Value, json};
 use sha2::{Digest, Sha256};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tracing::Instrument;
 use uuid::Uuid;
 
 mod agents;
@@ -74,6 +76,11 @@ pub struct AdminServiceBuilder {
 pub use crate::dto::*;
 
 impl AdminService {
+    pub async fn readiness(&self) -> Result<(), ApiError> {
+        self.registry.ping().await?;
+        Ok(())
+    }
+
     #[cfg(test)]
     pub fn new(
         registry: PostgresRegistry,
@@ -199,16 +206,33 @@ impl AdminServiceBuilder {
 }
 
 impl AdminService {
-    pub async fn export_workspace(
+    pub async fn export_workspace_catalog_snapshot(
         &self,
         workspace_id: &WorkspaceId,
-    ) -> Result<WorkspaceExportResponse, ApiError> {
+    ) -> Result<WorkspaceCatalogSnapshotResponse, ApiError> {
         let workspace = self.get_workspace(workspace_id).await?;
         let operations = self.list_operations(workspace_id).await?;
         let agents = self.list_agents(workspace_id).await?;
         let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?;
 
-        Ok(WorkspaceExportResponse {
+        Ok(WorkspaceCatalogSnapshotResponse {
+            kind: "workspace_catalog_snapshot".to_owned(),
+            format_version: "1".to_owned(),
+            restorable: false,
+            included: vec![
+                "workspace_settings".to_owned(),
+                "operation_summaries".to_owned(),
+                "agent_summaries".to_owned(),
+                "platform_api_key_metadata".to_owned(),
+            ],
+            excluded: vec![
+                "operation_versions_and_samples".to_owned(),
+                "agent_versions_and_bindings".to_owned(),
+                "secret_metadata_and_values".to_owned(),
+                "secret_values".to_owned(),
+                "invocation_logs_and_usage".to_owned(),
+                "authentication_sessions".to_owned(),
+            ],
             workspace,
             operations,
             agents,
@@ -239,9 +263,13 @@ impl AdminService {
             return Ok(None);
         };
 
-        let auth_profile = self
-            .registry
-            .get_auth_profile(workspace_id, auth_profile_id)
+        let span = Stage::AuthResolve.span();
+        let result = async {
+            let auth_profile = observe_db_query(
+                DbOperation::AuthProfileRead,
+                self.registry
+                    .get_auth_profile(workspace_id, auth_profile_id),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "load auth profile",
@@ -251,9 +279,20 @@ impl AdminService {
                 auth_profile_id: auth_profile_id.as_str().to_owned(),
             })?;
 
-        self.resolve_auth_profile(workspace_id, &auth_profile)
-            .await
-            .map(Some)
+            self.resolve_auth_profile(workspace_id, &auth_profile)
+                .await
+                .map(Some)
+        }
+        .instrument(span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&span),
+            Err(_) => {
+                StageOutcome::Error.record(&span);
+                ErrorCategory::Configuration.record(&span);
+            }
+        }
+        result
     }
 
     async fn resolve_auth_profile(
@@ -265,40 +304,46 @@ impl AdminService {
         let used_at = OffsetDateTime::now_utc();
 
         for secret_id in auth_profile.config.secret_ids() {
-            let secret = self
-                .registry
-                .get_secret(workspace_id, secret_id)
-                .await
-                .map_err(|error| RuntimeError::SecretCrypto {
-                    operation: "load secret",
-                    details: error.to_string(),
-                })?
-                .ok_or_else(|| RuntimeError::MissingSecret {
-                    secret_id: secret_id.as_str().to_owned(),
-                })?;
-            let version = self
-                .registry
-                .get_current_secret_version(workspace_id, secret_id)
-                .await
-                .map_err(|error| RuntimeError::SecretCrypto {
-                    operation: "load current secret version",
-                    details: error.to_string(),
-                })?
-                .ok_or_else(|| RuntimeError::MissingSecretVersion {
-                    secret_id: secret_id.as_str().to_owned(),
-                    version: secret.secret.current_version,
-                })?;
+            let secret = observe_db_query(
+                DbOperation::SecretRead,
+                self.registry.get_secret(workspace_id, secret_id),
+            )
+            .await
+            .map_err(|error| RuntimeError::SecretCrypto {
+                operation: "load secret",
+                details: error.to_string(),
+            })?
+            .ok_or_else(|| RuntimeError::MissingSecret {
+                secret_id: secret_id.as_str().to_owned(),
+            })?;
+            let version = observe_db_query(
+                DbOperation::SecretRead,
+                self.registry
+                    .get_current_secret_version(workspace_id, secret_id),
+            )
+            .await
+            .map_err(|error| RuntimeError::SecretCrypto {
+                operation: "load current secret version",
+                details: error.to_string(),
+            })?
+            .ok_or_else(|| RuntimeError::MissingSecretVersion {
+                secret_id: secret_id.as_str().to_owned(),
+                version: secret.secret.current_version,
+            })?;
             let plaintext = self.secret_crypto.decrypt(
                 &version.secret_version.key_version,
                 &version.secret_version.ciphertext,
             )?;
-            self.registry
-                .touch_secret(workspace_id, secret_id, &used_at)
-                .await
-                .map_err(|error| RuntimeError::SecretCrypto {
-                    operation: "touch secret",
-                    details: error.to_string(),
-                })?;
+            observe_db_query(
+                DbOperation::SecretTouch,
+                self.registry
+                    .touch_secret(workspace_id, secret_id, &used_at),
+            )
+            .await
+            .map_err(|error| RuntimeError::SecretCrypto {
+                operation: "touch secret",
+                details: error.to_string(),
+            })?;
             secrets.insert(secret_id.clone(), plaintext);
         }
 
@@ -412,7 +457,7 @@ impl AdminService {
     async fn record_invocation(
         &self,
         request: InvocationRecordRequest<'_>,
-    ) -> Result<(), ApiError> {
+    ) -> InvocationHistoryWriteOutcome {
         let log = InvocationLog {
             id: InvocationLogId::new(new_prefixed_id("log")),
             workspace_id: request.workspace_id.clone(),
@@ -432,11 +477,70 @@ impl AdminService {
             created_at: OffsetDateTime::now_utc(),
         };
 
-        self.registry
-            .create_invocation_log(CreateInvocationLogRequest { log: &log })
-            .await?;
+        let history_span = crank_trace::Stage::HistoryWrite.span();
+        let (outcome, db_span) = async {
+            let db_span = crank_trace::Stage::DbQuery
+                .db_span(crank_trace::DbOperation::InvocationHistoryWrite)
+                .expect("database stage");
+            let outcome = self
+                .registry
+                .create_invocation_log(CreateInvocationLogRequest { log: &log })
+                .instrument(db_span.clone())
+                .await;
+            (outcome, db_span)
+        }
+        .instrument(history_span.clone())
+        .await;
+        match outcome {
+            InvocationHistoryWriteOutcome::Recorded => {
+                crank_trace::StageOutcome::Success.record(&db_span);
+                crank_trace::StageOutcome::Success.record(&history_span);
+            }
+            InvocationHistoryWriteOutcome::Lost(_) => {
+                crank_trace::StageOutcome::Error.record(&db_span);
+                crank_trace::ErrorCategory::Database.record(&db_span);
+                crank_trace::StageOutcome::Error.record(&history_span);
+                crank_trace::ErrorCategory::History.record(&history_span);
+            }
+        }
+        drop(db_span);
+        drop(history_span);
+        observe_invocation_history_outcome(
+            outcome,
+            request.request_id,
+            request.status,
+            "admin_test_run",
+        );
+        outcome
+    }
+}
 
-        Ok(())
+fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: crank_core::InvocationStatus,
+    source: &'static str,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    tracing::warn!(
+        name: "admin.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source,
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
+    match status {
+        crank_core::InvocationStatus::Ok => "ok",
+        crank_core::InvocationStatus::Error => "error",
     }
 }
 
@@ -543,6 +647,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
         RuntimeError::RestAdapter(_) => "rest_error",
         RuntimeError::ProtocolAdapter(_) => "adapter_error",
         RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
@@ -761,7 +869,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule {
 #[cfg(test)]
 #[allow(clippy::items_after_test_module)]
 mod tests {
-    use super::{validate_profile_display_name, validate_profile_email};
+    use std::{
+        io,
+        sync::{Arc, Mutex},
+    };
+
+    use crank_core::InvocationStatus;
+    use crank_observability::{
+        ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+        operational_incident_total,
+    };
+    use crank_registry::{
+        InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
+    };
+    use serde_json::Value;
+    use tracing_subscriber::fmt::MakeWriter;
+
+    use super::{
+        observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email,
+    };
 
     #[test]
     fn validates_profile_identity_fields() {
@@ -782,6 +908,81 @@ mod tests {
         assert!(validate_profile_display_name(&"x".repeat(81)).is_err());
         assert!(validate_profile_email("owner <html>@crank.local").is_err());
     }
+
+    #[test]
+    fn emits_bounded_history_loss_incident() {
+        let writer = SharedLogWriter::default();
+        let subscriber = crank_observability::build_subscriber(
+            ObservabilityConfig::new(
+                ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+                "info",
+                RedactionLimits::default(),
+            ),
+            writer.clone(),
+        )
+        .unwrap();
+        let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let _guard = tracing::dispatcher::set_default(&dispatch);
+
+        observe_invocation_history_outcome(
+            InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
+                category: InvocationHistoryLossCategory::InvalidRecord,
+            }),
+            Some("req_admin_dc08"),
+            InvocationStatus::Error,
+            "admin_test_run",
+        );
+
+        let output = writer.output();
+        assert!(!output.contains("dc08-canary-secret"));
+        let event: Value = output
+            .lines()
+            .map(|line| serde_json::from_str(line).unwrap())
+            .find(|event: &Value| event["event"] == "admin.invocation_history.lost")
+            .unwrap();
+        assert_eq!(event["request_id"], "req_admin_dc08");
+        assert_eq!(event["fields"]["source"], "admin_test_run");
+        assert_eq!(event["fields"]["invocation_status"], "error");
+        assert_eq!(event["fields"]["error_category"], "invalid_record");
+        assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+    }
+
+    #[derive(Clone, Default)]
+    struct SharedLogWriter {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl SharedLogWriter {
+        fn output(&self) -> String {
+            String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+        }
+    }
+
+    impl<'a> MakeWriter<'a> for SharedLogWriter {
+        type Writer = SharedLogGuard;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            SharedLogGuard {
+                buffer: Arc::clone(&self.buffer),
+            }
+        }
+    }
+
+    struct SharedLogGuard {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl io::Write for SharedLogGuard {
+        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+            self.buffer.lock().unwrap().extend_from_slice(bytes);
+            Ok(bytes.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
 }
 
 fn enrich_operation_summary(

apps/admin-api/src/service/agents.rs

@@ -296,7 +296,12 @@ impl AdminService {
                 bindings: &[],
             })
             .await?;
-        info!(agent_id = %agent_id.as_str(), version = 1, "agent created");
+        info!(
+            name: "admin.agent.created",
+            agent_id = %agent_id.as_str(),
+            version = 1,
+            "agent created"
+        );
 
         Ok(CreatedAgentResponse {
             agent_id: agent_id.as_str().to_owned(),
@@ -417,6 +422,7 @@ impl AdminService {
             })
             .await?;
         info!(
+            name: "admin.agent.bindings_saved",
             agent_id = %agent_id.as_str(),
             version = current_version.version,
             binding_count = bindings.len(),
@@ -531,7 +537,12 @@ impl AdminService {
                 published_by: None,
             })
             .await?;
-        info!(agent_id = %agent_id.as_str(), version, "agent published");
+        info!(
+            name: "admin.agent.published",
+            agent_id = %agent_id.as_str(),
+            version,
+            "agent published"
+        );
 
         Ok(PublishAgentResponse {
             agent_id: agent_id.as_str().to_owned(),
@@ -589,7 +600,11 @@ impl AdminService {
         self.registry
             .unpublish_agent(workspace_id, agent_id, &updated_at)
             .await?;
-        info!(agent_id = %agent_id.as_str(), "agent moved to draft");
+        info!(
+            name: "admin.agent.unpublished",
+            agent_id = %agent_id.as_str(),
+            "agent moved to draft"
+        );
 
         Ok(AgentMutationResult {
             agent_id: agent_id.as_str().to_owned(),
@@ -609,7 +624,11 @@ impl AdminService {
         self.registry
             .archive_agent(workspace_id, agent_id, &updated_at)
             .await?;
-        info!(agent_id = %agent_id.as_str(), "agent archived");
+        info!(
+            name: "admin.agent.archived",
+            agent_id = %agent_id.as_str(),
+            "agent archived"
+        );
 
         Ok(AgentMutationResult {
             agent_id: agent_id.as_str().to_owned(),

apps/admin-api/src/service/auth.rs

@@ -22,7 +22,7 @@ impl AdminService {
         )?;
         let user_id = self
             .registry
-            .upsert_bootstrap_user(
+            .ensure_bootstrap_user(
                 &self.auth_settings.bootstrap_admin.email,
                 &self.auth_settings.bootstrap_admin.display_name,
                 &password_hash,
@@ -227,6 +227,7 @@ impl AdminService {
     pub async fn change_password(
         &self,
         user_id: &crank_core::UserId,
+        current_session_id: &UserSessionId,
         payload: ChangePasswordPayload,
     ) -> Result<(), ApiError> {
         if payload.new_password.len() < 12 {
@@ -257,7 +258,11 @@ impl AdminService {
         let password_hash =
             hash_password(&payload.new_password, &self.auth_settings.password_pepper)?;
         self.registry
-            .update_user_password(user_id, &password_hash)
+            .update_user_password_and_revoke_other_sessions(
+                user_id,
+                current_session_id,
+                &password_hash,
+            )
             .await?;
 
         Ok(())

apps/admin-api/src/service/demo.rs

@@ -131,6 +131,7 @@ impl AdminService {
             Ok(()) => Ok(()),
             Err(RegistryError::OperationHasPublishedAgentBindings { .. }) => {
                 tracing::warn!(
+                    name: "admin.demo_operation.cleanup_skipped",
                     operation_id = %operation_id.as_str(),
                     "legacy demo operation is still bound to a published agent; leaving it in place"
                 );
@@ -335,7 +336,7 @@ impl AdminService {
             }),
             response_preview: demo_rest_response_sample(),
         })
-        .await?;
+        .await;
         Ok(())
     }
 }

apps/admin-api/src/service/import_export.rs

@@ -110,6 +110,7 @@ impl AdminService {
                         warnings,
                     };
                     info!(
+                        name: "admin.operation.imported",
                         operation_id = %response.operation_id,
                         version = response.version,
                         "operation imported by upsert"
@@ -125,6 +126,7 @@ impl AdminService {
                         warnings,
                     };
                     info!(
+                        name: "admin.operation.imported",
                         operation_id = %response.operation_id,
                         version = response.version,
                         "operation imported by upsert"

apps/admin-api/src/service/imports.rs

@@ -8,9 +8,11 @@ use crank_import::rest::{
     ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
 };
 use crank_registry::{
-    CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus,
+    ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind,
+    ImportJobStatus, ImportOperationDraft,
 };
 use serde_json::json;
+use sha2::{Digest, Sha256};
 use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
 use tracing::{info, instrument};
 
@@ -50,7 +52,7 @@ impl AdminService {
                 kind: ImportJobKind::OpenApi,
                 source_format: &preview.source.format,
                 source_version: preview.source.version.as_deref(),
-                status: ImportJobStatus::Completed,
+                status: ImportJobStatus::Pending,
                 preview_payload: &preview_payload,
                 created_at: &now,
                 expires_at: &expires_at,
@@ -99,9 +101,13 @@ impl AdminService {
             return Err(ApiError::validation("import job kind is not openapi"));
         }
 
-        let preview: crank_import::rest::ImportPreview =
-            serde_json::from_value(job.preview_payload.clone())
-                .map_err(|error| ApiError::internal(error.to_string()))?;
+        let stored_preview = job
+            .preview_payload
+            .get("preview")
+            .cloned()
+            .unwrap_or_else(|| job.preview_payload.clone());
+        let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview)
+            .map_err(|error| ApiError::internal(error.to_string()))?;
         let selected = payload
             .selected_operation_keys
             .iter()
@@ -120,10 +126,8 @@ impl AdminService {
             }
         }
 
-        let mut created = Vec::new();
         let mut skipped = Vec::new();
-        let mut findings = Vec::new();
-        let mut created_ids = Vec::new();
+        let mut operations = Vec::new();
 
         for operation_key in selected {
             let Some(candidate) = candidates.get(&operation_key) else {
@@ -137,44 +141,7 @@ impl AdminService {
             let mut draft =
                 operation_draft_from_candidate(candidate, payload.server_url.as_deref());
             attach_import_findings(&mut draft, candidate);
-            if let Some(existing_name) = self
-                .find_operation_by_name(workspace_id, &draft.name)
-                .await?
-                .map(|operation| operation.name)
-            {
-                if payload.conflict_mode == "skip" {
-                    skipped.push(OpenApiImportSkippedOperation {
-                        operation_key: candidate.key.clone(),
-                        name: draft.name.clone(),
-                        reason: "operation with this name already exists".to_owned(),
-                    });
-                    findings.push(ImportFinding {
-                        code: "operation_name_conflict".to_owned(),
-                        severity: ImportFindingSeverity::Warning,
-                        message: format!(
-                            "Операция {} уже существует и была пропущена.",
-                            draft.name
-                        ),
-                        operation_key: Some(candidate.key.clone()),
-                    });
-                    continue;
-                }
-
-                let renamed = self
-                    .next_available_operation_name(workspace_id, &draft.name)
-                    .await?;
-                findings.push(ImportFinding {
-                    code: "operation_name_renamed".to_owned(),
-                    severity: ImportFindingSeverity::Info,
-                    message: format!(
-                        "Операция {existing_name} уже существует, новый черновик создан как {renamed}."
-                    ),
-                    operation_key: Some(candidate.key.clone()),
-                });
-                draft.name = renamed;
-            }
-
-            let payload = OperationPayload {
+            let operation = self.new_operation_snapshot(OperationPayload {
                 name: draft.name.clone(),
                 display_name: draft.display_name.clone(),
                 category: draft.category,
@@ -197,27 +164,74 @@ impl AdminService {
                 },
                 tool_description: draft.tool_description,
                 wizard_state: draft.wizard_state,
-            };
-            let result = self.create_operation(workspace_id, payload).await?;
-            created_ids.push(result.operation_id.clone());
-            created.push(OpenApiImportCreatedOperation {
-                operation_id: result.operation_id,
-                name: draft.name,
-                version: result.version,
+            })?;
+            operations.push(ImportOperationDraft {
+                operation_key: candidate.key.clone(),
+                operation,
             });
         }
 
         let finished_at = OffsetDateTime::now_utc();
-        self.registry
-            .finish_import_job(FinishImportJobRequest {
+        let application_key = openapi_application_key(&payload)?;
+        let conflict_mode = if payload.conflict_mode == "skip" {
+            ImportConflictMode::Skip
+        } else {
+            ImportConflictMode::Rename
+        };
+        let applied = self
+            .registry
+            .apply_import_job(ApplyImportJobRequest {
                 id: job_id,
-                status: ImportJobStatus::Completed,
-                created_operation_ids: &json!(created_ids),
-                error_text: None,
+                workspace_id,
+                application_key: &application_key,
+                conflict_mode,
+                operations: &operations,
                 finished_at: &finished_at,
             })
             .await?;
+
+        let created = applied
+            .created
+            .iter()
+            .map(|operation| OpenApiImportCreatedOperation {
+                operation_id: operation.operation_id.as_str().to_owned(),
+                name: operation.name.clone(),
+                version: operation.version,
+            })
+            .collect::<Vec<_>>();
+        let mut findings = applied
+            .created
+            .iter()
+            .filter_map(|operation| {
+                operation.renamed_from.as_ref().map(|previous_name| ImportFinding {
+                    code: "operation_name_renamed".to_owned(),
+                    severity: ImportFindingSeverity::Info,
+                    message: format!(
+                        "Операция {previous_name} уже существует, новый черновик создан как {}.",
+                        operation.name
+                    ),
+                    operation_key: Some(operation.operation_key.clone()),
+                })
+            })
+            .collect::<Vec<_>>();
+        for operation in applied.skipped {
+            skipped.push(OpenApiImportSkippedOperation {
+                operation_key: operation.operation_key.clone(),
+                name: operation.name.clone(),
+                reason: "operation with this name already exists".to_owned(),
+            });
+            findings.push(ImportFinding {
+                code: operation.reason,
+                severity: ImportFindingSeverity::Warning,
+                message: format!(
+                    "Операция {} уже существует и была пропущена.",
+                    operation.name
+                ),
+                operation_key: Some(operation.operation_key),
+            });
+        }
         info!(
+            name: "admin.openapi_import.completed",
             created = created.len(),
             skipped = skipped.len(),
             "openapi import created drafts"
@@ -229,25 +243,21 @@ impl AdminService {
             findings,
         })
     }
+}
 
-    async fn next_available_operation_name(
-        &self,
-        workspace_id: &WorkspaceId,
-        base_name: &str,
-    ) -> Result<String, ApiError> {
-        for index in 2.. {
-            let candidate = format!("{base_name}_{index}");
-            if self
-                .find_operation_by_name(workspace_id, &candidate)
-                .await?
-                .is_none()
-            {
-                return Ok(candidate);
-            }
-        }
-
-        unreachable!()
-    }
+fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
+    let selected_operation_keys = payload
+        .selected_operation_keys
+        .iter()
+        .cloned()
+        .collect::<BTreeSet<_>>();
+    let canonical = serde_json::to_vec(&json!({
+        "selected_operation_keys": selected_operation_keys,
+        "server_url": payload.server_url.as_deref(),
+        "conflict_mode": payload.conflict_mode.as_str(),
+    }))
+    .map_err(|error| ApiError::internal(error.to_string()))?;
+    Ok(format!("{:x}", Sha256::digest(canonical)))
 }
 
 fn attach_import_findings(

apps/admin-api/src/service/observability.rs

@@ -19,6 +19,16 @@ use crate::{
 };
 
 impl AdminService {
+    pub async fn cleanup_invocation_logs_before(
+        &self,
+        cutoff: OffsetDateTime,
+    ) -> Result<u64, ApiError> {
+        self.registry
+            .delete_invocation_logs_before(cutoff)
+            .await
+            .map_err(ApiError::from)
+    }
+
     #[instrument(skip(self))]
     pub async fn list_logs(
         &self,

apps/admin-api/src/service/operations.rs

@@ -141,7 +141,6 @@ impl AdminService {
         workspace_id: &WorkspaceId,
         payload: OperationPayload,
     ) -> Result<CreatedOperationResponse, ApiError> {
-        self.validate_operation_payload(&payload)?;
         self.ensure_workspace_exists(workspace_id).await?;
 
         if self
@@ -155,10 +154,36 @@ impl AdminService {
             ));
         }
 
+        let snapshot = self.new_operation_snapshot(payload)?;
+        let operation_id = snapshot.id.clone();
+
+        self.registry
+            .create_operation(workspace_id, &snapshot, None)
+            .await?;
+        info!(
+            name: "admin.operation.created",
+            operation_id = %operation_id.as_str(),
+            version = 1,
+            "operation created"
+        );
+
+        Ok(CreatedOperationResponse {
+            operation_id: operation_id.as_str().to_owned(),
+            workspace_id: workspace_id.as_str().to_owned(),
+            version: 1,
+            status: OperationStatus::Draft,
+            updated_at: format_timestamp(snapshot.updated_at),
+        })
+    }
+
+    pub(super) fn new_operation_snapshot(
+        &self,
+        payload: OperationPayload,
+    ) -> Result<RegistryOperation, ApiError> {
+        self.validate_operation_payload(&payload)?;
         let now = OffsetDateTime::now_utc();
-        let operation_id = OperationId::new(new_prefixed_id("op"));
-        let snapshot = RegistryOperation {
-            id: operation_id.clone(),
+        Ok(RegistryOperation {
+            id: OperationId::new(new_prefixed_id("op")),
             name: payload.name,
             display_name: payload.display_name,
             category: payload.category,
@@ -183,19 +208,6 @@ impl AdminService {
             created_at: now,
             updated_at: now,
             published_at: None,
-        };
-
-        self.registry
-            .create_operation(workspace_id, &snapshot, None)
-            .await?;
-        info!(operation_id = %operation_id.as_str(), version = 1, "operation created");
-
-        Ok(CreatedOperationResponse {
-            operation_id: operation_id.as_str().to_owned(),
-            workspace_id: workspace_id.as_str().to_owned(),
-            version: 1,
-            status: OperationStatus::Draft,
-            updated_at: format_timestamp(snapshot.updated_at),
         })
     }
 
@@ -279,7 +291,12 @@ impl AdminService {
                 created_by: None,
             })
             .await?;
-        info!(operation_id = %operation_id.as_str(), version, "operation version created");
+        info!(
+            name: "admin.operation.version_created",
+            operation_id = %operation_id.as_str(),
+            version,
+            "operation version created"
+        );
 
         Ok(CreatedOperationResponse {
             operation_id: operation_id.as_str().to_owned(),
@@ -364,7 +381,12 @@ impl AdminService {
                 published_by: None,
             })
             .await?;
-        info!(operation_id = %operation_id.as_str(), version, "operation published");
+        info!(
+            name: "admin.operation.published",
+            operation_id = %operation_id.as_str(),
+            version,
+            "operation published"
+        );
 
         Ok(PublishResponse {
             operation_id: operation_id.as_str().to_owned(),
@@ -431,37 +453,44 @@ impl AdminService {
             .await?;
         let runtime = RuntimeOperation::from(record.snapshot.clone());
         let mode = ExecutionMode::Unary;
-        let request_preview =
-            match build_request_preview(&record.snapshot.input_mapping, &payload.input) {
-                Ok(preview) => preview,
-                Err(error) => {
-                    self.record_invocation(InvocationRecordRequest {
-                        workspace_id,
-                        agent_id: None,
-                        operation: &record.snapshot,
-                        request_id: Some(request_id),
-                        source: InvocationSource::AdminTestRun,
-                        level: InvocationLevel::Error,
-                        status: InvocationStatus::Error,
-                        message: "mapping preview failed".to_owned(),
-                        status_code: None,
-                        error_kind: Some("mapping".to_owned()),
-                        duration_ms: 0,
-                        request_preview: Value::Null,
-                        response_preview: Value::Null,
-                    })
-                    .await?;
-                    return Ok(TestRunResult {
-                        ok: false,
-                        mode,
-                        request_preview: Value::Null,
-                        response_preview: Value::Null,
-                        errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping(
-                            error,
-                        ))],
-                    });
-                }
-            };
+        let preview_span = crank_trace::Stage::RuntimeArgumentsMap.span();
+        let preview_result = preview_span
+            .in_scope(|| build_request_preview(&record.snapshot.input_mapping, &payload.input));
+        let request_preview = match preview_result {
+            Ok(preview) => preview,
+            Err(error) => {
+                crank_trace::StageOutcome::Error.record(&preview_span);
+                crank_trace::ErrorCategory::Mapping.record(&preview_span);
+                drop(preview_span);
+                self.record_invocation(InvocationRecordRequest {
+                    workspace_id,
+                    agent_id: None,
+                    operation: &record.snapshot,
+                    request_id: Some(request_id),
+                    source: InvocationSource::AdminTestRun,
+                    level: InvocationLevel::Error,
+                    status: InvocationStatus::Error,
+                    message: "mapping preview failed".to_owned(),
+                    status_code: None,
+                    error_kind: Some("mapping".to_owned()),
+                    duration_ms: 0,
+                    request_preview: Value::Null,
+                    response_preview: Value::Null,
+                })
+                .await;
+                return Ok(TestRunResult {
+                    ok: false,
+                    mode,
+                    request_preview: Value::Null,
+                    response_preview: Value::Null,
+                    errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping(
+                        error,
+                    ))],
+                });
+            }
+        };
+        crank_trace::StageOutcome::Success.record(&preview_span);
+        drop(preview_span);
 
         let resolved_auth = self
             .resolve_operation_auth(workspace_id, &runtime.execution_config)
@@ -497,7 +526,7 @@ impl AdminService {
                     request_preview: request_preview.clone(),
                     response_preview: response_preview.clone(),
                 })
-                .await?;
+                .await;
                 Ok(TestRunResult {
                     ok: true,
                     mode,
@@ -524,7 +553,7 @@ impl AdminService {
                     request_preview: request_preview.clone(),
                     response_preview: Value::Null,
                 })
-                .await?;
+                .await;
                 Ok(TestRunResult {
                     ok: false,
                     mode,

apps/admin-api/src/service/samples.rs

@@ -50,6 +50,7 @@ impl AdminService {
             .save_sample_metadata(SaveSampleMetadataRequest { sample: &metadata })
             .await?;
         info!(
+            name: "admin.sample.saved",
             operation_id = %operation_id.as_str(),
             sample_id = %metadata.id.as_str(),
             version,
@@ -119,7 +120,11 @@ impl AdminService {
             input_mapping,
             output_mapping,
         };
-        info!(operation_id = %operation_id.as_str(), "draft generated from samples");
+        info!(
+            name: "admin.operation_draft.generated",
+            operation_id = %operation_id.as_str(),
+            "draft generated from samples"
+        );
 
         Ok(result)
     }

apps/admin-api/src/service/secrets.rs

@@ -92,7 +92,11 @@ impl AdminService {
                 created_by,
             })
             .await?;
-        info!(secret_id = %secret.id.as_str(), "secret created");
+        info!(
+            name: "admin.secret.created",
+            secret_id = %secret.id.as_str(),
+            "secret created"
+        );
 
         Ok(secret)
     }
@@ -126,7 +130,11 @@ impl AdminService {
                 created_by,
             })
             .await?;
-        info!(secret_id = %secret_id.as_str(), "secret rotated");
+        info!(
+            name: "admin.secret.rotated",
+            secret_id = %secret_id.as_str(),
+            "secret rotated"
+        );
 
         self.get_secret(workspace_id, secret_id).await
     }
@@ -152,7 +160,11 @@ impl AdminService {
             .into());
         }
         self.registry.delete_secret(workspace_id, secret_id).await?;
-        info!(secret_id = %secret_id.as_str(), "secret deleted");
+        info!(
+            name: "admin.secret.deleted",
+            secret_id = %secret_id.as_str(),
+            "secret deleted"
+        );
         Ok(())
     }
 
@@ -201,7 +213,11 @@ impl AdminService {
                 profile: &profile,
             })
             .await?;
-        info!(auth_profile_id = %profile.id.as_str(), "auth profile created");
+        info!(
+            name: "admin.auth_profile.created",
+            auth_profile_id = %profile.id.as_str(),
+            "auth profile created"
+        );
 
         Ok(profile)
     }

apps/admin-api/src/service/upstreams.rs

@@ -75,7 +75,11 @@ impl AdminService {
                 upstream: &upstream,
             })
             .await?;
-        info!(upstream_id = %upstream.id.as_str(), "workspace upstream saved");
+        info!(
+            name: "admin.upstream.saved",
+            upstream_id = %upstream.id.as_str(),
+            "workspace upstream saved"
+        );
 
         Ok(upstream)
     }

apps/admin-api/tests/integration/community_access_usage.rs

@@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind};
 use serde_json::{Value, json};
 use serial_test::serial;
 use tokio::net::TcpListener;
+use uuid::Version;
 
 use admin_api::{
     app::build_app,
@@ -40,6 +41,8 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
 const TEST_SESSION_SECRET: &str = "test-session-secret";
 const TEST_MASTER_KEY: &str = "test-master-key";
 
+mod history_loss;
+
 struct TestServer {
     base_url: String,
     shutdown: Option<tokio::sync::oneshot::Sender<()>>,
@@ -385,6 +388,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() {
         exported["workspace"]["workspace"]["id"],
         DEFAULT_WORKSPACE_ID
     );
+    assert_eq!(exported["kind"], "workspace_catalog_snapshot");
+    assert_eq!(exported["format_version"], "1");
+    assert_eq!(exported["restorable"], false);
+    assert_eq!(
+        exported["included"],
+        json!([
+            "workspace_settings",
+            "operation_summaries",
+            "agent_summaries",
+            "platform_api_key_metadata"
+        ])
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("secret_values"))
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("invocation_logs_and_usage"))
+    );
     assert!(exported.get("memberships").is_none());
     assert!(exported.get("invitations").is_none());
 
@@ -574,6 +601,25 @@ async fn updates_profile_and_changes_password() {
         .unwrap()
         .to_owned();
     let client = authorized_client(&base_url).await;
+    let second_client = reqwest::Client::builder()
+        .cookie_store(true)
+        .build()
+        .unwrap();
+    let second_login = second_client
+        .post(format!("{root_url}/api/auth/login"))
+        .json(&json!({
+            "email": TEST_AUTH_EMAIL,
+            "password": TEST_AUTH_PASSWORD,
+        }))
+        .send()
+        .await
+        .unwrap();
+    let second_login_status = second_login.status();
+    let second_login_body = second_login.text().await.unwrap();
+    assert!(
+        second_login_status.is_success(),
+        "second login failed with {second_login_status}: {second_login_body}"
+    );
 
     let profile = assert_success_json(
         client
@@ -615,6 +661,21 @@ async fn updates_profile_and_changes_password() {
         .status();
     assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);
 
+    let current_session_status = client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    let other_session_status = second_client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    assert_eq!(current_session_status, reqwest::StatusCode::OK);
+    assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
+
     let relogin_client = reqwest::Client::builder()
         .cookie_store(true)
         .build()
@@ -855,7 +916,10 @@ async fn generates_request_id_for_test_run_invocations() {
         .unwrap()
         .to_owned();
 
-    assert!(!request_id.is_empty());
+    assert_eq!(
+        uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
     response.error_for_status().unwrap();
 
     let logs = client

apps/admin-api/tests/integration/community_access_usage/history_loss.rs

@@ -0,0 +1,141 @@
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+use axum::{Json, Router, extract::State, routing::post};
+use tokio::{net::TcpListener, sync::Notify};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn preserves_external_success_when_invocation_history_is_lost() {
+    let registry = test_registry().await;
+    let registry_for_failure = registry.clone();
+    let storage_root = test_storage_root("observability_history_loss");
+    let upstream = spawn_blocking_upstream_server().await;
+    let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
+    let client = authorized_client(&base_url).await;
+
+    let created = client
+        .post(format!("{base_url}/operations"))
+        .json(&test_operation_payload(
+            &upstream.base_url,
+            "crm_history_loss",
+        ))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    let operation_id = created["operation_id"].as_str().unwrap().to_owned();
+    let request_client = client.clone();
+    let request_url = format!("{base_url}/operations/{operation_id}/test-runs");
+    let before = crank_observability::operational_incident_total(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+
+    let request = tokio::spawn(async move {
+        request_client
+            .post(request_url)
+            .header("x-request-id", "req_dc08_admin")
+            .json(&json!({
+                "version": 1,
+                "input": { "email": "dc08-canary-secret@example.com" }
+            }))
+            .send()
+            .await
+            .unwrap()
+    });
+
+    upstream.started.notified().await;
+    registry_for_failure
+        .delete_operation(
+            &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
+            &OperationId::new(operation_id.clone()),
+        )
+        .await
+        .unwrap();
+    upstream.release.notify_one();
+
+    let response = request.await.unwrap();
+    assert_eq!(response.status(), reqwest::StatusCode::OK);
+    assert_eq!(
+        response.headers()["x-request-id"].to_str().unwrap(),
+        "req_dc08_admin"
+    );
+    let body = response.json::<Value>().await.unwrap();
+    assert_eq!(body["ok"], true);
+    assert_eq!(body["response_preview"]["id"], "lead_123");
+    assert_eq!(upstream.calls.load(Ordering::SeqCst), 1);
+    assert!(
+        crank_observability::operational_incident_total(
+            crank_observability::OperationalIncident::InvocationHistoryLost
+        ) > before
+    );
+
+    let logs = client
+        .get(format!("{base_url}/logs?period=7d"))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    assert!(logs["items"].as_array().unwrap().is_empty());
+}
+
+struct BlockingUpstream {
+    base_url: String,
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+#[derive(Clone)]
+struct BlockingUpstreamState {
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+async fn spawn_blocking_upstream_server() -> BlockingUpstream {
+    let state = BlockingUpstreamState {
+        started: Arc::new(Notify::new()),
+        release: Arc::new(Notify::new()),
+        calls: Arc::new(AtomicUsize::new(0)),
+    };
+    let app = Router::new()
+        .route("/crm/leads", post(blocking_create_lead))
+        .with_state(state.clone());
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    BlockingUpstream {
+        base_url: format!("http://{address}"),
+        started: state.started,
+        release: state.release,
+        calls: state.calls,
+    }
+}
+
+async fn blocking_create_lead(
+    State(state): State<BlockingUpstreamState>,
+    Json(payload): Json<Value>,
+) -> Json<Value> {
+    state.calls.fetch_add(1, Ordering::SeqCst);
+    state.started.notify_one();
+    state.release.notified().await;
+
+    Json(json!({
+        "id": "lead_123",
+        "status": "created",
+        "email": payload["email"]
+    }))
+}

apps/admin-api/tests/integration/openapi_import.rs

@@ -1,5 +1,6 @@
 use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
 use crank_core::WorkspaceId;
+use crank_registry::ImportJobStatus;
 use serial_test::serial;
 
 use super::common::{
@@ -42,7 +43,7 @@ paths:
 async fn previews_openapi_and_creates_draft_operations() {
     let registry = test_registry().await;
     let service = test_service(
-        registry,
+        registry.clone(),
         test_storage_root("openapi_import"),
         test_auth_settings(),
         test_secret_crypto(),
@@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() {
         preview.preview.groups[0].operations[0].suggested_name,
         "latest_rates"
     );
+    let preview_job = registry
+        .get_import_job(&workspace_id, &preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(preview_job.status, ImportJobStatus::Pending);
 
     let created = service
         .create_openapi_import(
@@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() {
             .any(|finding| finding.code == "openapi_import.weak_tool_description")
     );
 
+    let skip_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let skipped = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &skip_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(skipped.skipped[0].name, "latest_rates");
     assert_eq!(skipped.findings[0].code, "operation_name_conflict");
 
+    let rename_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let renamed = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &rename_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(renamed.created[0].name, "latest_rates_2");
     assert_eq!(renamed.findings[0].code, "operation_name_renamed");
 }
+
+#[tokio::test]
+#[serial]
+async fn concurrent_openapi_import_replays_the_same_atomic_result() {
+    let registry = test_registry().await;
+    let service = test_service(
+        registry,
+        test_storage_root("openapi_import_replay"),
+        test_auth_settings(),
+        test_secret_crypto(),
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    let job_id = preview.job_id.as_str().into();
+    let payload = OpenApiImportCreatePayload {
+        selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+        server_url: Some("https://api.frankfurter.dev".to_owned()),
+        conflict_mode: "rename".to_owned(),
+    };
+
+    let (first, second) = tokio::join!(
+        service.create_openapi_import(&workspace_id, &job_id, payload.clone()),
+        service.create_openapi_import(&workspace_id, &job_id, payload),
+    );
+    let first = first.unwrap();
+    let second = second.unwrap();
+
+    assert_eq!(first.created.len(), 1);
+    assert_eq!(second.created.len(), 1);
+    assert_eq!(
+        first.created[0].operation_id,
+        second.created[0].operation_id
+    );
+    assert_eq!(first.created[0].name, second.created[0].name);
+    assert_eq!(
+        service.list_operations(&workspace_id).await.unwrap().len(),
+        1
+    );
+
+    let conflicting_replay = service
+        .create_openapi_import(
+            &workspace_id,
+            &job_id,
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+                server_url: Some("https://api.frankfurter.dev".to_owned()),
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await;
+    assert!(conflicting_replay.is_err());
+}

apps/admin-api/tests/integration/request_context.rs

@@ -0,0 +1,211 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
+use axum::{
+    Router,
+    body::Body,
+    http::{HeaderMap, Request, StatusCode},
+    routing::get,
+};
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
+};
+use opentelemetry::{global, trace::TracerProvider as _};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
+use tower::ServiceExt;
+use tracing::instrument::WithSubscriber;
+use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
+use uuid::Version;
+
+#[tokio::test(flavor = "current_thread")]
+async fn logs_request_completion_and_rejects_untrusted_values() {
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = probe_app();
+
+    let response = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch.clone())
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(
+        response.headers()[REQUEST_ID_HEADER.as_str()]
+            .to_str()
+            .unwrap(),
+        "req_admin_trace_123"
+    );
+    let event: serde_json::Value = writer
+        .output()
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
+        .unwrap();
+    assert_eq!(event["request_id"], "req_admin_trace_123");
+    assert_eq!(event["fields"]["status"], 200);
+
+    let invalid_response = app
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "bad,value")
+                .header("traceparent", "canary-invalid-traceparent")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch)
+        .await
+        .unwrap();
+    let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
+        .to_str()
+        .unwrap();
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
+    assert!(!writer.output().contains("canary-invalid-traceparent"));
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn covers_valid_invalid_and_absent_traceparent() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("admin-request-context-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = trace_probe_app();
+
+    let valid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header(
+                        "traceparent",
+                        "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+                    )
+                    .header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .with_subscriber(dispatch.clone())
+            .await
+            .unwrap(),
+    );
+    let invalid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header("traceparent", "canary-invalid-traceparent")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .with_subscriber(dispatch.clone())
+            .await
+            .unwrap(),
+    );
+    let absent = observed_trace_id(
+        app.oneshot(
+            Request::builder()
+                .uri("/trace")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch)
+        .await
+        .unwrap(),
+    );
+
+    assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c");
+    assert_ne!(invalid, valid);
+    assert_ne!(absent, valid);
+    assert_ne!(invalid, absent);
+    provider.shutdown().unwrap();
+}
+
+fn probe_app() -> Router {
+    Router::new()
+        .route("/probe", get(|| async { "ok" }))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+fn trace_probe_app() -> Router {
+    Router::new()
+        .route("/trace", get(observed_traceparent))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+async fn observed_traceparent() -> HeaderMap {
+    let mut trace_headers = HeaderMap::new();
+    inject_current_trace_context(&mut trace_headers);
+    let mut response_headers = HeaderMap::new();
+    if let Some(traceparent) = trace_headers.remove("traceparent") {
+        response_headers.insert("x-observed-traceparent", traceparent);
+    }
+    response_headers
+}
+
+fn observed_trace_id(response: axum::response::Response) -> String {
+    let traceparent = response.headers()["x-observed-traceparent"]
+        .to_str()
+        .unwrap();
+    traceparent[3..35].to_owned()
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

apps/admin-api/tests/request_context.rs

@@ -0,0 +1,2 @@
+#[path = "integration/request_context.rs"]
+mod request_context;