наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

crates/crank-observability/src/otlp.rs

@@ -0,0 +1,954 @@
+use std::{collections::HashMap, env, fmt, time::Duration};
+
+use axum::http::{HeaderName, HeaderValue};
+use opentelemetry::{
+    KeyValue, Value,
+    trace::{Status, TracerProvider as _},
+};
+use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig, WithHttpConfig};
+use opentelemetry_sdk::{
+    Resource,
+    error::OTelSdkResult,
+    trace::{
+        BatchConfigBuilder, BatchSpanProcessor, SdkTracer, SdkTracerProvider, SpanData,
+        SpanExporter as SpanExporterTrait,
+    },
+};
+use percent_encoding::percent_decode_str;
+use thiserror::Error;
+use url::Url;
+
+use crate::ServiceIdentity;
+
+const DEFAULT_EXPORT_TIMEOUT: Duration = Duration::from_secs(10);
+const DEFAULT_MAX_QUEUE_SIZE: usize = 2_048;
+const DEFAULT_MAX_EXPORT_BATCH_SIZE: usize = 512;
+const DEFAULT_SCHEDULE_DELAY: Duration = Duration::from_secs(5);
+const DEFAULT_BATCH_EXPORT_TIMEOUT: Duration = Duration::from_secs(30);
+const MAX_QUEUE_SIZE: usize = 65_536;
+const MAX_DURATION: Duration = Duration::from_secs(300);
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct OtlpBatchConfig {
+    max_queue_size: usize,
+    max_export_batch_size: usize,
+    scheduled_delay: Duration,
+    export_timeout: Duration,
+}
+
+impl OtlpBatchConfig {
+    pub fn try_new(
+        max_queue_size: usize,
+        max_export_batch_size: usize,
+        scheduled_delay: Duration,
+        export_timeout: Duration,
+    ) -> Result<Self, OtlpTraceConfigError> {
+        let valid = max_queue_size > 0
+            && max_queue_size <= MAX_QUEUE_SIZE
+            && max_export_batch_size > 0
+            && max_export_batch_size <= max_queue_size
+            && duration_is_bounded(scheduled_delay)
+            && duration_is_bounded(export_timeout);
+        if !valid {
+            return Err(OtlpTraceConfigError::InvalidBatchLimits);
+        }
+
+        Ok(Self {
+            max_queue_size,
+            max_export_batch_size,
+            scheduled_delay,
+            export_timeout,
+        })
+    }
+
+    pub fn max_queue_size(&self) -> usize {
+        self.max_queue_size
+    }
+
+    pub fn max_export_batch_size(&self) -> usize {
+        self.max_export_batch_size
+    }
+
+    pub fn scheduled_delay(&self) -> Duration {
+        self.scheduled_delay
+    }
+
+    pub fn export_timeout(&self) -> Duration {
+        self.export_timeout
+    }
+
+    fn sdk_config(&self) -> opentelemetry_sdk::trace::BatchConfig {
+        BatchConfigBuilder::default()
+            .with_max_queue_size(self.max_queue_size)
+            .with_max_export_batch_size(self.max_export_batch_size)
+            .with_scheduled_delay(self.scheduled_delay)
+            .build()
+    }
+}
+
+impl Default for OtlpBatchConfig {
+    fn default() -> Self {
+        Self {
+            max_queue_size: DEFAULT_MAX_QUEUE_SIZE,
+            max_export_batch_size: DEFAULT_MAX_EXPORT_BATCH_SIZE,
+            scheduled_delay: DEFAULT_SCHEDULE_DELAY,
+            export_timeout: DEFAULT_BATCH_EXPORT_TIMEOUT,
+        }
+    }
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct OtlpTraceConfig {
+    endpoint: Option<String>,
+    export_timeout: Duration,
+    batch: OtlpBatchConfig,
+    headers: HashMap<String, String>,
+}
+
+impl fmt::Debug for OtlpTraceConfig {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("OtlpTraceConfig")
+            .field("enabled", &self.is_enabled())
+            .field("export_timeout", &self.export_timeout)
+            .field("batch", &self.batch)
+            .field("header_count", &self.headers.len())
+            .finish()
+    }
+}
+
+impl OtlpTraceConfig {
+    pub fn from_env() -> Result<Self, OtlpTraceConfigError> {
+        OtlpEnvSettings::from_env()?.into_config()
+    }
+
+    fn from_settings(settings: OtlpEnvSettings) -> Result<Self, OtlpTraceConfigError> {
+        let endpoint = match settings.traces_endpoint {
+            Some(endpoint) => Some(validate_endpoint(endpoint, EndpointKind::Trace)?),
+            None => settings
+                .generic_endpoint
+                .map(|endpoint| validate_endpoint(endpoint, EndpointKind::Generic))
+                .transpose()?,
+        };
+        if endpoint.is_none() {
+            return Ok(Self {
+                endpoint: None,
+                export_timeout: DEFAULT_EXPORT_TIMEOUT,
+                batch: OtlpBatchConfig::default(),
+                headers: HashMap::new(),
+            });
+        }
+
+        let protocol = settings.traces_protocol.or(settings.generic_protocol);
+        let export_timeout = match settings.traces_timeout {
+            Some(timeout) => duration_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", Some(timeout))?,
+            None => duration_env("OTEL_EXPORTER_OTLP_TIMEOUT", settings.generic_timeout)?,
+        }
+        .unwrap_or(DEFAULT_EXPORT_TIMEOUT);
+        let batch = OtlpBatchConfig::try_new(
+            usize_env("OTEL_BSP_MAX_QUEUE_SIZE", settings.max_queue_size)?
+                .unwrap_or(DEFAULT_MAX_QUEUE_SIZE),
+            usize_env(
+                "OTEL_BSP_MAX_EXPORT_BATCH_SIZE",
+                settings.max_export_batch_size,
+            )?
+            .unwrap_or(DEFAULT_MAX_EXPORT_BATCH_SIZE),
+            duration_env("OTEL_BSP_SCHEDULE_DELAY", settings.scheduled_delay)?
+                .unwrap_or(DEFAULT_SCHEDULE_DELAY),
+            duration_env("OTEL_BSP_EXPORT_TIMEOUT", settings.batch_export_timeout)?
+                .unwrap_or(DEFAULT_BATCH_EXPORT_TIMEOUT),
+        )?;
+        let headers = settings
+            .traces_headers
+            .filter(|value| !value.is_empty())
+            .or(settings.generic_headers.filter(|value| !value.is_empty()))
+            .map(|value| parse_headers(&value))
+            .transpose()?
+            .unwrap_or_default();
+
+        Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, headers)
+    }
+
+    pub fn try_new(
+        endpoint: Option<String>,
+        protocol: Option<String>,
+        export_timeout: Duration,
+        batch: OtlpBatchConfig,
+    ) -> Result<Self, OtlpTraceConfigError> {
+        Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, HashMap::new())
+    }
+
+    fn try_new_with_headers(
+        endpoint: Option<String>,
+        protocol: Option<String>,
+        export_timeout: Duration,
+        batch: OtlpBatchConfig,
+        headers: HashMap<String, String>,
+    ) -> Result<Self, OtlpTraceConfigError> {
+        let endpoint = endpoint
+            .map(|endpoint| validate_endpoint(endpoint, EndpointKind::Trace))
+            .transpose()?;
+        if endpoint.is_some() && protocol.as_deref().unwrap_or("http/protobuf") != "http/protobuf" {
+            return Err(OtlpTraceConfigError::UnsupportedProtocol);
+        }
+        if !duration_is_bounded(export_timeout) {
+            return Err(OtlpTraceConfigError::InvalidDuration {
+                field: "OTEL_EXPORTER_OTLP_TIMEOUT",
+            });
+        }
+
+        Ok(Self {
+            endpoint,
+            export_timeout,
+            batch,
+            headers,
+        })
+    }
+
+    pub fn is_enabled(&self) -> bool {
+        self.endpoint.is_some()
+    }
+
+    pub fn export_timeout(&self) -> Duration {
+        self.export_timeout
+    }
+
+    pub fn batch(&self) -> &OtlpBatchConfig {
+        &self.batch
+    }
+
+    fn effective_export_timeout(&self) -> Duration {
+        self.export_timeout.min(self.batch.export_timeout)
+    }
+
+    #[cfg(test)]
+    fn endpoint(&self) -> Option<&str> {
+        self.endpoint.as_deref()
+    }
+
+    #[cfg(test)]
+    fn header(&self, name: &str) -> Option<&str> {
+        self.headers.get(name).map(String::as_str)
+    }
+}
+
+#[derive(Default)]
+struct OtlpEnvSettings {
+    traces_endpoint: Option<String>,
+    generic_endpoint: Option<String>,
+    traces_protocol: Option<String>,
+    generic_protocol: Option<String>,
+    traces_timeout: Option<String>,
+    generic_timeout: Option<String>,
+    traces_headers: Option<String>,
+    generic_headers: Option<String>,
+    max_queue_size: Option<String>,
+    max_export_batch_size: Option<String>,
+    scheduled_delay: Option<String>,
+    batch_export_timeout: Option<String>,
+}
+
+impl OtlpEnvSettings {
+    fn from_env() -> Result<Self, OtlpTraceConfigError> {
+        Ok(Self {
+            traces_endpoint: optional_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")?,
+            generic_endpoint: optional_env("OTEL_EXPORTER_OTLP_ENDPOINT")?,
+            traces_protocol: optional_env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")?,
+            generic_protocol: optional_env("OTEL_EXPORTER_OTLP_PROTOCOL")?,
+            traces_timeout: optional_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT")?,
+            generic_timeout: optional_env("OTEL_EXPORTER_OTLP_TIMEOUT")?,
+            traces_headers: optional_env("OTEL_EXPORTER_OTLP_TRACES_HEADERS")?,
+            generic_headers: optional_env("OTEL_EXPORTER_OTLP_HEADERS")?,
+            max_queue_size: optional_env("OTEL_BSP_MAX_QUEUE_SIZE")?,
+            max_export_batch_size: optional_env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE")?,
+            scheduled_delay: optional_env("OTEL_BSP_SCHEDULE_DELAY")?,
+            batch_export_timeout: optional_env("OTEL_BSP_EXPORT_TIMEOUT")?,
+        })
+    }
+
+    fn into_config(self) -> Result<OtlpTraceConfig, OtlpTraceConfigError> {
+        OtlpTraceConfig::from_settings(self)
+    }
+}
+
+#[derive(Debug, Error, Eq, PartialEq)]
+pub enum OtlpTraceConfigError {
+    #[error("OTLP environment variable is not valid UTF-8: {field}")]
+    InvalidEnvironmentEncoding { field: &'static str },
+    #[error("OTLP trace endpoint is invalid: {reason}")]
+    InvalidEndpoint { reason: &'static str },
+    #[error("OTLP trace protocol must be http/protobuf")]
+    UnsupportedProtocol,
+    #[error("OTLP numeric setting is invalid: {field}")]
+    InvalidNumber { field: &'static str },
+    #[error("OTLP duration setting is invalid: {field}")]
+    InvalidDuration { field: &'static str },
+    #[error("OTLP batch limits are invalid")]
+    InvalidBatchLimits,
+    #[error("OTLP trace headers are invalid")]
+    InvalidHeaders,
+}
+
+#[derive(Debug, Error)]
+pub enum OtlpTraceError {
+    #[error("failed to configure OTLP trace exporter")]
+    ExporterConfiguration,
+}
+
+pub fn build_tracer_provider(
+    identity: &ServiceIdentity,
+    config: &OtlpTraceConfig,
+) -> Result<Option<(SdkTracerProvider, SdkTracer)>, OtlpTraceError> {
+    let Some(endpoint) = config.endpoint.as_deref() else {
+        return Ok(None);
+    };
+    let exporter = SpanExporter::builder()
+        .with_http()
+        .with_protocol(Protocol::HttpBinary)
+        .with_endpoint(endpoint)
+        .with_timeout(config.effective_export_timeout())
+        .with_headers(config.headers.clone())
+        .build()
+        .map_err(|_| OtlpTraceError::ExporterConfiguration)?;
+    let processor = BatchSpanProcessor::builder(ObservedSpanExporter(exporter))
+        .with_batch_config(config.batch.sdk_config())
+        .build();
+    let resource = Resource::builder_empty()
+        .with_attributes([
+            KeyValue::new("service.name", identity.service().to_owned()),
+            KeyValue::new("service.version", identity.version().to_owned()),
+            KeyValue::new(
+                "deployment.environment.name",
+                identity.environment().to_owned(),
+            ),
+        ])
+        .build();
+    let provider = SdkTracerProvider::builder()
+        .with_span_processor(processor)
+        .with_resource(resource)
+        .build();
+    let tracer = provider.tracer("crank");
+
+    Ok(Some((provider, tracer)))
+}
+
+#[derive(Debug)]
+struct ObservedSpanExporter(SpanExporter);
+
+impl SpanExporterTrait for ObservedSpanExporter {
+    async fn export(&self, mut batch: Vec<SpanData>) -> OTelSdkResult {
+        sanitize_trace_batch(&mut batch);
+        let result = self.0.export(batch).await;
+        if result.is_err() {
+            metrics::counter!(
+                "crank_telemetry_export_failures_total",
+                "signal_type" => "trace",
+                "exporter" => "otlp"
+            )
+            .increment(1);
+        }
+        result
+    }
+
+    fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult {
+        self.0.shutdown_with_timeout(timeout)
+    }
+
+    fn force_flush(&self) -> OTelSdkResult {
+        self.0.force_flush()
+    }
+
+    fn set_resource(&mut self, resource: &Resource) {
+        self.0.set_resource(resource);
+    }
+}
+
+fn sanitize_trace_batch(batch: &mut Vec<SpanData>) {
+    batch.retain(|span| is_allowed_span_name(span.name.as_ref()));
+    for span in batch {
+        let original_attribute_count = span.attributes.len();
+        span.attributes.retain(is_allowed_span_attribute);
+        span.dropped_attributes_count = span
+            .dropped_attributes_count
+            .saturating_add((original_attribute_count - span.attributes.len()) as u32);
+        span.events = Default::default();
+        span.links = Default::default();
+        if matches!(span.status, Status::Error { .. }) {
+            span.status = Status::error("");
+        }
+    }
+}
+
+fn is_allowed_span_name(name: &str) -> bool {
+    matches!(
+        name,
+        "http.request"
+            | "mcp.request"
+            | "mcp.rate_limit"
+            | "mcp.access.check"
+            | "mcp.catalog.load"
+            | "mcp.tools.resolve"
+            | "approval.check"
+            | "runtime.execute"
+            | "runtime.arguments.map"
+            | "runtime.idempotency"
+            | "upstream.http"
+            | "runtime.response.transform"
+            | "auth.resolve"
+            | "approval.recovery"
+            | "history.write"
+            | "db.query"
+    )
+}
+
+fn is_allowed_span_attribute(attribute: &KeyValue) -> bool {
+    let Value::String(value) = &attribute.value else {
+        return false;
+    };
+    let value = value.as_str();
+    match attribute.key.as_str() {
+        "request_id" => crate::RequestId::is_valid(value),
+        "stage" => is_allowed_span_name(value),
+        "outcome" => matches!(
+            value,
+            "success"
+                | "error"
+                | "allowed"
+                | "denied"
+                | "required"
+                | "replay"
+                | "execute"
+                | "skipped"
+                | "cache_hit"
+        ),
+        "error.category" => matches!(
+            value,
+            "access"
+                | "rate_limit"
+                | "catalog"
+                | "approval"
+                | "idempotency"
+                | "schema"
+                | "mapping"
+                | "upstream"
+                | "transformation"
+                | "history"
+                | "database"
+                | "concurrency"
+                | "configuration"
+                | "internal"
+        ),
+        "db.system" => value == "postgresql",
+        "db.operation" => matches!(
+            value,
+            "machine_access.read"
+                | "machine_access.touch"
+                | "catalog.load"
+                | "approval.read"
+                | "approval.write"
+                | "auth_profile.read"
+                | "secret.read"
+                | "secret.touch"
+                | "invocation_history.write"
+        ),
+        _ => false,
+    }
+}
+
+#[derive(Clone, Copy)]
+enum EndpointKind {
+    Trace,
+    Generic,
+}
+
+fn validate_endpoint(endpoint: String, kind: EndpointKind) -> Result<String, OtlpTraceConfigError> {
+    let mut url = Url::parse(&endpoint).map_err(|_| OtlpTraceConfigError::InvalidEndpoint {
+        reason: "invalid URL",
+    })?;
+    if !matches!(url.scheme(), "http" | "https") {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "unsupported scheme",
+        });
+    }
+    if url.host_str().is_none() {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "host is required",
+        });
+    }
+    if !url.username().is_empty() || url.password().is_some() {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "credentials are forbidden",
+        });
+    }
+    if url.query().is_some() || url.fragment().is_some() {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "query and fragment are forbidden",
+        });
+    }
+    if matches!(kind, EndpointKind::Generic) {
+        let path = url.path().trim_end_matches('/');
+        url.set_path(&format!("{path}/v1/traces"));
+    }
+
+    Ok(url.into())
+}
+
+fn parse_headers(value: &str) -> Result<HashMap<String, String>, OtlpTraceConfigError> {
+    value
+        .split_terminator(',')
+        .map(str::trim)
+        .filter(|item| !item.is_empty())
+        .try_fold(HashMap::new(), |mut headers, item| {
+            let (name, encoded_value) = item
+                .split_once('=')
+                .ok_or(OtlpTraceConfigError::InvalidHeaders)?;
+            let name = HeaderName::from_bytes(name.trim().as_bytes())
+                .map_err(|_| OtlpTraceConfigError::InvalidHeaders)?;
+            let value = percent_decode_str(encoded_value.trim())
+                .decode_utf8()
+                .map_err(|_| OtlpTraceConfigError::InvalidHeaders)?
+                .into_owned();
+            if value.is_empty() || HeaderValue::from_str(&value).is_err() {
+                return Err(OtlpTraceConfigError::InvalidHeaders);
+            }
+            headers.insert(name.as_str().to_owned(), value);
+            Ok(headers)
+        })
+}
+
+fn optional_env(field: &'static str) -> Result<Option<String>, OtlpTraceConfigError> {
+    match env::var(field) {
+        Ok(value) if value.is_empty() => Ok(None),
+        Ok(value) => Ok(Some(value)),
+        Err(env::VarError::NotPresent) => Ok(None),
+        Err(env::VarError::NotUnicode(_)) => {
+            Err(OtlpTraceConfigError::InvalidEnvironmentEncoding { field })
+        }
+    }
+}
+
+fn usize_env(
+    field: &'static str,
+    value: Option<String>,
+) -> Result<Option<usize>, OtlpTraceConfigError> {
+    value
+        .map(|value| {
+            value
+                .parse()
+                .map_err(|_| OtlpTraceConfigError::InvalidNumber { field })
+        })
+        .transpose()
+}
+
+fn duration_env(
+    field: &'static str,
+    value: Option<String>,
+) -> Result<Option<Duration>, OtlpTraceConfigError> {
+    value
+        .map(|value| {
+            value
+                .parse::<u64>()
+                .map(Duration::from_millis)
+                .map_err(|_| OtlpTraceConfigError::InvalidDuration { field })
+        })
+        .transpose()
+}
+
+fn duration_is_bounded(duration: Duration) -> bool {
+    !duration.is_zero() && duration <= MAX_DURATION
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        io::{Read, Write},
+        net::TcpListener,
+        sync::mpsc,
+        thread,
+        time::{Duration, Instant},
+    };
+
+    use axum::{
+        Router,
+        body::{Body, to_bytes},
+        extract::Request,
+        middleware::Next,
+        response::Response,
+        routing::get,
+    };
+    use opentelemetry::{
+        KeyValue,
+        trace::{Span as _, Status, Tracer as _},
+    };
+    use opentelemetry_proto::tonic::{
+        collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value,
+    };
+    use prost::Message;
+    use tower::ServiceExt;
+    use tracing::{Instrument, info_span};
+    use tracing_subscriber::layer::SubscriberExt;
+
+    use super::{OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_tracer_provider};
+    use crate::ServiceIdentity;
+
+    #[test]
+    fn signal_specific_settings_override_generic_settings() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/custom".to_owned()),
+            generic_endpoint: Some("https://generic.example.test/otel".to_owned()),
+            traces_protocol: Some("http/protobuf".to_owned()),
+            generic_protocol: Some("grpc".to_owned()),
+            traces_timeout: Some("2500".to_owned()),
+            generic_timeout: Some("invalid-unused-fallback".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(
+            config.endpoint(),
+            Some("https://traces.example.test/custom")
+        );
+        assert_eq!(config.export_timeout(), Duration::from_millis(2500));
+    }
+
+    #[test]
+    fn disabled_export_ignores_inactive_settings() {
+        let config = OtlpEnvSettings {
+            traces_protocol: Some("grpc".to_owned()),
+            generic_protocol: Some("grpc".to_owned()),
+            traces_timeout: Some("invalid".to_owned()),
+            generic_timeout: Some("invalid".to_owned()),
+            max_queue_size: Some("invalid".to_owned()),
+            max_export_batch_size: Some("invalid".to_owned()),
+            scheduled_delay: Some("invalid".to_owned()),
+            batch_export_timeout: Some("invalid".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert!(!config.is_enabled());
+    }
+
+    #[test]
+    fn empty_signal_headers_use_generic_headers() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
+            traces_headers: Some(String::new()),
+            generic_headers: Some(
+                "authorization=Bearer%20canary-token,x-tenant=community".to_owned(),
+            ),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(config.header("authorization"), Some("Bearer canary-token"));
+        assert_eq!(config.header("x-tenant"), Some("community"));
+        assert!(!format!("{config:?}").contains("canary-token"));
+    }
+
+    #[test]
+    fn invalid_headers_return_a_safe_error() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
+            traces_headers: Some("authorization=canary-secret%0Ainjected".to_owned()),
+            ..OtlpEnvSettings::default()
+        };
+
+        let error = config.into_config().unwrap_err();
+        assert!(matches!(error, super::OtlpTraceConfigError::InvalidHeaders));
+        assert!(!error.to_string().contains("canary-secret"));
+    }
+
+    #[test]
+    fn stricter_batch_timeout_bounds_http_export() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
+            traces_protocol: Some("http/protobuf".to_owned()),
+            traces_timeout: Some("9000".to_owned()),
+            batch_export_timeout: Some("2500".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(
+            config.effective_export_timeout(),
+            Duration::from_millis(2500)
+        );
+    }
+
+    #[test]
+    fn generic_endpoint_receives_standard_trace_path() {
+        let config = OtlpEnvSettings {
+            generic_endpoint: Some("https://generic.example.test/otel/".to_owned()),
+            generic_protocol: Some("http/protobuf".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(
+            config.endpoint(),
+            Some("https://generic.example.test/otel/v1/traces")
+        );
+    }
+
+    #[test]
+    fn disabled_export_does_not_build_a_provider() {
+        let config = OtlpTraceConfig::try_new(
+            None,
+            None,
+            Duration::from_secs(1),
+            OtlpBatchConfig::default(),
+        )
+        .unwrap();
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap();
+
+        assert!(build_tracer_provider(&identity, &config).unwrap().is_none());
+    }
+
+    #[test]
+    fn real_http_protobuf_export_contains_resource_and_trace() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let (request_tx, request_rx) = mpsc::sync_channel(1);
+        let server = thread::spawn(move || {
+            let (mut stream, _) = listener.accept().unwrap();
+            let request = read_http_request(&mut stream);
+            stream
+                .write_all(
+                    b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
+                )
+                .unwrap();
+            request_tx.send(request).unwrap();
+        });
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some(format!("http://{address}/v1/traces")),
+            traces_protocol: Some("http/protobuf".to_owned()),
+            traces_timeout: Some("2000".to_owned()),
+            traces_headers: Some(String::new()),
+            generic_headers: Some("authorization=Bearer%20canary-token".to_owned()),
+            max_queue_size: Some("16".to_owned()),
+            max_export_batch_size: Some("8".to_owned()),
+            scheduled_delay: Some("10".to_owned()),
+            batch_export_timeout: Some("2000".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "integration-test").unwrap();
+        let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
+        let mut span = tracer.start("http.request");
+        let trace_id = span.span_context().trace_id().to_bytes();
+        span.set_attribute(KeyValue::new("request_id", "req_otlp_contract"));
+        span.set_attribute(KeyValue::new("authorization", "Bearer canary-span-secret"));
+        span.add_event(
+            "canary-span-event",
+            vec![KeyValue::new("payload", "canary-span-secret")],
+        );
+        span.set_status(Status::error("canary-span-secret"));
+        span.end();
+
+        provider.force_flush().unwrap();
+        provider.shutdown().unwrap();
+        let request = request_rx.recv_timeout(Duration::from_secs(2)).unwrap();
+        server.join().unwrap();
+        let (headers, body) = split_http_request(&request);
+
+        assert!(headers.contains("POST /v1/traces HTTP/1.1"));
+        assert!(
+            headers
+                .to_ascii_lowercase()
+                .contains("content-type: application/x-protobuf")
+        );
+        assert!(
+            headers
+                .to_ascii_lowercase()
+                .contains("authorization: bearer canary-token")
+        );
+        let export = ExportTraceServiceRequest::decode(body).unwrap();
+        let resource_spans = export.resource_spans.first().unwrap();
+        let attributes = &resource_spans.resource.as_ref().unwrap().attributes;
+        assert_eq!(
+            string_attribute(attributes, "service.name"),
+            Some("admin-api")
+        );
+        assert_eq!(
+            string_attribute(attributes, "service.version"),
+            Some("0.3.1")
+        );
+        assert_eq!(
+            string_attribute(attributes, "deployment.environment.name"),
+            Some("integration-test")
+        );
+        assert_eq!(
+            resource_spans.scope_spans[0].spans[0].trace_id.as_slice(),
+            trace_id
+        );
+        let exported_span = &resource_spans.scope_spans[0].spans[0];
+        assert_eq!(exported_span.name, "http.request");
+        assert_eq!(
+            string_attribute(&exported_span.attributes, "request_id"),
+            Some("req_otlp_contract")
+        );
+        assert!(
+            exported_span
+                .attributes
+                .iter()
+                .all(|attribute| attribute.key != "authorization")
+        );
+        assert!(exported_span.events.is_empty());
+        assert_eq!(
+            exported_span
+                .status
+                .as_ref()
+                .map(|status| status.message.as_str()),
+            Some("")
+        );
+        assert!(
+            !body
+                .windows(b"canary-span-secret".len())
+                .any(|window| { window == b"canary-span-secret" })
+        );
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn unavailable_receiver_does_not_change_product_result() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = thread::spawn(move || {
+            let (stream, _) = listener.accept().unwrap();
+            drop(stream);
+        });
+        let config = OtlpTraceConfig::try_new(
+            Some(format!("http://{address}/v1/traces")),
+            Some("http/protobuf".to_owned()),
+            Duration::from_millis(250),
+            OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(250))
+                .unwrap(),
+        )
+        .unwrap();
+        let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "fault-test").unwrap();
+        let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
+        let subscriber =
+            tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+        let app = Router::new()
+            .route("/product", get(|| async { "product-success" }))
+            .layer(axum::middleware::from_fn(trace_product_request));
+
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .uri("/product")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        let status = response.status();
+        let body = to_bytes(response.into_body(), 64).await.unwrap();
+
+        assert_eq!(status, axum::http::StatusCode::OK);
+        assert_eq!(body.as_ref(), b"product-success");
+        assert!(provider.force_flush().is_err());
+        let _ = provider.shutdown();
+        server.join().unwrap();
+    }
+
+    async fn trace_product_request(request: Request, next: Next) -> Response {
+        next.run(request)
+            .instrument(info_span!(target: "crank::trace", "http.request"))
+            .await
+    }
+
+    #[test]
+    fn hanging_receiver_respects_the_stricter_export_timeout() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = thread::spawn(move || {
+            let (stream, _) = listener.accept().unwrap();
+            thread::sleep(Duration::from_millis(750));
+            drop(stream);
+        });
+        let config = OtlpTraceConfig::try_new(
+            Some(format!("http://{address}/v1/traces")),
+            Some("http/protobuf".to_owned()),
+            Duration::from_secs(2),
+            OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(100))
+                .unwrap(),
+        )
+        .unwrap();
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "timeout-test").unwrap();
+        let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
+        let mut span = tracer.start("http.request");
+        span.end();
+        let started_at = Instant::now();
+
+        assert!(provider.force_flush().is_err());
+        assert!(started_at.elapsed() < Duration::from_millis(500));
+        let _ = provider.shutdown();
+        server.join().unwrap();
+    }
+
+    fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
+        stream
+            .set_read_timeout(Some(Duration::from_secs(2)))
+            .unwrap();
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 4096];
+        loop {
+            let read = stream.read(&mut buffer).unwrap();
+            request.extend_from_slice(&buffer[..read]);
+            let Some(header_end) = find_bytes(&request, b"\r\n\r\n") else {
+                continue;
+            };
+            let headers = String::from_utf8_lossy(&request[..header_end]);
+            let content_length = headers
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("content-length")
+                        .then(|| value.trim().parse::<usize>().ok())
+                        .flatten()
+                })
+                .unwrap_or(0);
+            if request.len() >= header_end + 4 + content_length {
+                return request;
+            }
+        }
+    }
+
+    fn split_http_request(request: &[u8]) -> (&str, &[u8]) {
+        let header_end = find_bytes(request, b"\r\n\r\n").unwrap();
+        (
+            std::str::from_utf8(&request[..header_end]).unwrap(),
+            &request[header_end + 4..],
+        )
+    }
+
+    fn string_attribute<'a>(
+        attributes: &'a [opentelemetry_proto::tonic::common::v1::KeyValue],
+        key: &str,
+    ) -> Option<&'a str> {
+        attributes.iter().find_map(|attribute| {
+            let value = attribute.value.as_ref()?.value.as_ref()?;
+            (attribute.key == key)
+                .then_some(value)
+                .and_then(|value| match value {
+                    any_value::Value::StringValue(value) => Some(value.as_str()),
+                    _ => None,
+                })
+        })
+    }
+
+    fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+        haystack
+            .windows(needle.len())
+            .position(|candidate| candidate == needle)
+    }
+}