998 lines
33 KiB
Rust
998 lines
33 KiB
Rust
use std::{collections::HashMap, fmt, time::Duration};
|
|
|
|
use axum::http::{HeaderName, HeaderValue};
|
|
use opentelemetry::{
|
|
KeyValue, Value,
|
|
trace::{Status, TracerProvider as _},
|
|
};
|
|
use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig, WithHttpConfig};
|
|
use opentelemetry_sdk::{
|
|
Resource,
|
|
error::OTelSdkResult,
|
|
trace::{
|
|
BatchConfigBuilder, BatchSpanProcessor, SdkTracer, SdkTracerProvider, SpanData,
|
|
SpanExporter as SpanExporterTrait,
|
|
},
|
|
};
|
|
use percent_encoding::percent_decode_str;
|
|
use thiserror::Error;
|
|
use url::Url;
|
|
|
|
use crate::ServiceIdentity;
|
|
|
|
const DEFAULT_EXPORT_TIMEOUT: Duration = Duration::from_secs(10);
|
|
const DEFAULT_MAX_QUEUE_SIZE: usize = 2_048;
|
|
const DEFAULT_MAX_EXPORT_BATCH_SIZE: usize = 512;
|
|
const DEFAULT_SCHEDULE_DELAY: Duration = Duration::from_secs(5);
|
|
const DEFAULT_BATCH_EXPORT_TIMEOUT: Duration = Duration::from_secs(30);
|
|
const MAX_QUEUE_SIZE: usize = 65_536;
|
|
const MAX_DURATION: Duration = Duration::from_secs(300);
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct OtlpBatchConfig {
|
|
max_queue_size: usize,
|
|
max_export_batch_size: usize,
|
|
scheduled_delay: Duration,
|
|
export_timeout: Duration,
|
|
}
|
|
|
|
impl OtlpBatchConfig {
|
|
pub fn try_new(
|
|
max_queue_size: usize,
|
|
max_export_batch_size: usize,
|
|
scheduled_delay: Duration,
|
|
export_timeout: Duration,
|
|
) -> Result<Self, OtlpTraceConfigError> {
|
|
let valid = max_queue_size > 0
|
|
&& max_queue_size <= MAX_QUEUE_SIZE
|
|
&& max_export_batch_size > 0
|
|
&& max_export_batch_size <= max_queue_size
|
|
&& duration_is_bounded(scheduled_delay)
|
|
&& duration_is_bounded(export_timeout);
|
|
if !valid {
|
|
return Err(OtlpTraceConfigError::InvalidBatchLimits);
|
|
}
|
|
|
|
Ok(Self {
|
|
max_queue_size,
|
|
max_export_batch_size,
|
|
scheduled_delay,
|
|
export_timeout,
|
|
})
|
|
}
|
|
|
|
pub fn max_queue_size(&self) -> usize {
|
|
self.max_queue_size
|
|
}
|
|
|
|
pub fn max_export_batch_size(&self) -> usize {
|
|
self.max_export_batch_size
|
|
}
|
|
|
|
pub fn scheduled_delay(&self) -> Duration {
|
|
self.scheduled_delay
|
|
}
|
|
|
|
pub fn export_timeout(&self) -> Duration {
|
|
self.export_timeout
|
|
}
|
|
|
|
fn sdk_config(&self) -> opentelemetry_sdk::trace::BatchConfig {
|
|
BatchConfigBuilder::default()
|
|
.with_max_queue_size(self.max_queue_size)
|
|
.with_max_export_batch_size(self.max_export_batch_size)
|
|
.with_scheduled_delay(self.scheduled_delay)
|
|
.build()
|
|
}
|
|
}
|
|
|
|
impl Default for OtlpBatchConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
max_queue_size: DEFAULT_MAX_QUEUE_SIZE,
|
|
max_export_batch_size: DEFAULT_MAX_EXPORT_BATCH_SIZE,
|
|
scheduled_delay: DEFAULT_SCHEDULE_DELAY,
|
|
export_timeout: DEFAULT_BATCH_EXPORT_TIMEOUT,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Eq, PartialEq)]
|
|
pub struct OtlpTraceConfig {
|
|
endpoint: Option<String>,
|
|
export_timeout: Duration,
|
|
batch: OtlpBatchConfig,
|
|
headers: HashMap<String, String>,
|
|
}
|
|
|
|
impl fmt::Debug for OtlpTraceConfig {
|
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
formatter
|
|
.debug_struct("OtlpTraceConfig")
|
|
.field("enabled", &self.is_enabled())
|
|
.field("export_timeout", &self.export_timeout)
|
|
.field("batch", &self.batch)
|
|
.field("header_count", &self.headers.len())
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
impl OtlpTraceConfig {
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn from_values(
|
|
generic_endpoint: Option<String>,
|
|
traces_endpoint: Option<String>,
|
|
generic_protocol: Option<String>,
|
|
traces_protocol: Option<String>,
|
|
generic_timeout: Option<String>,
|
|
traces_timeout: Option<String>,
|
|
generic_headers: Option<String>,
|
|
traces_headers: Option<String>,
|
|
max_queue_size: usize,
|
|
max_export_batch_size: usize,
|
|
scheduled_delay: String,
|
|
batch_export_timeout: String,
|
|
) -> Result<Self, OtlpTraceConfigError> {
|
|
OtlpEnvSettings {
|
|
traces_endpoint,
|
|
generic_endpoint,
|
|
traces_protocol,
|
|
generic_protocol,
|
|
traces_timeout,
|
|
generic_timeout,
|
|
traces_headers,
|
|
generic_headers,
|
|
max_queue_size: Some(max_queue_size.to_string()),
|
|
max_export_batch_size: Some(max_export_batch_size.to_string()),
|
|
scheduled_delay: Some(scheduled_delay),
|
|
batch_export_timeout: Some(batch_export_timeout),
|
|
}
|
|
.into_config()
|
|
}
|
|
|
|
fn from_settings(settings: OtlpEnvSettings) -> Result<Self, OtlpTraceConfigError> {
|
|
let endpoint = match settings.traces_endpoint {
|
|
Some(endpoint) => Some(validate_endpoint(endpoint, EndpointKind::Trace)?),
|
|
None => settings
|
|
.generic_endpoint
|
|
.map(|endpoint| validate_endpoint(endpoint, EndpointKind::Generic))
|
|
.transpose()?,
|
|
};
|
|
if endpoint.is_none() {
|
|
return Ok(Self {
|
|
endpoint: None,
|
|
export_timeout: DEFAULT_EXPORT_TIMEOUT,
|
|
batch: OtlpBatchConfig::default(),
|
|
headers: HashMap::new(),
|
|
});
|
|
}
|
|
|
|
let protocol = settings.traces_protocol.or(settings.generic_protocol);
|
|
let export_timeout = match settings.traces_timeout {
|
|
Some(timeout) => duration_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", Some(timeout))?,
|
|
None => duration_env("OTEL_EXPORTER_OTLP_TIMEOUT", settings.generic_timeout)?,
|
|
}
|
|
.unwrap_or(DEFAULT_EXPORT_TIMEOUT);
|
|
let batch = OtlpBatchConfig::try_new(
|
|
usize_env("OTEL_BSP_MAX_QUEUE_SIZE", settings.max_queue_size)?
|
|
.unwrap_or(DEFAULT_MAX_QUEUE_SIZE),
|
|
usize_env(
|
|
"OTEL_BSP_MAX_EXPORT_BATCH_SIZE",
|
|
settings.max_export_batch_size,
|
|
)?
|
|
.unwrap_or(DEFAULT_MAX_EXPORT_BATCH_SIZE),
|
|
duration_env("OTEL_BSP_SCHEDULE_DELAY", settings.scheduled_delay)?
|
|
.unwrap_or(DEFAULT_SCHEDULE_DELAY),
|
|
duration_env("OTEL_BSP_EXPORT_TIMEOUT", settings.batch_export_timeout)?
|
|
.unwrap_or(DEFAULT_BATCH_EXPORT_TIMEOUT),
|
|
)?;
|
|
let headers = settings
|
|
.traces_headers
|
|
.filter(|value| !value.is_empty())
|
|
.or(settings.generic_headers.filter(|value| !value.is_empty()))
|
|
.map(|value| parse_headers(&value))
|
|
.transpose()?
|
|
.unwrap_or_default();
|
|
|
|
Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, headers)
|
|
}
|
|
|
|
pub fn try_new(
|
|
endpoint: Option<String>,
|
|
protocol: Option<String>,
|
|
export_timeout: Duration,
|
|
batch: OtlpBatchConfig,
|
|
) -> Result<Self, OtlpTraceConfigError> {
|
|
Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, HashMap::new())
|
|
}
|
|
|
|
fn try_new_with_headers(
|
|
endpoint: Option<String>,
|
|
protocol: Option<String>,
|
|
export_timeout: Duration,
|
|
batch: OtlpBatchConfig,
|
|
headers: HashMap<String, String>,
|
|
) -> Result<Self, OtlpTraceConfigError> {
|
|
let endpoint = endpoint
|
|
.map(|endpoint| validate_endpoint(endpoint, EndpointKind::Trace))
|
|
.transpose()?;
|
|
if endpoint.is_some() && protocol.as_deref().unwrap_or("http/protobuf") != "http/protobuf" {
|
|
return Err(OtlpTraceConfigError::UnsupportedProtocol);
|
|
}
|
|
if !duration_is_bounded(export_timeout) {
|
|
return Err(OtlpTraceConfigError::InvalidDuration {
|
|
field: "OTEL_EXPORTER_OTLP_TIMEOUT",
|
|
});
|
|
}
|
|
|
|
Ok(Self {
|
|
endpoint,
|
|
export_timeout,
|
|
batch,
|
|
headers,
|
|
})
|
|
}
|
|
|
|
pub fn is_enabled(&self) -> bool {
|
|
self.endpoint.is_some()
|
|
}
|
|
|
|
pub fn export_timeout(&self) -> Duration {
|
|
self.export_timeout
|
|
}
|
|
|
|
pub fn batch(&self) -> &OtlpBatchConfig {
|
|
&self.batch
|
|
}
|
|
|
|
fn effective_export_timeout(&self) -> Duration {
|
|
self.export_timeout.min(self.batch.export_timeout)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn endpoint(&self) -> Option<&str> {
|
|
self.endpoint.as_deref()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn header(&self, name: &str) -> Option<&str> {
|
|
self.headers.get(name).map(String::as_str)
|
|
}
|
|
}
|
|
|
|
impl Default for OtlpTraceConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
endpoint: None,
|
|
export_timeout: DEFAULT_EXPORT_TIMEOUT,
|
|
batch: OtlpBatchConfig::default(),
|
|
headers: HashMap::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct OtlpEnvSettings {
|
|
traces_endpoint: Option<String>,
|
|
generic_endpoint: Option<String>,
|
|
traces_protocol: Option<String>,
|
|
generic_protocol: Option<String>,
|
|
traces_timeout: Option<String>,
|
|
generic_timeout: Option<String>,
|
|
traces_headers: Option<String>,
|
|
generic_headers: Option<String>,
|
|
max_queue_size: Option<String>,
|
|
max_export_batch_size: Option<String>,
|
|
scheduled_delay: Option<String>,
|
|
batch_export_timeout: Option<String>,
|
|
}
|
|
|
|
impl OtlpEnvSettings {
|
|
fn into_config(self) -> Result<OtlpTraceConfig, OtlpTraceConfigError> {
|
|
OtlpTraceConfig::from_settings(self)
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Error, Eq, PartialEq)]
|
|
pub enum OtlpTraceConfigError {
|
|
#[error("OTLP environment variable is not valid UTF-8: {field}")]
|
|
InvalidEnvironmentEncoding { field: &'static str },
|
|
#[error("OTLP trace endpoint is invalid: {reason}")]
|
|
InvalidEndpoint { reason: &'static str },
|
|
#[error("OTLP trace protocol must be http/protobuf")]
|
|
UnsupportedProtocol,
|
|
#[error("OTLP numeric setting is invalid: {field}")]
|
|
InvalidNumber { field: &'static str },
|
|
#[error("OTLP duration setting is invalid: {field}")]
|
|
InvalidDuration { field: &'static str },
|
|
#[error("OTLP batch limits are invalid")]
|
|
InvalidBatchLimits,
|
|
#[error("OTLP trace headers are invalid")]
|
|
InvalidHeaders,
|
|
}
|
|
|
|
#[derive(Debug, Error)]
|
|
pub enum OtlpTraceError {
|
|
#[error("failed to configure OTLP trace exporter")]
|
|
ExporterConfiguration,
|
|
}
|
|
|
|
pub fn build_tracer_provider(
|
|
identity: &ServiceIdentity,
|
|
config: &OtlpTraceConfig,
|
|
) -> Result<Option<(SdkTracerProvider, SdkTracer)>, OtlpTraceError> {
|
|
let Some(endpoint) = config.endpoint.as_deref() else {
|
|
return Ok(None);
|
|
};
|
|
let exporter = SpanExporter::builder()
|
|
.with_http()
|
|
.with_protocol(Protocol::HttpBinary)
|
|
.with_endpoint(endpoint)
|
|
.with_timeout(config.effective_export_timeout())
|
|
.with_headers(config.headers.clone())
|
|
.build()
|
|
.map_err(|_| OtlpTraceError::ExporterConfiguration)?;
|
|
let processor = BatchSpanProcessor::builder(ObservedSpanExporter(exporter))
|
|
.with_batch_config(config.batch.sdk_config())
|
|
.build();
|
|
let resource = trace_resource(identity);
|
|
let provider = SdkTracerProvider::builder()
|
|
.with_span_processor(processor)
|
|
.with_resource(resource)
|
|
.build();
|
|
let tracer = provider.tracer("crank");
|
|
|
|
Ok(Some((provider, tracer)))
|
|
}
|
|
|
|
pub(crate) fn build_local_tracer_provider(
|
|
identity: &ServiceIdentity,
|
|
) -> (SdkTracerProvider, SdkTracer) {
|
|
let provider = SdkTracerProvider::builder()
|
|
.with_resource(trace_resource(identity))
|
|
.build();
|
|
let tracer = provider.tracer("crank");
|
|
(provider, tracer)
|
|
}
|
|
|
|
fn trace_resource(identity: &ServiceIdentity) -> Resource {
|
|
Resource::builder_empty()
|
|
.with_attributes([
|
|
KeyValue::new("service.name", identity.service().to_owned()),
|
|
KeyValue::new("service.version", identity.version().to_owned()),
|
|
KeyValue::new(
|
|
"deployment.environment.name",
|
|
identity.environment().to_owned(),
|
|
),
|
|
])
|
|
.build()
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
struct ObservedSpanExporter(SpanExporter);
|
|
|
|
impl SpanExporterTrait for ObservedSpanExporter {
|
|
async fn export(&self, mut batch: Vec<SpanData>) -> OTelSdkResult {
|
|
sanitize_trace_batch(&mut batch);
|
|
let result = self.0.export(batch).await;
|
|
if result.is_err() {
|
|
crank_metrics::record_export_failure(
|
|
crank_metrics::SignalType::Trace,
|
|
crank_metrics::Exporter::Otlp,
|
|
);
|
|
}
|
|
result
|
|
}
|
|
|
|
fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult {
|
|
self.0.shutdown_with_timeout(timeout)
|
|
}
|
|
|
|
fn force_flush(&self) -> OTelSdkResult {
|
|
self.0.force_flush()
|
|
}
|
|
|
|
fn set_resource(&mut self, resource: &Resource) {
|
|
self.0.set_resource(resource);
|
|
}
|
|
}
|
|
|
|
fn sanitize_trace_batch(batch: &mut Vec<SpanData>) {
|
|
batch.retain(|span| is_allowed_span_name(span.name.as_ref()));
|
|
for span in batch {
|
|
let original_attribute_count = span.attributes.len();
|
|
span.attributes.retain(is_allowed_span_attribute);
|
|
span.dropped_attributes_count = span
|
|
.dropped_attributes_count
|
|
.saturating_add((original_attribute_count - span.attributes.len()) as u32);
|
|
span.events = Default::default();
|
|
span.links = Default::default();
|
|
if matches!(span.status, Status::Error { .. }) {
|
|
span.status = Status::error("");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn is_allowed_span_name(name: &str) -> bool {
|
|
matches!(
|
|
name,
|
|
"http.request"
|
|
| "mcp.request"
|
|
| "mcp.rate_limit"
|
|
| "mcp.access.check"
|
|
| "mcp.catalog.load"
|
|
| "mcp.tools.resolve"
|
|
| "approval.check"
|
|
| "runtime.execute"
|
|
| "runtime.arguments.map"
|
|
| "runtime.idempotency"
|
|
| "upstream.http"
|
|
| "runtime.response.transform"
|
|
| "auth.resolve"
|
|
| "approval.recovery"
|
|
| "history.write"
|
|
| "db.query"
|
|
)
|
|
}
|
|
|
|
fn is_allowed_span_attribute(attribute: &KeyValue) -> bool {
|
|
let Value::String(value) = &attribute.value else {
|
|
return false;
|
|
};
|
|
let value = value.as_str();
|
|
match attribute.key.as_str() {
|
|
"request_id" => is_valid_request_id(value),
|
|
"stage" => is_allowed_span_name(value),
|
|
"outcome" => matches!(
|
|
value,
|
|
"success"
|
|
| "error"
|
|
| "allowed"
|
|
| "denied"
|
|
| "required"
|
|
| "replay"
|
|
| "execute"
|
|
| "skipped"
|
|
| "cache_hit"
|
|
),
|
|
"error.category" => matches!(
|
|
value,
|
|
"access"
|
|
| "rate_limit"
|
|
| "catalog"
|
|
| "approval"
|
|
| "idempotency"
|
|
| "schema"
|
|
| "mapping"
|
|
| "upstream"
|
|
| "transformation"
|
|
| "history"
|
|
| "database"
|
|
| "concurrency"
|
|
| "configuration"
|
|
| "internal"
|
|
),
|
|
"db.system" => value == "postgresql",
|
|
"db.operation" => matches!(
|
|
value,
|
|
"machine_access.read"
|
|
| "machine_access.touch"
|
|
| "catalog.load"
|
|
| "approval.read"
|
|
| "approval.write"
|
|
| "auth_profile.read"
|
|
| "secret.read"
|
|
| "secret.touch"
|
|
| "invocation_history.write"
|
|
),
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
fn is_valid_request_id(value: &str) -> bool {
|
|
!value.is_empty()
|
|
&& value.len() <= 128
|
|
&& value
|
|
.bytes()
|
|
.all(|byte| byte.is_ascii_graphic() && byte != b',' && byte != b';')
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum EndpointKind {
|
|
Trace,
|
|
Generic,
|
|
}
|
|
|
|
fn validate_endpoint(endpoint: String, kind: EndpointKind) -> Result<String, OtlpTraceConfigError> {
|
|
let mut url = Url::parse(&endpoint).map_err(|_| OtlpTraceConfigError::InvalidEndpoint {
|
|
reason: "invalid URL",
|
|
})?;
|
|
if !matches!(url.scheme(), "http" | "https") {
|
|
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
|
reason: "unsupported scheme",
|
|
});
|
|
}
|
|
if url.host_str().is_none() {
|
|
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
|
reason: "host is required",
|
|
});
|
|
}
|
|
if !url.username().is_empty() || url.password().is_some() {
|
|
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
|
reason: "credentials are forbidden",
|
|
});
|
|
}
|
|
if url.query().is_some() || url.fragment().is_some() {
|
|
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
|
reason: "query and fragment are forbidden",
|
|
});
|
|
}
|
|
if matches!(kind, EndpointKind::Generic) {
|
|
let path = url.path().trim_end_matches('/');
|
|
url.set_path(&format!("{path}/v1/traces"));
|
|
}
|
|
|
|
Ok(url.into())
|
|
}
|
|
|
|
fn parse_headers(value: &str) -> Result<HashMap<String, String>, OtlpTraceConfigError> {
|
|
value
|
|
.split_terminator(',')
|
|
.map(str::trim)
|
|
.filter(|item| !item.is_empty())
|
|
.try_fold(HashMap::new(), |mut headers, item| {
|
|
let (name, encoded_value) = item
|
|
.split_once('=')
|
|
.ok_or(OtlpTraceConfigError::InvalidHeaders)?;
|
|
let name = HeaderName::from_bytes(name.trim().as_bytes())
|
|
.map_err(|_| OtlpTraceConfigError::InvalidHeaders)?;
|
|
let value = percent_decode_str(encoded_value.trim())
|
|
.decode_utf8()
|
|
.map_err(|_| OtlpTraceConfigError::InvalidHeaders)?
|
|
.into_owned();
|
|
if value.is_empty() || HeaderValue::from_str(&value).is_err() {
|
|
return Err(OtlpTraceConfigError::InvalidHeaders);
|
|
}
|
|
headers.insert(name.as_str().to_owned(), value);
|
|
Ok(headers)
|
|
})
|
|
}
|
|
|
|
fn usize_env(
|
|
field: &'static str,
|
|
value: Option<String>,
|
|
) -> Result<Option<usize>, OtlpTraceConfigError> {
|
|
value
|
|
.map(|value| {
|
|
value
|
|
.parse()
|
|
.map_err(|_| OtlpTraceConfigError::InvalidNumber { field })
|
|
})
|
|
.transpose()
|
|
}
|
|
|
|
fn duration_env(
|
|
field: &'static str,
|
|
value: Option<String>,
|
|
) -> Result<Option<Duration>, OtlpTraceConfigError> {
|
|
value
|
|
.map(|value| {
|
|
value
|
|
.parse::<u64>()
|
|
.map(Duration::from_millis)
|
|
.map_err(|_| OtlpTraceConfigError::InvalidDuration { field })
|
|
})
|
|
.transpose()
|
|
}
|
|
|
|
fn duration_is_bounded(duration: Duration) -> bool {
|
|
!duration.is_zero() && duration <= MAX_DURATION
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::{
|
|
io::{Read, Write},
|
|
net::TcpListener,
|
|
sync::mpsc,
|
|
thread,
|
|
time::{Duration, Instant},
|
|
};
|
|
|
|
use axum::{
|
|
Router,
|
|
body::{Body, to_bytes},
|
|
extract::Request,
|
|
middleware::Next,
|
|
response::Response,
|
|
routing::get,
|
|
};
|
|
use opentelemetry::{
|
|
KeyValue,
|
|
trace::{Span as _, Status, Tracer as _},
|
|
};
|
|
use opentelemetry_proto::tonic::{
|
|
collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value,
|
|
};
|
|
use prost::Message;
|
|
use tower::ServiceExt;
|
|
use tracing::{Instrument, info_span};
|
|
use tracing_subscriber::layer::SubscriberExt;
|
|
|
|
use super::{
|
|
OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_local_tracer_provider,
|
|
build_tracer_provider,
|
|
};
|
|
use crate::ServiceIdentity;
|
|
|
|
#[test]
|
|
fn signal_specific_settings_override_generic_settings() {
|
|
let config = OtlpEnvSettings {
|
|
traces_endpoint: Some("https://traces.example.test/custom".to_owned()),
|
|
generic_endpoint: Some("https://generic.example.test/otel".to_owned()),
|
|
traces_protocol: Some("http/protobuf".to_owned()),
|
|
generic_protocol: Some("grpc".to_owned()), // community-scope: allow=grpc
|
|
traces_timeout: Some("2500".to_owned()),
|
|
generic_timeout: Some("invalid-unused-fallback".to_owned()),
|
|
..OtlpEnvSettings::default()
|
|
}
|
|
.into_config()
|
|
.unwrap();
|
|
|
|
assert_eq!(
|
|
config.endpoint(),
|
|
Some("https://traces.example.test/custom")
|
|
);
|
|
assert_eq!(config.export_timeout(), Duration::from_millis(2500));
|
|
}
|
|
|
|
#[test]
|
|
fn disabled_export_ignores_inactive_settings() {
|
|
let config = OtlpEnvSettings {
|
|
traces_protocol: Some("grpc".to_owned()), // community-scope: allow=grpc
|
|
generic_protocol: Some("grpc".to_owned()), // community-scope: allow=grpc
|
|
traces_timeout: Some("invalid".to_owned()),
|
|
generic_timeout: Some("invalid".to_owned()),
|
|
max_queue_size: Some("invalid".to_owned()),
|
|
max_export_batch_size: Some("invalid".to_owned()),
|
|
scheduled_delay: Some("invalid".to_owned()),
|
|
batch_export_timeout: Some("invalid".to_owned()),
|
|
..OtlpEnvSettings::default()
|
|
}
|
|
.into_config()
|
|
.unwrap();
|
|
|
|
assert!(!config.is_enabled());
|
|
}
|
|
|
|
#[test]
|
|
fn empty_signal_headers_use_generic_headers() {
|
|
let config = OtlpEnvSettings {
|
|
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
|
|
traces_headers: Some(String::new()),
|
|
generic_headers: Some(
|
|
"authorization=Bearer%20canary-token,x-scope=community".to_owned(),
|
|
),
|
|
..OtlpEnvSettings::default()
|
|
}
|
|
.into_config()
|
|
.unwrap();
|
|
|
|
assert_eq!(config.header("authorization"), Some("Bearer canary-token"));
|
|
assert_eq!(config.header("x-scope"), Some("community"));
|
|
assert!(!format!("{config:?}").contains("canary-token"));
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_headers_return_a_safe_error() {
|
|
let config = OtlpEnvSettings {
|
|
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
|
|
traces_headers: Some("authorization=canary-secret%0Ainjected".to_owned()),
|
|
..OtlpEnvSettings::default()
|
|
};
|
|
|
|
let error = config.into_config().unwrap_err();
|
|
assert!(matches!(error, super::OtlpTraceConfigError::InvalidHeaders));
|
|
assert!(!error.to_string().contains("canary-secret"));
|
|
}
|
|
|
|
#[test]
|
|
fn stricter_batch_timeout_bounds_http_export() {
|
|
let config = OtlpEnvSettings {
|
|
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
|
|
traces_protocol: Some("http/protobuf".to_owned()),
|
|
traces_timeout: Some("9000".to_owned()),
|
|
batch_export_timeout: Some("2500".to_owned()),
|
|
..OtlpEnvSettings::default()
|
|
}
|
|
.into_config()
|
|
.unwrap();
|
|
|
|
assert_eq!(
|
|
config.effective_export_timeout(),
|
|
Duration::from_millis(2500)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn generic_endpoint_receives_standard_trace_path() {
|
|
let config = OtlpEnvSettings {
|
|
generic_endpoint: Some("https://generic.example.test/otel/".to_owned()),
|
|
generic_protocol: Some("http/protobuf".to_owned()),
|
|
..OtlpEnvSettings::default()
|
|
}
|
|
.into_config()
|
|
.unwrap();
|
|
|
|
assert_eq!(
|
|
config.endpoint(),
|
|
Some("https://generic.example.test/otel/v1/traces")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn disabled_export_does_not_build_a_provider() {
|
|
let config = OtlpTraceConfig::try_new(
|
|
None,
|
|
None,
|
|
Duration::from_secs(1),
|
|
OtlpBatchConfig::default(),
|
|
)
|
|
.unwrap();
|
|
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap();
|
|
|
|
assert!(build_tracer_provider(&identity, &config).unwrap().is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn local_provider_creates_valid_context_without_an_exporter() {
|
|
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap();
|
|
let (provider, tracer) = build_local_tracer_provider(&identity);
|
|
let span = tracer.start("http.request");
|
|
assert!(span.span_context().is_valid());
|
|
provider.shutdown().unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn real_http_protobuf_export_contains_resource_and_trace() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let (request_tx, request_rx) = mpsc::sync_channel(1);
|
|
let server = thread::spawn(move || {
|
|
let (mut stream, _) = listener.accept().unwrap();
|
|
let request = read_http_request(&mut stream);
|
|
stream
|
|
.write_all(
|
|
b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
|
|
)
|
|
.unwrap();
|
|
request_tx.send(request).unwrap();
|
|
});
|
|
let config = OtlpEnvSettings {
|
|
traces_endpoint: Some(format!("http://{address}/v1/traces")),
|
|
traces_protocol: Some("http/protobuf".to_owned()),
|
|
traces_timeout: Some("2000".to_owned()),
|
|
traces_headers: Some(String::new()),
|
|
generic_headers: Some("authorization=Bearer%20canary-token".to_owned()),
|
|
max_queue_size: Some("16".to_owned()),
|
|
max_export_batch_size: Some("8".to_owned()),
|
|
scheduled_delay: Some("10".to_owned()),
|
|
batch_export_timeout: Some("2000".to_owned()),
|
|
..OtlpEnvSettings::default()
|
|
}
|
|
.into_config()
|
|
.unwrap();
|
|
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "integration-test").unwrap();
|
|
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
|
|
let mut span = tracer.start("http.request");
|
|
let trace_id = span.span_context().trace_id().to_bytes();
|
|
span.set_attribute(KeyValue::new("request_id", "req_otlp_contract"));
|
|
span.set_attribute(KeyValue::new("authorization", "Bearer canary-span-secret"));
|
|
span.add_event(
|
|
"canary-span-event",
|
|
vec![KeyValue::new("payload", "canary-span-secret")],
|
|
);
|
|
span.set_status(Status::error("canary-span-secret"));
|
|
span.end();
|
|
|
|
provider.force_flush().unwrap();
|
|
provider.shutdown().unwrap();
|
|
let request = request_rx.recv_timeout(Duration::from_secs(2)).unwrap();
|
|
server.join().unwrap();
|
|
let (headers, body) = split_http_request(&request);
|
|
|
|
assert!(headers.contains("POST /v1/traces HTTP/1.1"));
|
|
assert!(
|
|
headers
|
|
.to_ascii_lowercase()
|
|
.contains("content-type: application/x-protobuf")
|
|
);
|
|
assert!(
|
|
headers
|
|
.to_ascii_lowercase()
|
|
.contains("authorization: bearer canary-token")
|
|
);
|
|
let export = ExportTraceServiceRequest::decode(body).unwrap();
|
|
let resource_spans = export.resource_spans.first().unwrap();
|
|
let attributes = &resource_spans.resource.as_ref().unwrap().attributes;
|
|
assert_eq!(
|
|
string_attribute(attributes, "service.name"),
|
|
Some("admin-api")
|
|
);
|
|
assert_eq!(
|
|
string_attribute(attributes, "service.version"),
|
|
Some("0.3.1")
|
|
);
|
|
assert_eq!(
|
|
string_attribute(attributes, "deployment.environment.name"),
|
|
Some("integration-test")
|
|
);
|
|
assert_eq!(
|
|
resource_spans.scope_spans[0].spans[0].trace_id.as_slice(),
|
|
trace_id
|
|
);
|
|
let exported_span = &resource_spans.scope_spans[0].spans[0];
|
|
assert_eq!(exported_span.name, "http.request");
|
|
assert_eq!(
|
|
string_attribute(&exported_span.attributes, "request_id"),
|
|
Some("req_otlp_contract")
|
|
);
|
|
assert!(
|
|
exported_span
|
|
.attributes
|
|
.iter()
|
|
.all(|attribute| attribute.key != "authorization")
|
|
);
|
|
assert!(exported_span.events.is_empty());
|
|
assert_eq!(
|
|
exported_span
|
|
.status
|
|
.as_ref()
|
|
.map(|status| status.message.as_str()),
|
|
Some("")
|
|
);
|
|
assert!(
|
|
!body
|
|
.windows(b"canary-span-secret".len())
|
|
.any(|window| { window == b"canary-span-secret" })
|
|
);
|
|
}
|
|
|
|
#[tokio::test(flavor = "current_thread")]
|
|
async fn unavailable_receiver_does_not_change_product_result() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = thread::spawn(move || {
|
|
let (stream, _) = listener.accept().unwrap();
|
|
drop(stream);
|
|
});
|
|
let config = OtlpTraceConfig::try_new(
|
|
Some(format!("http://{address}/v1/traces")),
|
|
Some("http/protobuf".to_owned()),
|
|
Duration::from_millis(250),
|
|
OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(250))
|
|
.unwrap(),
|
|
)
|
|
.unwrap();
|
|
let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "fault-test").unwrap();
|
|
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
|
|
let subscriber =
|
|
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
|
let dispatch = tracing::Dispatch::new(subscriber);
|
|
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
|
let app = Router::new()
|
|
.route("/product", get(|| async { "product-success" }))
|
|
.layer(axum::middleware::from_fn(trace_product_request));
|
|
|
|
let response = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/product")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let status = response.status();
|
|
let body = to_bytes(response.into_body(), 64).await.unwrap();
|
|
|
|
assert_eq!(status, axum::http::StatusCode::OK);
|
|
assert_eq!(body.as_ref(), b"product-success");
|
|
assert!(provider.force_flush().is_err());
|
|
let _ = provider.shutdown();
|
|
server.join().unwrap();
|
|
}
|
|
|
|
async fn trace_product_request(request: Request, next: Next) -> Response {
|
|
next.run(request)
|
|
.instrument(info_span!(target: "crank::trace", "http.request"))
|
|
.await
|
|
}
|
|
|
|
#[test]
|
|
fn hanging_receiver_respects_the_stricter_export_timeout() {
|
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = thread::spawn(move || {
|
|
let (stream, _) = listener.accept().unwrap();
|
|
thread::sleep(Duration::from_millis(750));
|
|
drop(stream);
|
|
});
|
|
let config = OtlpTraceConfig::try_new(
|
|
Some(format!("http://{address}/v1/traces")),
|
|
Some("http/protobuf".to_owned()),
|
|
Duration::from_secs(2),
|
|
OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(100))
|
|
.unwrap(),
|
|
)
|
|
.unwrap();
|
|
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "timeout-test").unwrap();
|
|
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
|
|
let mut span = tracer.start("http.request");
|
|
span.end();
|
|
let started_at = Instant::now();
|
|
|
|
assert!(provider.force_flush().is_err());
|
|
assert!(started_at.elapsed() < Duration::from_millis(500));
|
|
let _ = provider.shutdown();
|
|
server.join().unwrap();
|
|
}
|
|
|
|
fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
|
|
stream
|
|
.set_read_timeout(Some(Duration::from_secs(2)))
|
|
.unwrap();
|
|
let mut request = Vec::new();
|
|
let mut buffer = [0_u8; 4096];
|
|
loop {
|
|
let read = stream.read(&mut buffer).unwrap();
|
|
request.extend_from_slice(&buffer[..read]);
|
|
let Some(header_end) = find_bytes(&request, b"\r\n\r\n") else {
|
|
continue;
|
|
};
|
|
let headers = String::from_utf8_lossy(&request[..header_end]);
|
|
let content_length = headers
|
|
.lines()
|
|
.find_map(|line| {
|
|
let (name, value) = line.split_once(':')?;
|
|
name.eq_ignore_ascii_case("content-length")
|
|
.then(|| value.trim().parse::<usize>().ok())
|
|
.flatten()
|
|
})
|
|
.unwrap_or(0);
|
|
if request.len() >= header_end + 4 + content_length {
|
|
return request;
|
|
}
|
|
}
|
|
}
|
|
|
|
fn split_http_request(request: &[u8]) -> (&str, &[u8]) {
|
|
let header_end = find_bytes(request, b"\r\n\r\n").unwrap();
|
|
(
|
|
std::str::from_utf8(&request[..header_end]).unwrap(),
|
|
&request[header_end + 4..],
|
|
)
|
|
}
|
|
|
|
fn string_attribute<'a>(
|
|
attributes: &'a [opentelemetry_proto::tonic::common::v1::KeyValue],
|
|
key: &str,
|
|
) -> Option<&'a str> {
|
|
attributes.iter().find_map(|attribute| {
|
|
let value = attribute.value.as_ref()?.value.as_ref()?;
|
|
(attribute.key == key)
|
|
.then_some(value)
|
|
.and_then(|value| match value {
|
|
any_value::Value::StringValue(value) => Some(value.as_str()),
|
|
_ => None,
|
|
})
|
|
})
|
|
}
|
|
|
|
fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
|
haystack
|
|
.windows(needle.len())
|
|
.position(|candidate| candidate == needle)
|
|
}
|
|
}
|