bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat: harden community production foundation through story 1.5

Browse Source
This commit is contained in:
bsodfather
2026-08-14 00:21:59 +03:00
parent c30461cc92
commit f6fc2e5c9b
161 changed files with 16758 additions and 2515 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/crank-observability/Cargo.toml
-1
View File
@@ -27,7 +27,6 @@ tracing.workspace = true
tracing-opentelemetry.workspace = true
tracing-subscriber.workspace = true
url.workspace = true
uuid.workspace = true
[dev-dependencies]
opentelemetry-proto.workspace = true
crates/crank-observability/src/config.rs
+18 -61
View File
@@ -1,10 +1,8 @@
use std::env;
use thiserror::Error;
use tracing_subscriber::EnvFilter;
use crate::RedactionLimits;
const DEFAULT_ENVIRONMENT: &str = "development";
const MAX_IDENTITY_LABEL_BYTES: usize = 64;
#[derive(Clone, Debug, Eq, PartialEq)]
@@ -52,6 +50,20 @@ pub struct ObservabilityConfig {
}
impl ObservabilityConfig {
pub fn try_new(
identity: ServiceIdentity,
filter: impl Into<String>,
redaction_limits: RedactionLimits,
) -> Result<Self, ObservabilityConfigError> {
let filter = filter.into();
EnvFilter::try_new(&filter).map_err(|_| ObservabilityConfigError::InvalidFilter)?;
Ok(Self {
identity,
filter,
redaction_limits,
})
}
pub fn new(
identity: ServiceIdentity,
filter: impl Into<String>,
@@ -64,26 +76,6 @@ impl ObservabilityConfig {
}
}
pub fn from_env(
service: &'static str,
version: &'static str,
default_filter: &'static str,
) -> Result<Self, ObservabilityConfigError> {
let environment = env_value_or_default(
"CRANK_ENVIRONMENT",
env::var("CRANK_ENVIRONMENT"),
DEFAULT_ENVIRONMENT,
)?;
let filter = env_value_or_default(
"CRANK_LOG_LEVEL",
env::var("CRANK_LOG_LEVEL"),
default_filter,
)?;
let identity = ServiceIdentity::try_new(service, version, environment)?;
Ok(Self::new(identity, filter, RedactionLimits::default()))
}
pub(crate) fn into_parts(self) -> (ServiceIdentity, String, RedactionLimits) {
(self.identity, self.filter, self.redaction_limits)
}
@@ -101,22 +93,8 @@ impl ObservabilityConfig {
pub enum ObservabilityConfigError {
#[error("invalid observability identity field: {field}")]
InvalidIdentity { field: &'static str },
#[error("observability environment variable is not valid UTF-8: {field}")]
InvalidEnvironmentEncoding { field: &'static str },
}
fn env_value_or_default(
field: &'static str,
value: Result<String, env::VarError>,
default: &'static str,
) -> Result<String, ObservabilityConfigError> {
match value {
Ok(value) => Ok(value),
Err(env::VarError::NotPresent) => Ok(default.to_owned()),
Err(env::VarError::NotUnicode(_)) => {
Err(ObservabilityConfigError::InvalidEnvironmentEncoding { field })
}
}
#[error("invalid observability log filter")]
InvalidFilter,
}
fn validate_label(field: &'static str, value: &str) -> Result<(), ObservabilityConfigError> {
@@ -135,9 +113,7 @@ fn validate_label(field: &'static str, value: &str) -> Result<(), ObservabilityC
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use super::{ObservabilityConfigError, ServiceIdentity, env_value_or_default};
use super::ServiceIdentity;
#[test]
fn accepts_release_and_environment_labels() {
@@ -148,23 +124,4 @@ mod tests {
assert_eq!(identity.version(), "0.3.1+build.7");
assert_eq!(identity.environment(), "production");
}
#[test]
fn rejects_non_utf8_environment_values() {
let error = env_value_or_default(
"CRANK_ENVIRONMENT",
Err(std::env::VarError::NotUnicode(OsString::from(
"invalid-environment",
))),
"development",
)
.expect_err("non-UTF-8 values must not be replaced with defaults");
assert!(matches!(
error,
ObservabilityConfigError::InvalidEnvironmentEncoding {
field: "CRANK_ENVIRONMENT"
}
));
}
}
crates/crank-observability/src/correlation.rs
-51
View File
@@ -1,51 +0,0 @@
use std::fmt;
use axum::http::HeaderMap;
use uuid::Uuid;
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct RequestId(String);
impl RequestId {
pub const MAX_LEN: usize = 128;
const HEADER_NAME: &'static str = "x-request-id";
pub fn resolve(candidate: Option<&str>) -> Self {
candidate
.filter(|value| Self::is_valid(value))
.map(|value| Self(value.to_owned()))
.unwrap_or_else(|| Self(Uuid::now_v7().to_string()))
}
pub fn resolve_from_headers(headers: &HeaderMap) -> Self {
let mut values = headers.get_all(Self::HEADER_NAME).iter();
let candidate = values.next();
if values.next().is_some() {
return Self::resolve(None);
}
Self::resolve(candidate.and_then(|value| value.to_str().ok()))
}
pub fn is_valid(value: &str) -> bool {
!value.is_empty()
&& value.len() <= Self::MAX_LEN
&& value
.bytes()
.all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
}
pub fn as_str(&self) -> &str {
&self.0
}
pub fn into_string(self) -> String {
self.0
}
}
impl fmt::Display for RequestId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
crates/crank-observability/src/error_reporting.rs
+57 -37
View File
@@ -1,7 +1,7 @@
use std::{
borrow::Cow,
collections::BTreeMap,
env, fmt,
fmt,
future::Future,
time::{Duration, SystemTime},
};
@@ -13,11 +13,8 @@ use sentry::{
};
use thiserror::Error;
use crate::{
RedactionLimits, ServiceIdentity, propagation::current_trace_id, redaction::truncate_string,
};
use crate::{RedactionLimits, ServiceIdentity, propagation::current_trace_id};
const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
const CRITICAL_ERROR_MESSAGE: &str = "critical error";
const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
// Sentry serializes SystemTime as a finite f64; this keeps conservative fixed headroom.
@@ -25,6 +22,7 @@ const MAX_SERIALIZED_TIMESTAMP_BYTES: usize = 32;
tokio::task_local! {
static REQUEST_ID: String;
static TRACE_ID: String;
}
pub struct SentryConfig {
@@ -43,14 +41,6 @@ impl SentryConfig {
Ok(Self { dsn: Some(dsn) })
}
pub fn from_env() -> Result<Self, SentryConfigError> {
match env::var(SENTRY_DSN_ENV) {
Ok(value) => Self::parse(Some(&value)),
Err(env::VarError::NotPresent) => Self::parse(None),
Err(env::VarError::NotUnicode(_)) => Err(SentryConfigError::InvalidEnvironmentEncoding),
}
}
pub fn enabled(&self) -> bool {
self.dsn.is_some()
}
@@ -69,8 +59,6 @@ impl fmt::Debug for SentryConfig {
pub enum SentryConfigError {
#[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")]
InvalidDsn,
#[error("CRANK_SENTRY_DSN is not valid UTF-8")]
InvalidEnvironmentEncoding,
#[error("critical error event budget cannot hold the required fields")]
EventBudgetTooSmall,
}
@@ -123,11 +111,43 @@ pub fn capture_critical_error(category: CriticalErrorCategory) {
});
}
pub async fn with_request_correlation<F>(request_id: String, future: F) -> F::Output
pub async fn with_request_correlation<F>(
request_id: String,
trace_id: String,
future: F,
) -> F::Output
where
F: Future,
{
REQUEST_ID.scope(request_id, future).await
if !valid_request_id(&request_id) || !valid_trace_id(&trace_id) {
return future.await;
}
REQUEST_ID
.scope(request_id, TRACE_ID.scope(trace_id, future))
.await
}
fn valid_request_id(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 128
&& value
.bytes()
.all(|byte| (0x21..=0x7e).contains(&byte) && byte != b',' && byte != b';')
}
fn valid_trace_id(value: &str) -> bool {
value.len() == 32
&& value != "00000000000000000000000000000000"
&& value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
pub fn current_request_correlation() -> (Option<String>, Option<String>) {
(
REQUEST_ID.try_with(Clone::clone).ok(),
TRACE_ID.try_with(Clone::clone).ok(),
)
}
pub(crate) fn init_sentry(
@@ -184,16 +204,7 @@ fn sanitize_event(
CriticalErrorCategory::Panic
}
});
let mut tags = correlation_tags()
.into_iter()
.map(|(key, value)| (key, truncate_string(&value, limits.max_string_bytes)))
.collect::<BTreeMap<_, _>>();
for key in ["request_id", "trace_id"] {
if let Some(value) = event.tags.get(key) {
tags.entry(key.to_owned())
.or_insert_with(|| truncate_string(value, limits.max_string_bytes));
}
}
let mut tags = correlation_tags().into_iter().collect::<BTreeMap<_, _>>();
tags.insert("service".to_owned(), identity.service().to_owned());
tags.insert("category".to_owned(), category.as_str().to_owned());
@@ -222,19 +233,21 @@ fn correlation_tags() -> BTreeMap<String, String> {
if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) {
tags.insert("request_id".to_owned(), request_id);
}
if let Some(trace_id) = current_trace_id() {
if let Ok(trace_id) = TRACE_ID.try_with(Clone::clone) {
tags.insert("trace_id".to_owned(), trace_id);
} else if let Some(trace_id) = current_trace_id() {
tags.insert("trace_id".to_owned(), trace_id);
}
tags
}
fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
fn enforce_event_budget(event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
if serialized_event_len(&event) <= max_event_bytes {
return event;
}
event.tags.remove("request_id");
event.tags.remove("trace_id");
// Startup validation reserves enough room for maximum canonical IDs. They
// are never evicted from a support event to satisfy a byte budget.
debug_assert!(serialized_event_len(&event) <= max_event_bytes);
event
}
@@ -257,7 +270,7 @@ fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionL
CriticalErrorCategory::ALL
.into_iter()
.map(|category| {
let event = sanitize_event(
let mut event = sanitize_event(
Event {
tags: BTreeMap::from([("category".to_owned(), category.as_str().to_owned())]),
timestamp: SystemTime::UNIX_EPOCH,
@@ -266,6 +279,8 @@ fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionL
identity,
unbounded_limits,
);
event.tags.insert("request_id".to_owned(), "r".repeat(128));
event.tags.insert("trace_id".to_owned(), "a".repeat(32));
serialized_event_len(&event)
.saturating_add(MAX_SERIALIZED_TIMESTAMP_BYTES.saturating_sub(1))
})
@@ -439,11 +454,15 @@ mod tests {
let events = sentry::test::with_captured_events_options(
|| {
tracing::dispatcher::with_default(&dispatch, || {
runtime.block_on(with_request_correlation("request-123".to_owned(), async {
let span = tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
capture_critical_error(CriticalErrorCategory::DataIntegrity);
}));
runtime.block_on(with_request_correlation(
"request-123".to_owned(),
"0af7651916cd43dd8448eb211c80319c".to_owned(),
async {
let span = tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
capture_critical_error(CriticalErrorCategory::DataIntegrity);
},
));
});
},
options,
@@ -496,6 +515,7 @@ mod tests {
tracing::dispatcher::with_default(&dispatch, || {
runtime.block_on(with_request_correlation(
"panic-request-123".to_owned(),
"0af7651916cd43dd8448eb211c80319c".to_owned(),
async {
let span =
tracing::info_span!(target: "crank::trace", "http.request");
crates/crank-observability/src/lib.rs
+1 -3
View File
@@ -1,5 +1,4 @@
mod config;
mod correlation;
mod error_reporting;
mod incidents;
mod instrumentation;
@@ -12,13 +11,12 @@ mod redaction;
mod schema;
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
pub use correlation::RequestId;
pub use crank_metrics::{
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
};
pub use error_reporting::{
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
with_request_correlation,
current_request_correlation, with_request_correlation,
};
pub use incidents::{OperationalIncident, operational_incident_total, record_operational_incident};
pub use instrumentation::{record_db_pool_connections, record_http_request};
crates/crank-observability/src/lifecycle.rs
+30 -11
View File
@@ -6,29 +6,45 @@ use tracing_subscriber::util::SubscriberInitExt;
use crate::{
MetricsConfig, MetricsSurface, MetricsSurfaceError, ObservabilityConfig,
ObservabilityConfigError, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError,
RedactionLimitsError, SentryConfig, SentryConfigError, error_reporting::init_sentry,
instrumentation::register_metric_schema, logging::build_subscriber_with_tracer,
otlp::build_tracer_provider, prometheus::install_prometheus_recorder,
RedactionLimitsError, SentryConfig, SentryConfigError,
error_reporting::init_sentry,
instrumentation::register_metric_schema,
logging::build_subscriber_with_tracer,
otlp::{build_local_tracer_provider, build_tracer_provider},
prometheus::install_prometheus_recorder,
propagation::install_trace_context_propagator,
};
#[must_use = "observability resources must be retained until process shutdown"]
pub struct ObservabilityLifecycle {
metrics_handle: metrics_exporter_prometheus::PrometheusHandle,
tracer_provider: Option<opentelemetry_sdk::trace::SdkTracerProvider>,
_tracer_provider: opentelemetry_sdk::trace::SdkTracerProvider,
trace_export_enabled: bool,
sentry_guard: Option<sentry::ClientInitGuard>,
}
impl ObservabilityLifecycle {
pub fn init(config: ObservabilityConfig) -> Result<Self, ObservabilityInitError> {
Self::init_with_exporters(
config,
SentryConfig::parse(None)?,
OtlpTraceConfig::default(),
)
}
pub fn init_with_exporters(
config: ObservabilityConfig,
sentry_config: SentryConfig,
trace_config: OtlpTraceConfig,
) -> Result<Self, ObservabilityInitError> {
let identity = config.identity().clone();
let redaction_limits = config.redaction_limits();
let sentry_config = SentryConfig::from_env()?;
let trace_config = OtlpTraceConfig::from_env()?;
let tracing = build_tracer_provider(&identity, &trace_config)?;
let tracer = tracing.as_ref().map(|(_, tracer)| tracer.clone());
let exported_tracing = build_tracer_provider(&identity, &trace_config)?;
let trace_export_enabled = exported_tracing.is_some();
let (tracer_provider, tracer) =
exported_tracing.unwrap_or_else(|| build_local_tracer_provider(&identity));
install_trace_context_propagator();
build_subscriber_with_tracer(config, io::stdout, tracer)?
build_subscriber_with_tracer(config, io::stdout, Some(tracer))?
.try_init()
.map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?;
let metrics_handle = install_prometheus_recorder(&identity)?;
@@ -37,7 +53,8 @@ impl ObservabilityLifecycle {
Ok(Self {
metrics_handle,
tracer_provider: tracing.map(|(provider, _)| provider),
_tracer_provider: tracer_provider,
trace_export_enabled,
sentry_guard,
})
}
@@ -47,7 +64,7 @@ impl ObservabilityLifecycle {
}
pub fn traces_enabled(&self) -> bool {
self.tracer_provider.is_some()
self.trace_export_enabled
}
pub fn critical_errors_enabled(&self) -> bool {
@@ -77,6 +94,8 @@ pub enum ObservabilityInitError {
InvalidRedactionLimits(#[from] RedactionLimitsError),
#[error("invalid log filter")]
InvalidFilter,
#[error("log event budget cannot hold canonical correlation fields")]
LogEventBudgetTooSmall,
#[error("global tracing subscriber is already initialized")]
SubscriberAlreadyInitialized,
#[error(transparent)]
crates/crank-observability/src/logging.rs
+43 -18
View File
@@ -13,6 +13,7 @@ use tracing_subscriber::{
use crate::{
ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity,
current_request_correlation,
propagation::current_trace_id,
redaction::{redact_value, truncate_string},
schema::LogEnvelope,
@@ -38,6 +39,9 @@ where
{
let (identity, filter, limits) = config.into_parts();
limits.validate()?;
if required_correlated_log_budget(&identity) > limits.max_event_bytes {
return Err(ObservabilityInitError::LogEventBudgetTooSmall);
}
let filter = EnvFilter::try_new(filter).map_err(|_| ObservabilityInitError::InvalidFilter)?;
let formatter = JsonEventFormatter::new(identity, limits);
let fmt_layer = tracing_subscriber::fmt::layer()
@@ -58,6 +62,22 @@ where
.with(otel_layer))
}
fn required_correlated_log_budget(identity: &ServiceIdentity) -> usize {
let envelope = LogEnvelope {
timestamp: "9999-12-31T23:59:59.999999999Z".to_owned(),
level: "ERROR".to_owned(),
service: identity.service().to_owned(),
version: identity.version().to_owned(),
environment: identity.environment().to_owned(),
target: "0123456789abcdef".to_owned(),
event: "0123456789abcdef".to_owned(),
request_id: Some("r".repeat(128)),
trace_id: Some("a".repeat(32)),
fields: Map::from_iter([("truncated".to_owned(), Value::Bool(true))]),
};
serde_json::to_vec(&envelope).map_or(usize::MAX, |value| value.len().saturating_add(1))
}
#[derive(Clone, Debug)]
struct JsonEventFormatter {
identity: ServiceIdentity,
@@ -75,10 +95,10 @@ impl JsonEventFormatter {
event.record(&mut visitor);
let mut raw_fields = visitor.fields;
let request_id = take_correlation_id(&mut raw_fields, "request_id")
.map(|value| truncate_string(&value, self.limits.max_string_bytes));
.or_else(|| current_request_correlation().0);
let trace_id = take_correlation_id(&mut raw_fields, "trace_id")
.or_else(current_trace_id)
.map(|value| truncate_string(&value, self.limits.max_string_bytes));
.or_else(|| current_request_correlation().1)
.or_else(current_trace_id);
let cleaned = redact_value(&Value::Object(raw_fields), self.limits);
let fields = cleaned.as_object().cloned().unwrap_or_default();
let timestamp = OffsetDateTime::now_utc()
@@ -110,19 +130,11 @@ impl JsonEventFormatter {
let fallback_string_limit = self.limits.max_string_bytes.min(64);
envelope.target = truncate_string(&envelope.target, fallback_string_limit);
envelope.event = truncate_string(&envelope.event, fallback_string_limit);
envelope.request_id = envelope
.request_id
.map(|value| truncate_string(&value, fallback_string_limit));
envelope.trace_id = envelope
.trace_id
.map(|value| truncate_string(&value, fallback_string_limit));
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
if serialized.len() <= line_budget {
return Ok(serialized);
}
envelope.request_id = None;
envelope.trace_id = None;
envelope.target = truncate_string(&envelope.target, 16);
envelope.event = truncate_string(&envelope.event, 16);
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
@@ -202,14 +214,27 @@ impl Visit for JsonFieldVisitor {
}
fn take_correlation_id(fields: &mut Map<String, Value>, name: &str) -> Option<String> {
let value = fields.remove(name)?;
let value = match value {
Value::String(value) => value,
Value::Number(value) => value.to_string(),
Value::Bool(value) => value.to_string(),
Value::Null | Value::Array(_) | Value::Object(_) => return None,
let Value::String(value) = fields.remove(name)? else {
return None;
};
(!value.is_empty()).then_some(value)
let valid = match name {
"trace_id" => {
value.len() == 32
&& value != "00000000000000000000000000000000"
&& value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
"request_id" | "correlation_id" => {
!value.is_empty()
&& value.len() <= 128
&& value
.bytes()
.all(|byte| byte.is_ascii_graphic() && byte != b',' && byte != b';')
}
_ => false,
};
valid.then_some(value)
}
fn is_correlation_field(name: &str) -> bool {
crates/crank-observability/src/otlp.rs
+87 -42
View File
@@ -1,4 +1,4 @@
use std::{collections::HashMap, env, fmt, time::Duration};
use std::{collections::HashMap, fmt, time::Duration};
use axum::http::{HeaderName, HeaderValue};
use opentelemetry::{
@@ -118,8 +118,36 @@ impl fmt::Debug for OtlpTraceConfig {
}
impl OtlpTraceConfig {
pub fn from_env() -> Result<Self, OtlpTraceConfigError> {
OtlpEnvSettings::from_env()?.into_config()
#[allow(clippy::too_many_arguments)]
pub fn from_values(
generic_endpoint: Option<String>,
traces_endpoint: Option<String>,
generic_protocol: Option<String>,
traces_protocol: Option<String>,
generic_timeout: Option<String>,
traces_timeout: Option<String>,
generic_headers: Option<String>,
traces_headers: Option<String>,
max_queue_size: usize,
max_export_batch_size: usize,
scheduled_delay: String,
batch_export_timeout: String,
) -> Result<Self, OtlpTraceConfigError> {
OtlpEnvSettings {
traces_endpoint,
generic_endpoint,
traces_protocol,
generic_protocol,
traces_timeout,
generic_timeout,
traces_headers,
generic_headers,
max_queue_size: Some(max_queue_size.to_string()),
max_export_batch_size: Some(max_export_batch_size.to_string()),
scheduled_delay: Some(scheduled_delay),
batch_export_timeout: Some(batch_export_timeout),
}
.into_config()
}
fn from_settings(settings: OtlpEnvSettings) -> Result<Self, OtlpTraceConfigError> {
@@ -232,6 +260,17 @@ impl OtlpTraceConfig {
}
}
impl Default for OtlpTraceConfig {
fn default() -> Self {
Self {
endpoint: None,
export_timeout: DEFAULT_EXPORT_TIMEOUT,
batch: OtlpBatchConfig::default(),
headers: HashMap::new(),
}
}
}
#[derive(Default)]
struct OtlpEnvSettings {
traces_endpoint: Option<String>,
@@ -249,23 +288,6 @@ struct OtlpEnvSettings {
}
impl OtlpEnvSettings {
fn from_env() -> Result<Self, OtlpTraceConfigError> {
Ok(Self {
traces_endpoint: optional_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")?,
generic_endpoint: optional_env("OTEL_EXPORTER_OTLP_ENDPOINT")?,
traces_protocol: optional_env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")?,
generic_protocol: optional_env("OTEL_EXPORTER_OTLP_PROTOCOL")?,
traces_timeout: optional_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT")?,
generic_timeout: optional_env("OTEL_EXPORTER_OTLP_TIMEOUT")?,
traces_headers: optional_env("OTEL_EXPORTER_OTLP_TRACES_HEADERS")?,
generic_headers: optional_env("OTEL_EXPORTER_OTLP_HEADERS")?,
max_queue_size: optional_env("OTEL_BSP_MAX_QUEUE_SIZE")?,
max_export_batch_size: optional_env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE")?,
scheduled_delay: optional_env("OTEL_BSP_SCHEDULE_DELAY")?,
batch_export_timeout: optional_env("OTEL_BSP_EXPORT_TIMEOUT")?,
})
}
fn into_config(self) -> Result<OtlpTraceConfig, OtlpTraceConfigError> {
OtlpTraceConfig::from_settings(self)
}
@@ -313,7 +335,28 @@ pub fn build_tracer_provider(
let processor = BatchSpanProcessor::builder(ObservedSpanExporter(exporter))
.with_batch_config(config.batch.sdk_config())
.build();
let resource = Resource::builder_empty()
let resource = trace_resource(identity);
let provider = SdkTracerProvider::builder()
.with_span_processor(processor)
.with_resource(resource)
.build();
let tracer = provider.tracer("crank");
Ok(Some((provider, tracer)))
}
pub(crate) fn build_local_tracer_provider(
identity: &ServiceIdentity,
) -> (SdkTracerProvider, SdkTracer) {
let provider = SdkTracerProvider::builder()
.with_resource(trace_resource(identity))
.build();
let tracer = provider.tracer("crank");
(provider, tracer)
}
fn trace_resource(identity: &ServiceIdentity) -> Resource {
Resource::builder_empty()
.with_attributes([
KeyValue::new("service.name", identity.service().to_owned()),
KeyValue::new("service.version", identity.version().to_owned()),
@@ -322,14 +365,7 @@ pub fn build_tracer_provider(
identity.environment().to_owned(),
),
])
.build();
let provider = SdkTracerProvider::builder()
.with_span_processor(processor)
.with_resource(resource)
.build();
let tracer = provider.tracer("crank");
Ok(Some((provider, tracer)))
.build()
}
#[derive(Debug)]
@@ -405,7 +441,7 @@ fn is_allowed_span_attribute(attribute: &KeyValue) -> bool {
};
let value = value.as_str();
match attribute.key.as_str() {
"request_id" => crate::RequestId::is_valid(value),
"request_id" => is_valid_request_id(value),
"stage" => is_allowed_span_name(value),
"outcome" => matches!(
value,
@@ -453,6 +489,14 @@ fn is_allowed_span_attribute(attribute: &KeyValue) -> bool {
}
}
fn is_valid_request_id(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 128
&& value
.bytes()
.all(|byte| byte.is_ascii_graphic() && byte != b',' && byte != b';')
}
#[derive(Clone, Copy)]
enum EndpointKind {
Trace,
@@ -514,17 +558,6 @@ fn parse_headers(value: &str) -> Result<HashMap<String, String>, OtlpTraceConfig
})
}
fn optional_env(field: &'static str) -> Result<Option<String>, OtlpTraceConfigError> {
match env::var(field) {
Ok(value) if value.is_empty() => Ok(None),
Ok(value) => Ok(Some(value)),
Err(env::VarError::NotPresent) => Ok(None),
Err(env::VarError::NotUnicode(_)) => {
Err(OtlpTraceConfigError::InvalidEnvironmentEncoding { field })
}
}
}
fn usize_env(
field: &'static str,
value: Option<String>,
@@ -586,7 +619,10 @@ mod tests {
use tracing::{Instrument, info_span};
use tracing_subscriber::layer::SubscriberExt;
use super::{OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_tracer_provider};
use super::{
OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_local_tracer_provider,
build_tracer_provider,
};
use crate::ServiceIdentity;
#[test]
@@ -708,6 +744,15 @@ mod tests {
assert!(build_tracer_provider(&identity, &config).unwrap().is_none());
}
#[test]
fn local_provider_creates_valid_context_without_an_exporter() {
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap();
let (provider, tracer) = build_local_tracer_provider(&identity);
let span = tracer.start("http.request");
assert!(span.span_context().is_valid());
provider.shutdown().unwrap();
}
#[test]
fn real_http_protobuf_export_contains_resource_and_trace() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
crates/crank-observability/src/prometheus.rs
+1 -44
View File
@@ -1,4 +1,4 @@
use std::{env, net::SocketAddr};
use std::net::SocketAddr;
use axum::{
Router,
@@ -19,8 +19,6 @@ use tokio::net::TcpListener;
use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
const METRICS_ENABLED_ENV: &str = "CRANK_METRICS_ENABLED";
const METRICS_TOKEN_ENV: &str = "CRANK_METRICS_BEARER_TOKEN";
const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
#[derive(Clone)]
@@ -62,33 +60,6 @@ impl MetricsConfig {
})
}
pub fn from_env(
bind_env: &'static str,
default_bind: SocketAddr,
) -> Result<Self, MetricsConfigError> {
let enabled = parse_enabled(env::var(METRICS_ENABLED_ENV))?;
let bind_addr = match env::var(bind_env) {
Ok(raw) => raw
.parse()
.map_err(|_| MetricsConfigError::InvalidBindAddress { field: bind_env })?,
Err(env::VarError::NotPresent) => default_bind,
Err(env::VarError::NotUnicode(_)) => {
return Err(MetricsConfigError::InvalidEnvironmentEncoding { field: bind_env });
}
};
let bearer_token = match env::var(METRICS_TOKEN_ENV) {
Ok(token) => Some(token),
Err(env::VarError::NotPresent) => None,
Err(env::VarError::NotUnicode(_)) => {
return Err(MetricsConfigError::InvalidEnvironmentEncoding {
field: METRICS_TOKEN_ENV,
});
}
};
Self::new(enabled, bind_addr, bearer_token)
}
pub fn enabled(&self) -> bool {
self.enabled
}
@@ -280,17 +251,3 @@ fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
fn token_digest(token: &[u8]) -> [u8; 32] {
Sha256::digest(token).into()
}
fn parse_enabled(value: Result<String, env::VarError>) -> Result<bool, MetricsConfigError> {
match value {
Ok(raw) => match raw.to_ascii_lowercase().as_str() {
"true" | "1" => Ok(true),
"false" | "0" => Ok(false),
_ => Err(MetricsConfigError::InvalidEnabledFlag),
},
Err(env::VarError::NotPresent) => Ok(true),
Err(env::VarError::NotUnicode(_)) => Err(MetricsConfigError::InvalidEnvironmentEncoding {
field: METRICS_ENABLED_ENV,
}),
}
}
crates/crank-observability/tests/correlation.rs
-74
View File
@@ -1,74 +0,0 @@
use axum::http::{HeaderMap, HeaderValue};
use crank_observability::RequestId;
use uuid::Version;
#[test]
fn preserves_valid_opaque_request_id() {
let request_id = RequestId::resolve(Some("req_test-123/abc"));
assert_eq!(request_id.as_str(), "req_test-123/abc");
}
#[test]
fn replaces_missing_and_invalid_values_with_uuid_v7() {
for candidate in [
None,
Some(""),
Some("bad value"),
Some(" leading"),
Some("trailing "),
Some("bad,value"),
Some("bad;value"),
Some("я"),
] {
let request_id = RequestId::resolve(candidate);
let parsed = uuid::Uuid::parse_str(request_id.as_str()).expect("generated UUID");
assert_eq!(parsed.get_version(), Some(Version::SortRand));
}
}
#[test]
fn rejects_values_over_the_shared_limit() {
let oversized = "x".repeat(RequestId::MAX_LEN + 1);
let request_id = RequestId::resolve(Some(&oversized));
assert_ne!(request_id.as_str(), oversized);
assert_eq!(
uuid::Uuid::parse_str(request_id.as_str())
.expect("generated UUID")
.get_version(),
Some(Version::SortRand)
);
}
#[test]
fn resolves_exactly_one_header_value_and_rejects_ambiguous_values() {
let mut single = HeaderMap::new();
single.insert(
"x-request-id",
HeaderValue::from_static("opaque-request-id"),
);
assert_eq!(
RequestId::resolve_from_headers(&single).as_str(),
"opaque-request-id"
);
let mut ambiguous = HeaderMap::new();
ambiguous.append("x-request-id", HeaderValue::from_static("first-request-id"));
ambiguous.append(
"x-request-id",
HeaderValue::from_static("second-request-id"),
);
let generated = RequestId::resolve_from_headers(&ambiguous);
assert_ne!(generated.as_str(), "first-request-id");
assert_ne!(generated.as_str(), "second-request-id");
assert_eq!(
uuid::Uuid::parse_str(generated.as_str())
.expect("generated UUID")
.get_version(),
Some(Version::SortRand)
);
}
crates/crank-observability/tests/json_logging.rs
+6 -19
View File
@@ -132,11 +132,11 @@ fn correlation_fields_are_distinct_and_only_present_when_recorded() {
tracing::info!(
name: "admin.request.completed",
request_id = "req-123",
trace_id = "trace-456"
trace_id = "0af7651916cd43dd8448eb211c80319c"
);
});
assert_eq!(present[0]["request_id"], "req-123");
assert_eq!(present[0]["trace_id"], "trace-456");
assert_eq!(present[0]["trace_id"], "0af7651916cd43dd8448eb211c80319c");
assert!(present[0]["fields"].get("request_id").is_none());
assert!(present[0]["fields"].get("trace_id").is_none());
@@ -148,7 +148,7 @@ fn correlation_fields_are_distinct_and_only_present_when_recorded() {
}
#[test]
fn correlation_fields_preserve_scalar_display_values_before_field_limits() {
fn correlation_fields_accept_valid_strings_and_reject_non_string_values() {
let limits = RedactionLimits {
max_object_fields: 1,
..RedactionLimits::default()
@@ -164,7 +164,7 @@ fn correlation_fields_preserve_scalar_display_values_before_field_limits() {
});
assert_eq!(events[0]["request_id"], "123");
assert_eq!(events[0]["trace_id"], "true");
assert!(events[0].get("trace_id").is_none());
}
#[test]
@@ -321,7 +321,7 @@ fn subscriber_rejects_limits_that_cannot_hold_an_event() {
}
#[test]
fn minimum_event_budget_handles_maximum_identity_labels() {
fn event_budget_rejects_maximum_identity_labels_when_correlation_cannot_fit() {
let writer = SharedWriter::default();
let config = ObservabilityConfig::new(
ServiceIdentity::try_new("s".repeat(64), "v".repeat(64), "e".repeat(64))
@@ -332,20 +332,7 @@ fn minimum_event_budget_handles_maximum_identity_labels() {
..RedactionLimits::default()
},
);
let subscriber =
build_subscriber(config, writer.clone()).expect("minimum valid budget must be usable");
tracing::subscriber::with_default(subscriber, || {
tracing::info!(
name: "event-name-that-is-intentionally-longer-than-the-fallback-limit",
description = %"x".repeat(4096),
);
});
let output = writer.output();
assert!(output.len() <= 512);
assert_eq!(output.lines().count(), 1);
serde_json::from_str::<Value>(output.trim_end()).expect("bounded line must remain valid JSON");
assert!(build_subscriber(config, writer).is_err());
}
#[test]
crates/crank-observability/tests/prometheus.rs
+2
View File
@@ -97,6 +97,8 @@ fn schema_is_closed_and_uses_fixed_duration_buckets() {
"agent_id",
"operation_id",
"request_id",
"trace_id",
"correlation_id",
"url",
"error_message",
"text",
crates/crank-observability/tests/request_correlation.rs
+60
View File
@@ -0,0 +1,60 @@
use std::sync::Arc;
use crank_observability::{current_request_correlation, with_request_correlation};
#[tokio::test]
async fn concurrent_request_correlation_is_task_local() {
let barrier = Arc::new(tokio::sync::Barrier::new(2));
let first = observe(
"request-first",
"0af7651916cd43dd8448eb211c80319c",
Arc::clone(&barrier),
);
let second = observe(
"request-second",
"1af7651916cd43dd8448eb211c80319c",
barrier,
);
let (first, second) = tokio::join!(first, second);
assert_eq!(
first,
(
Some("request-first".to_owned()),
Some("0af7651916cd43dd8448eb211c80319c".to_owned()),
)
);
assert_eq!(
second,
(
Some("request-second".to_owned()),
Some("1af7651916cd43dd8448eb211c80319c".to_owned()),
)
);
assert_eq!(current_request_correlation(), (None, None));
}
#[tokio::test]
async fn invalid_correlation_strings_never_enter_task_local_state() {
let observed = with_request_correlation(
"bad request id".to_owned(),
"CANARY-NOT-A-TRACE-ID".to_owned(),
async { current_request_correlation() },
)
.await;
assert_eq!(observed, (None, None));
}
async fn observe(
request_id: &str,
trace_id: &str,
barrier: Arc<tokio::sync::Barrier>,
) -> (Option<String>, Option<String>) {
with_request_correlation(request_id.to_owned(), trace_id.to_owned(), async move {
barrier.wait().await;
tokio::task::yield_now().await;
current_request_correlation()
})
.await
}