наблюдаемость: завершить базовый контур Community
Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
[package]
|
||||
name = "crank-observability"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
publish.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
axum.workspace = true
|
||||
metrics.workspace = true
|
||||
metrics-exporter-prometheus.workspace = true
|
||||
opentelemetry.workspace = true
|
||||
opentelemetry-otlp.workspace = true
|
||||
opentelemetry_sdk.workspace = true
|
||||
percent-encoding.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sentry.workspace = true
|
||||
sha2.workspace = true
|
||||
subtle.workspace = true
|
||||
thiserror.workspace = true
|
||||
time.workspace = true
|
||||
tokio = { workspace = true, features = ["net"] }
|
||||
tracing.workspace = true
|
||||
tracing-opentelemetry.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
url.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
opentelemetry-proto.workspace = true
|
||||
prost.workspace = true
|
||||
sentry = { workspace = true, features = ["test"] }
|
||||
tower.workspace = true
|
||||
@@ -0,0 +1,170 @@
|
||||
use std::env;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::RedactionLimits;
|
||||
|
||||
const DEFAULT_ENVIRONMENT: &str = "development";
|
||||
const MAX_IDENTITY_LABEL_BYTES: usize = 64;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ServiceIdentity {
|
||||
service: String,
|
||||
version: String,
|
||||
environment: String,
|
||||
}
|
||||
|
||||
impl ServiceIdentity {
|
||||
pub fn try_new(
|
||||
service: impl Into<String>,
|
||||
version: impl Into<String>,
|
||||
environment: impl Into<String>,
|
||||
) -> Result<Self, ObservabilityConfigError> {
|
||||
let identity = Self {
|
||||
service: service.into(),
|
||||
version: version.into(),
|
||||
environment: environment.into(),
|
||||
};
|
||||
validate_label("service", &identity.service)?;
|
||||
validate_label("version", &identity.version)?;
|
||||
validate_label("environment", &identity.environment)?;
|
||||
Ok(identity)
|
||||
}
|
||||
|
||||
pub fn service(&self) -> &str {
|
||||
&self.service
|
||||
}
|
||||
|
||||
pub fn version(&self) -> &str {
|
||||
&self.version
|
||||
}
|
||||
|
||||
pub fn environment(&self) -> &str {
|
||||
&self.environment
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ObservabilityConfig {
|
||||
identity: ServiceIdentity,
|
||||
filter: String,
|
||||
redaction_limits: RedactionLimits,
|
||||
}
|
||||
|
||||
impl ObservabilityConfig {
|
||||
pub fn new(
|
||||
identity: ServiceIdentity,
|
||||
filter: impl Into<String>,
|
||||
redaction_limits: RedactionLimits,
|
||||
) -> Self {
|
||||
Self {
|
||||
identity,
|
||||
filter: filter.into(),
|
||||
redaction_limits,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_env(
|
||||
service: &'static str,
|
||||
version: &'static str,
|
||||
default_filter: &'static str,
|
||||
) -> Result<Self, ObservabilityConfigError> {
|
||||
let environment = env_value_or_default(
|
||||
"CRANK_ENVIRONMENT",
|
||||
env::var("CRANK_ENVIRONMENT"),
|
||||
DEFAULT_ENVIRONMENT,
|
||||
)?;
|
||||
let filter = env_value_or_default(
|
||||
"CRANK_LOG_LEVEL",
|
||||
env::var("CRANK_LOG_LEVEL"),
|
||||
default_filter,
|
||||
)?;
|
||||
let identity = ServiceIdentity::try_new(service, version, environment)?;
|
||||
|
||||
Ok(Self::new(identity, filter, RedactionLimits::default()))
|
||||
}
|
||||
|
||||
pub(crate) fn into_parts(self) -> (ServiceIdentity, String, RedactionLimits) {
|
||||
(self.identity, self.filter, self.redaction_limits)
|
||||
}
|
||||
|
||||
pub(crate) fn identity(&self) -> &ServiceIdentity {
|
||||
&self.identity
|
||||
}
|
||||
|
||||
pub(crate) fn redaction_limits(&self) -> RedactionLimits {
|
||||
self.redaction_limits
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ObservabilityConfigError {
|
||||
#[error("invalid observability identity field: {field}")]
|
||||
InvalidIdentity { field: &'static str },
|
||||
#[error("observability environment variable is not valid UTF-8: {field}")]
|
||||
InvalidEnvironmentEncoding { field: &'static str },
|
||||
}
|
||||
|
||||
fn env_value_or_default(
|
||||
field: &'static str,
|
||||
value: Result<String, env::VarError>,
|
||||
default: &'static str,
|
||||
) -> Result<String, ObservabilityConfigError> {
|
||||
match value {
|
||||
Ok(value) => Ok(value),
|
||||
Err(env::VarError::NotPresent) => Ok(default.to_owned()),
|
||||
Err(env::VarError::NotUnicode(_)) => {
|
||||
Err(ObservabilityConfigError::InvalidEnvironmentEncoding { field })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_label(field: &'static str, value: &str) -> Result<(), ObservabilityConfigError> {
|
||||
let valid = !value.is_empty()
|
||||
&& value.len() <= MAX_IDENTITY_LABEL_BYTES
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+'));
|
||||
|
||||
if valid {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ObservabilityConfigError::InvalidIdentity { field })
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::ffi::OsString;
|
||||
|
||||
use super::{ObservabilityConfigError, ServiceIdentity, env_value_or_default};
|
||||
|
||||
#[test]
|
||||
fn accepts_release_and_environment_labels() {
|
||||
let identity = ServiceIdentity::try_new("admin-api", "0.3.1+build.7", "production")
|
||||
.expect("identity must be valid");
|
||||
|
||||
assert_eq!(identity.service(), "admin-api");
|
||||
assert_eq!(identity.version(), "0.3.1+build.7");
|
||||
assert_eq!(identity.environment(), "production");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_utf8_environment_values() {
|
||||
let error = env_value_or_default(
|
||||
"CRANK_ENVIRONMENT",
|
||||
Err(std::env::VarError::NotUnicode(OsString::from(
|
||||
"invalid-environment",
|
||||
))),
|
||||
"development",
|
||||
)
|
||||
.expect_err("non-UTF-8 values must not be replaced with defaults");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
ObservabilityConfigError::InvalidEnvironmentEncoding {
|
||||
field: "CRANK_ENVIRONMENT"
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
use std::fmt;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
||||
pub struct RequestId(String);
|
||||
|
||||
impl RequestId {
|
||||
pub const MAX_LEN: usize = 128;
|
||||
|
||||
pub fn resolve(candidate: Option<&str>) -> Self {
|
||||
candidate
|
||||
.filter(|value| Self::is_valid(value))
|
||||
.map(|value| Self(value.to_owned()))
|
||||
.unwrap_or_else(|| Self(Uuid::now_v7().to_string()))
|
||||
}
|
||||
|
||||
pub fn is_valid(value: &str) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= Self::MAX_LEN
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
|
||||
pub fn into_string(self) -> String {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for RequestId {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration};
|
||||
|
||||
use sentry::{
|
||||
ClientInitGuard, ClientOptions,
|
||||
protocol::{Event, Level},
|
||||
types::Dsn,
|
||||
};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::{
|
||||
RedactionLimits, ServiceIdentity, propagation::current_trace_id, redaction::truncate_string,
|
||||
};
|
||||
|
||||
const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
|
||||
const CRITICAL_ERROR_MESSAGE: &str = "critical error";
|
||||
const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
tokio::task_local! {
|
||||
static REQUEST_ID: String;
|
||||
}
|
||||
|
||||
pub struct SentryConfig {
|
||||
dsn: Option<Dsn>,
|
||||
}
|
||||
|
||||
impl SentryConfig {
|
||||
pub fn parse(value: Option<&str>) -> Result<Self, SentryConfigError> {
|
||||
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return Ok(Self { dsn: None });
|
||||
};
|
||||
|
||||
let dsn = value
|
||||
.parse::<Dsn>()
|
||||
.map_err(|_| SentryConfigError::InvalidDsn)?;
|
||||
Ok(Self { dsn: Some(dsn) })
|
||||
}
|
||||
|
||||
pub fn from_env() -> Result<Self, SentryConfigError> {
|
||||
match env::var(SENTRY_DSN_ENV) {
|
||||
Ok(value) => Self::parse(Some(&value)),
|
||||
Err(env::VarError::NotPresent) => Self::parse(None),
|
||||
Err(env::VarError::NotUnicode(_)) => Err(SentryConfigError::InvalidEnvironmentEncoding),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn enabled(&self) -> bool {
|
||||
self.dsn.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for SentryConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("SentryConfig")
|
||||
.field("enabled", &self.enabled())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SentryConfigError {
|
||||
#[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")]
|
||||
InvalidDsn,
|
||||
#[error("CRANK_SENTRY_DSN is not valid UTF-8")]
|
||||
InvalidEnvironmentEncoding,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum CriticalErrorCategory {
|
||||
Panic,
|
||||
Startup,
|
||||
Internal,
|
||||
DataIntegrity,
|
||||
}
|
||||
|
||||
impl CriticalErrorCategory {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Panic => "panic",
|
||||
Self::Startup => "startup",
|
||||
Self::Internal => "internal",
|
||||
Self::DataIntegrity => "data_integrity",
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"panic" => Some(Self::Panic),
|
||||
"startup" => Some(Self::Startup),
|
||||
"internal" => Some(Self::Internal),
|
||||
"data_integrity" => Some(Self::DataIntegrity),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn capture_critical_error(category: CriticalErrorCategory) {
|
||||
let mut tags = correlation_tags();
|
||||
tags.insert("category".to_owned(), category.as_str().to_owned());
|
||||
sentry::capture_event(Event {
|
||||
level: Level::Error,
|
||||
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
|
||||
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
|
||||
tags,
|
||||
..Event::default()
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn with_request_correlation<F>(request_id: String, future: F) -> F::Output
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
REQUEST_ID.scope(request_id, future).await
|
||||
}
|
||||
|
||||
pub(crate) fn init_sentry(
|
||||
identity: &ServiceIdentity,
|
||||
limits: RedactionLimits,
|
||||
config: SentryConfig,
|
||||
) -> Option<ClientInitGuard> {
|
||||
let dsn = config.dsn?;
|
||||
let identity = identity.clone();
|
||||
let options = client_options(identity, limits);
|
||||
Some(sentry::init((dsn, options)))
|
||||
}
|
||||
|
||||
fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
|
||||
let release = identity.version().to_owned();
|
||||
let environment = identity.environment().to_owned();
|
||||
let service = identity.service().to_owned();
|
||||
let sanitizer_identity = identity.clone();
|
||||
|
||||
let mut options = ClientOptions::default();
|
||||
options.release = Some(Cow::Owned(release));
|
||||
options.environment = Some(Cow::Owned(environment));
|
||||
options.server_name = Some(Cow::Owned(service));
|
||||
options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled;
|
||||
options.max_breadcrumbs = 0;
|
||||
options.attach_stacktrace = false;
|
||||
options.send_default_pii = false;
|
||||
options.before_send = Some(std::sync::Arc::new(move |event| {
|
||||
Some(sanitize_event(event, &sanitizer_identity, limits))
|
||||
}));
|
||||
options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT;
|
||||
options.auto_session_tracking = false;
|
||||
options.enable_logs = false;
|
||||
options.enable_metrics = false;
|
||||
options
|
||||
}
|
||||
|
||||
fn sanitize_event(
|
||||
event: Event<'static>,
|
||||
identity: &ServiceIdentity,
|
||||
limits: RedactionLimits,
|
||||
) -> Event<'static> {
|
||||
let category = event
|
||||
.tags
|
||||
.get("category")
|
||||
.and_then(|value| CriticalErrorCategory::parse(value))
|
||||
.unwrap_or_else(|| {
|
||||
if event.exception.is_empty() {
|
||||
CriticalErrorCategory::Internal
|
||||
} else {
|
||||
CriticalErrorCategory::Panic
|
||||
}
|
||||
});
|
||||
let mut tags = correlation_tags()
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key, truncate_string(&value, limits.max_string_bytes)))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for key in ["request_id", "trace_id"] {
|
||||
if let Some(value) = event.tags.get(key) {
|
||||
tags.entry(key.to_owned())
|
||||
.or_insert_with(|| truncate_string(value, limits.max_string_bytes));
|
||||
}
|
||||
}
|
||||
tags.insert("service".to_owned(), identity.service().to_owned());
|
||||
tags.insert("category".to_owned(), category.as_str().to_owned());
|
||||
|
||||
enforce_event_budget(
|
||||
Event {
|
||||
event_id: event.event_id,
|
||||
level: Level::Error,
|
||||
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
|
||||
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
|
||||
timestamp: event.timestamp,
|
||||
server_name: Some(Cow::Owned(identity.service().to_owned())),
|
||||
release: Some(Cow::Owned(identity.version().to_owned())),
|
||||
environment: Some(Cow::Owned(identity.environment().to_owned())),
|
||||
tags,
|
||||
..Event::default()
|
||||
},
|
||||
limits.max_event_bytes,
|
||||
)
|
||||
}
|
||||
|
||||
fn correlation_tags() -> BTreeMap<String, String> {
|
||||
let mut tags = BTreeMap::new();
|
||||
if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) {
|
||||
tags.insert("request_id".to_owned(), request_id);
|
||||
}
|
||||
if let Some(trace_id) = current_trace_id() {
|
||||
tags.insert("trace_id".to_owned(), trace_id);
|
||||
}
|
||||
tags
|
||||
}
|
||||
|
||||
fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
|
||||
if serialized_event_len(&event) <= max_event_bytes {
|
||||
return event;
|
||||
}
|
||||
|
||||
event.tags.remove("request_id");
|
||||
event.tags.remove("trace_id");
|
||||
event
|
||||
}
|
||||
|
||||
fn serialized_event_len(event: &Event<'_>) -> usize {
|
||||
serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
use opentelemetry::trace::TracerProvider as _;
|
||||
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||
use sentry::{
|
||||
Envelope, Hub,
|
||||
protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values},
|
||||
};
|
||||
|
||||
use super::{
|
||||
CriticalErrorCategory, capture_critical_error, client_options, sanitize_event,
|
||||
with_request_correlation,
|
||||
};
|
||||
use crate::{
|
||||
ObservabilityConfig, RedactionLimits, ServiceIdentity,
|
||||
logging::build_subscriber_with_tracer,
|
||||
};
|
||||
|
||||
fn identity() -> ServiceIdentity {
|
||||
ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_disables_non_error_telemetry() {
|
||||
let options = client_options(identity(), RedactionLimits::default());
|
||||
|
||||
assert_eq!(options.max_breadcrumbs, 0);
|
||||
assert!(!options.attach_stacktrace);
|
||||
assert!(!options.send_default_pii);
|
||||
assert!(!options.auto_session_tracking);
|
||||
assert!(!options.enable_logs);
|
||||
assert!(!options.enable_metrics);
|
||||
assert!(options.before_send.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitizer_uses_a_strict_allowlist() {
|
||||
let mut tags = BTreeMap::new();
|
||||
tags.insert("category".to_owned(), "data_integrity".to_owned());
|
||||
tags.insert("secret".to_owned(), "must-not-leak".to_owned());
|
||||
let mut contexts = BTreeMap::new();
|
||||
contexts.insert(
|
||||
"secret".to_owned(),
|
||||
Context::Other(BTreeMap::from([(
|
||||
"token".to_owned(),
|
||||
Value::String("must-not-leak".to_owned()),
|
||||
)])),
|
||||
);
|
||||
let event = sentry::protocol::Event {
|
||||
message: Some("password=must-not-leak".to_owned()),
|
||||
request: Some(Request::default()),
|
||||
user: Some(User::default()),
|
||||
breadcrumbs: Values {
|
||||
values: vec![Breadcrumb::default()],
|
||||
},
|
||||
exception: Values {
|
||||
values: vec![Exception {
|
||||
value: Some("must-not-leak".to_owned()),
|
||||
..Exception::default()
|
||||
}],
|
||||
},
|
||||
contexts,
|
||||
extra: BTreeMap::from([(
|
||||
"payload".to_owned(),
|
||||
Value::String("must-not-leak".to_owned()),
|
||||
)]),
|
||||
tags,
|
||||
..sentry::protocol::Event::default()
|
||||
};
|
||||
|
||||
let cleaned = sanitize_event(event, &identity(), RedactionLimits::default());
|
||||
let serialized = serde_json::to_string(&cleaned).expect("serialize event");
|
||||
|
||||
assert_eq!(cleaned.message.as_deref(), Some("critical error"));
|
||||
assert_eq!(
|
||||
cleaned.tags.get("category").map(String::as_str),
|
||||
Some(CriticalErrorCategory::DataIntegrity.as_str())
|
||||
);
|
||||
assert!(cleaned.request.is_none());
|
||||
assert!(cleaned.user.is_none());
|
||||
assert!(cleaned.breadcrumbs.is_empty());
|
||||
assert!(cleaned.exception.is_empty());
|
||||
assert!(cleaned.contexts.is_empty());
|
||||
assert!(cleaned.extra.is_empty());
|
||||
assert!(!serialized.contains("must-not-leak"));
|
||||
assert!(!serialized.contains("password"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitizer_honours_total_event_budget() {
|
||||
let limits = RedactionLimits {
|
||||
max_string_bytes: 8 * 1024,
|
||||
max_event_bytes: 512,
|
||||
..RedactionLimits::default()
|
||||
};
|
||||
let event = sentry::protocol::Event {
|
||||
tags: BTreeMap::from([
|
||||
("category".to_owned(), "internal".to_owned()),
|
||||
("request_id".to_owned(), "r".repeat(8 * 1024)),
|
||||
("trace_id".to_owned(), "t".repeat(8 * 1024)),
|
||||
]),
|
||||
..sentry::protocol::Event::default()
|
||||
};
|
||||
|
||||
let cleaned = sanitize_event(event, &identity(), limits);
|
||||
let serialized = serde_json::to_vec(&cleaned).expect("serialize event");
|
||||
|
||||
assert!(serialized.len() <= limits.max_event_bytes);
|
||||
assert_eq!(
|
||||
cleaned.tags.get("category").map(String::as_str),
|
||||
Some("internal")
|
||||
);
|
||||
assert_eq!(
|
||||
cleaned.tags.get("service").map(String::as_str),
|
||||
Some("admin-api")
|
||||
);
|
||||
assert!(!cleaned.tags.contains_key("request_id"));
|
||||
assert!(!cleaned.tags.contains_key("trace_id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expected_application_errors_do_not_create_critical_events() {
|
||||
let options =
|
||||
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
|
||||
let events = sentry::test::with_captured_events_options(
|
||||
|| tracing::error!("ordinary product error"),
|
||||
options,
|
||||
);
|
||||
|
||||
assert!(events.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_critical_error_is_correlated_and_sanitized() {
|
||||
let options =
|
||||
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.expect("runtime");
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let tracer = provider.tracer("critical-error-test");
|
||||
let subscriber = build_subscriber_with_tracer(
|
||||
ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
|
||||
std::io::sink,
|
||||
Some(tracer),
|
||||
)
|
||||
.expect("subscriber");
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let events = sentry::test::with_captured_events_options(
|
||||
|| {
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
runtime.block_on(with_request_correlation("request-123".to_owned(), async {
|
||||
let span = tracing::info_span!(target: "crank::trace", "http.request");
|
||||
let _span_guard = span.enter();
|
||||
capture_critical_error(CriticalErrorCategory::DataIntegrity);
|
||||
}));
|
||||
});
|
||||
},
|
||||
options,
|
||||
);
|
||||
|
||||
assert_eq!(events.len(), 1);
|
||||
let event = &events[0];
|
||||
assert_eq!(event.message.as_deref(), Some("critical error"));
|
||||
assert_eq!(
|
||||
event.tags.get("category").map(String::as_str),
|
||||
Some("data_integrity")
|
||||
);
|
||||
assert_eq!(
|
||||
event.tags.get("request_id").map(String::as_str),
|
||||
Some("request-123")
|
||||
);
|
||||
assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
|
||||
assert_eq!(event.release.as_deref(), Some("1.2.3"));
|
||||
assert_eq!(event.environment.as_deref(), Some("test"));
|
||||
assert_eq!(event.server_name.as_deref(), Some("admin-api"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn panic_creates_exactly_one_sanitized_critical_event() {
|
||||
let options =
|
||||
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
|
||||
let events = sentry::test::with_captured_events_options(
|
||||
|| {
|
||||
let result = std::panic::catch_unwind(|| {
|
||||
panic!("password=must-not-leak");
|
||||
});
|
||||
assert!(result.is_err());
|
||||
},
|
||||
options,
|
||||
);
|
||||
|
||||
assert_eq!(events.len(), 1);
|
||||
let event = &events[0];
|
||||
assert_eq!(
|
||||
event.tags.get("category").map(String::as_str),
|
||||
Some("panic")
|
||||
);
|
||||
let serialized = serde_json::to_string(event).expect("serialize event");
|
||||
assert!(!serialized.contains("must-not-leak"));
|
||||
assert!(!serialized.contains("password"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn receiver_failure_does_not_change_product_result_or_recurse() {
|
||||
struct DroppingTransport {
|
||||
attempts: AtomicUsize,
|
||||
}
|
||||
|
||||
impl sentry::Transport for DroppingTransport {
|
||||
fn send_envelope(&self, _envelope: Envelope) {
|
||||
self.attempts.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
let transport = Arc::new(DroppingTransport {
|
||||
attempts: AtomicUsize::new(0),
|
||||
});
|
||||
let mut options =
|
||||
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
|
||||
options.dsn = Some(
|
||||
"https://public@example.invalid/1"
|
||||
.parse()
|
||||
.expect("valid test DSN"),
|
||||
);
|
||||
options.transport = Some(Arc::new(transport.clone()));
|
||||
let client = Arc::new(sentry::Client::from(options));
|
||||
let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default())));
|
||||
|
||||
let product_result = Hub::run(hub, || {
|
||||
capture_critical_error(CriticalErrorCategory::Internal);
|
||||
42
|
||||
});
|
||||
|
||||
assert_eq!(product_result, 42);
|
||||
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum OperationalIncident {
|
||||
InvocationHistoryLost,
|
||||
}
|
||||
|
||||
static INVOCATION_HISTORY_LOST_TOTAL: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub fn record_operational_incident(incident: OperationalIncident) {
|
||||
let counter = counter(incident);
|
||||
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
|
||||
value.checked_add(1)
|
||||
});
|
||||
match incident {
|
||||
OperationalIncident::InvocationHistoryLost => {
|
||||
metrics::counter!("crank_invocation_history_lost_total").increment(1);
|
||||
metrics::counter!(
|
||||
"crank_telemetry_export_failures_total",
|
||||
"signal_type" => "invocation_history",
|
||||
"exporter" => "postgres"
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn operational_incident_total(incident: OperationalIncident) -> u64 {
|
||||
counter(incident).load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn counter(incident: OperationalIncident) -> &'static AtomicU64 {
|
||||
match incident {
|
||||
OperationalIncident::InvocationHistoryLost => &INVOCATION_HISTORY_LOST_TOTAL,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use axum::{
|
||||
extract::{MatchedPath, Request},
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use metrics::{Gauge, Unit};
|
||||
|
||||
use crate::{MetricKind, MetricUnit, metric_schema};
|
||||
|
||||
pub async fn record_http_request(request: Request, next: Next) -> Response {
|
||||
let route = request
|
||||
.extensions()
|
||||
.get::<MatchedPath>()
|
||||
.map_or("unmatched", MatchedPath::as_str)
|
||||
.to_owned();
|
||||
let method = normalized_http_method(request.method().as_str());
|
||||
let started_at = Instant::now();
|
||||
let _inflight = GaugeGuard::increment("crank_http_inflight");
|
||||
|
||||
let response = next.run(request).await;
|
||||
let status_class = status_class(response.status().as_u16());
|
||||
|
||||
metrics::counter!(
|
||||
"crank_http_requests_total",
|
||||
"route" => route.clone(),
|
||||
"method" => method,
|
||||
"status_class" => status_class
|
||||
)
|
||||
.increment(1);
|
||||
metrics::histogram!(
|
||||
"crank_http_request_duration_seconds",
|
||||
"route" => route,
|
||||
"method" => method
|
||||
)
|
||||
.record(started_at.elapsed().as_secs_f64());
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
pub fn record_db_pool_connections(total: u32, idle: usize) {
|
||||
let idle = idle.min(total as usize) as f64;
|
||||
metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(idle);
|
||||
metrics::gauge!("crank_db_pool_connections", "state" => "used").set(f64::from(total) - idle);
|
||||
}
|
||||
|
||||
pub(crate) fn register_metric_schema() {
|
||||
for definition in metric_schema() {
|
||||
let unit = match definition.unit {
|
||||
MetricUnit::Count => Unit::Count,
|
||||
MetricUnit::Seconds => Unit::Seconds,
|
||||
};
|
||||
match definition.kind {
|
||||
MetricKind::Counter => {
|
||||
metrics::describe_counter!(definition.name, unit, definition.description);
|
||||
}
|
||||
MetricKind::Gauge => {
|
||||
metrics::describe_gauge!(definition.name, unit, definition.description);
|
||||
}
|
||||
MetricKind::Histogram => {
|
||||
metrics::describe_histogram!(definition.name, unit, definition.description);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metrics::gauge!("crank_http_inflight").set(0.0);
|
||||
metrics::gauge!("crank_mcp_active_sessions").set(0.0);
|
||||
metrics::gauge!("crank_runtime_inflight").set(0.0);
|
||||
metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(0.0);
|
||||
metrics::gauge!("crank_db_pool_connections", "state" => "used").set(0.0);
|
||||
metrics::gauge!("crank_catalog_tools").set(0.0);
|
||||
metrics::gauge!("crank_catalog_estimated_context_tokens").set(0.0);
|
||||
metrics::gauge!("crank_catalog_warnings").set(0.0);
|
||||
}
|
||||
|
||||
fn normalized_http_method(method: &str) -> &'static str {
|
||||
match method {
|
||||
"GET" => "GET",
|
||||
"POST" => "POST",
|
||||
"PUT" => "PUT",
|
||||
"PATCH" => "PATCH",
|
||||
"DELETE" => "DELETE",
|
||||
"OPTIONS" => "OPTIONS",
|
||||
"HEAD" => "HEAD",
|
||||
"CONNECT" => "CONNECT",
|
||||
"TRACE" => "TRACE",
|
||||
_ => "OTHER",
|
||||
}
|
||||
}
|
||||
|
||||
fn status_class(status: u16) -> &'static str {
|
||||
match status {
|
||||
100..=199 => "1xx",
|
||||
200..=299 => "2xx",
|
||||
300..=399 => "3xx",
|
||||
400..=499 => "4xx",
|
||||
500..=599 => "5xx",
|
||||
_ => "other",
|
||||
}
|
||||
}
|
||||
|
||||
struct GaugeGuard {
|
||||
gauge: Gauge,
|
||||
}
|
||||
|
||||
impl GaugeGuard {
|
||||
fn increment(name: &'static str) -> Self {
|
||||
let gauge = metrics::gauge!(name);
|
||||
gauge.increment(1.0);
|
||||
Self { gauge }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GaugeGuard {
|
||||
fn drop(&mut self) {
|
||||
self.gauge.decrement(1.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{normalized_http_method, status_class};
|
||||
|
||||
#[test]
|
||||
fn normalizes_unbounded_http_values() {
|
||||
assert_eq!(normalized_http_method("GET"), "GET");
|
||||
assert_eq!(normalized_http_method("CUSTOM-user-controlled"), "OTHER");
|
||||
assert_eq!(status_class(204), "2xx");
|
||||
assert_eq!(status_class(429), "4xx");
|
||||
assert_eq!(status_class(999), "other");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
mod config;
|
||||
mod correlation;
|
||||
mod error_reporting;
|
||||
mod incidents;
|
||||
mod instrumentation;
|
||||
mod lifecycle;
|
||||
mod logging;
|
||||
mod metrics_schema;
|
||||
mod otlp;
|
||||
mod prometheus;
|
||||
mod propagation;
|
||||
mod redaction;
|
||||
mod schema;
|
||||
|
||||
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
|
||||
pub use correlation::RequestId;
|
||||
pub use error_reporting::{
|
||||
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
|
||||
with_request_correlation,
|
||||
};
|
||||
pub use incidents::{OperationalIncident, operational_incident_total, record_operational_incident};
|
||||
pub use instrumentation::{record_db_pool_connections, record_http_request};
|
||||
pub use lifecycle::{ObservabilityInitError, ObservabilityLifecycle, init};
|
||||
pub use logging::build_subscriber;
|
||||
pub use metrics_schema::{
|
||||
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
|
||||
};
|
||||
pub use otlp::{
|
||||
OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError, build_tracer_provider,
|
||||
};
|
||||
pub use prometheus::{
|
||||
MetricsConfig, MetricsConfigError, MetricsServeError, MetricsSurface, MetricsSurfaceError,
|
||||
};
|
||||
pub use propagation::{inject_current_trace_context, set_remote_trace_parent};
|
||||
pub use redaction::{
|
||||
REDACTED_MARKER, RedactionLimits, RedactionLimitsError, SafeJsonError, redact_value, safe_json,
|
||||
};
|
||||
@@ -0,0 +1,90 @@
|
||||
use std::{fmt, io};
|
||||
|
||||
use thiserror::Error;
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
|
||||
use crate::{
|
||||
MetricsConfig, MetricsSurface, MetricsSurfaceError, ObservabilityConfig,
|
||||
ObservabilityConfigError, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError,
|
||||
RedactionLimitsError, SentryConfig, SentryConfigError, error_reporting::init_sentry,
|
||||
instrumentation::register_metric_schema, logging::build_subscriber_with_tracer,
|
||||
otlp::build_tracer_provider, prometheus::install_prometheus_recorder,
|
||||
propagation::install_trace_context_propagator,
|
||||
};
|
||||
|
||||
#[must_use = "observability resources must be retained until process shutdown"]
|
||||
pub struct ObservabilityLifecycle {
|
||||
metrics_handle: metrics_exporter_prometheus::PrometheusHandle,
|
||||
tracer_provider: Option<opentelemetry_sdk::trace::SdkTracerProvider>,
|
||||
sentry_guard: Option<sentry::ClientInitGuard>,
|
||||
}
|
||||
|
||||
impl ObservabilityLifecycle {
|
||||
pub fn init(config: ObservabilityConfig) -> Result<Self, ObservabilityInitError> {
|
||||
let identity = config.identity().clone();
|
||||
let redaction_limits = config.redaction_limits();
|
||||
let sentry_config = SentryConfig::from_env()?;
|
||||
let trace_config = OtlpTraceConfig::from_env()?;
|
||||
let tracing = build_tracer_provider(&identity, &trace_config)?;
|
||||
let tracer = tracing.as_ref().map(|(_, tracer)| tracer.clone());
|
||||
install_trace_context_propagator();
|
||||
build_subscriber_with_tracer(config, io::stdout, tracer)?
|
||||
.try_init()
|
||||
.map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?;
|
||||
let metrics_handle = install_prometheus_recorder(&identity)?;
|
||||
register_metric_schema();
|
||||
let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config);
|
||||
|
||||
Ok(Self {
|
||||
metrics_handle,
|
||||
tracer_provider: tracing.map(|(provider, _)| provider),
|
||||
sentry_guard,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn metrics_surface(&self, config: MetricsConfig) -> MetricsSurface {
|
||||
MetricsSurface::new(config, self.metrics_handle.clone())
|
||||
}
|
||||
|
||||
pub fn traces_enabled(&self) -> bool {
|
||||
self.tracer_provider.is_some()
|
||||
}
|
||||
|
||||
pub fn critical_errors_enabled(&self) -> bool {
|
||||
self.sentry_guard.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ObservabilityLifecycle {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ObservabilityLifecycle")
|
||||
.field("traces_enabled", &self.traces_enabled())
|
||||
.field("critical_errors_enabled", &self.critical_errors_enabled())
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn init(config: ObservabilityConfig) -> Result<ObservabilityLifecycle, ObservabilityInitError> {
|
||||
ObservabilityLifecycle::init(config)
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ObservabilityInitError {
|
||||
#[error(transparent)]
|
||||
InvalidConfig(#[from] ObservabilityConfigError),
|
||||
#[error(transparent)]
|
||||
InvalidRedactionLimits(#[from] RedactionLimitsError),
|
||||
#[error("invalid log filter")]
|
||||
InvalidFilter,
|
||||
#[error("global tracing subscriber is already initialized")]
|
||||
SubscriberAlreadyInitialized,
|
||||
#[error(transparent)]
|
||||
Metrics(#[from] MetricsSurfaceError),
|
||||
#[error(transparent)]
|
||||
OtlpConfig(#[from] OtlpTraceConfigError),
|
||||
#[error(transparent)]
|
||||
Otlp(#[from] OtlpTraceError),
|
||||
#[error(transparent)]
|
||||
SentryConfig(#[from] SentryConfigError),
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
use std::fmt;
|
||||
|
||||
use serde_json::{Map, Number, Value};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tracing::{Event, Subscriber, field::Visit};
|
||||
use tracing_subscriber::{
|
||||
EnvFilter, Layer,
|
||||
filter::filter_fn,
|
||||
fmt::{FmtContext, FormatEvent, FormatFields, MakeWriter, format::Writer},
|
||||
layer::SubscriberExt,
|
||||
registry::LookupSpan,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity,
|
||||
propagation::current_trace_id,
|
||||
redaction::{redact_value, truncate_string},
|
||||
schema::LogEnvelope,
|
||||
};
|
||||
|
||||
pub fn build_subscriber<W>(
|
||||
config: ObservabilityConfig,
|
||||
writer: W,
|
||||
) -> Result<impl Subscriber + Send + Sync, ObservabilityInitError>
|
||||
where
|
||||
W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static,
|
||||
{
|
||||
build_subscriber_with_tracer(config, writer, None)
|
||||
}
|
||||
|
||||
pub(crate) fn build_subscriber_with_tracer<W>(
|
||||
config: ObservabilityConfig,
|
||||
writer: W,
|
||||
tracer: Option<opentelemetry_sdk::trace::SdkTracer>,
|
||||
) -> Result<impl Subscriber + Send + Sync, ObservabilityInitError>
|
||||
where
|
||||
W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static,
|
||||
{
|
||||
let (identity, filter, limits) = config.into_parts();
|
||||
limits.validate()?;
|
||||
let filter = EnvFilter::try_new(filter).map_err(|_| ObservabilityInitError::InvalidFilter)?;
|
||||
let formatter = JsonEventFormatter::new(identity, limits);
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_ansi(false)
|
||||
.event_format(formatter)
|
||||
.with_writer(writer)
|
||||
.with_filter(filter);
|
||||
let otel_layer = tracer.map(|tracer| {
|
||||
tracing_opentelemetry::layer()
|
||||
.with_tracer(tracer)
|
||||
.with_filter(filter_fn(|metadata| {
|
||||
metadata.is_span() && metadata.target() == "crank::trace"
|
||||
}))
|
||||
});
|
||||
|
||||
Ok(tracing_subscriber::registry()
|
||||
.with(fmt_layer)
|
||||
.with(otel_layer))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct JsonEventFormatter {
|
||||
identity: ServiceIdentity,
|
||||
limits: RedactionLimits,
|
||||
}
|
||||
|
||||
impl JsonEventFormatter {
|
||||
fn new(identity: ServiceIdentity, limits: RedactionLimits) -> Self {
|
||||
Self { identity, limits }
|
||||
}
|
||||
|
||||
fn envelope(&self, event: &Event<'_>) -> Result<LogEnvelope, fmt::Error> {
|
||||
let metadata = event.metadata();
|
||||
let mut visitor = JsonFieldVisitor::default();
|
||||
event.record(&mut visitor);
|
||||
let mut raw_fields = visitor.fields;
|
||||
let request_id = take_correlation_id(&mut raw_fields, "request_id")
|
||||
.map(|value| truncate_string(&value, self.limits.max_string_bytes));
|
||||
let trace_id = take_correlation_id(&mut raw_fields, "trace_id")
|
||||
.or_else(current_trace_id)
|
||||
.map(|value| truncate_string(&value, self.limits.max_string_bytes));
|
||||
let cleaned = redact_value(&Value::Object(raw_fields), self.limits);
|
||||
let fields = cleaned.as_object().cloned().unwrap_or_default();
|
||||
let timestamp = OffsetDateTime::now_utc()
|
||||
.format(&Rfc3339)
|
||||
.map_err(|_| fmt::Error)?;
|
||||
|
||||
Ok(LogEnvelope {
|
||||
timestamp,
|
||||
level: metadata.level().as_str().to_owned(),
|
||||
service: self.identity.service().to_owned(),
|
||||
version: self.identity.version().to_owned(),
|
||||
environment: self.identity.environment().to_owned(),
|
||||
target: truncate_string(metadata.target(), self.limits.max_string_bytes),
|
||||
event: truncate_string(metadata.name(), self.limits.max_string_bytes),
|
||||
request_id,
|
||||
trace_id,
|
||||
fields,
|
||||
})
|
||||
}
|
||||
|
||||
fn serialize_bounded(&self, mut envelope: LogEnvelope) -> Result<String, fmt::Error> {
|
||||
let line_budget = self.limits.max_event_bytes.saturating_sub(1);
|
||||
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
|
||||
if serialized.len() <= line_budget {
|
||||
return Ok(serialized);
|
||||
}
|
||||
|
||||
envelope.fields = Map::from_iter([("truncated".to_owned(), Value::Bool(true))]);
|
||||
let fallback_string_limit = self.limits.max_string_bytes.min(64);
|
||||
envelope.target = truncate_string(&envelope.target, fallback_string_limit);
|
||||
envelope.event = truncate_string(&envelope.event, fallback_string_limit);
|
||||
envelope.request_id = envelope
|
||||
.request_id
|
||||
.map(|value| truncate_string(&value, fallback_string_limit));
|
||||
envelope.trace_id = envelope
|
||||
.trace_id
|
||||
.map(|value| truncate_string(&value, fallback_string_limit));
|
||||
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
|
||||
if serialized.len() <= line_budget {
|
||||
return Ok(serialized);
|
||||
}
|
||||
|
||||
envelope.request_id = None;
|
||||
envelope.trace_id = None;
|
||||
envelope.target = truncate_string(&envelope.target, 16);
|
||||
envelope.event = truncate_string(&envelope.event, 16);
|
||||
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
|
||||
(serialized.len() <= line_budget)
|
||||
.then_some(serialized)
|
||||
.ok_or(fmt::Error)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, N> FormatEvent<S, N> for JsonEventFormatter
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
N: for<'writer> FormatFields<'writer> + 'static,
|
||||
{
|
||||
fn format_event(
|
||||
&self,
|
||||
_ctx: &FmtContext<'_, S, N>,
|
||||
mut writer: Writer<'_>,
|
||||
event: &Event<'_>,
|
||||
) -> fmt::Result {
|
||||
let serialized = self.serialize_bounded(self.envelope(event)?)?;
|
||||
writer.write_str(&serialized)?;
|
||||
writer.write_char('\n')
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct JsonFieldVisitor {
|
||||
fields: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl JsonFieldVisitor {
|
||||
fn insert(&mut self, field: &tracing::field::Field, value: Value) {
|
||||
self.fields.insert(field.name().to_owned(), value);
|
||||
}
|
||||
}
|
||||
|
||||
impl Visit for JsonFieldVisitor {
|
||||
fn record_i64(&mut self, field: &tracing::field::Field, value: i64) {
|
||||
self.insert(field, Value::Number(value.into()));
|
||||
}
|
||||
|
||||
fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
|
||||
self.insert(field, Value::Number(value.into()));
|
||||
}
|
||||
|
||||
fn record_bool(&mut self, field: &tracing::field::Field, value: bool) {
|
||||
self.insert(field, Value::Bool(value));
|
||||
}
|
||||
|
||||
fn record_f64(&mut self, field: &tracing::field::Field, value: f64) {
|
||||
let value = Number::from_f64(value)
|
||||
.map(Value::Number)
|
||||
.unwrap_or(Value::Null);
|
||||
self.insert(field, value);
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.insert(field, Value::String(value.to_owned()));
|
||||
}
|
||||
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
|
||||
let rendered = format!("{value:?}");
|
||||
let value = if is_correlation_field(field.name()) {
|
||||
Value::String(debug_scalar(&rendered))
|
||||
} else {
|
||||
match serde_json::from_str(&rendered) {
|
||||
Ok(value @ (Value::Object(_) | Value::Array(_))) => value,
|
||||
_ if field.name() == "message" || is_safe_display_scalar(&rendered) => {
|
||||
Value::String(rendered)
|
||||
}
|
||||
_ => Value::String(crate::REDACTED_MARKER.to_owned()),
|
||||
}
|
||||
};
|
||||
self.insert(field, value);
|
||||
}
|
||||
}
|
||||
|
||||
fn take_correlation_id(fields: &mut Map<String, Value>, name: &str) -> Option<String> {
|
||||
let value = fields.remove(name)?;
|
||||
let value = match value {
|
||||
Value::String(value) => value,
|
||||
Value::Number(value) => value.to_string(),
|
||||
Value::Bool(value) => value.to_string(),
|
||||
Value::Null | Value::Array(_) | Value::Object(_) => return None,
|
||||
};
|
||||
(!value.is_empty()).then_some(value)
|
||||
}
|
||||
|
||||
fn is_correlation_field(name: &str) -> bool {
|
||||
matches!(name, "request_id" | "trace_id" | "correlation_id")
|
||||
}
|
||||
|
||||
fn debug_scalar(rendered: &str) -> String {
|
||||
serde_json::from_str::<String>(rendered).unwrap_or_else(|_| rendered.to_owned())
|
||||
}
|
||||
|
||||
fn is_safe_display_scalar(rendered: &str) -> bool {
|
||||
!rendered.is_empty()
|
||||
&& rendered.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'+' | b'@')
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use opentelemetry::{global, trace::TracerProvider as _};
|
||||
use opentelemetry_sdk::{
|
||||
error::OTelSdkResult,
|
||||
propagation::TraceContextPropagator,
|
||||
trace::{SdkTracerProvider, SpanData, SpanExporter},
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
use super::build_subscriber_with_tracer;
|
||||
use crate::{
|
||||
ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn trace_spans_ignore_the_log_level_filter() {
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let tracer = provider.tracer("trace-filter-test");
|
||||
let subscriber = build_subscriber_with_tracer(test_config("warn"), io::sink, Some(tracer))
|
||||
.expect("subscriber must build");
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let span = tracing::info_span!(target: "crank::trace", "http.request");
|
||||
let _span_guard = span.enter();
|
||||
let mut headers = axum::http::HeaderMap::new();
|
||||
|
||||
assert!(inject_current_trace_context(&mut headers));
|
||||
assert!(headers.contains_key("traceparent"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn otel_layer_does_not_export_events() {
|
||||
let exported = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = SdkTracerProvider::builder()
|
||||
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
|
||||
.build();
|
||||
let tracer = provider.tracer("event-filter-test");
|
||||
let subscriber = build_subscriber_with_tracer(test_config("info"), io::sink, Some(tracer))
|
||||
.expect("subscriber must build");
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
let span = tracing::info_span!(target: "crank::trace", "http.request");
|
||||
let _span_guard = span.enter();
|
||||
info!(password = "canary-secret", "sensitive event");
|
||||
});
|
||||
provider.force_flush().expect("span must be exported");
|
||||
|
||||
let spans = exported.lock().expect("capture lock");
|
||||
assert_eq!(spans.len(), 1);
|
||||
assert!(spans[0].events.is_empty());
|
||||
assert!(
|
||||
!format!("{:?}", spans[0])
|
||||
.as_bytes()
|
||||
.windows(b"canary-secret".len())
|
||||
.any(|window| window == b"canary-secret")
|
||||
);
|
||||
}
|
||||
|
||||
fn test_config(filter: &str) -> ObservabilityConfig {
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
|
||||
filter,
|
||||
RedactionLimits::default(),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
||||
|
||||
impl SpanExporter for CapturingExporter {
|
||||
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
|
||||
self.0.lock().expect("capture lock").extend(batch);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricKind {
|
||||
Counter,
|
||||
Gauge,
|
||||
Histogram,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricUnit {
|
||||
Count,
|
||||
Seconds,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct MetricDefinition {
|
||||
pub name: &'static str,
|
||||
pub kind: MetricKind,
|
||||
pub unit: MetricUnit,
|
||||
pub labels: &'static [&'static str],
|
||||
pub description: &'static str,
|
||||
}
|
||||
|
||||
pub const DURATION_BUCKETS_SECONDS: &[f64] = &[
|
||||
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
|
||||
];
|
||||
|
||||
const METRIC_SCHEMA: &[MetricDefinition] = &[
|
||||
counter(
|
||||
"crank_http_requests_total",
|
||||
&["route", "method", "status_class"],
|
||||
"Total HTTP requests.",
|
||||
),
|
||||
histogram(
|
||||
"crank_http_request_duration_seconds",
|
||||
&["route", "method"],
|
||||
"HTTP request duration in seconds.",
|
||||
),
|
||||
gauge(
|
||||
"crank_http_inflight",
|
||||
&[],
|
||||
"HTTP requests currently being processed.",
|
||||
),
|
||||
counter(
|
||||
"crank_mcp_requests_total",
|
||||
&["method", "response_mode", "outcome"],
|
||||
"Total MCP JSON-RPC requests.",
|
||||
),
|
||||
gauge(
|
||||
"crank_mcp_active_sessions",
|
||||
&[],
|
||||
"Active MCP transport sessions.",
|
||||
),
|
||||
counter(
|
||||
"crank_tool_invocations_total",
|
||||
&["source", "outcome", "error_kind"],
|
||||
"Total tool invocations.",
|
||||
),
|
||||
histogram(
|
||||
"crank_tool_invocation_duration_seconds",
|
||||
&["source", "outcome"],
|
||||
"Tool invocation duration in seconds.",
|
||||
),
|
||||
counter(
|
||||
"crank_upstream_requests_total",
|
||||
&["operation_kind", "outcome"],
|
||||
"Total upstream requests.",
|
||||
),
|
||||
histogram(
|
||||
"crank_upstream_request_duration_seconds",
|
||||
&["operation_kind", "outcome"],
|
||||
"Upstream request duration in seconds.",
|
||||
),
|
||||
gauge(
|
||||
"crank_runtime_inflight",
|
||||
&[],
|
||||
"Runtime executions currently in progress.",
|
||||
),
|
||||
counter(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
&["stage"],
|
||||
"Runtime executions rejected by a bounded limit.",
|
||||
),
|
||||
gauge(
|
||||
"crank_db_pool_connections",
|
||||
&["state"],
|
||||
"PostgreSQL pool connections by state.",
|
||||
),
|
||||
gauge(
|
||||
"crank_catalog_tools",
|
||||
&[],
|
||||
"Tools in the current published catalog.",
|
||||
),
|
||||
gauge(
|
||||
"crank_catalog_estimated_context_tokens",
|
||||
&[],
|
||||
"Estimated context tokens in the current published catalog.",
|
||||
),
|
||||
gauge(
|
||||
"crank_catalog_warnings",
|
||||
&[],
|
||||
"Warnings in the current published catalog.",
|
||||
),
|
||||
counter(
|
||||
"crank_invocation_history_lost_total",
|
||||
&[],
|
||||
"Invocation history records lost after an action completed.",
|
||||
),
|
||||
counter(
|
||||
"crank_telemetry_export_failures_total",
|
||||
&["signal_type", "exporter"],
|
||||
"Telemetry export failures.",
|
||||
),
|
||||
];
|
||||
|
||||
pub const fn metric_schema() -> &'static [MetricDefinition] {
|
||||
METRIC_SCHEMA
|
||||
}
|
||||
|
||||
const fn counter(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Counter,
|
||||
unit: MetricUnit::Count,
|
||||
labels,
|
||||
description,
|
||||
}
|
||||
}
|
||||
|
||||
const fn gauge(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Gauge,
|
||||
unit: MetricUnit::Count,
|
||||
labels,
|
||||
description,
|
||||
}
|
||||
}
|
||||
|
||||
const fn histogram(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Histogram,
|
||||
unit: MetricUnit::Seconds,
|
||||
labels,
|
||||
description,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,954 @@
|
||||
use std::{collections::HashMap, env, fmt, time::Duration};
|
||||
|
||||
use axum::http::{HeaderName, HeaderValue};
|
||||
use opentelemetry::{
|
||||
KeyValue, Value,
|
||||
trace::{Status, TracerProvider as _},
|
||||
};
|
||||
use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig, WithHttpConfig};
|
||||
use opentelemetry_sdk::{
|
||||
Resource,
|
||||
error::OTelSdkResult,
|
||||
trace::{
|
||||
BatchConfigBuilder, BatchSpanProcessor, SdkTracer, SdkTracerProvider, SpanData,
|
||||
SpanExporter as SpanExporterTrait,
|
||||
},
|
||||
};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use thiserror::Error;
|
||||
use url::Url;
|
||||
|
||||
use crate::ServiceIdentity;
|
||||
|
||||
const DEFAULT_EXPORT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const DEFAULT_MAX_QUEUE_SIZE: usize = 2_048;
|
||||
const DEFAULT_MAX_EXPORT_BATCH_SIZE: usize = 512;
|
||||
const DEFAULT_SCHEDULE_DELAY: Duration = Duration::from_secs(5);
|
||||
const DEFAULT_BATCH_EXPORT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_QUEUE_SIZE: usize = 65_536;
|
||||
const MAX_DURATION: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct OtlpBatchConfig {
|
||||
max_queue_size: usize,
|
||||
max_export_batch_size: usize,
|
||||
scheduled_delay: Duration,
|
||||
export_timeout: Duration,
|
||||
}
|
||||
|
||||
impl OtlpBatchConfig {
|
||||
pub fn try_new(
|
||||
max_queue_size: usize,
|
||||
max_export_batch_size: usize,
|
||||
scheduled_delay: Duration,
|
||||
export_timeout: Duration,
|
||||
) -> Result<Self, OtlpTraceConfigError> {
|
||||
let valid = max_queue_size > 0
|
||||
&& max_queue_size <= MAX_QUEUE_SIZE
|
||||
&& max_export_batch_size > 0
|
||||
&& max_export_batch_size <= max_queue_size
|
||||
&& duration_is_bounded(scheduled_delay)
|
||||
&& duration_is_bounded(export_timeout);
|
||||
if !valid {
|
||||
return Err(OtlpTraceConfigError::InvalidBatchLimits);
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
max_queue_size,
|
||||
max_export_batch_size,
|
||||
scheduled_delay,
|
||||
export_timeout,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn max_queue_size(&self) -> usize {
|
||||
self.max_queue_size
|
||||
}
|
||||
|
||||
pub fn max_export_batch_size(&self) -> usize {
|
||||
self.max_export_batch_size
|
||||
}
|
||||
|
||||
pub fn scheduled_delay(&self) -> Duration {
|
||||
self.scheduled_delay
|
||||
}
|
||||
|
||||
pub fn export_timeout(&self) -> Duration {
|
||||
self.export_timeout
|
||||
}
|
||||
|
||||
fn sdk_config(&self) -> opentelemetry_sdk::trace::BatchConfig {
|
||||
BatchConfigBuilder::default()
|
||||
.with_max_queue_size(self.max_queue_size)
|
||||
.with_max_export_batch_size(self.max_export_batch_size)
|
||||
.with_scheduled_delay(self.scheduled_delay)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for OtlpBatchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_queue_size: DEFAULT_MAX_QUEUE_SIZE,
|
||||
max_export_batch_size: DEFAULT_MAX_EXPORT_BATCH_SIZE,
|
||||
scheduled_delay: DEFAULT_SCHEDULE_DELAY,
|
||||
export_timeout: DEFAULT_BATCH_EXPORT_TIMEOUT,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
pub struct OtlpTraceConfig {
|
||||
endpoint: Option<String>,
|
||||
export_timeout: Duration,
|
||||
batch: OtlpBatchConfig,
|
||||
headers: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for OtlpTraceConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("OtlpTraceConfig")
|
||||
.field("enabled", &self.is_enabled())
|
||||
.field("export_timeout", &self.export_timeout)
|
||||
.field("batch", &self.batch)
|
||||
.field("header_count", &self.headers.len())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl OtlpTraceConfig {
|
||||
pub fn from_env() -> Result<Self, OtlpTraceConfigError> {
|
||||
OtlpEnvSettings::from_env()?.into_config()
|
||||
}
|
||||
|
||||
fn from_settings(settings: OtlpEnvSettings) -> Result<Self, OtlpTraceConfigError> {
|
||||
let endpoint = match settings.traces_endpoint {
|
||||
Some(endpoint) => Some(validate_endpoint(endpoint, EndpointKind::Trace)?),
|
||||
None => settings
|
||||
.generic_endpoint
|
||||
.map(|endpoint| validate_endpoint(endpoint, EndpointKind::Generic))
|
||||
.transpose()?,
|
||||
};
|
||||
if endpoint.is_none() {
|
||||
return Ok(Self {
|
||||
endpoint: None,
|
||||
export_timeout: DEFAULT_EXPORT_TIMEOUT,
|
||||
batch: OtlpBatchConfig::default(),
|
||||
headers: HashMap::new(),
|
||||
});
|
||||
}
|
||||
|
||||
let protocol = settings.traces_protocol.or(settings.generic_protocol);
|
||||
let export_timeout = match settings.traces_timeout {
|
||||
Some(timeout) => duration_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", Some(timeout))?,
|
||||
None => duration_env("OTEL_EXPORTER_OTLP_TIMEOUT", settings.generic_timeout)?,
|
||||
}
|
||||
.unwrap_or(DEFAULT_EXPORT_TIMEOUT);
|
||||
let batch = OtlpBatchConfig::try_new(
|
||||
usize_env("OTEL_BSP_MAX_QUEUE_SIZE", settings.max_queue_size)?
|
||||
.unwrap_or(DEFAULT_MAX_QUEUE_SIZE),
|
||||
usize_env(
|
||||
"OTEL_BSP_MAX_EXPORT_BATCH_SIZE",
|
||||
settings.max_export_batch_size,
|
||||
)?
|
||||
.unwrap_or(DEFAULT_MAX_EXPORT_BATCH_SIZE),
|
||||
duration_env("OTEL_BSP_SCHEDULE_DELAY", settings.scheduled_delay)?
|
||||
.unwrap_or(DEFAULT_SCHEDULE_DELAY),
|
||||
duration_env("OTEL_BSP_EXPORT_TIMEOUT", settings.batch_export_timeout)?
|
||||
.unwrap_or(DEFAULT_BATCH_EXPORT_TIMEOUT),
|
||||
)?;
|
||||
let headers = settings
|
||||
.traces_headers
|
||||
.filter(|value| !value.is_empty())
|
||||
.or(settings.generic_headers.filter(|value| !value.is_empty()))
|
||||
.map(|value| parse_headers(&value))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
|
||||
Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, headers)
|
||||
}
|
||||
|
||||
pub fn try_new(
|
||||
endpoint: Option<String>,
|
||||
protocol: Option<String>,
|
||||
export_timeout: Duration,
|
||||
batch: OtlpBatchConfig,
|
||||
) -> Result<Self, OtlpTraceConfigError> {
|
||||
Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, HashMap::new())
|
||||
}
|
||||
|
||||
fn try_new_with_headers(
|
||||
endpoint: Option<String>,
|
||||
protocol: Option<String>,
|
||||
export_timeout: Duration,
|
||||
batch: OtlpBatchConfig,
|
||||
headers: HashMap<String, String>,
|
||||
) -> Result<Self, OtlpTraceConfigError> {
|
||||
let endpoint = endpoint
|
||||
.map(|endpoint| validate_endpoint(endpoint, EndpointKind::Trace))
|
||||
.transpose()?;
|
||||
if endpoint.is_some() && protocol.as_deref().unwrap_or("http/protobuf") != "http/protobuf" {
|
||||
return Err(OtlpTraceConfigError::UnsupportedProtocol);
|
||||
}
|
||||
if !duration_is_bounded(export_timeout) {
|
||||
return Err(OtlpTraceConfigError::InvalidDuration {
|
||||
field: "OTEL_EXPORTER_OTLP_TIMEOUT",
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
endpoint,
|
||||
export_timeout,
|
||||
batch,
|
||||
headers,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
self.endpoint.is_some()
|
||||
}
|
||||
|
||||
pub fn export_timeout(&self) -> Duration {
|
||||
self.export_timeout
|
||||
}
|
||||
|
||||
pub fn batch(&self) -> &OtlpBatchConfig {
|
||||
&self.batch
|
||||
}
|
||||
|
||||
fn effective_export_timeout(&self) -> Duration {
|
||||
self.export_timeout.min(self.batch.export_timeout)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn endpoint(&self) -> Option<&str> {
|
||||
self.endpoint.as_deref()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers.get(name).map(String::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct OtlpEnvSettings {
|
||||
traces_endpoint: Option<String>,
|
||||
generic_endpoint: Option<String>,
|
||||
traces_protocol: Option<String>,
|
||||
generic_protocol: Option<String>,
|
||||
traces_timeout: Option<String>,
|
||||
generic_timeout: Option<String>,
|
||||
traces_headers: Option<String>,
|
||||
generic_headers: Option<String>,
|
||||
max_queue_size: Option<String>,
|
||||
max_export_batch_size: Option<String>,
|
||||
scheduled_delay: Option<String>,
|
||||
batch_export_timeout: Option<String>,
|
||||
}
|
||||
|
||||
impl OtlpEnvSettings {
|
||||
fn from_env() -> Result<Self, OtlpTraceConfigError> {
|
||||
Ok(Self {
|
||||
traces_endpoint: optional_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")?,
|
||||
generic_endpoint: optional_env("OTEL_EXPORTER_OTLP_ENDPOINT")?,
|
||||
traces_protocol: optional_env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")?,
|
||||
generic_protocol: optional_env("OTEL_EXPORTER_OTLP_PROTOCOL")?,
|
||||
traces_timeout: optional_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT")?,
|
||||
generic_timeout: optional_env("OTEL_EXPORTER_OTLP_TIMEOUT")?,
|
||||
traces_headers: optional_env("OTEL_EXPORTER_OTLP_TRACES_HEADERS")?,
|
||||
generic_headers: optional_env("OTEL_EXPORTER_OTLP_HEADERS")?,
|
||||
max_queue_size: optional_env("OTEL_BSP_MAX_QUEUE_SIZE")?,
|
||||
max_export_batch_size: optional_env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE")?,
|
||||
scheduled_delay: optional_env("OTEL_BSP_SCHEDULE_DELAY")?,
|
||||
batch_export_timeout: optional_env("OTEL_BSP_EXPORT_TIMEOUT")?,
|
||||
})
|
||||
}
|
||||
|
||||
fn into_config(self) -> Result<OtlpTraceConfig, OtlpTraceConfigError> {
|
||||
OtlpTraceConfig::from_settings(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error, Eq, PartialEq)]
|
||||
pub enum OtlpTraceConfigError {
|
||||
#[error("OTLP environment variable is not valid UTF-8: {field}")]
|
||||
InvalidEnvironmentEncoding { field: &'static str },
|
||||
#[error("OTLP trace endpoint is invalid: {reason}")]
|
||||
InvalidEndpoint { reason: &'static str },
|
||||
#[error("OTLP trace protocol must be http/protobuf")]
|
||||
UnsupportedProtocol,
|
||||
#[error("OTLP numeric setting is invalid: {field}")]
|
||||
InvalidNumber { field: &'static str },
|
||||
#[error("OTLP duration setting is invalid: {field}")]
|
||||
InvalidDuration { field: &'static str },
|
||||
#[error("OTLP batch limits are invalid")]
|
||||
InvalidBatchLimits,
|
||||
#[error("OTLP trace headers are invalid")]
|
||||
InvalidHeaders,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum OtlpTraceError {
|
||||
#[error("failed to configure OTLP trace exporter")]
|
||||
ExporterConfiguration,
|
||||
}
|
||||
|
||||
pub fn build_tracer_provider(
|
||||
identity: &ServiceIdentity,
|
||||
config: &OtlpTraceConfig,
|
||||
) -> Result<Option<(SdkTracerProvider, SdkTracer)>, OtlpTraceError> {
|
||||
let Some(endpoint) = config.endpoint.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let exporter = SpanExporter::builder()
|
||||
.with_http()
|
||||
.with_protocol(Protocol::HttpBinary)
|
||||
.with_endpoint(endpoint)
|
||||
.with_timeout(config.effective_export_timeout())
|
||||
.with_headers(config.headers.clone())
|
||||
.build()
|
||||
.map_err(|_| OtlpTraceError::ExporterConfiguration)?;
|
||||
let processor = BatchSpanProcessor::builder(ObservedSpanExporter(exporter))
|
||||
.with_batch_config(config.batch.sdk_config())
|
||||
.build();
|
||||
let resource = Resource::builder_empty()
|
||||
.with_attributes([
|
||||
KeyValue::new("service.name", identity.service().to_owned()),
|
||||
KeyValue::new("service.version", identity.version().to_owned()),
|
||||
KeyValue::new(
|
||||
"deployment.environment.name",
|
||||
identity.environment().to_owned(),
|
||||
),
|
||||
])
|
||||
.build();
|
||||
let provider = SdkTracerProvider::builder()
|
||||
.with_span_processor(processor)
|
||||
.with_resource(resource)
|
||||
.build();
|
||||
let tracer = provider.tracer("crank");
|
||||
|
||||
Ok(Some((provider, tracer)))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ObservedSpanExporter(SpanExporter);
|
||||
|
||||
impl SpanExporterTrait for ObservedSpanExporter {
|
||||
async fn export(&self, mut batch: Vec<SpanData>) -> OTelSdkResult {
|
||||
sanitize_trace_batch(&mut batch);
|
||||
let result = self.0.export(batch).await;
|
||||
if result.is_err() {
|
||||
metrics::counter!(
|
||||
"crank_telemetry_export_failures_total",
|
||||
"signal_type" => "trace",
|
||||
"exporter" => "otlp"
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult {
|
||||
self.0.shutdown_with_timeout(timeout)
|
||||
}
|
||||
|
||||
fn force_flush(&self) -> OTelSdkResult {
|
||||
self.0.force_flush()
|
||||
}
|
||||
|
||||
fn set_resource(&mut self, resource: &Resource) {
|
||||
self.0.set_resource(resource);
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_trace_batch(batch: &mut Vec<SpanData>) {
|
||||
batch.retain(|span| is_allowed_span_name(span.name.as_ref()));
|
||||
for span in batch {
|
||||
let original_attribute_count = span.attributes.len();
|
||||
span.attributes.retain(is_allowed_span_attribute);
|
||||
span.dropped_attributes_count = span
|
||||
.dropped_attributes_count
|
||||
.saturating_add((original_attribute_count - span.attributes.len()) as u32);
|
||||
span.events = Default::default();
|
||||
span.links = Default::default();
|
||||
if matches!(span.status, Status::Error { .. }) {
|
||||
span.status = Status::error("");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_allowed_span_name(name: &str) -> bool {
|
||||
matches!(
|
||||
name,
|
||||
"http.request"
|
||||
| "mcp.request"
|
||||
| "mcp.rate_limit"
|
||||
| "mcp.access.check"
|
||||
| "mcp.catalog.load"
|
||||
| "mcp.tools.resolve"
|
||||
| "approval.check"
|
||||
| "runtime.execute"
|
||||
| "runtime.arguments.map"
|
||||
| "runtime.idempotency"
|
||||
| "upstream.http"
|
||||
| "runtime.response.transform"
|
||||
| "auth.resolve"
|
||||
| "approval.recovery"
|
||||
| "history.write"
|
||||
| "db.query"
|
||||
)
|
||||
}
|
||||
|
||||
fn is_allowed_span_attribute(attribute: &KeyValue) -> bool {
|
||||
let Value::String(value) = &attribute.value else {
|
||||
return false;
|
||||
};
|
||||
let value = value.as_str();
|
||||
match attribute.key.as_str() {
|
||||
"request_id" => crate::RequestId::is_valid(value),
|
||||
"stage" => is_allowed_span_name(value),
|
||||
"outcome" => matches!(
|
||||
value,
|
||||
"success"
|
||||
| "error"
|
||||
| "allowed"
|
||||
| "denied"
|
||||
| "required"
|
||||
| "replay"
|
||||
| "execute"
|
||||
| "skipped"
|
||||
| "cache_hit"
|
||||
),
|
||||
"error.category" => matches!(
|
||||
value,
|
||||
"access"
|
||||
| "rate_limit"
|
||||
| "catalog"
|
||||
| "approval"
|
||||
| "idempotency"
|
||||
| "schema"
|
||||
| "mapping"
|
||||
| "upstream"
|
||||
| "transformation"
|
||||
| "history"
|
||||
| "database"
|
||||
| "concurrency"
|
||||
| "configuration"
|
||||
| "internal"
|
||||
),
|
||||
"db.system" => value == "postgresql",
|
||||
"db.operation" => matches!(
|
||||
value,
|
||||
"machine_access.read"
|
||||
| "machine_access.touch"
|
||||
| "catalog.load"
|
||||
| "approval.read"
|
||||
| "approval.write"
|
||||
| "auth_profile.read"
|
||||
| "secret.read"
|
||||
| "secret.touch"
|
||||
| "invocation_history.write"
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum EndpointKind {
|
||||
Trace,
|
||||
Generic,
|
||||
}
|
||||
|
||||
fn validate_endpoint(endpoint: String, kind: EndpointKind) -> Result<String, OtlpTraceConfigError> {
|
||||
let mut url = Url::parse(&endpoint).map_err(|_| OtlpTraceConfigError::InvalidEndpoint {
|
||||
reason: "invalid URL",
|
||||
})?;
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
||||
reason: "unsupported scheme",
|
||||
});
|
||||
}
|
||||
if url.host_str().is_none() {
|
||||
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
||||
reason: "host is required",
|
||||
});
|
||||
}
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
||||
reason: "credentials are forbidden",
|
||||
});
|
||||
}
|
||||
if url.query().is_some() || url.fragment().is_some() {
|
||||
return Err(OtlpTraceConfigError::InvalidEndpoint {
|
||||
reason: "query and fragment are forbidden",
|
||||
});
|
||||
}
|
||||
if matches!(kind, EndpointKind::Generic) {
|
||||
let path = url.path().trim_end_matches('/');
|
||||
url.set_path(&format!("{path}/v1/traces"));
|
||||
}
|
||||
|
||||
Ok(url.into())
|
||||
}
|
||||
|
||||
fn parse_headers(value: &str) -> Result<HashMap<String, String>, OtlpTraceConfigError> {
|
||||
value
|
||||
.split_terminator(',')
|
||||
.map(str::trim)
|
||||
.filter(|item| !item.is_empty())
|
||||
.try_fold(HashMap::new(), |mut headers, item| {
|
||||
let (name, encoded_value) = item
|
||||
.split_once('=')
|
||||
.ok_or(OtlpTraceConfigError::InvalidHeaders)?;
|
||||
let name = HeaderName::from_bytes(name.trim().as_bytes())
|
||||
.map_err(|_| OtlpTraceConfigError::InvalidHeaders)?;
|
||||
let value = percent_decode_str(encoded_value.trim())
|
||||
.decode_utf8()
|
||||
.map_err(|_| OtlpTraceConfigError::InvalidHeaders)?
|
||||
.into_owned();
|
||||
if value.is_empty() || HeaderValue::from_str(&value).is_err() {
|
||||
return Err(OtlpTraceConfigError::InvalidHeaders);
|
||||
}
|
||||
headers.insert(name.as_str().to_owned(), value);
|
||||
Ok(headers)
|
||||
})
|
||||
}
|
||||
|
||||
fn optional_env(field: &'static str) -> Result<Option<String>, OtlpTraceConfigError> {
|
||||
match env::var(field) {
|
||||
Ok(value) if value.is_empty() => Ok(None),
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(env::VarError::NotPresent) => Ok(None),
|
||||
Err(env::VarError::NotUnicode(_)) => {
|
||||
Err(OtlpTraceConfigError::InvalidEnvironmentEncoding { field })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn usize_env(
|
||||
field: &'static str,
|
||||
value: Option<String>,
|
||||
) -> Result<Option<usize>, OtlpTraceConfigError> {
|
||||
value
|
||||
.map(|value| {
|
||||
value
|
||||
.parse()
|
||||
.map_err(|_| OtlpTraceConfigError::InvalidNumber { field })
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn duration_env(
|
||||
field: &'static str,
|
||||
value: Option<String>,
|
||||
) -> Result<Option<Duration>, OtlpTraceConfigError> {
|
||||
value
|
||||
.map(|value| {
|
||||
value
|
||||
.parse::<u64>()
|
||||
.map(Duration::from_millis)
|
||||
.map_err(|_| OtlpTraceConfigError::InvalidDuration { field })
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn duration_is_bounded(duration: Duration) -> bool {
|
||||
!duration.is_zero() && duration <= MAX_DURATION
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
net::TcpListener,
|
||||
sync::mpsc,
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
extract::Request,
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
routing::get,
|
||||
};
|
||||
use opentelemetry::{
|
||||
KeyValue,
|
||||
trace::{Span as _, Status, Tracer as _},
|
||||
};
|
||||
use opentelemetry_proto::tonic::{
|
||||
collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value,
|
||||
};
|
||||
use prost::Message;
|
||||
use tower::ServiceExt;
|
||||
use tracing::{Instrument, info_span};
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
|
||||
use super::{OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_tracer_provider};
|
||||
use crate::ServiceIdentity;
|
||||
|
||||
#[test]
|
||||
fn signal_specific_settings_override_generic_settings() {
|
||||
let config = OtlpEnvSettings {
|
||||
traces_endpoint: Some("https://traces.example.test/custom".to_owned()),
|
||||
generic_endpoint: Some("https://generic.example.test/otel".to_owned()),
|
||||
traces_protocol: Some("http/protobuf".to_owned()),
|
||||
generic_protocol: Some("grpc".to_owned()),
|
||||
traces_timeout: Some("2500".to_owned()),
|
||||
generic_timeout: Some("invalid-unused-fallback".to_owned()),
|
||||
..OtlpEnvSettings::default()
|
||||
}
|
||||
.into_config()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
config.endpoint(),
|
||||
Some("https://traces.example.test/custom")
|
||||
);
|
||||
assert_eq!(config.export_timeout(), Duration::from_millis(2500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_export_ignores_inactive_settings() {
|
||||
let config = OtlpEnvSettings {
|
||||
traces_protocol: Some("grpc".to_owned()),
|
||||
generic_protocol: Some("grpc".to_owned()),
|
||||
traces_timeout: Some("invalid".to_owned()),
|
||||
generic_timeout: Some("invalid".to_owned()),
|
||||
max_queue_size: Some("invalid".to_owned()),
|
||||
max_export_batch_size: Some("invalid".to_owned()),
|
||||
scheduled_delay: Some("invalid".to_owned()),
|
||||
batch_export_timeout: Some("invalid".to_owned()),
|
||||
..OtlpEnvSettings::default()
|
||||
}
|
||||
.into_config()
|
||||
.unwrap();
|
||||
|
||||
assert!(!config.is_enabled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_signal_headers_use_generic_headers() {
|
||||
let config = OtlpEnvSettings {
|
||||
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
|
||||
traces_headers: Some(String::new()),
|
||||
generic_headers: Some(
|
||||
"authorization=Bearer%20canary-token,x-tenant=community".to_owned(),
|
||||
),
|
||||
..OtlpEnvSettings::default()
|
||||
}
|
||||
.into_config()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.header("authorization"), Some("Bearer canary-token"));
|
||||
assert_eq!(config.header("x-tenant"), Some("community"));
|
||||
assert!(!format!("{config:?}").contains("canary-token"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_headers_return_a_safe_error() {
|
||||
let config = OtlpEnvSettings {
|
||||
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
|
||||
traces_headers: Some("authorization=canary-secret%0Ainjected".to_owned()),
|
||||
..OtlpEnvSettings::default()
|
||||
};
|
||||
|
||||
let error = config.into_config().unwrap_err();
|
||||
assert!(matches!(error, super::OtlpTraceConfigError::InvalidHeaders));
|
||||
assert!(!error.to_string().contains("canary-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stricter_batch_timeout_bounds_http_export() {
|
||||
let config = OtlpEnvSettings {
|
||||
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
|
||||
traces_protocol: Some("http/protobuf".to_owned()),
|
||||
traces_timeout: Some("9000".to_owned()),
|
||||
batch_export_timeout: Some("2500".to_owned()),
|
||||
..OtlpEnvSettings::default()
|
||||
}
|
||||
.into_config()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
config.effective_export_timeout(),
|
||||
Duration::from_millis(2500)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_endpoint_receives_standard_trace_path() {
|
||||
let config = OtlpEnvSettings {
|
||||
generic_endpoint: Some("https://generic.example.test/otel/".to_owned()),
|
||||
generic_protocol: Some("http/protobuf".to_owned()),
|
||||
..OtlpEnvSettings::default()
|
||||
}
|
||||
.into_config()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
config.endpoint(),
|
||||
Some("https://generic.example.test/otel/v1/traces")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_export_does_not_build_a_provider() {
|
||||
let config = OtlpTraceConfig::try_new(
|
||||
None,
|
||||
None,
|
||||
Duration::from_secs(1),
|
||||
OtlpBatchConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap();
|
||||
|
||||
assert!(build_tracer_provider(&identity, &config).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_http_protobuf_export_contains_resource_and_trace() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let (request_tx, request_rx) = mpsc::sync_channel(1);
|
||||
let server = thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_http_request(&mut stream);
|
||||
stream
|
||||
.write_all(
|
||||
b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
|
||||
)
|
||||
.unwrap();
|
||||
request_tx.send(request).unwrap();
|
||||
});
|
||||
let config = OtlpEnvSettings {
|
||||
traces_endpoint: Some(format!("http://{address}/v1/traces")),
|
||||
traces_protocol: Some("http/protobuf".to_owned()),
|
||||
traces_timeout: Some("2000".to_owned()),
|
||||
traces_headers: Some(String::new()),
|
||||
generic_headers: Some("authorization=Bearer%20canary-token".to_owned()),
|
||||
max_queue_size: Some("16".to_owned()),
|
||||
max_export_batch_size: Some("8".to_owned()),
|
||||
scheduled_delay: Some("10".to_owned()),
|
||||
batch_export_timeout: Some("2000".to_owned()),
|
||||
..OtlpEnvSettings::default()
|
||||
}
|
||||
.into_config()
|
||||
.unwrap();
|
||||
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "integration-test").unwrap();
|
||||
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
|
||||
let mut span = tracer.start("http.request");
|
||||
let trace_id = span.span_context().trace_id().to_bytes();
|
||||
span.set_attribute(KeyValue::new("request_id", "req_otlp_contract"));
|
||||
span.set_attribute(KeyValue::new("authorization", "Bearer canary-span-secret"));
|
||||
span.add_event(
|
||||
"canary-span-event",
|
||||
vec![KeyValue::new("payload", "canary-span-secret")],
|
||||
);
|
||||
span.set_status(Status::error("canary-span-secret"));
|
||||
span.end();
|
||||
|
||||
provider.force_flush().unwrap();
|
||||
provider.shutdown().unwrap();
|
||||
let request = request_rx.recv_timeout(Duration::from_secs(2)).unwrap();
|
||||
server.join().unwrap();
|
||||
let (headers, body) = split_http_request(&request);
|
||||
|
||||
assert!(headers.contains("POST /v1/traces HTTP/1.1"));
|
||||
assert!(
|
||||
headers
|
||||
.to_ascii_lowercase()
|
||||
.contains("content-type: application/x-protobuf")
|
||||
);
|
||||
assert!(
|
||||
headers
|
||||
.to_ascii_lowercase()
|
||||
.contains("authorization: bearer canary-token")
|
||||
);
|
||||
let export = ExportTraceServiceRequest::decode(body).unwrap();
|
||||
let resource_spans = export.resource_spans.first().unwrap();
|
||||
let attributes = &resource_spans.resource.as_ref().unwrap().attributes;
|
||||
assert_eq!(
|
||||
string_attribute(attributes, "service.name"),
|
||||
Some("admin-api")
|
||||
);
|
||||
assert_eq!(
|
||||
string_attribute(attributes, "service.version"),
|
||||
Some("0.3.1")
|
||||
);
|
||||
assert_eq!(
|
||||
string_attribute(attributes, "deployment.environment.name"),
|
||||
Some("integration-test")
|
||||
);
|
||||
assert_eq!(
|
||||
resource_spans.scope_spans[0].spans[0].trace_id.as_slice(),
|
||||
trace_id
|
||||
);
|
||||
let exported_span = &resource_spans.scope_spans[0].spans[0];
|
||||
assert_eq!(exported_span.name, "http.request");
|
||||
assert_eq!(
|
||||
string_attribute(&exported_span.attributes, "request_id"),
|
||||
Some("req_otlp_contract")
|
||||
);
|
||||
assert!(
|
||||
exported_span
|
||||
.attributes
|
||||
.iter()
|
||||
.all(|attribute| attribute.key != "authorization")
|
||||
);
|
||||
assert!(exported_span.events.is_empty());
|
||||
assert_eq!(
|
||||
exported_span
|
||||
.status
|
||||
.as_ref()
|
||||
.map(|status| status.message.as_str()),
|
||||
Some("")
|
||||
);
|
||||
assert!(
|
||||
!body
|
||||
.windows(b"canary-span-secret".len())
|
||||
.any(|window| { window == b"canary-span-secret" })
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn unavailable_receiver_does_not_change_product_result() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
let (stream, _) = listener.accept().unwrap();
|
||||
drop(stream);
|
||||
});
|
||||
let config = OtlpTraceConfig::try_new(
|
||||
Some(format!("http://{address}/v1/traces")),
|
||||
Some("http/protobuf".to_owned()),
|
||||
Duration::from_millis(250),
|
||||
OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(250))
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "fault-test").unwrap();
|
||||
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
|
||||
let subscriber =
|
||||
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let app = Router::new()
|
||||
.route("/product", get(|| async { "product-success" }))
|
||||
.layer(axum::middleware::from_fn(trace_product_request));
|
||||
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/product")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 64).await.unwrap();
|
||||
|
||||
assert_eq!(status, axum::http::StatusCode::OK);
|
||||
assert_eq!(body.as_ref(), b"product-success");
|
||||
assert!(provider.force_flush().is_err());
|
||||
let _ = provider.shutdown();
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
async fn trace_product_request(request: Request, next: Next) -> Response {
|
||||
next.run(request)
|
||||
.instrument(info_span!(target: "crank::trace", "http.request"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hanging_receiver_respects_the_stricter_export_timeout() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
let (stream, _) = listener.accept().unwrap();
|
||||
thread::sleep(Duration::from_millis(750));
|
||||
drop(stream);
|
||||
});
|
||||
let config = OtlpTraceConfig::try_new(
|
||||
Some(format!("http://{address}/v1/traces")),
|
||||
Some("http/protobuf".to_owned()),
|
||||
Duration::from_secs(2),
|
||||
OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(100))
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "timeout-test").unwrap();
|
||||
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
|
||||
let mut span = tracer.start("http.request");
|
||||
span.end();
|
||||
let started_at = Instant::now();
|
||||
|
||||
assert!(provider.force_flush().is_err());
|
||||
assert!(started_at.elapsed() < Duration::from_millis(500));
|
||||
let _ = provider.shutdown();
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 4096];
|
||||
loop {
|
||||
let read = stream.read(&mut buffer).unwrap();
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
let Some(header_end) = find_bytes(&request, b"\r\n\r\n") else {
|
||||
continue;
|
||||
};
|
||||
let headers = String::from_utf8_lossy(&request[..header_end]);
|
||||
let content_length = headers
|
||||
.lines()
|
||||
.find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
name.eq_ignore_ascii_case("content-length")
|
||||
.then(|| value.trim().parse::<usize>().ok())
|
||||
.flatten()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
if request.len() >= header_end + 4 + content_length {
|
||||
return request;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn split_http_request(request: &[u8]) -> (&str, &[u8]) {
|
||||
let header_end = find_bytes(request, b"\r\n\r\n").unwrap();
|
||||
(
|
||||
std::str::from_utf8(&request[..header_end]).unwrap(),
|
||||
&request[header_end + 4..],
|
||||
)
|
||||
}
|
||||
|
||||
fn string_attribute<'a>(
|
||||
attributes: &'a [opentelemetry_proto::tonic::common::v1::KeyValue],
|
||||
key: &str,
|
||||
) -> Option<&'a str> {
|
||||
attributes.iter().find_map(|attribute| {
|
||||
let value = attribute.value.as_ref()?.value.as_ref()?;
|
||||
(attribute.key == key)
|
||||
.then_some(value)
|
||||
.and_then(|value| match value {
|
||||
any_value::Value::StringValue(value) => Some(value.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.position(|candidate| candidate == needle)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
use std::{env, net::SocketAddr};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{Request, State},
|
||||
http::{
|
||||
HeaderMap, StatusCode,
|
||||
header::{self, HeaderValue},
|
||||
},
|
||||
middleware::{self, Next},
|
||||
response::{IntoResponse, Response},
|
||||
routing::get,
|
||||
};
|
||||
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle, PrometheusRecorder};
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
use thiserror::Error;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
|
||||
|
||||
const METRICS_ENABLED_ENV: &str = "CRANK_METRICS_ENABLED";
|
||||
const METRICS_TOKEN_ENV: &str = "CRANK_METRICS_BEARER_TOKEN";
|
||||
const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MetricsConfig {
|
||||
enabled: bool,
|
||||
bind_addr: SocketAddr,
|
||||
token_digest: Option<[u8; 32]>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for MetricsConfig {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("MetricsConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field("bind_addr", &self.bind_addr)
|
||||
.field("authentication_configured", &self.token_digest.is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl MetricsConfig {
|
||||
pub fn new(
|
||||
enabled: bool,
|
||||
bind_addr: SocketAddr,
|
||||
bearer_token: Option<String>,
|
||||
) -> Result<Self, MetricsConfigError> {
|
||||
let token_digest = bearer_token
|
||||
.filter(|token| !token.is_empty())
|
||||
.map(|token| token_digest(token.as_bytes()));
|
||||
|
||||
if enabled && !bind_addr.ip().is_loopback() && token_digest.is_none() {
|
||||
return Err(MetricsConfigError::MissingTokenForExternalBind);
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
bind_addr,
|
||||
token_digest,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn from_env(
|
||||
bind_env: &'static str,
|
||||
default_bind: SocketAddr,
|
||||
) -> Result<Self, MetricsConfigError> {
|
||||
let enabled = parse_enabled(env::var(METRICS_ENABLED_ENV))?;
|
||||
let bind_addr = match env::var(bind_env) {
|
||||
Ok(raw) => raw
|
||||
.parse()
|
||||
.map_err(|_| MetricsConfigError::InvalidBindAddress { field: bind_env })?,
|
||||
Err(env::VarError::NotPresent) => default_bind,
|
||||
Err(env::VarError::NotUnicode(_)) => {
|
||||
return Err(MetricsConfigError::InvalidEnvironmentEncoding { field: bind_env });
|
||||
}
|
||||
};
|
||||
let bearer_token = match env::var(METRICS_TOKEN_ENV) {
|
||||
Ok(token) => Some(token),
|
||||
Err(env::VarError::NotPresent) => None,
|
||||
Err(env::VarError::NotUnicode(_)) => {
|
||||
return Err(MetricsConfigError::InvalidEnvironmentEncoding {
|
||||
field: METRICS_TOKEN_ENV,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
Self::new(enabled, bind_addr, bearer_token)
|
||||
}
|
||||
|
||||
pub fn enabled(&self) -> bool {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
pub fn bind_addr(&self) -> SocketAddr {
|
||||
self.bind_addr
|
||||
}
|
||||
|
||||
pub fn requires_authentication(&self) -> bool {
|
||||
!self.bind_addr.ip().is_loopback()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum MetricsConfigError {
|
||||
#[error("metrics environment variable is not valid UTF-8: {field}")]
|
||||
InvalidEnvironmentEncoding { field: &'static str },
|
||||
#[error("metrics bind address is invalid: {field}")]
|
||||
InvalidBindAddress { field: &'static str },
|
||||
#[error("metrics enabled flag must be one of true, false, 1, 0")]
|
||||
InvalidEnabledFlag,
|
||||
#[error("external metrics bind requires a bearer token")]
|
||||
MissingTokenForExternalBind,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MetricsState {
|
||||
handle: PrometheusHandle,
|
||||
token_digest: Option<[u8; 32]>,
|
||||
requires_authentication: bool,
|
||||
}
|
||||
|
||||
pub struct MetricsSurface {
|
||||
config: MetricsConfig,
|
||||
state: MetricsState,
|
||||
_recorder: Option<PrometheusRecorder>,
|
||||
}
|
||||
|
||||
impl MetricsSurface {
|
||||
pub(crate) fn new(config: MetricsConfig, handle: PrometheusHandle) -> Self {
|
||||
Self {
|
||||
state: MetricsState {
|
||||
handle,
|
||||
token_digest: config.token_digest,
|
||||
requires_authentication: config.requires_authentication(),
|
||||
},
|
||||
config,
|
||||
_recorder: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn for_test(
|
||||
config: MetricsConfig,
|
||||
identity: ServiceIdentity,
|
||||
) -> Result<Self, MetricsSurfaceError> {
|
||||
let recorder = prometheus_builder(&identity)?.build_recorder();
|
||||
let handle = recorder.handle();
|
||||
let mut surface = Self::new(config, handle);
|
||||
surface._recorder = Some(recorder);
|
||||
Ok(surface)
|
||||
}
|
||||
|
||||
pub fn router(&self) -> Router {
|
||||
Router::new()
|
||||
.route("/metrics", get(render_metrics))
|
||||
.route("/health", get(metrics_health))
|
||||
.layer(middleware::from_fn_with_state(
|
||||
self.state.clone(),
|
||||
authorize_metrics,
|
||||
))
|
||||
.with_state(self.state.clone())
|
||||
}
|
||||
|
||||
pub async fn bind(self) -> Result<MetricsServer, MetricsServeError> {
|
||||
let listener = TcpListener::bind(self.config.bind_addr)
|
||||
.await
|
||||
.map_err(|_| MetricsServeError::Bind)?;
|
||||
Ok(MetricsServer {
|
||||
listener,
|
||||
router: self.router(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MetricsServer {
|
||||
listener: TcpListener,
|
||||
router: Router,
|
||||
}
|
||||
|
||||
impl MetricsServer {
|
||||
pub async fn serve(self) -> Result<(), MetricsServeError> {
|
||||
axum::serve(self.listener, self.router)
|
||||
.await
|
||||
.map_err(|_| MetricsServeError::Serve)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum MetricsSurfaceError {
|
||||
#[error("failed to configure Prometheus recorder")]
|
||||
RecorderConfiguration,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum MetricsServeError {
|
||||
#[error("failed to bind metrics listener")]
|
||||
Bind,
|
||||
#[error("metrics listener stopped unexpectedly")]
|
||||
Serve,
|
||||
}
|
||||
|
||||
pub(crate) fn install_prometheus_recorder(
|
||||
identity: &ServiceIdentity,
|
||||
) -> Result<PrometheusHandle, MetricsSurfaceError> {
|
||||
prometheus_builder(identity)?
|
||||
.install_recorder()
|
||||
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
|
||||
}
|
||||
|
||||
fn prometheus_builder(
|
||||
identity: &ServiceIdentity,
|
||||
) -> Result<PrometheusBuilder, MetricsSurfaceError> {
|
||||
PrometheusBuilder::new()
|
||||
.set_buckets(DURATION_BUCKETS_SECONDS)
|
||||
.map(|builder| {
|
||||
builder
|
||||
.add_global_label("service", identity.service())
|
||||
.add_global_label("version", identity.version())
|
||||
.add_global_label("environment", identity.environment())
|
||||
})
|
||||
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
|
||||
}
|
||||
|
||||
async fn render_metrics(State(state): State<MetricsState>) -> Response {
|
||||
let mut response = state.handle.render().into_response();
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static(PROMETHEUS_CONTENT_TYPE),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
async fn metrics_health() -> impl IntoResponse {
|
||||
(StatusCode::OK, "ok\n")
|
||||
}
|
||||
|
||||
async fn authorize_metrics(
|
||||
State(state): State<MetricsState>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
if !state.requires_authentication {
|
||||
return next.run(request).await;
|
||||
}
|
||||
|
||||
let authorized = bearer_token(request.headers())
|
||||
.map(token_digest)
|
||||
.zip(state.token_digest)
|
||||
.is_some_and(|(actual, expected)| bool::from(actual.ct_eq(&expected)));
|
||||
|
||||
if authorized {
|
||||
next.run(request).await
|
||||
} else {
|
||||
StatusCode::UNAUTHORIZED.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
|
||||
headers
|
||||
.get(header::AUTHORIZATION)?
|
||||
.as_bytes()
|
||||
.strip_prefix(b"Bearer ")
|
||||
.filter(|token| !token.is_empty())
|
||||
}
|
||||
|
||||
fn token_digest(token: &[u8]) -> [u8; 32] {
|
||||
Sha256::digest(token).into()
|
||||
}
|
||||
|
||||
fn parse_enabled(value: Result<String, env::VarError>) -> Result<bool, MetricsConfigError> {
|
||||
match value {
|
||||
Ok(raw) => match raw.to_ascii_lowercase().as_str() {
|
||||
"true" | "1" => Ok(true),
|
||||
"false" | "0" => Ok(false),
|
||||
_ => Err(MetricsConfigError::InvalidEnabledFlag),
|
||||
},
|
||||
Err(env::VarError::NotPresent) => Ok(true),
|
||||
Err(env::VarError::NotUnicode(_)) => Err(MetricsConfigError::InvalidEnvironmentEncoding {
|
||||
field: METRICS_ENABLED_ENV,
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
use axum::http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use opentelemetry::{
|
||||
Context, global,
|
||||
propagation::{Extractor, Injector},
|
||||
trace::TraceContextExt,
|
||||
};
|
||||
use opentelemetry_sdk::propagation::TraceContextPropagator;
|
||||
use tracing::Span;
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
|
||||
pub fn set_remote_trace_parent(span: &Span, headers: &HeaderMap) -> bool {
|
||||
let context =
|
||||
global::get_text_map_propagator(|propagator| propagator.extract(&HeaderExtractor(headers)));
|
||||
let span_context = context.span().span_context().clone();
|
||||
if !span_context.is_valid() || !span_context.is_remote() {
|
||||
return false;
|
||||
}
|
||||
|
||||
span.set_parent(context).is_ok()
|
||||
}
|
||||
|
||||
pub fn inject_current_trace_context(headers: &mut HeaderMap) -> bool {
|
||||
let context = Span::current().context();
|
||||
if !context.span().span_context().is_valid() {
|
||||
return false;
|
||||
}
|
||||
|
||||
global::get_text_map_propagator(|propagator| {
|
||||
propagator.inject_context(&context, &mut HeaderInjector(headers));
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn install_trace_context_propagator() {
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
}
|
||||
|
||||
struct HeaderExtractor<'a>(&'a HeaderMap);
|
||||
|
||||
impl Extractor for HeaderExtractor<'_> {
|
||||
fn get(&self, key: &str) -> Option<&str> {
|
||||
self.0.get(key).and_then(|value| value.to_str().ok())
|
||||
}
|
||||
|
||||
fn keys(&self) -> Vec<&str> {
|
||||
self.0.keys().map(HeaderName::as_str).collect()
|
||||
}
|
||||
}
|
||||
|
||||
struct HeaderInjector<'a>(&'a mut HeaderMap);
|
||||
|
||||
impl Injector for HeaderInjector<'_> {
|
||||
fn set(&mut self, key: &str, value: String) {
|
||||
let Ok(name) = HeaderName::try_from(key) else {
|
||||
return;
|
||||
};
|
||||
let Ok(value) = HeaderValue::try_from(value) else {
|
||||
return;
|
||||
};
|
||||
self.0.insert(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn current_trace_id() -> Option<String> {
|
||||
let context: Context = Span::current().context();
|
||||
let span_context = context.span().span_context().clone();
|
||||
span_context
|
||||
.is_valid()
|
||||
.then(|| span_context.trace_id().to_string())
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
use serde_json::{Map, Value};
|
||||
use thiserror::Error;
|
||||
|
||||
pub const REDACTED_MARKER: &str = "[REDACTED]";
|
||||
const TRUNCATED_MARKER: &str = "[TRUNCATED]";
|
||||
const MIN_EVENT_BYTES: usize = 512;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct RedactionLimits {
|
||||
pub max_string_bytes: usize,
|
||||
pub max_array_items: usize,
|
||||
pub max_object_fields: usize,
|
||||
pub max_depth: usize,
|
||||
pub max_event_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for RedactionLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_string_bytes: 1024,
|
||||
max_array_items: 32,
|
||||
max_object_fields: 64,
|
||||
max_depth: 8,
|
||||
max_event_bytes: 16 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RedactionLimits {
|
||||
pub fn validate(self) -> Result<(), RedactionLimitsError> {
|
||||
for (field, value, minimum) in [
|
||||
(
|
||||
"max_string_bytes",
|
||||
self.max_string_bytes,
|
||||
TRUNCATED_MARKER.len(),
|
||||
),
|
||||
("max_array_items", self.max_array_items, 1),
|
||||
("max_object_fields", self.max_object_fields, 1),
|
||||
("max_depth", self.max_depth, 1),
|
||||
("max_event_bytes", self.max_event_bytes, MIN_EVENT_BYTES),
|
||||
] {
|
||||
if value < minimum {
|
||||
return Err(RedactionLimitsError::TooSmall { field, minimum });
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum RedactionLimitsError {
|
||||
#[error("invalid redaction limit {field}: minimum is {minimum}")]
|
||||
TooSmall { field: &'static str, minimum: usize },
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SafeJsonError {
|
||||
#[error(transparent)]
|
||||
InvalidLimits(#[from] RedactionLimitsError),
|
||||
#[error(transparent)]
|
||||
Serialization(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
pub fn redact_value(value: &Value, limits: RedactionLimits) -> Value {
|
||||
redact_at_depth(value, limits, 0)
|
||||
}
|
||||
|
||||
pub fn safe_json(value: &Value, limits: RedactionLimits) -> Result<String, SafeJsonError> {
|
||||
limits.validate()?;
|
||||
let serialized = serde_json::to_string(&redact_value(value, limits))?;
|
||||
if serialized.len() <= limits.max_event_bytes {
|
||||
return Ok(serialized);
|
||||
}
|
||||
|
||||
Ok(serde_json::to_string(&serde_json::json!({
|
||||
"truncated": true
|
||||
}))?)
|
||||
}
|
||||
|
||||
pub(crate) fn truncate_string(value: &str, max_bytes: usize) -> String {
|
||||
if value.len() <= max_bytes {
|
||||
return value.to_owned();
|
||||
}
|
||||
if max_bytes == 0 {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let marker = if max_bytes >= TRUNCATED_MARKER.len() {
|
||||
TRUNCATED_MARKER
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let content_budget = max_bytes.saturating_sub(marker.len());
|
||||
let mut boundary = content_budget.min(value.len());
|
||||
while boundary > 0 && !value.is_char_boundary(boundary) {
|
||||
boundary -= 1;
|
||||
}
|
||||
|
||||
let mut truncated = String::with_capacity(max_bytes);
|
||||
truncated.push_str(&value[..boundary]);
|
||||
if marker.is_empty() {
|
||||
let mut marker_boundary = max_bytes.min(TRUNCATED_MARKER.len());
|
||||
while marker_boundary > 0 && !TRUNCATED_MARKER.is_char_boundary(marker_boundary) {
|
||||
marker_boundary -= 1;
|
||||
}
|
||||
truncated.clear();
|
||||
truncated.push_str(&TRUNCATED_MARKER[..marker_boundary]);
|
||||
} else {
|
||||
truncated.push_str(marker);
|
||||
}
|
||||
truncated
|
||||
}
|
||||
|
||||
fn redact_at_depth(value: &Value, limits: RedactionLimits, depth: usize) -> Value {
|
||||
if depth >= limits.max_depth {
|
||||
return Value::String(TRUNCATED_MARKER.to_owned());
|
||||
}
|
||||
|
||||
match value {
|
||||
Value::Null | Value::Bool(_) | Value::Number(_) => value.clone(),
|
||||
Value::String(value) => Value::String(truncate_string(value, limits.max_string_bytes)),
|
||||
Value::Array(values) => redact_array(values, limits, depth),
|
||||
Value::Object(values) => redact_object(values, limits, depth),
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_array(values: &[Value], limits: RedactionLimits, depth: usize) -> Value {
|
||||
if limits.max_array_items == 0 {
|
||||
return Value::Array(Vec::new());
|
||||
}
|
||||
|
||||
let truncated = values.len() > limits.max_array_items;
|
||||
let value_limit = if truncated {
|
||||
limits.max_array_items.saturating_sub(1)
|
||||
} else {
|
||||
limits.max_array_items
|
||||
};
|
||||
let mut output: Vec<_> = values
|
||||
.iter()
|
||||
.take(value_limit)
|
||||
.map(|value| redact_at_depth(value, limits, depth + 1))
|
||||
.collect();
|
||||
if truncated {
|
||||
output.push(Value::String(TRUNCATED_MARKER.to_owned()));
|
||||
}
|
||||
Value::Array(output)
|
||||
}
|
||||
|
||||
fn redact_object(values: &Map<String, Value>, limits: RedactionLimits, depth: usize) -> Value {
|
||||
if limits.max_object_fields == 0 {
|
||||
return Value::Object(Map::new());
|
||||
}
|
||||
|
||||
let truncated = values.len() > limits.max_object_fields;
|
||||
let value_limit = if truncated {
|
||||
limits.max_object_fields.saturating_sub(1)
|
||||
} else {
|
||||
limits.max_object_fields
|
||||
};
|
||||
let mut output = Map::new();
|
||||
|
||||
for (index, (key, value)) in values.iter().take(value_limit).enumerate() {
|
||||
let cleaned = if is_sensitive_key(key) {
|
||||
Value::String(REDACTED_MARKER.to_owned())
|
||||
} else if is_url_key(key) {
|
||||
value
|
||||
.as_str()
|
||||
.map(sanitize_url)
|
||||
.map(|value| truncate_string(&value, limits.max_string_bytes))
|
||||
.map(Value::String)
|
||||
.unwrap_or_else(|| redact_at_depth(value, limits, depth + 1))
|
||||
} else {
|
||||
redact_at_depth(value, limits, depth + 1)
|
||||
};
|
||||
output.insert(
|
||||
bounded_object_key(key, limits.max_string_bytes, index),
|
||||
cleaned,
|
||||
);
|
||||
}
|
||||
if truncated {
|
||||
output.insert(
|
||||
"_truncated".to_owned(),
|
||||
Value::String(TRUNCATED_MARKER.to_owned()),
|
||||
);
|
||||
}
|
||||
|
||||
Value::Object(output)
|
||||
}
|
||||
|
||||
fn normalized_key(key: &str) -> String {
|
||||
key.bytes()
|
||||
.filter(|byte| !matches!(byte, b'_' | b'-' | b'.'))
|
||||
.map(|byte| byte.to_ascii_lowercase() as char)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_sensitive_key(key: &str) -> bool {
|
||||
let key = normalized_key(key);
|
||||
let exact_match = matches!(
|
||||
key.as_str(),
|
||||
"password"
|
||||
| "passwd"
|
||||
| "secret"
|
||||
| "token"
|
||||
| "apikey"
|
||||
| "accesskey"
|
||||
| "secretkey"
|
||||
| "authorization"
|
||||
| "proxyauthorization"
|
||||
| "cookie"
|
||||
| "setcookie"
|
||||
| "query"
|
||||
| "querystring"
|
||||
| "rawquery"
|
||||
| "urlquery"
|
||||
| "payload"
|
||||
| "body"
|
||||
| "requestbody"
|
||||
| "arguments"
|
||||
| "result"
|
||||
| "response"
|
||||
| "context"
|
||||
| "error"
|
||||
| "errormessage"
|
||||
);
|
||||
let contains_high_risk_name = [
|
||||
"password",
|
||||
"passwd",
|
||||
"secret",
|
||||
"token",
|
||||
"apikey",
|
||||
"accesskey",
|
||||
"authorization",
|
||||
"cookie",
|
||||
]
|
||||
.iter()
|
||||
.any(|part| key.contains(part));
|
||||
|
||||
exact_match
|
||||
|| contains_high_risk_name
|
||||
|| key.ends_with("payload")
|
||||
|| key.ends_with("body")
|
||||
|| key.ends_with("arguments")
|
||||
|| key.ends_with("result")
|
||||
|| key.ends_with("response")
|
||||
|| key.ends_with("query")
|
||||
|| key.ends_with("context")
|
||||
|| key.starts_with("query")
|
||||
}
|
||||
|
||||
fn is_url_key(key: &str) -> bool {
|
||||
let key = normalized_key(key);
|
||||
matches!(
|
||||
key.as_str(),
|
||||
"url" | "uri" | "endpoint" | "endpointurl" | "endpointuri" | "requesturl" | "targeturl"
|
||||
) || key.ends_with("url")
|
||||
|| key.ends_with("uri")
|
||||
|| key.ends_with("endpoint")
|
||||
}
|
||||
|
||||
fn sanitize_url(value: &str) -> String {
|
||||
let without_query = value
|
||||
.find(['?', '#'])
|
||||
.map(|index| &value[..index])
|
||||
.unwrap_or(value);
|
||||
let Some(scheme_end) = without_query.find("://") else {
|
||||
return without_query.to_owned();
|
||||
};
|
||||
let authority_start = scheme_end + 3;
|
||||
let authority_end = without_query[authority_start..]
|
||||
.find('/')
|
||||
.map(|index| authority_start + index)
|
||||
.unwrap_or(without_query.len());
|
||||
let authority = &without_query[authority_start..authority_end];
|
||||
let Some(userinfo_end) = authority.rfind('@') else {
|
||||
return without_query.to_owned();
|
||||
};
|
||||
|
||||
format!(
|
||||
"{}{}",
|
||||
&without_query[..authority_start],
|
||||
&without_query[authority_start + userinfo_end + 1..]
|
||||
)
|
||||
}
|
||||
|
||||
fn bounded_object_key(key: &str, max_bytes: usize, index: usize) -> String {
|
||||
if key.len() <= max_bytes {
|
||||
return key.to_owned();
|
||||
}
|
||||
|
||||
truncate_string(&format!("_truncated_key_{index}"), max_bytes)
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct LogEnvelope {
|
||||
pub timestamp: String,
|
||||
pub level: String,
|
||||
pub service: String,
|
||||
pub version: String,
|
||||
pub environment: String,
|
||||
pub target: String,
|
||||
pub event: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub request_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub trace_id: Option<String>,
|
||||
pub fields: Map<String, Value>,
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
use crank_observability::RequestId;
|
||||
use uuid::Version;
|
||||
|
||||
#[test]
|
||||
fn preserves_valid_opaque_request_id() {
|
||||
let request_id = RequestId::resolve(Some("req_test-123/abc"));
|
||||
|
||||
assert_eq!(request_id.as_str(), "req_test-123/abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaces_missing_and_invalid_values_with_uuid_v7() {
|
||||
for candidate in [
|
||||
None,
|
||||
Some(""),
|
||||
Some("bad value"),
|
||||
Some(" leading"),
|
||||
Some("trailing "),
|
||||
Some("bad,value"),
|
||||
Some("bad;value"),
|
||||
Some("я"),
|
||||
] {
|
||||
let request_id = RequestId::resolve(candidate);
|
||||
let parsed = uuid::Uuid::parse_str(request_id.as_str()).expect("generated UUID");
|
||||
|
||||
assert_eq!(parsed.get_version(), Some(Version::SortRand));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_values_over_the_shared_limit() {
|
||||
let oversized = "x".repeat(RequestId::MAX_LEN + 1);
|
||||
let request_id = RequestId::resolve(Some(&oversized));
|
||||
|
||||
assert_ne!(request_id.as_str(), oversized);
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(request_id.as_str())
|
||||
.expect("generated UUID")
|
||||
.get_version(),
|
||||
Some(Version::SortRand)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
use crank_observability::{CriticalErrorCategory, SentryConfig, SentryConfigError};
|
||||
|
||||
#[test]
|
||||
fn missing_or_blank_dsn_disables_critical_error_channel() {
|
||||
assert!(!SentryConfig::parse(None).expect("missing DSN").enabled());
|
||||
assert!(!SentryConfig::parse(Some("")).expect("empty DSN").enabled());
|
||||
assert!(
|
||||
!SentryConfig::parse(Some(" "))
|
||||
.expect("blank DSN")
|
||||
.enabled()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_explicit_dsn_is_rejected_without_echoing_the_value() {
|
||||
let secret_value = "not-a-dsn?token=control-secret";
|
||||
let error = SentryConfig::parse(Some(secret_value)).expect_err("invalid DSN must fail");
|
||||
|
||||
assert!(matches!(error, SentryConfigError::InvalidDsn));
|
||||
assert!(!error.to_string().contains(secret_value));
|
||||
assert!(!error.to_string().contains("control-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn critical_error_categories_are_closed_and_stable() {
|
||||
assert_eq!(CriticalErrorCategory::Panic.as_str(), "panic");
|
||||
assert_eq!(CriticalErrorCategory::Startup.as_str(), "startup");
|
||||
assert_eq!(CriticalErrorCategory::Internal.as_str(), "internal");
|
||||
assert_eq!(
|
||||
CriticalErrorCategory::DataIntegrity.as_str(),
|
||||
"data_integrity"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
middleware,
|
||||
routing::get,
|
||||
};
|
||||
use crank_observability::{
|
||||
MetricsConfig, ObservabilityConfig, RedactionLimits, ServiceIdentity, record_http_request,
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_metrics_use_matched_routes_and_closed_labels() {
|
||||
let identity =
|
||||
ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity");
|
||||
let lifecycle = crank_observability::init(ObservabilityConfig::new(
|
||||
identity,
|
||||
"off",
|
||||
RedactionLimits::default(),
|
||||
))
|
||||
.expect("observability lifecycle");
|
||||
let config = MetricsConfig::new(
|
||||
true,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
|
||||
None,
|
||||
)
|
||||
.expect("loopback metrics");
|
||||
let metrics = lifecycle.metrics_surface(config).router();
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/documents/{document_id}",
|
||||
get(|| async { StatusCode::NO_CONTENT }),
|
||||
)
|
||||
.layer(middleware::from_fn(record_http_request));
|
||||
|
||||
let sensitive_path_segment = "customer-secret-document-id";
|
||||
for index in 0..100 {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::get(format!("/documents/{sensitive_path_segment}-{index}"))
|
||||
.body(Body::empty())
|
||||
.expect("request"),
|
||||
)
|
||||
.await
|
||||
.expect("response");
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::get("/unknown/customer-controlled-path")
|
||||
.body(Body::empty())
|
||||
.expect("request"),
|
||||
)
|
||||
.await
|
||||
.expect("response");
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
|
||||
let response = metrics
|
||||
.oneshot(
|
||||
Request::get("/metrics")
|
||||
.body(Body::empty())
|
||||
.expect("request"),
|
||||
)
|
||||
.await
|
||||
.expect("metrics response");
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("bounded metrics body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("utf-8 metrics");
|
||||
|
||||
assert!(body.contains("crank_http_requests_total"));
|
||||
assert!(body.contains("route=\"/documents/{document_id}\""));
|
||||
assert!(body.contains("method=\"GET\""));
|
||||
assert!(body.contains("status_class=\"2xx\""));
|
||||
assert!(body.contains("crank_http_request_duration_seconds_bucket"));
|
||||
assert!(!body.contains(sensitive_path_segment));
|
||||
assert_eq!(
|
||||
body.lines()
|
||||
.filter(|line| {
|
||||
line.starts_with("crank_http_requests_total{")
|
||||
&& line.contains("route=\"/documents/{document_id}\"")
|
||||
})
|
||||
.count(),
|
||||
1,
|
||||
"different entity ids must not create additional series"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
use crank_observability::{
|
||||
OperationalIncident, operational_incident_total, record_operational_incident,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn history_loss_counter_has_no_dynamic_dimensions() {
|
||||
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
|
||||
|
||||
record_operational_incident(OperationalIncident::InvocationHistoryLost);
|
||||
|
||||
assert!(
|
||||
operational_incident_total(OperationalIncident::InvocationHistoryLost) > before,
|
||||
"the closed incident counter must increase"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
use std::{
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber, safe_json,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().expect("test writer lock").clone())
|
||||
.expect("log output must be UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedWriter {
|
||||
type Writer = SharedWriterGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedWriterGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedWriterGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedWriterGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer
|
||||
.lock()
|
||||
.map_err(|_| io::Error::other("test writer lock poisoned"))?
|
||||
.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn capture(service: &'static str, emit: impl FnOnce()) -> Vec<Value> {
|
||||
capture_with_limits(service, RedactionLimits::default(), emit)
|
||||
}
|
||||
|
||||
fn capture_with_limits(
|
||||
service: &'static str,
|
||||
limits: RedactionLimits,
|
||||
emit: impl FnOnce(),
|
||||
) -> Vec<Value> {
|
||||
let writer = SharedWriter::default();
|
||||
let config = ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new(service, "0.3.1", "test").expect("valid test identity"),
|
||||
"info",
|
||||
limits,
|
||||
);
|
||||
let subscriber =
|
||||
build_subscriber(config, writer.clone()).expect("test subscriber must be built");
|
||||
|
||||
tracing::subscriber::with_default(subscriber, emit);
|
||||
|
||||
writer
|
||||
.output()
|
||||
.lines()
|
||||
.map(|line| {
|
||||
assert!(!line.contains('\u{1b}'), "ANSI is forbidden: {line}");
|
||||
serde_json::from_str(line).expect("every line must be one JSON object")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_contract_is_identical_for_both_services() {
|
||||
let outputs = ["admin-api", "mcp-server"].map(|service| {
|
||||
capture(service, || {
|
||||
tracing::info!(
|
||||
name: "service.started",
|
||||
target: "crank::startup",
|
||||
port = 3101_u64,
|
||||
"service started"
|
||||
);
|
||||
})
|
||||
});
|
||||
|
||||
for (service, events) in ["admin-api", "mcp-server"].into_iter().zip(outputs.iter()) {
|
||||
assert_eq!(events.len(), 1);
|
||||
let event = &events[0];
|
||||
assert_eq!(event["service"], service);
|
||||
assert_eq!(event["version"], "0.3.1");
|
||||
assert_eq!(event["environment"], "test");
|
||||
assert_eq!(event["level"], "INFO");
|
||||
assert_eq!(event["target"], "crank::startup");
|
||||
assert_eq!(event["event"], "service.started");
|
||||
assert!(event["fields"].is_object());
|
||||
assert_eq!(event["fields"]["port"], 3101);
|
||||
assert_eq!(event["fields"]["message"], "service started");
|
||||
let timestamp = event["timestamp"].as_str().expect("timestamp string");
|
||||
let parsed =
|
||||
OffsetDateTime::parse(timestamp, &Rfc3339).expect("timestamp must be RFC 3339");
|
||||
assert_eq!(parsed.offset(), time::UtcOffset::UTC);
|
||||
}
|
||||
|
||||
let first_keys: Vec<_> = outputs[0][0]
|
||||
.as_object()
|
||||
.expect("object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let second_keys: Vec<_> = outputs[1][0]
|
||||
.as_object()
|
||||
.expect("object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
assert_eq!(first_keys, second_keys);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correlation_fields_are_distinct_and_only_present_when_recorded() {
|
||||
let present = capture("admin-api", || {
|
||||
tracing::info!(
|
||||
name: "admin.request.completed",
|
||||
request_id = "req-123",
|
||||
trace_id = "trace-456"
|
||||
);
|
||||
});
|
||||
assert_eq!(present[0]["request_id"], "req-123");
|
||||
assert_eq!(present[0]["trace_id"], "trace-456");
|
||||
assert!(present[0]["fields"].get("request_id").is_none());
|
||||
assert!(present[0]["fields"].get("trace_id").is_none());
|
||||
|
||||
let absent = capture("mcp-server", || {
|
||||
tracing::info!(name: "mcp.request.completed", status = 200_u64);
|
||||
});
|
||||
assert!(absent[0].get("request_id").is_none());
|
||||
assert!(absent[0].get("trace_id").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correlation_fields_preserve_scalar_display_values_before_field_limits() {
|
||||
let limits = RedactionLimits {
|
||||
max_object_fields: 1,
|
||||
..RedactionLimits::default()
|
||||
};
|
||||
let request_id = "123";
|
||||
let events = capture_with_limits("admin-api", limits, || {
|
||||
tracing::info!(
|
||||
name: "admin.request.completed",
|
||||
alpha = "field that consumes the object budget",
|
||||
request_id = %request_id,
|
||||
trace_id = true,
|
||||
);
|
||||
});
|
||||
|
||||
assert_eq!(events[0]["request_id"], "123");
|
||||
assert_eq!(events[0]["trace_id"], "true");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_correlation_fields_are_omitted() {
|
||||
let events = capture("admin-api", || {
|
||||
tracing::info!(
|
||||
name: "admin.request.completed",
|
||||
request_id = "",
|
||||
trace_id = ""
|
||||
);
|
||||
});
|
||||
|
||||
assert!(events[0].get("request_id").is_none());
|
||||
assert!(events[0].get("trace_id").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formatter_redacts_fields_before_serialization() {
|
||||
let context = safe_json(
|
||||
&serde_json::json!({
|
||||
"nested": {
|
||||
"access_token": "nested-canary-secret",
|
||||
"endpoint": "https://example.test/private?key=nested-canary-secret"
|
||||
}
|
||||
}),
|
||||
RedactionLimits::default(),
|
||||
)
|
||||
.expect("safe nested context");
|
||||
let events = capture("admin-api", || {
|
||||
tracing::warn!(
|
||||
name: "admin.request.rejected",
|
||||
password = "canary-secret",
|
||||
url = "https://example.test/path?token=canary-secret",
|
||||
safe_fields = %context,
|
||||
unsafe_context = ?serde_json::json!({"password": "debug-canary-secret"}),
|
||||
error_code = "invalid_request"
|
||||
);
|
||||
});
|
||||
let serialized = serde_json::to_string(&events[0]).expect("event JSON");
|
||||
|
||||
assert_eq!(events[0]["fields"]["password"], "[REDACTED]");
|
||||
assert_eq!(events[0]["fields"]["url"], "https://example.test/path");
|
||||
assert_eq!(
|
||||
events[0]["fields"]["safe_fields"]["nested"]["access_token"],
|
||||
"[REDACTED]"
|
||||
);
|
||||
assert_eq!(
|
||||
events[0]["fields"]["safe_fields"]["nested"]["endpoint"],
|
||||
"https://example.test/private"
|
||||
);
|
||||
assert_eq!(events[0]["fields"]["unsafe_context"], "[REDACTED]");
|
||||
assert_eq!(events[0]["fields"]["error_code"], "invalid_request");
|
||||
assert!(!serialized.contains("canary-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arbitrary_debug_text_is_never_written_verbatim() {
|
||||
struct Credentials;
|
||||
|
||||
impl std::fmt::Debug for Credentials {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("Credentials { password: \"debug-canary-secret\" }")
|
||||
}
|
||||
}
|
||||
|
||||
let events = capture("admin-api", || {
|
||||
tracing::warn!(
|
||||
name: "admin.debug.inspected",
|
||||
details = ?Credentials
|
||||
);
|
||||
});
|
||||
let serialized = serde_json::to_string(&events[0]).expect("event JSON");
|
||||
|
||||
assert_eq!(events[0]["fields"]["details"], "[REDACTED]");
|
||||
assert!(!serialized.contains("debug-canary-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound_sensitive_event_fields_are_redacted() {
|
||||
let events = capture("admin-api", || {
|
||||
tracing::warn!(
|
||||
name: "admin.request.rejected",
|
||||
client_api_key = "client-canary-secret",
|
||||
authorization_header = "Bearer auth-canary-secret",
|
||||
response_body = "response-canary-secret",
|
||||
tool_arguments = "argument-canary-secret",
|
||||
query_params = "query-canary-secret",
|
||||
);
|
||||
});
|
||||
let serialized = serde_json::to_string(&events[0]).expect("event JSON");
|
||||
|
||||
for key in [
|
||||
"client_api_key",
|
||||
"authorization_header",
|
||||
"response_body",
|
||||
"tool_arguments",
|
||||
"query_params",
|
||||
] {
|
||||
assert_eq!(events[0]["fields"][key], "[REDACTED]");
|
||||
}
|
||||
assert!(!serialized.contains("canary-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_event_falls_back_to_valid_bounded_json() {
|
||||
let limits = RedactionLimits {
|
||||
max_event_bytes: 512,
|
||||
..RedactionLimits::default()
|
||||
};
|
||||
let events = capture_with_limits("admin-api", limits, || {
|
||||
tracing::info!(
|
||||
name: "admin.payload.inspected",
|
||||
description = %"x".repeat(1024)
|
||||
);
|
||||
});
|
||||
let serialized = serde_json::to_vec(&events[0]).expect("bounded event JSON");
|
||||
|
||||
assert!(serialized.len() < limits.max_event_bytes);
|
||||
assert_eq!(events[0]["fields"]["truncated"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_json_honours_the_total_event_budget() {
|
||||
let limits = RedactionLimits {
|
||||
max_event_bytes: 512,
|
||||
..RedactionLimits::default()
|
||||
};
|
||||
|
||||
let serialized = safe_json(
|
||||
&serde_json::json!({"description": "x".repeat(4096)}),
|
||||
limits,
|
||||
)
|
||||
.expect("safe JSON must remain serializable");
|
||||
|
||||
assert!(serialized.len() <= limits.max_event_bytes);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(&serialized).expect("valid JSON")["truncated"],
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subscriber_rejects_limits_that_cannot_hold_an_event() {
|
||||
let config = ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
|
||||
"info",
|
||||
RedactionLimits {
|
||||
max_event_bytes: 16,
|
||||
..RedactionLimits::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(build_subscriber(config, SharedWriter::default()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimum_event_budget_handles_maximum_identity_labels() {
|
||||
let writer = SharedWriter::default();
|
||||
let config = ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("s".repeat(64), "v".repeat(64), "e".repeat(64))
|
||||
.expect("maximum identity labels are valid"),
|
||||
"info",
|
||||
RedactionLimits {
|
||||
max_event_bytes: 512,
|
||||
..RedactionLimits::default()
|
||||
},
|
||||
);
|
||||
let subscriber =
|
||||
build_subscriber(config, writer.clone()).expect("minimum valid budget must be usable");
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
tracing::info!(
|
||||
name: "event-name-that-is-intentionally-longer-than-the-fallback-limit",
|
||||
description = %"x".repeat(4096),
|
||||
);
|
||||
});
|
||||
|
||||
let output = writer.output();
|
||||
assert!(output.len() <= 512);
|
||||
assert_eq!(output.lines().count(), 1);
|
||||
serde_json::from_str::<Value>(output.trim_end()).expect("bounded line must remain valid JSON");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn env_filter_is_applied_and_invalid_filter_is_safe() {
|
||||
let writer = SharedWriter::default();
|
||||
let config = ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
|
||||
"warn",
|
||||
RedactionLimits::default(),
|
||||
);
|
||||
let subscriber =
|
||||
build_subscriber(config, writer.clone()).expect("test subscriber must be built");
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
tracing::info!(name: "filtered.info", "filtered");
|
||||
tracing::warn!(name: "visible.warning", "visible");
|
||||
});
|
||||
let output = writer.output();
|
||||
|
||||
assert!(!output.contains("filtered.info"));
|
||||
assert!(output.contains("visible.warning"));
|
||||
|
||||
let invalid = ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
|
||||
"[not a valid filter",
|
||||
RedactionLimits::default(),
|
||||
);
|
||||
let error = build_subscriber(invalid, SharedWriter::default())
|
||||
.err()
|
||||
.expect("invalid filter must fail");
|
||||
assert_eq!(error.to_string(), "invalid log filter");
|
||||
assert!(!error.to_string().contains("not a valid filter"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_identity_rejects_empty_or_unsafe_labels() {
|
||||
for (service, version, environment) in [
|
||||
("", "0.3.1", "test"),
|
||||
("admin api", "0.3.1", "test"),
|
||||
("admin-api", "", "test"),
|
||||
("admin-api", "0.3.1", "prod\nsecret"),
|
||||
] {
|
||||
assert!(ServiceIdentity::try_new(service, version, environment).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity, init,
|
||||
};
|
||||
|
||||
fn config() -> ObservabilityConfig {
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"),
|
||||
"info",
|
||||
RedactionLimits::default(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_global_initialization_returns_typed_error() {
|
||||
let _lifecycle = init(config()).expect("first initialization must succeed");
|
||||
let error = init(config()).expect_err("second initialization must fail");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
ObservabilityInitError::SubscriberAlreadyInitialized
|
||||
));
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use crank_observability::{OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError};
|
||||
|
||||
#[test]
|
||||
fn absent_endpoint_disables_export_without_background_resources() {
|
||||
let config = OtlpTraceConfig::try_new(
|
||||
None,
|
||||
None,
|
||||
Duration::from_secs(10),
|
||||
OtlpBatchConfig::default(),
|
||||
)
|
||||
.expect("missing endpoint must be valid");
|
||||
|
||||
assert!(!config.is_enabled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_config_accepts_only_bounded_http_protobuf() {
|
||||
let config = OtlpTraceConfig::try_new(
|
||||
Some("https://collector.example.test/v1/traces".to_owned()),
|
||||
Some("http/protobuf".to_owned()),
|
||||
Duration::from_secs(3),
|
||||
OtlpBatchConfig::try_new(256, 64, Duration::from_millis(500), Duration::from_secs(3))
|
||||
.unwrap(),
|
||||
)
|
||||
.expect("bounded HTTP protobuf config must be valid");
|
||||
|
||||
assert!(config.is_enabled());
|
||||
assert_eq!(config.export_timeout(), Duration::from_secs(3));
|
||||
assert_eq!(config.batch().max_queue_size(), 256);
|
||||
assert_eq!(config.batch().max_export_batch_size(), 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_values_return_safe_typed_errors() {
|
||||
let secret_endpoint = "https://user:canary-secret@collector.example.test/v1/traces";
|
||||
let error = OtlpTraceConfig::try_new(
|
||||
Some(secret_endpoint.to_owned()),
|
||||
Some("grpc".to_owned()),
|
||||
Duration::ZERO,
|
||||
OtlpBatchConfig::default(),
|
||||
)
|
||||
.expect_err("credentials in endpoint must be rejected");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
OtlpTraceConfigError::InvalidEndpoint { .. }
|
||||
));
|
||||
assert!(!error.to_string().contains(secret_endpoint));
|
||||
assert!(!error.to_string().contains("canary-secret"));
|
||||
|
||||
let error = OtlpBatchConfig::try_new(8, 9, Duration::from_millis(1), Duration::from_secs(1))
|
||||
.expect_err("batch cannot exceed queue");
|
||||
assert!(matches!(error, OtlpTraceConfigError::InvalidBatchLimits));
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use crank_observability::{
|
||||
DURATION_BUCKETS_SECONDS, MetricsConfig, MetricsConfigError, MetricsSurface, ServiceIdentity,
|
||||
metric_schema,
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
fn identity() -> ServiceIdentity {
|
||||
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loopback_is_allowed_without_a_token() {
|
||||
let config = MetricsConfig::new(
|
||||
true,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
|
||||
None,
|
||||
)
|
||||
.expect("loopback metrics must be safe by default");
|
||||
|
||||
assert_eq!(config.bind_addr().to_string(), "127.0.0.1:9464");
|
||||
assert!(!config.requires_authentication());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_loopback_without_a_token_is_rejected_without_secret_data() {
|
||||
let error = MetricsConfig::new(
|
||||
true,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9464),
|
||||
None,
|
||||
)
|
||||
.expect_err("external metrics must require authentication");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
MetricsConfigError::MissingTokenForExternalBind
|
||||
));
|
||||
assert!(!error.to_string().contains("token="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_is_closed_and_uses_fixed_duration_buckets() {
|
||||
let schema = metric_schema();
|
||||
let names: Vec<_> = schema.iter().map(|metric| metric.name).collect();
|
||||
|
||||
assert!(names.contains(&"crank_http_requests_total"));
|
||||
assert!(names.contains(&"crank_http_request_duration_seconds"));
|
||||
assert!(names.contains(&"crank_mcp_requests_total"));
|
||||
assert!(names.contains(&"crank_tool_invocations_total"));
|
||||
assert!(names.contains(&"crank_runtime_inflight"));
|
||||
assert!(names.contains(&"crank_db_pool_connections"));
|
||||
assert!(names.contains(&"crank_catalog_tools"));
|
||||
assert!(names.contains(&"crank_invocation_history_lost_total"));
|
||||
assert!(names.contains(&"crank_telemetry_export_failures_total"));
|
||||
|
||||
for metric in schema {
|
||||
for forbidden in [
|
||||
"workspace",
|
||||
"agent_id",
|
||||
"operation_id",
|
||||
"request_id",
|
||||
"url",
|
||||
"error_message",
|
||||
"text",
|
||||
] {
|
||||
assert!(
|
||||
!metric.labels.contains(&forbidden),
|
||||
"{} exposes forbidden label {forbidden}",
|
||||
metric.name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
&[
|
||||
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn external_surface_protects_both_routes_and_exposes_nothing_else() {
|
||||
let token = "metrics-canary-secret";
|
||||
let config = MetricsConfig::new(
|
||||
true,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9464),
|
||||
Some(token.to_owned()),
|
||||
)
|
||||
.expect("external metrics with token");
|
||||
let surface = MetricsSurface::for_test(config, identity()).expect("test metrics surface");
|
||||
let app = surface.router();
|
||||
|
||||
for path in ["/metrics", "/health"] {
|
||||
let unauthorized = app
|
||||
.clone()
|
||||
.oneshot(Request::get(path).body(Body::empty()).expect("request"))
|
||||
.await
|
||||
.expect("response");
|
||||
assert_eq!(unauthorized.status(), StatusCode::UNAUTHORIZED);
|
||||
|
||||
let authorized = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::get(path)
|
||||
.header(header::AUTHORIZATION, format!("Bearer {token}"))
|
||||
.body(Body::empty())
|
||||
.expect("request"),
|
||||
)
|
||||
.await
|
||||
.expect("response");
|
||||
assert_eq!(authorized.status(), StatusCode::OK);
|
||||
let body = to_bytes(authorized.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("bounded body");
|
||||
assert!(!String::from_utf8_lossy(&body).contains(token));
|
||||
}
|
||||
|
||||
let absent = app
|
||||
.oneshot(
|
||||
Request::get("/api/operations")
|
||||
.header(header::AUTHORIZATION, format!("Bearer {token}"))
|
||||
.body(Body::empty())
|
||||
.expect("request"),
|
||||
)
|
||||
.await
|
||||
.expect("response");
|
||||
assert_eq!(absent.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use crank_observability::{inject_current_trace_context, set_remote_trace_parent};
|
||||
use opentelemetry::{global, trace::TracerProvider as _};
|
||||
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
|
||||
use tracing::info_span;
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
|
||||
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
|
||||
|
||||
#[test]
|
||||
fn valid_remote_parent_is_continued_and_request_id_is_unrelated() {
|
||||
with_trace_dispatch(|| {
|
||||
let mut incoming = HeaderMap::new();
|
||||
incoming.insert(
|
||||
"traceparent",
|
||||
HeaderValue::from_static("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
|
||||
);
|
||||
incoming.insert("x-request-id", HeaderValue::from_static("req-unrelated"));
|
||||
|
||||
let span = info_span!("http.request", request_id = "req-unrelated");
|
||||
assert!(set_remote_trace_parent(&span, &incoming));
|
||||
let _guard = span.enter();
|
||||
|
||||
let mut outgoing = HeaderMap::new();
|
||||
assert!(inject_current_trace_context(&mut outgoing));
|
||||
let propagated = outgoing["traceparent"].to_str().unwrap();
|
||||
assert_eq!(&propagated[3..35], REMOTE_TRACE_ID);
|
||||
assert!(!propagated.contains("req-unrelated"));
|
||||
assert!(!outgoing.contains_key("baggage"));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_parent_is_ignored_and_a_new_trace_is_created() {
|
||||
with_trace_dispatch(|| {
|
||||
let mut incoming = HeaderMap::new();
|
||||
incoming.insert(
|
||||
"traceparent",
|
||||
HeaderValue::from_static("canary-invalid-traceparent"),
|
||||
);
|
||||
|
||||
let span = info_span!("mcp.request");
|
||||
assert!(!set_remote_trace_parent(&span, &incoming));
|
||||
let _guard = span.enter();
|
||||
|
||||
let mut outgoing = HeaderMap::new();
|
||||
assert!(inject_current_trace_context(&mut outgoing));
|
||||
let propagated = outgoing["traceparent"].to_str().unwrap();
|
||||
assert!(propagated.starts_with("00-"));
|
||||
assert!(!propagated.contains("canary-invalid-traceparent"));
|
||||
});
|
||||
}
|
||||
|
||||
fn with_trace_dispatch(test: impl FnOnce()) {
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let tracer = provider.tracer("propagation-test");
|
||||
let subscriber =
|
||||
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
|
||||
test();
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
use crank_observability::{RedactionLimits, redact_value};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const REDACTED: &str = "[REDACTED]";
|
||||
const TRUNCATED: &str = "[TRUNCATED]";
|
||||
|
||||
#[test]
|
||||
fn sensitive_keys_are_redacted_case_insensitively() {
|
||||
let keys = [
|
||||
"password",
|
||||
"PassWord",
|
||||
"api_key",
|
||||
"access-token",
|
||||
"authorization",
|
||||
"Proxy.Authorization",
|
||||
"cookie",
|
||||
"set_cookie",
|
||||
"payload",
|
||||
"request_body",
|
||||
"arguments",
|
||||
"result",
|
||||
"response",
|
||||
];
|
||||
|
||||
for key in keys {
|
||||
let cleaned = redact_value(&json!({ key: "canary-secret" }), RedactionLimits::default());
|
||||
assert_eq!(cleaned[key], REDACTED, "key {key} was not redacted");
|
||||
assert!(!cleaned.to_string().contains("canary-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_sensitive_key_variants_are_redacted() {
|
||||
for canonical in [
|
||||
"password",
|
||||
"api_key",
|
||||
"access_key",
|
||||
"secret_key",
|
||||
"proxy_authorization",
|
||||
"set_cookie",
|
||||
"query_string",
|
||||
"request_body",
|
||||
] {
|
||||
for separator in ["_", "-", "."] {
|
||||
let variant = canonical
|
||||
.split('_')
|
||||
.collect::<Vec<_>>()
|
||||
.join(separator)
|
||||
.to_ascii_uppercase();
|
||||
let cleaned = redact_value(
|
||||
&json!({ variant.clone(): "canary-secret" }),
|
||||
RedactionLimits::default(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
cleaned[&variant], REDACTED,
|
||||
"key {variant} was not redacted"
|
||||
);
|
||||
assert!(!cleaned.to_string().contains("canary-secret"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound_sensitive_key_names_are_redacted() {
|
||||
for key in [
|
||||
"client_api_key",
|
||||
"aws_access_key_id",
|
||||
"http_authorization_header",
|
||||
"response_body",
|
||||
"tool_arguments",
|
||||
"query_params",
|
||||
"secret_value",
|
||||
] {
|
||||
let cleaned = redact_value(&json!({ key: "canary-secret" }), RedactionLimits::default());
|
||||
|
||||
assert_eq!(cleaned[key], REDACTED, "key {key} was not redacted");
|
||||
assert!(!cleaned.to_string().contains("canary-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_query_and_fragment_are_removed_at_every_depth() {
|
||||
let input = json!({
|
||||
"url": "https://example.test/path?token=canary-secret#fragment",
|
||||
"nested": [{
|
||||
"endpoint_uri": "https://example.test/other?q=canary-secret"
|
||||
}]
|
||||
});
|
||||
|
||||
let cleaned = redact_value(&input, RedactionLimits::default());
|
||||
|
||||
assert_eq!(cleaned["url"], "https://example.test/path");
|
||||
assert_eq!(
|
||||
cleaned["nested"][0]["endpoint_uri"],
|
||||
"https://example.test/other"
|
||||
);
|
||||
assert!(!cleaned.to_string().contains("canary-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_credentials_and_compound_endpoint_fields_are_removed() {
|
||||
let input = json!({
|
||||
"upstream_endpoint": "https://user:canary-secret@example.test/path?token=canary-secret",
|
||||
});
|
||||
|
||||
let cleaned = redact_value(&input, RedactionLimits::default());
|
||||
|
||||
assert_eq!(cleaned["upstream_endpoint"], "https://example.test/path");
|
||||
assert!(!cleaned.to_string().contains("user"));
|
||||
assert!(!cleaned.to_string().contains("canary-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_values_and_collections_respect_all_limits() {
|
||||
let limits = RedactionLimits {
|
||||
max_string_bytes: 16,
|
||||
max_array_items: 3,
|
||||
max_object_fields: 3,
|
||||
max_depth: 2,
|
||||
max_event_bytes: 256,
|
||||
};
|
||||
let input = json!({
|
||||
"long": "абвгдежзийклмнопрсту",
|
||||
"array": [1, 2, 3, 4, 5],
|
||||
"object": {"a": 1, "b": 2, "c": 3, "d": 4},
|
||||
"nested": {"level2": {"level3": "must not survive"}}
|
||||
});
|
||||
|
||||
let cleaned = redact_value(&input, limits);
|
||||
let object = cleaned
|
||||
.as_object()
|
||||
.expect("cleaned root must remain an object");
|
||||
|
||||
assert!(object.len() <= limits.max_object_fields);
|
||||
assert!(
|
||||
cleaned["long"]
|
||||
.as_str()
|
||||
.map(|value| value.len() <= limits.max_string_bytes)
|
||||
.unwrap_or(true)
|
||||
);
|
||||
assert!(
|
||||
cleaned["array"]
|
||||
.as_array()
|
||||
.map(|value| value.len() <= limits.max_array_items)
|
||||
.unwrap_or(true)
|
||||
);
|
||||
assert!(!cleaned.to_string().contains("must not survive"));
|
||||
assert!(cleaned.to_string().contains(TRUNCATED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_preserves_utf8_and_never_reveals_secret_fragments() {
|
||||
let input = json!({
|
||||
"secret_key": "секретное-значение",
|
||||
"description": "я".repeat(2048),
|
||||
});
|
||||
|
||||
let cleaned = redact_value(&input, RedactionLimits::default());
|
||||
let serialized = serde_json::to_string(&cleaned).expect("cleaned value must be valid JSON");
|
||||
|
||||
assert_eq!(cleaned["secret_key"], REDACTED);
|
||||
assert!(!serialized.contains("секретное"));
|
||||
assert!(cleaned["description"].as_str().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacted_value_does_not_mutate_input() {
|
||||
let input = json!({"password": "canary-secret"});
|
||||
let original = input.clone();
|
||||
|
||||
let _ = redact_value(&input, RedactionLimits::default());
|
||||
|
||||
assert_eq!(input, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_keys_respect_the_string_limit() {
|
||||
let limits = RedactionLimits {
|
||||
max_string_bytes: 16,
|
||||
..RedactionLimits::default()
|
||||
};
|
||||
let long_key = format!("field-{}", "x".repeat(128));
|
||||
|
||||
let cleaned = redact_value(&json!({ long_key: "value" }), limits);
|
||||
|
||||
assert!(
|
||||
cleaned
|
||||
.as_object()
|
||||
.expect("cleaned object")
|
||||
.keys()
|
||||
.all(|key| key.len() <= limits.max_string_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limits_have_finite_safe_defaults() {
|
||||
let limits = RedactionLimits::default();
|
||||
|
||||
assert_eq!(limits.max_string_bytes, 1024);
|
||||
assert_eq!(limits.max_array_items, 32);
|
||||
assert_eq!(limits.max_object_fields, 64);
|
||||
assert_eq!(limits.max_depth, 8);
|
||||
assert_eq!(limits.max_event_bytes, 16 * 1024);
|
||||
assert!(Value::Null.is_null());
|
||||
}
|
||||
Reference in New Issue
Block a user