исправить: закрыть ревью критических ошибок
CI / Rust Checks (pull_request) Successful in 6m4s
CI / UI Checks (pull_request) Successful in 5s
CI / Community Image Smoke (pull_request) Successful in 4m27s
CI / Frontend E2E (pull_request) Successful in 5m19s
CI / Deploy (pull_request) Has been skipped
CI / Rust Checks (push) Successful in 6m5s
CI / UI Checks (push) Successful in 5s
CI / Community Image Smoke (push) Successful in 1m5s
CI / Frontend E2E (push) Successful in 3m54s
CI / Deploy (push) Failing after 45s
CI / Rust Checks (pull_request) Successful in 6m4s
CI / UI Checks (pull_request) Successful in 5s
CI / Community Image Smoke (pull_request) Successful in 4m27s
CI / Frontend E2E (pull_request) Successful in 5m19s
CI / Deploy (pull_request) Has been skipped
CI / Rust Checks (push) Successful in 6m5s
CI / UI Checks (push) Successful in 5s
CI / Community Image Smoke (push) Successful in 1m5s
CI / Frontend E2E (push) Successful in 3m54s
CI / Deploy (push) Failing after 45s
This commit is contained in:
+11
-10
@@ -21,6 +21,8 @@ use sqlx::postgres::PgConnectOptions;
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::{info, warn};
|
||||
|
||||
const MAX_INVOCATION_LOG_RETENTION_DAYS: i64 = 36_500;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let observability = crank_observability::init(ObservabilityConfig::from_env(
|
||||
@@ -106,8 +108,7 @@ async fn run(
|
||||
.with_outbound_http_policy(outbound_http_policy)
|
||||
.with_identity_provider(std::sync::Arc::new(identity_provider))
|
||||
.build();
|
||||
let invocation_log_retention_days =
|
||||
positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
|
||||
let invocation_log_retention_days = invocation_log_retention_days_from_env()?;
|
||||
service.bootstrap_admin_user().await?;
|
||||
if env_flag("CRANK_DEMO_SEED") {
|
||||
service.seed_demo_assets().await?;
|
||||
@@ -159,17 +160,17 @@ async fn run(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn positive_i64_from_env(
|
||||
name: &'static str,
|
||||
default: i64,
|
||||
) -> Result<i64, Box<dyn std::error::Error>> {
|
||||
let value = match env::var(name) {
|
||||
fn invocation_log_retention_days_from_env() -> Result<i64, Box<dyn std::error::Error>> {
|
||||
const NAME: &str = "CRANK_INVOCATION_LOG_RETENTION_DAYS";
|
||||
let value = match env::var(NAME) {
|
||||
Ok(raw) => raw.parse::<i64>()?,
|
||||
Err(env::VarError::NotPresent) => default,
|
||||
Err(env::VarError::NotPresent) => 30,
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if value <= 0 {
|
||||
return Err(format!("{name} must be greater than zero").into());
|
||||
if !(1..=MAX_INVOCATION_LOG_RETENTION_DAYS).contains(&value) {
|
||||
return Err(
|
||||
format!("{NAME} must be between 1 and {MAX_INVOCATION_LOG_RETENTION_DAYS}").into(),
|
||||
);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use axum::{
|
||||
extract::Request,
|
||||
extract::{MatchedPath, Request},
|
||||
http::{HeaderName, HeaderValue},
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
@@ -19,7 +19,11 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
|
||||
request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
|
||||
};
|
||||
let method = request.method().clone();
|
||||
let path = request.uri().path().to_owned();
|
||||
let route = request
|
||||
.extensions()
|
||||
.get::<MatchedPath>()
|
||||
.map_or("unmatched", MatchedPath::as_str)
|
||||
.to_owned();
|
||||
let span = info_span!(
|
||||
target: "crank::trace",
|
||||
"http.request",
|
||||
@@ -34,7 +38,7 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
|
||||
name: "admin.request.completed",
|
||||
request_id = %context.request_id,
|
||||
method = %method,
|
||||
path,
|
||||
route,
|
||||
status = response.status().as_u16(),
|
||||
"admin request completed"
|
||||
);
|
||||
|
||||
@@ -65,6 +65,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
|
||||
.unwrap();
|
||||
assert_eq!(event["request_id"], "req_admin_trace_123");
|
||||
assert_eq!(event["fields"]["status"], 200);
|
||||
assert_eq!(event["fields"]["route"], "/probe");
|
||||
|
||||
let invalid_response = app
|
||||
.oneshot(
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::BTreeMap,
|
||||
env, fmt,
|
||||
future::Future,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use sentry::{
|
||||
ClientInitGuard, ClientOptions,
|
||||
@@ -14,6 +20,8 @@ use crate::{
|
||||
const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
|
||||
const CRITICAL_ERROR_MESSAGE: &str = "critical error";
|
||||
const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
// Sentry serializes SystemTime as a finite f64; this keeps conservative fixed headroom.
|
||||
const MAX_SERIALIZED_TIMESTAMP_BYTES: usize = 32;
|
||||
|
||||
tokio::task_local! {
|
||||
static REQUEST_ID: String;
|
||||
@@ -63,6 +71,8 @@ pub enum SentryConfigError {
|
||||
InvalidDsn,
|
||||
#[error("CRANK_SENTRY_DSN is not valid UTF-8")]
|
||||
InvalidEnvironmentEncoding,
|
||||
#[error("critical error event budget cannot hold the required fields")]
|
||||
EventBudgetTooSmall,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -74,6 +84,13 @@ pub enum CriticalErrorCategory {
|
||||
}
|
||||
|
||||
impl CriticalErrorCategory {
|
||||
const ALL: [Self; 4] = [
|
||||
Self::Panic,
|
||||
Self::Startup,
|
||||
Self::Internal,
|
||||
Self::DataIntegrity,
|
||||
];
|
||||
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Panic => "panic",
|
||||
@@ -117,11 +134,14 @@ pub(crate) fn init_sentry(
|
||||
identity: &ServiceIdentity,
|
||||
limits: RedactionLimits,
|
||||
config: SentryConfig,
|
||||
) -> Option<ClientInitGuard> {
|
||||
let dsn = config.dsn?;
|
||||
) -> Result<Option<ClientInitGuard>, SentryConfigError> {
|
||||
let Some(dsn) = config.dsn else {
|
||||
return Ok(None);
|
||||
};
|
||||
validate_critical_event_budget(identity, limits)?;
|
||||
let identity = identity.clone();
|
||||
let options = client_options(identity, limits);
|
||||
Some(sentry::init((dsn, options)))
|
||||
Ok(Some(sentry::init((dsn, options))))
|
||||
}
|
||||
|
||||
fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
|
||||
@@ -181,7 +201,10 @@ fn sanitize_event(
|
||||
Event {
|
||||
event_id: event.event_id,
|
||||
level: Level::Error,
|
||||
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
|
||||
fingerprint: Cow::Owned(vec![
|
||||
Cow::Owned(identity.service().to_owned()),
|
||||
Cow::Borrowed(category.as_str()),
|
||||
]),
|
||||
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
|
||||
timestamp: event.timestamp,
|
||||
server_name: Some(Cow::Owned(identity.service().to_owned())),
|
||||
@@ -212,9 +235,44 @@ fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Ev
|
||||
|
||||
event.tags.remove("request_id");
|
||||
event.tags.remove("trace_id");
|
||||
debug_assert!(serialized_event_len(&event) <= max_event_bytes);
|
||||
event
|
||||
}
|
||||
|
||||
fn validate_critical_event_budget(
|
||||
identity: &ServiceIdentity,
|
||||
limits: RedactionLimits,
|
||||
) -> Result<(), SentryConfigError> {
|
||||
if required_critical_event_budget(identity, limits) > limits.max_event_bytes {
|
||||
return Err(SentryConfigError::EventBudgetTooSmall);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionLimits) -> usize {
|
||||
let unbounded_limits = RedactionLimits {
|
||||
max_event_bytes: usize::MAX,
|
||||
..limits
|
||||
};
|
||||
CriticalErrorCategory::ALL
|
||||
.into_iter()
|
||||
.map(|category| {
|
||||
let event = sanitize_event(
|
||||
Event {
|
||||
tags: BTreeMap::from([("category".to_owned(), category.as_str().to_owned())]),
|
||||
timestamp: SystemTime::UNIX_EPOCH,
|
||||
..Event::default()
|
||||
},
|
||||
identity,
|
||||
unbounded_limits,
|
||||
);
|
||||
serialized_event_len(&event)
|
||||
.saturating_add(MAX_SERIALIZED_TIMESTAMP_BYTES.saturating_sub(1))
|
||||
})
|
||||
.max()
|
||||
.unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
fn serialized_event_len(event: &Event<'_>) -> usize {
|
||||
serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
|
||||
}
|
||||
@@ -223,10 +281,12 @@ fn serialized_event_len(event: &Event<'_>) -> usize {
|
||||
mod tests {
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
panic::AssertUnwindSafe,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use opentelemetry::trace::TracerProvider as _;
|
||||
@@ -237,7 +297,8 @@ mod tests {
|
||||
};
|
||||
|
||||
use super::{
|
||||
CriticalErrorCategory, capture_critical_error, client_options, sanitize_event,
|
||||
CriticalErrorCategory, capture_critical_error, client_options,
|
||||
required_critical_event_budget, sanitize_event, validate_critical_event_budget,
|
||||
with_request_correlation,
|
||||
};
|
||||
use crate::{
|
||||
@@ -395,6 +456,14 @@ mod tests {
|
||||
event.tags.get("category").map(String::as_str),
|
||||
Some("data_integrity")
|
||||
);
|
||||
assert_eq!(
|
||||
event
|
||||
.fingerprint
|
||||
.iter()
|
||||
.map(AsRef::as_ref)
|
||||
.collect::<Vec<&str>>(),
|
||||
vec!["admin-api", "data_integrity"]
|
||||
);
|
||||
assert_eq!(
|
||||
event.tags.get("request_id").map(String::as_str),
|
||||
Some("request-123")
|
||||
@@ -409,11 +478,33 @@ mod tests {
|
||||
fn panic_creates_exactly_one_sanitized_critical_event() {
|
||||
let options =
|
||||
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.expect("runtime");
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let tracer = provider.tracer("panic-correlation-test");
|
||||
let subscriber = build_subscriber_with_tracer(
|
||||
ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
|
||||
std::io::sink,
|
||||
Some(tracer),
|
||||
)
|
||||
.expect("subscriber");
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let events = sentry::test::with_captured_events_options(
|
||||
|| {
|
||||
let result = std::panic::catch_unwind(|| {
|
||||
let result = std::panic::catch_unwind(AssertUnwindSafe(|| {
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
runtime.block_on(with_request_correlation(
|
||||
"panic-request-123".to_owned(),
|
||||
async {
|
||||
let span =
|
||||
tracing::info_span!(target: "crank::trace", "http.request");
|
||||
let _span_guard = span.enter();
|
||||
panic!("password=must-not-leak");
|
||||
},
|
||||
));
|
||||
});
|
||||
}));
|
||||
assert!(result.is_err());
|
||||
},
|
||||
options,
|
||||
@@ -425,24 +516,34 @@ mod tests {
|
||||
event.tags.get("category").map(String::as_str),
|
||||
Some("panic")
|
||||
);
|
||||
assert_eq!(
|
||||
event.tags.get("request_id").map(String::as_str),
|
||||
Some("panic-request-123")
|
||||
);
|
||||
assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
|
||||
let serialized = serde_json::to_string(event).expect("serialize event");
|
||||
assert!(!serialized.contains("must-not-leak"));
|
||||
assert!(!serialized.contains("password"));
|
||||
provider.shutdown().expect("provider shutdown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn receiver_failure_does_not_change_product_result_or_recurse() {
|
||||
struct DroppingTransport {
|
||||
struct UnavailableTransport {
|
||||
attempts: AtomicUsize,
|
||||
}
|
||||
|
||||
impl sentry::Transport for DroppingTransport {
|
||||
impl sentry::Transport for UnavailableTransport {
|
||||
fn send_envelope(&self, _envelope: Envelope) {
|
||||
self.attempts.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn flush(&self, _timeout: Duration) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
let transport = Arc::new(DroppingTransport {
|
||||
let transport = Arc::new(UnavailableTransport {
|
||||
attempts: AtomicUsize::new(0),
|
||||
});
|
||||
let mut options =
|
||||
@@ -454,7 +555,10 @@ mod tests {
|
||||
);
|
||||
options.transport = Some(Arc::new(transport.clone()));
|
||||
let client = Arc::new(sentry::Client::from(options));
|
||||
let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default())));
|
||||
let hub = Arc::new(Hub::new(
|
||||
Some(Arc::clone(&client)),
|
||||
Arc::new(Default::default()),
|
||||
));
|
||||
|
||||
let product_result = Hub::run(hub, || {
|
||||
capture_critical_error(CriticalErrorCategory::Internal);
|
||||
@@ -463,5 +567,58 @@ mod tests {
|
||||
|
||||
assert_eq!(product_result, 42);
|
||||
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
|
||||
assert!(!client.close(Some(Duration::from_millis(10))));
|
||||
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn critical_event_budget_rejects_identity_that_does_not_fit() {
|
||||
let maximum_label = "a".repeat(64);
|
||||
let identity =
|
||||
ServiceIdentity::try_new(maximum_label.clone(), maximum_label.clone(), maximum_label)
|
||||
.expect("maximum identity");
|
||||
let limits = RedactionLimits {
|
||||
max_event_bytes: 512,
|
||||
..RedactionLimits::default()
|
||||
};
|
||||
|
||||
assert!(matches!(
|
||||
validate_critical_event_budget(&identity, limits),
|
||||
Err(super::SentryConfigError::EventBudgetTooSmall)
|
||||
));
|
||||
validate_critical_event_budget(
|
||||
&identity,
|
||||
RedactionLimits {
|
||||
max_event_bytes: 1024,
|
||||
..limits
|
||||
},
|
||||
)
|
||||
.expect("larger budget must hold the required identity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn critical_event_budget_covers_every_category() {
|
||||
let identity = identity();
|
||||
let limits = RedactionLimits::default();
|
||||
let required_budget = required_critical_event_budget(&identity, limits);
|
||||
|
||||
validate_critical_event_budget(
|
||||
&identity,
|
||||
RedactionLimits {
|
||||
max_event_bytes: required_budget,
|
||||
..limits
|
||||
},
|
||||
)
|
||||
.expect("exact required budget must be accepted");
|
||||
assert!(matches!(
|
||||
validate_critical_event_budget(
|
||||
&identity,
|
||||
RedactionLimits {
|
||||
max_event_bytes: required_budget - 1,
|
||||
..limits
|
||||
},
|
||||
),
|
||||
Err(super::SentryConfigError::EventBudgetTooSmall)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ impl ObservabilityLifecycle {
|
||||
.map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?;
|
||||
let metrics_handle = install_prometheus_recorder(&identity)?;
|
||||
register_metric_schema();
|
||||
let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config);
|
||||
let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config)?;
|
||||
|
||||
Ok(Self {
|
||||
metrics_handle,
|
||||
|
||||
@@ -99,6 +99,11 @@ CRANK_SENTRY_DSN=https://public-key@errors.example.com/1
|
||||
- доступные `request_id` и `trace_id`;
|
||||
- статическое сообщение без исходного текста ошибки.
|
||||
|
||||
События группируются по паре `service` и закрытой категории, поэтому одинаковая
|
||||
ошибка `admin-api` и `mcp-server` не объединяется в один инцидент. Если
|
||||
обязательные поля идентичности не помещаются в заданный предел события, запуск
|
||||
останавливается безопасной типизированной ошибкой до приёма запросов.
|
||||
|
||||
До отправки удаляются request, user, breadcrumbs, URL, query, cookie,
|
||||
authorization, payload, произвольные contexts и extra, а также исходный текст
|
||||
panic или ошибки. Performance tracing, журналы, показатели и отслеживание
|
||||
|
||||
@@ -151,6 +151,11 @@ CRANK_LOG_LEVEL=info
|
||||
Внешний сборщик журналов не обязателен. Его отсутствие не влияет на `/health`
|
||||
и `/ready`.
|
||||
|
||||
`CRANK_INVOCATION_LOG_RETENTION_DAYS` задаёт срок хранения подробной истории
|
||||
вызовов в днях. Допустимый диапазон: от `1` до `36500`, значение по умолчанию:
|
||||
`30`. Значение вне диапазона останавливает запуск до создания фоновой задачи
|
||||
очистки.
|
||||
|
||||
## Критические ошибки
|
||||
|
||||
- `CRANK_SENTRY_DSN` — DSN внешнего Sentry-совместимого приёмника.
|
||||
|
||||
Reference in New Issue
Block a user