исправить: закрыть ревью критических ошибок
CI / Rust Checks (pull_request) Successful in 6m4s
CI / UI Checks (pull_request) Successful in 5s
CI / Community Image Smoke (pull_request) Successful in 4m27s
CI / Frontend E2E (pull_request) Successful in 5m19s
CI / Deploy (pull_request) Has been skipped
CI / Rust Checks (push) Successful in 6m5s
CI / UI Checks (push) Successful in 5s
CI / Community Image Smoke (push) Successful in 1m5s
CI / Frontend E2E (push) Successful in 3m54s
CI / Deploy (push) Failing after 45s

This commit is contained in:
2026-07-31 09:31:38 +03:00
parent 9b1a739e39
commit c30461cc92
7 changed files with 200 additions and 27 deletions
+11 -10
View File
@@ -21,6 +21,8 @@ use sqlx::postgres::PgConnectOptions;
use tokio::net::TcpListener;
use tracing::{info, warn};
const MAX_INVOCATION_LOG_RETENTION_DAYS: i64 = 36_500;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let observability = crank_observability::init(ObservabilityConfig::from_env(
@@ -106,8 +108,7 @@ async fn run(
.with_outbound_http_policy(outbound_http_policy)
.with_identity_provider(std::sync::Arc::new(identity_provider))
.build();
let invocation_log_retention_days =
positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
let invocation_log_retention_days = invocation_log_retention_days_from_env()?;
service.bootstrap_admin_user().await?;
if env_flag("CRANK_DEMO_SEED") {
service.seed_demo_assets().await?;
@@ -159,17 +160,17 @@ async fn run(
Ok(())
}
fn positive_i64_from_env(
name: &'static str,
default: i64,
) -> Result<i64, Box<dyn std::error::Error>> {
let value = match env::var(name) {
fn invocation_log_retention_days_from_env() -> Result<i64, Box<dyn std::error::Error>> {
const NAME: &str = "CRANK_INVOCATION_LOG_RETENTION_DAYS";
let value = match env::var(NAME) {
Ok(raw) => raw.parse::<i64>()?,
Err(env::VarError::NotPresent) => default,
Err(env::VarError::NotPresent) => 30,
Err(error) => return Err(error.into()),
};
if value <= 0 {
return Err(format!("{name} must be greater than zero").into());
if !(1..=MAX_INVOCATION_LOG_RETENTION_DAYS).contains(&value) {
return Err(
format!("{NAME} must be between 1 and {MAX_INVOCATION_LOG_RETENTION_DAYS}").into(),
);
}
Ok(value)
}
+7 -3
View File
@@ -1,5 +1,5 @@
use axum::{
extract::Request,
extract::{MatchedPath, Request},
http::{HeaderName, HeaderValue},
middleware::Next,
response::Response,
@@ -19,7 +19,11 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
};
let method = request.method().clone();
let path = request.uri().path().to_owned();
let route = request
.extensions()
.get::<MatchedPath>()
.map_or("unmatched", MatchedPath::as_str)
.to_owned();
let span = info_span!(
target: "crank::trace",
"http.request",
@@ -34,7 +38,7 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
name: "admin.request.completed",
request_id = %context.request_id,
method = %method,
path,
route,
status = response.status().as_u16(),
"admin request completed"
);
@@ -65,6 +65,7 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
.unwrap();
assert_eq!(event["request_id"], "req_admin_trace_123");
assert_eq!(event["fields"]["status"], 200);
assert_eq!(event["fields"]["route"], "/probe");
let invalid_response = app
.oneshot(
+168 -11
View File
@@ -1,4 +1,10 @@
use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration};
use std::{
borrow::Cow,
collections::BTreeMap,
env, fmt,
future::Future,
time::{Duration, SystemTime},
};
use sentry::{
ClientInitGuard, ClientOptions,
@@ -14,6 +20,8 @@ use crate::{
const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
const CRITICAL_ERROR_MESSAGE: &str = "critical error";
const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
// Sentry serializes SystemTime as a finite f64; this keeps conservative fixed headroom.
const MAX_SERIALIZED_TIMESTAMP_BYTES: usize = 32;
tokio::task_local! {
static REQUEST_ID: String;
@@ -63,6 +71,8 @@ pub enum SentryConfigError {
InvalidDsn,
#[error("CRANK_SENTRY_DSN is not valid UTF-8")]
InvalidEnvironmentEncoding,
#[error("critical error event budget cannot hold the required fields")]
EventBudgetTooSmall,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -74,6 +84,13 @@ pub enum CriticalErrorCategory {
}
impl CriticalErrorCategory {
const ALL: [Self; 4] = [
Self::Panic,
Self::Startup,
Self::Internal,
Self::DataIntegrity,
];
pub const fn as_str(self) -> &'static str {
match self {
Self::Panic => "panic",
@@ -117,11 +134,14 @@ pub(crate) fn init_sentry(
identity: &ServiceIdentity,
limits: RedactionLimits,
config: SentryConfig,
) -> Option<ClientInitGuard> {
let dsn = config.dsn?;
) -> Result<Option<ClientInitGuard>, SentryConfigError> {
let Some(dsn) = config.dsn else {
return Ok(None);
};
validate_critical_event_budget(identity, limits)?;
let identity = identity.clone();
let options = client_options(identity, limits);
Some(sentry::init((dsn, options)))
Ok(Some(sentry::init((dsn, options))))
}
fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
@@ -181,7 +201,10 @@ fn sanitize_event(
Event {
event_id: event.event_id,
level: Level::Error,
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
fingerprint: Cow::Owned(vec![
Cow::Owned(identity.service().to_owned()),
Cow::Borrowed(category.as_str()),
]),
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
timestamp: event.timestamp,
server_name: Some(Cow::Owned(identity.service().to_owned())),
@@ -212,9 +235,44 @@ fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Ev
event.tags.remove("request_id");
event.tags.remove("trace_id");
debug_assert!(serialized_event_len(&event) <= max_event_bytes);
event
}
fn validate_critical_event_budget(
identity: &ServiceIdentity,
limits: RedactionLimits,
) -> Result<(), SentryConfigError> {
if required_critical_event_budget(identity, limits) > limits.max_event_bytes {
return Err(SentryConfigError::EventBudgetTooSmall);
}
Ok(())
}
fn required_critical_event_budget(identity: &ServiceIdentity, limits: RedactionLimits) -> usize {
let unbounded_limits = RedactionLimits {
max_event_bytes: usize::MAX,
..limits
};
CriticalErrorCategory::ALL
.into_iter()
.map(|category| {
let event = sanitize_event(
Event {
tags: BTreeMap::from([("category".to_owned(), category.as_str().to_owned())]),
timestamp: SystemTime::UNIX_EPOCH,
..Event::default()
},
identity,
unbounded_limits,
);
serialized_event_len(&event)
.saturating_add(MAX_SERIALIZED_TIMESTAMP_BYTES.saturating_sub(1))
})
.max()
.unwrap_or(usize::MAX)
}
fn serialized_event_len(event: &Event<'_>) -> usize {
serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
}
@@ -223,10 +281,12 @@ fn serialized_event_len(event: &Event<'_>) -> usize {
mod tests {
use std::{
collections::BTreeMap,
panic::AssertUnwindSafe,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use opentelemetry::trace::TracerProvider as _;
@@ -237,7 +297,8 @@ mod tests {
};
use super::{
CriticalErrorCategory, capture_critical_error, client_options, sanitize_event,
CriticalErrorCategory, capture_critical_error, client_options,
required_critical_event_budget, sanitize_event, validate_critical_event_budget,
with_request_correlation,
};
use crate::{
@@ -395,6 +456,14 @@ mod tests {
event.tags.get("category").map(String::as_str),
Some("data_integrity")
);
assert_eq!(
event
.fingerprint
.iter()
.map(AsRef::as_ref)
.collect::<Vec<&str>>(),
vec!["admin-api", "data_integrity"]
);
assert_eq!(
event.tags.get("request_id").map(String::as_str),
Some("request-123")
@@ -409,11 +478,33 @@ mod tests {
fn panic_creates_exactly_one_sanitized_critical_event() {
let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("runtime");
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("panic-correlation-test");
let subscriber = build_subscriber_with_tracer(
ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
std::io::sink,
Some(tracer),
)
.expect("subscriber");
let dispatch = tracing::Dispatch::new(subscriber);
let events = sentry::test::with_captured_events_options(
|| {
let result = std::panic::catch_unwind(|| {
let result = std::panic::catch_unwind(AssertUnwindSafe(|| {
tracing::dispatcher::with_default(&dispatch, || {
runtime.block_on(with_request_correlation(
"panic-request-123".to_owned(),
async {
let span =
tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
panic!("password=must-not-leak");
},
));
});
}));
assert!(result.is_err());
},
options,
@@ -425,24 +516,34 @@ mod tests {
event.tags.get("category").map(String::as_str),
Some("panic")
);
assert_eq!(
event.tags.get("request_id").map(String::as_str),
Some("panic-request-123")
);
assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
let serialized = serde_json::to_string(event).expect("serialize event");
assert!(!serialized.contains("must-not-leak"));
assert!(!serialized.contains("password"));
provider.shutdown().expect("provider shutdown");
}
#[test]
fn receiver_failure_does_not_change_product_result_or_recurse() {
struct DroppingTransport {
struct UnavailableTransport {
attempts: AtomicUsize,
}
impl sentry::Transport for DroppingTransport {
impl sentry::Transport for UnavailableTransport {
fn send_envelope(&self, _envelope: Envelope) {
self.attempts.fetch_add(1, Ordering::Relaxed);
}
fn flush(&self, _timeout: Duration) -> bool {
false
}
}
let transport = Arc::new(DroppingTransport {
let transport = Arc::new(UnavailableTransport {
attempts: AtomicUsize::new(0),
});
let mut options =
@@ -454,7 +555,10 @@ mod tests {
);
options.transport = Some(Arc::new(transport.clone()));
let client = Arc::new(sentry::Client::from(options));
let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default())));
let hub = Arc::new(Hub::new(
Some(Arc::clone(&client)),
Arc::new(Default::default()),
));
let product_result = Hub::run(hub, || {
capture_critical_error(CriticalErrorCategory::Internal);
@@ -463,5 +567,58 @@ mod tests {
assert_eq!(product_result, 42);
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
assert!(!client.close(Some(Duration::from_millis(10))));
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
}
#[test]
fn critical_event_budget_rejects_identity_that_does_not_fit() {
let maximum_label = "a".repeat(64);
let identity =
ServiceIdentity::try_new(maximum_label.clone(), maximum_label.clone(), maximum_label)
.expect("maximum identity");
let limits = RedactionLimits {
max_event_bytes: 512,
..RedactionLimits::default()
};
assert!(matches!(
validate_critical_event_budget(&identity, limits),
Err(super::SentryConfigError::EventBudgetTooSmall)
));
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: 1024,
..limits
},
)
.expect("larger budget must hold the required identity");
}
#[test]
fn critical_event_budget_covers_every_category() {
let identity = identity();
let limits = RedactionLimits::default();
let required_budget = required_critical_event_budget(&identity, limits);
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: required_budget,
..limits
},
)
.expect("exact required budget must be accepted");
assert!(matches!(
validate_critical_event_budget(
&identity,
RedactionLimits {
max_event_bytes: required_budget - 1,
..limits
},
),
Err(super::SentryConfigError::EventBudgetTooSmall)
));
}
}
+1 -1
View File
@@ -33,7 +33,7 @@ impl ObservabilityLifecycle {
.map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?;
let metrics_handle = install_prometheus_recorder(&identity)?;
register_metric_schema();
let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config);
let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config)?;
Ok(Self {
metrics_handle,
+5
View File
@@ -99,6 +99,11 @@ CRANK_SENTRY_DSN=https://public-key@errors.example.com/1
- доступные `request_id` и `trace_id`;
- статическое сообщение без исходного текста ошибки.
События группируются по паре `service` и закрытой категории, поэтому одинаковая
ошибка `admin-api` и `mcp-server` не объединяется в один инцидент. Если
обязательные поля идентичности не помещаются в заданный предел события, запуск
останавливается безопасной типизированной ошибкой до приёма запросов.
До отправки удаляются request, user, breadcrumbs, URL, query, cookie,
authorization, payload, произвольные contexts и extra, а также исходный текст
panic или ошибки. Performance tracing, журналы, показатели и отслеживание
+5
View File
@@ -151,6 +151,11 @@ CRANK_LOG_LEVEL=info
Внешний сборщик журналов не обязателен. Его отсутствие не влияет на `/health`
и `/ready`.
`CRANK_INVOCATION_LOG_RETENTION_DAYS` задаёт срок хранения подробной истории
вызовов в днях. Допустимый диапазон: от `1` до `36500`, значение по умолчанию:
`30`. Значение вне диапазона останавливает запуск до создания фоновой задачи
очистки.
## Критические ошибки
- `CRANK_SENTRY_DSN` — DSN внешнего Sentry-совместимого приёмника.