feat: freeze typed metrics registry and bounded exemplars
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
use std::{fs, process::ExitCode};
|
||||
|
||||
const SNAPSHOT: &str = "docs/schemas/metrics-registry-v1.json";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let rendered = match crank_metrics::render_metric_schema_json() {
|
||||
Ok(rendered) => rendered,
|
||||
Err(_) => {
|
||||
eprintln!("metrics_contract_invalid");
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
};
|
||||
let args = std::env::args().skip(1).collect::<Vec<_>>();
|
||||
if args.is_empty() {
|
||||
print!("{rendered}");
|
||||
return ExitCode::SUCCESS;
|
||||
}
|
||||
if args.as_slice() != ["--check"] {
|
||||
eprintln!("metrics_contract_invalid_command");
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
match fs::read_to_string(SNAPSHOT) {
|
||||
Ok(snapshot) if snapshot == rendered => {
|
||||
println!("metrics_contract_ok schema_version=1");
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
Ok(_) => {
|
||||
eprintln!("metrics_contract_drift");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("metrics_contract_missing");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Mutex, OnceLock},
|
||||
};
|
||||
|
||||
use crate::{DURATION_BUCKETS_SECONDS, max_exemplar_slots};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
pub struct ExemplarTraceId([u8; 32]);
|
||||
|
||||
impl ExemplarTraceId {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
if value.len() != 32
|
||||
|| value
|
||||
.bytes()
|
||||
.any(|byte| !byte.is_ascii_digit() && !(b'a'..=b'f').contains(&byte))
|
||||
|| value.bytes().all(|byte| byte == b'0')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut bytes = [0; 32];
|
||||
bytes.copy_from_slice(value.as_bytes());
|
||||
Some(Self(bytes))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
std::str::from_utf8(&self.0).expect("validated ASCII trace id")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ExemplarObservation {
|
||||
pub metric: &'static str,
|
||||
pub labels: Vec<(&'static str, &'static str)>,
|
||||
pub bucket_upper_bound: Option<f64>,
|
||||
pub value: f64,
|
||||
pub trace_id: ExemplarTraceId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
struct ExemplarKey {
|
||||
metric: &'static str,
|
||||
labels: Vec<(&'static str, &'static str)>,
|
||||
bucket_index: usize,
|
||||
}
|
||||
|
||||
fn store() -> &'static Mutex<BTreeMap<ExemplarKey, ExemplarObservation>> {
|
||||
static STORE: OnceLock<Mutex<BTreeMap<ExemplarKey, ExemplarObservation>>> = OnceLock::new();
|
||||
STORE.get_or_init(|| Mutex::new(BTreeMap::new()))
|
||||
}
|
||||
|
||||
pub(crate) fn record_exemplar(
|
||||
metric: &'static str,
|
||||
labels: Vec<(&'static str, &'static str)>,
|
||||
value: f64,
|
||||
trace_id: Option<ExemplarTraceId>,
|
||||
) {
|
||||
let Some(trace_id) = trace_id else { return };
|
||||
let bucket_index = DURATION_BUCKETS_SECONDS
|
||||
.iter()
|
||||
.position(|bound| value <= *bound)
|
||||
.unwrap_or(DURATION_BUCKETS_SECONDS.len());
|
||||
let observation = ExemplarObservation {
|
||||
metric,
|
||||
labels: labels.clone(),
|
||||
bucket_upper_bound: DURATION_BUCKETS_SECONDS.get(bucket_index).copied(),
|
||||
value,
|
||||
trace_id,
|
||||
};
|
||||
let Ok(mut observations) = store().lock() else {
|
||||
return;
|
||||
};
|
||||
let key = ExemplarKey {
|
||||
metric,
|
||||
labels,
|
||||
bucket_index,
|
||||
};
|
||||
if observations.contains_key(&key) || observations.len() < max_exemplar_slots() {
|
||||
observations.insert(key, observation);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn exemplar_snapshot() -> Vec<ExemplarObservation> {
|
||||
store()
|
||||
.lock()
|
||||
.map(|observations| observations.values().cloned().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn reset_exemplars_for_test() {
|
||||
if let Ok(mut observations) = store().lock() {
|
||||
observations.clear();
|
||||
}
|
||||
}
|
||||
@@ -4,10 +4,14 @@
|
||||
//! ключи и значения labels сосредоточены здесь, поэтому пользовательские
|
||||
//! идентификаторы и тексты нельзя случайно превратить во временные ряды.
|
||||
|
||||
mod exemplar;
|
||||
mod labels;
|
||||
mod record;
|
||||
mod schema;
|
||||
|
||||
pub use exemplar::{
|
||||
ExemplarObservation, ExemplarTraceId, exemplar_snapshot, reset_exemplars_for_test,
|
||||
};
|
||||
pub use labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
@@ -17,10 +21,17 @@ pub use labels::{
|
||||
pub use record::{
|
||||
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, initialize_gauges,
|
||||
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
|
||||
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
|
||||
set_db_pool_connections, set_mcp_active_sessions,
|
||||
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
|
||||
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
|
||||
record_tool_invocation, record_tool_invocation_with_exemplar, record_upstream_request,
|
||||
record_upstream_request_with_exemplar, set_catalog, set_db_pool_connections,
|
||||
set_mcp_active_sessions, set_mcp_session_metrics_fresh,
|
||||
};
|
||||
pub use schema::{
|
||||
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
|
||||
DURATION_BUCKETS_SECONDS, HTTP_ROUTE_DOMAIN, LabelClass, LabelDomain, MAX_EXPOSITION_BYTES,
|
||||
MAX_LOGICAL_SERIES_PER_PROCESS, MAX_METRIC_FAMILIES, MAX_PRODUCT_LABELS_PER_FAMILY,
|
||||
MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, MetricDefinition, MetricKind,
|
||||
MetricProcess, MetricService, MetricUnit, PROCESS_CONSTANT_LABELS, SchemaBudget, SchemaError,
|
||||
max_exemplar_slots, metric_schema, render_metric_schema_json, schema_budget, validate_budget,
|
||||
validate_schema,
|
||||
};
|
||||
|
||||
@@ -3,9 +3,9 @@ use std::time::{Duration, Instant};
|
||||
use metrics::Gauge;
|
||||
|
||||
use crate::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
|
||||
UpstreamOutcome,
|
||||
};
|
||||
|
||||
@@ -14,6 +14,16 @@ pub fn record_http_request(
|
||||
method: HttpMethod,
|
||||
status: HttpStatusClass,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_http_request_with_exemplar(route, method, status, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_http_request_with_exemplar(
|
||||
route: HttpRoute,
|
||||
method: HttpMethod,
|
||||
status: HttpStatusClass,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
metrics::counter!(
|
||||
"crank_http_requests_total",
|
||||
@@ -28,9 +38,30 @@ pub fn record_http_request(
|
||||
"method" => method.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_http_request_duration_seconds",
|
||||
vec![("route", route.as_str()), ("method", method.as_str())],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, outcome: McpOutcome) {
|
||||
pub fn record_mcp_request(
|
||||
method: McpMethod,
|
||||
response_mode: McpResponseMode,
|
||||
outcome: McpOutcome,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_mcp_request_with_exemplar(method, response_mode, outcome, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_mcp_request_with_exemplar(
|
||||
method: McpMethod,
|
||||
response_mode: McpResponseMode,
|
||||
outcome: McpOutcome,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
metrics::counter!(
|
||||
"crank_mcp_requests_total",
|
||||
"method" => method.as_str(),
|
||||
@@ -38,18 +69,54 @@ pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, out
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
metrics::histogram!(
|
||||
"crank_mcp_request_duration_seconds",
|
||||
"method" => method.as_str(),
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_mcp_request_duration_seconds",
|
||||
vec![("method", method.as_str()), ("outcome", outcome.as_str())],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn set_mcp_active_sessions(count: u64) {
|
||||
metrics::gauge!("crank_mcp_active_sessions").set(count as f64);
|
||||
}
|
||||
|
||||
pub fn set_mcp_session_metrics_fresh(fresh: bool) {
|
||||
metrics::gauge!("crank_mcp_session_metrics_fresh").set(if fresh { 1.0 } else { 0.0 });
|
||||
}
|
||||
|
||||
pub fn record_tool_invocation(
|
||||
source: InvocationSource,
|
||||
outcome: ToolOutcome,
|
||||
error_kind: ToolErrorKind,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_tool_invocation_with_exemplar(source, outcome, error_kind, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_tool_invocation_with_exemplar(
|
||||
source: InvocationSource,
|
||||
outcome: ToolOutcome,
|
||||
error_kind: ToolErrorKind,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
let error_kind = match outcome {
|
||||
ToolOutcome::Success => ToolErrorKind::None,
|
||||
ToolOutcome::Aborted => ToolErrorKind::Aborted,
|
||||
ToolOutcome::Error
|
||||
if matches!(error_kind, ToolErrorKind::None | ToolErrorKind::Aborted) =>
|
||||
{
|
||||
ToolErrorKind::ProtocolAdapter
|
||||
}
|
||||
ToolOutcome::Error => error_kind,
|
||||
};
|
||||
metrics::counter!(
|
||||
"crank_tool_invocations_total",
|
||||
"source" => source.as_str(),
|
||||
@@ -63,12 +130,27 @@ pub fn record_tool_invocation(
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_tool_invocation_duration_seconds",
|
||||
vec![("source", source.as_str()), ("outcome", outcome.as_str())],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_upstream_request(
|
||||
operation_kind: UpstreamOperationKind,
|
||||
outcome: UpstreamOutcome,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_upstream_request_with_exemplar(operation_kind, outcome, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_upstream_request_with_exemplar(
|
||||
operation_kind: UpstreamOperationKind,
|
||||
outcome: UpstreamOutcome,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
metrics::counter!(
|
||||
"crank_upstream_requests_total",
|
||||
@@ -82,11 +164,21 @@ pub fn record_upstream_request(
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_upstream_request_duration_seconds",
|
||||
vec![
|
||||
("operation_kind", operation_kind.as_str()),
|
||||
("outcome", outcome.as_str()),
|
||||
],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub struct ToolInvocationMetrics {
|
||||
source: InvocationSource,
|
||||
started_at: Option<Instant>,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
}
|
||||
|
||||
impl ToolInvocationMetrics {
|
||||
@@ -94,6 +186,18 @@ impl ToolInvocationMetrics {
|
||||
Self {
|
||||
source,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_with_exemplar(
|
||||
source: InvocationSource,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) -> Self {
|
||||
Self {
|
||||
source,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +207,13 @@ impl ToolInvocationMetrics {
|
||||
|
||||
fn finish(&mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) {
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
record_tool_invocation(self.source, outcome, error_kind, started_at.elapsed());
|
||||
record_tool_invocation_with_exemplar(
|
||||
self.source,
|
||||
outcome,
|
||||
error_kind,
|
||||
started_at.elapsed(),
|
||||
self.exemplar,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,6 +227,7 @@ impl Drop for ToolInvocationMetrics {
|
||||
pub struct UpstreamRequestMetrics {
|
||||
operation_kind: UpstreamOperationKind,
|
||||
started_at: Option<Instant>,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
}
|
||||
|
||||
impl UpstreamRequestMetrics {
|
||||
@@ -124,6 +235,18 @@ impl UpstreamRequestMetrics {
|
||||
Self {
|
||||
operation_kind,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_with_exemplar(
|
||||
operation_kind: UpstreamOperationKind,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) -> Self {
|
||||
Self {
|
||||
operation_kind,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +256,12 @@ impl UpstreamRequestMetrics {
|
||||
|
||||
fn finish(&mut self, outcome: UpstreamOutcome) {
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
record_upstream_request(self.operation_kind, outcome, started_at.elapsed());
|
||||
record_upstream_request_with_exemplar(
|
||||
self.operation_kind,
|
||||
outcome,
|
||||
started_at.elapsed(),
|
||||
self.exemplar,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -206,6 +334,7 @@ pub fn record_export_failure(signal: SignalType, exporter: Exporter) {
|
||||
pub fn initialize_gauges() {
|
||||
metrics::gauge!("crank_http_inflight").set(0.0);
|
||||
metrics::gauge!("crank_mcp_active_sessions").set(0.0);
|
||||
set_mcp_session_metrics_fresh(false);
|
||||
metrics::gauge!("crank_mcp_active_streams").set(0.0);
|
||||
metrics::gauge!("crank_runtime_inflight").set(0.0);
|
||||
set_db_pool_connections(DbPoolState::Idle, 0);
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
pub const METRIC_SCHEMA_VERSION: u32 = 1;
|
||||
pub const MAX_METRIC_FAMILIES: usize = 128;
|
||||
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 5_000;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 15_000;
|
||||
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricKind {
|
||||
Counter,
|
||||
@@ -5,129 +14,619 @@ pub enum MetricKind {
|
||||
Histogram,
|
||||
}
|
||||
|
||||
impl MetricKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Counter => "counter",
|
||||
Self::Gauge => "gauge",
|
||||
Self::Histogram => "histogram",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricUnit {
|
||||
Count,
|
||||
Seconds,
|
||||
}
|
||||
|
||||
impl MetricUnit {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Count => "count",
|
||||
Self::Seconds => "seconds",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricProcess {
|
||||
AdminApi,
|
||||
McpServer,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricService {
|
||||
AdminApi,
|
||||
McpServer,
|
||||
}
|
||||
|
||||
impl MetricService {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"admin-api" => Some(Self::AdminApi),
|
||||
"mcp-server" => Some(Self::McpServer),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MetricProcess {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::AdminApi => "admin_api",
|
||||
Self::McpServer => "mcp_server",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum LabelClass {
|
||||
ProductClosed,
|
||||
ProcessConstant,
|
||||
}
|
||||
|
||||
impl LabelClass {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ProductClosed => "product_closed",
|
||||
Self::ProcessConstant => "process_constant",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct LabelDomain {
|
||||
pub name: &'static str,
|
||||
pub class: LabelClass,
|
||||
pub values: &'static [&'static str],
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct MetricDefinition {
|
||||
pub name: &'static str,
|
||||
pub kind: MetricKind,
|
||||
pub unit: MetricUnit,
|
||||
pub labels: &'static [&'static str],
|
||||
pub label_domains: &'static [LabelDomain],
|
||||
pub buckets: &'static [f64],
|
||||
pub processes: &'static [MetricProcess],
|
||||
pub max_logical_series: usize,
|
||||
pub max_rendered_series: usize,
|
||||
pub description: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct SchemaBudget {
|
||||
pub logical_series: usize,
|
||||
pub rendered_series: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum SchemaError {
|
||||
TooManyFamilies,
|
||||
DuplicateMetric,
|
||||
InvalidMetric,
|
||||
DuplicateLabel,
|
||||
InvalidDomain,
|
||||
InvalidBuckets,
|
||||
InvalidCeiling,
|
||||
BudgetExceeded,
|
||||
}
|
||||
|
||||
pub const DURATION_BUCKETS_SECONDS: &[f64] = &[
|
||||
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
|
||||
];
|
||||
|
||||
pub const PROCESS_CONSTANT_LABELS: &[LabelDomain] = &[
|
||||
LabelDomain {
|
||||
name: "service",
|
||||
class: LabelClass::ProcessConstant,
|
||||
values: &["admin-api", "mcp-server"],
|
||||
},
|
||||
LabelDomain {
|
||||
name: "version",
|
||||
class: LabelClass::ProcessConstant,
|
||||
values: &["validated_release_identity"],
|
||||
},
|
||||
LabelDomain {
|
||||
name: "environment",
|
||||
class: LabelClass::ProcessConstant,
|
||||
values: &["validated_startup_value"],
|
||||
},
|
||||
];
|
||||
|
||||
const BOTH: &[MetricProcess] = &[MetricProcess::AdminApi, MetricProcess::McpServer];
|
||||
const MCP: &[MetricProcess] = &[MetricProcess::McpServer];
|
||||
|
||||
pub const HTTP_ROUTE_DOMAIN: &[&str] = &[
|
||||
"unmatched",
|
||||
"/health",
|
||||
"/ready",
|
||||
"/api/auth/login",
|
||||
"/api/auth/logout",
|
||||
"/api/auth/session",
|
||||
"/api/auth/profile",
|
||||
"/api/auth/password",
|
||||
"/api/admin/capabilities",
|
||||
"/api/admin/workspaces",
|
||||
"/api/admin/workspaces/{workspace_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/operations",
|
||||
"/api/admin/workspaces/{workspace_id}/imports/openapi/preview",
|
||||
"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/analyze-quality",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/import",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export",
|
||||
"/api/admin/workspaces/{workspace_id}/agents",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/auth-profiles",
|
||||
"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/upstreams",
|
||||
"/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/secrets",
|
||||
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate",
|
||||
"/api/admin/workspaces/{workspace_id}/export",
|
||||
"/api/admin/workspaces/{workspace_id}/logs",
|
||||
"/api/admin/workspaces/{workspace_id}/logs/{log_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/approvals",
|
||||
"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/usage",
|
||||
"/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}",
|
||||
"/v1/{workspace_slug}/{agent_slug}",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny",
|
||||
];
|
||||
const METHODS: &[&str] = &[
|
||||
"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER",
|
||||
];
|
||||
const STATUS: &[&str] = &["1xx", "2xx", "3xx", "4xx", "5xx", "other"];
|
||||
const MCP_METHODS: &[&str] = &[
|
||||
"initialize",
|
||||
"initialized",
|
||||
"ping",
|
||||
"tools_list",
|
||||
"tools_call",
|
||||
"notification",
|
||||
"unsupported",
|
||||
"response",
|
||||
"invalid",
|
||||
];
|
||||
const RESPONSE_MODES: &[&str] = &["json", "sse", "unknown"];
|
||||
const MCP_OUTCOMES: &[&str] = &[
|
||||
"success",
|
||||
"client_error",
|
||||
"server_error",
|
||||
"jsonrpc_error",
|
||||
"tool_error",
|
||||
"aborted",
|
||||
"other",
|
||||
];
|
||||
const SOURCES: &[&str] = &["internal", "admin_test_run", "agent_tool_call"];
|
||||
const TOOL_OUTCOMES: &[&str] = &["success", "error", "aborted"];
|
||||
const TOOL_ERRORS: &[&str] = &[
|
||||
"none",
|
||||
"schema",
|
||||
"mapping",
|
||||
"rest_adapter",
|
||||
"protocol_adapter",
|
||||
"unsupported_protocol",
|
||||
"unsupported_execution_mode",
|
||||
"concurrency_limit",
|
||||
"invalid_prepared_request",
|
||||
"confirmation_required",
|
||||
"invalid_confirmation_token",
|
||||
"confirmation_store",
|
||||
"idempotency_store",
|
||||
"idempotency_in_progress",
|
||||
"idempotency_conflict",
|
||||
"idempotency_outcome_unknown",
|
||||
"missing_auth_profile",
|
||||
"missing_secret",
|
||||
"missing_secret_version",
|
||||
"invalid_auth_secret",
|
||||
"secret_crypto",
|
||||
"aborted",
|
||||
];
|
||||
const OPERATION_KINDS: &[&str] = &["rest"];
|
||||
const UPSTREAM_OUTCOMES: &[&str] = &[
|
||||
"success",
|
||||
"client_error",
|
||||
"server_error",
|
||||
"unexpected_status",
|
||||
"timeout",
|
||||
"transport_error",
|
||||
"response_too_large",
|
||||
"rejected",
|
||||
"window_expired",
|
||||
"invalid_response",
|
||||
"invalid_request",
|
||||
"configuration",
|
||||
"aborted",
|
||||
];
|
||||
const LIMIT_STAGES: &[&str] = &["concurrency", "rate_limit", "mcp_stream"];
|
||||
const CACHE_OUTCOMES: &[&str] = &[
|
||||
"hit",
|
||||
"miss",
|
||||
"read_error",
|
||||
"decode_error",
|
||||
"evict_error",
|
||||
"stored",
|
||||
"write_error",
|
||||
];
|
||||
const IDEMPOTENCY_OUTCOMES: &[&str] = &[
|
||||
"execute",
|
||||
"replay",
|
||||
"completed",
|
||||
"conflict",
|
||||
"in_progress",
|
||||
"outcome_unknown",
|
||||
"store_unavailable",
|
||||
"error",
|
||||
];
|
||||
const CONFIRMATION_OUTCOMES: &[&str] = &[
|
||||
"approved",
|
||||
"required",
|
||||
"invalid_token",
|
||||
"store_unavailable",
|
||||
"error",
|
||||
];
|
||||
const DB_STATES: &[&str] = &["idle", "used"];
|
||||
const SIGNAL_TYPES: &[&str] = &["trace", "invocation_history"];
|
||||
const EXPORTERS: &[&str] = &["otlp", "postgres"];
|
||||
|
||||
const fn domain(name: &'static str, values: &'static [&'static str]) -> LabelDomain {
|
||||
LabelDomain {
|
||||
name,
|
||||
class: LabelClass::ProductClosed,
|
||||
values,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::too_many_arguments,
|
||||
reason = "the canonical descriptor keeps every metric contract dimension explicit"
|
||||
)]
|
||||
const fn metric(
|
||||
name: &'static str,
|
||||
kind: MetricKind,
|
||||
unit: MetricUnit,
|
||||
labels: &'static [&'static str],
|
||||
label_domains: &'static [LabelDomain],
|
||||
buckets: &'static [f64],
|
||||
processes: &'static [MetricProcess],
|
||||
logical: usize,
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
let rendered = if matches!(kind, MetricKind::Histogram) {
|
||||
logical * (buckets.len() + 3)
|
||||
} else {
|
||||
logical
|
||||
};
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind,
|
||||
unit,
|
||||
labels,
|
||||
label_domains,
|
||||
buckets,
|
||||
processes,
|
||||
max_logical_series: logical,
|
||||
max_rendered_series: rendered,
|
||||
description,
|
||||
}
|
||||
}
|
||||
|
||||
const METRIC_SCHEMA: &[MetricDefinition] = &[
|
||||
counter(
|
||||
metric(
|
||||
"crank_http_requests_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["route", "method", "status_class"],
|
||||
&[
|
||||
domain("route", HTTP_ROUTE_DOMAIN),
|
||||
domain("method", METHODS),
|
||||
domain("status_class", STATUS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
57 * 10 * 6,
|
||||
"Total HTTP requests.",
|
||||
),
|
||||
histogram(
|
||||
metric(
|
||||
"crank_http_request_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["route", "method"],
|
||||
&[
|
||||
domain("route", HTTP_ROUTE_DOMAIN),
|
||||
domain("method", METHODS),
|
||||
],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
BOTH,
|
||||
57 * 10,
|
||||
"HTTP request duration in seconds.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_http_inflight",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"HTTP requests currently being processed.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_mcp_requests_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["method", "response_mode", "outcome"],
|
||||
&[
|
||||
domain("method", MCP_METHODS),
|
||||
domain("response_mode", RESPONSE_MODES),
|
||||
domain("outcome", MCP_OUTCOMES),
|
||||
],
|
||||
&[],
|
||||
MCP,
|
||||
9 * 3 * 7,
|
||||
"Total MCP JSON-RPC requests.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_mcp_request_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["method", "outcome"],
|
||||
&[
|
||||
domain("method", MCP_METHODS),
|
||||
domain("outcome", MCP_OUTCOMES),
|
||||
],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
MCP,
|
||||
9 * 7,
|
||||
"MCP JSON-RPC request duration in seconds.",
|
||||
),
|
||||
metric(
|
||||
"crank_mcp_active_sessions",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Active persisted MCP transport sessions.",
|
||||
),
|
||||
gauge(
|
||||
"crank_mcp_active_streams",
|
||||
metric(
|
||||
"crank_mcp_session_metrics_fresh",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Whether the active-session gauge was refreshed successfully.",
|
||||
),
|
||||
metric(
|
||||
"crank_mcp_active_streams",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"MCP event streams currently being served.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_tool_invocations_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["source", "outcome", "error_kind"],
|
||||
&[
|
||||
domain("source", SOURCES),
|
||||
domain("outcome", TOOL_OUTCOMES),
|
||||
domain("error_kind", TOOL_ERRORS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
3 * 3 * 22,
|
||||
"Total tool invocations.",
|
||||
),
|
||||
histogram(
|
||||
metric(
|
||||
"crank_tool_invocation_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["source", "outcome"],
|
||||
&[domain("source", SOURCES), domain("outcome", TOOL_OUTCOMES)],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
BOTH,
|
||||
3 * 3,
|
||||
"Tool invocation duration in seconds.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_upstream_requests_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["operation_kind", "outcome"],
|
||||
&[
|
||||
domain("operation_kind", OPERATION_KINDS),
|
||||
domain("outcome", UPSTREAM_OUTCOMES),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
13,
|
||||
"Total upstream requests.",
|
||||
),
|
||||
histogram(
|
||||
metric(
|
||||
"crank_upstream_request_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["operation_kind", "outcome"],
|
||||
&[
|
||||
domain("operation_kind", OPERATION_KINDS),
|
||||
domain("outcome", UPSTREAM_OUTCOMES),
|
||||
],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
BOTH,
|
||||
13,
|
||||
"Upstream request duration in seconds.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_runtime_inflight",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"Runtime executions currently in progress.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["stage"],
|
||||
&[domain("stage", LIMIT_STAGES)],
|
||||
&[],
|
||||
BOTH,
|
||||
3,
|
||||
"Runtime executions rejected by a bounded limit.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_runtime_cache_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["outcome"],
|
||||
&[domain("outcome", CACHE_OUTCOMES)],
|
||||
&[],
|
||||
BOTH,
|
||||
7,
|
||||
"Runtime response cache outcomes.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_idempotency_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["outcome"],
|
||||
&[domain("outcome", IDEMPOTENCY_OUTCOMES)],
|
||||
&[],
|
||||
BOTH,
|
||||
8,
|
||||
"Runtime idempotency outcomes.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_confirmation_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["outcome"],
|
||||
&[domain("outcome", CONFIRMATION_OUTCOMES)],
|
||||
&[],
|
||||
BOTH,
|
||||
5,
|
||||
"Runtime confirmation outcomes.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_db_pool_connections",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&["state"],
|
||||
&[domain("state", DB_STATES)],
|
||||
&[],
|
||||
BOTH,
|
||||
2,
|
||||
"PostgreSQL pool connections by state.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_catalog_tools",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Tools in the current published catalog.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_catalog_estimated_context_tokens",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Estimated context tokens in the current published catalog.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_catalog_warnings",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Warnings in the current published catalog.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_invocation_history_lost_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"Invocation history records lost after an action completed.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_telemetry_export_failures_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["signal_type", "exporter"],
|
||||
&[
|
||||
domain("signal_type", SIGNAL_TYPES),
|
||||
domain("exporter", EXPORTERS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
4,
|
||||
"Telemetry export failures.",
|
||||
),
|
||||
];
|
||||
@@ -136,44 +635,176 @@ pub const fn metric_schema() -> &'static [MetricDefinition] {
|
||||
METRIC_SCHEMA
|
||||
}
|
||||
|
||||
const fn counter(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Counter,
|
||||
unit: MetricUnit::Count,
|
||||
labels,
|
||||
description,
|
||||
pub fn schema_budget() -> Result<SchemaBudget, SchemaError> {
|
||||
validate_schema(METRIC_SCHEMA)
|
||||
}
|
||||
|
||||
pub fn validate_budget(logical: usize, rendered: usize) -> Result<(), SchemaError> {
|
||||
if logical > MAX_LOGICAL_SERIES_PER_PROCESS || rendered > MAX_RENDERED_SERIES_PER_PROCESS {
|
||||
Err(SchemaError::BudgetExceeded)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn gauge(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Gauge,
|
||||
unit: MetricUnit::Count,
|
||||
labels,
|
||||
description,
|
||||
}
|
||||
pub fn max_exemplar_slots() -> usize {
|
||||
METRIC_SCHEMA
|
||||
.iter()
|
||||
.filter(|definition| definition.kind == MetricKind::Histogram)
|
||||
.map(|definition| definition.max_logical_series * (definition.buckets.len() + 1))
|
||||
.sum()
|
||||
}
|
||||
|
||||
const fn histogram(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Histogram,
|
||||
unit: MetricUnit::Seconds,
|
||||
labels,
|
||||
description,
|
||||
pub fn validate_schema(schema: &[MetricDefinition]) -> Result<SchemaBudget, SchemaError> {
|
||||
if schema.len() > MAX_METRIC_FAMILIES {
|
||||
return Err(SchemaError::TooManyFamilies);
|
||||
}
|
||||
let mut names = BTreeSet::new();
|
||||
let mut logical = 0usize;
|
||||
let mut rendered = 0usize;
|
||||
for item in schema {
|
||||
if !names.insert(item.name) {
|
||||
return Err(SchemaError::DuplicateMetric);
|
||||
}
|
||||
if !valid_metric_name(item.name)
|
||||
|| !valid_static_text(item.description)
|
||||
|| item.processes.is_empty()
|
||||
|| item.labels.len() > MAX_PRODUCT_LABELS_PER_FAMILY
|
||||
|| item.labels.len() != item.label_domains.len()
|
||||
{
|
||||
return Err(SchemaError::InvalidMetric);
|
||||
}
|
||||
let mut labels = BTreeSet::new();
|
||||
let mut computed = 1usize;
|
||||
for (label, domain) in item.labels.iter().zip(item.label_domains) {
|
||||
if *label != domain.name || !valid_label_name(label) || !labels.insert(*label) {
|
||||
return Err(SchemaError::DuplicateLabel);
|
||||
}
|
||||
if domain.class != LabelClass::ProductClosed
|
||||
|| domain.values.is_empty()
|
||||
|| domain.values.iter().any(|value| !valid_domain_value(value))
|
||||
|| domain.values.iter().copied().collect::<BTreeSet<_>>().len()
|
||||
!= domain.values.len()
|
||||
{
|
||||
return Err(SchemaError::InvalidDomain);
|
||||
}
|
||||
computed = computed
|
||||
.checked_mul(domain.values.len())
|
||||
.ok_or(SchemaError::BudgetExceeded)?;
|
||||
}
|
||||
if item.kind == MetricKind::Histogram {
|
||||
if item.buckets.is_empty()
|
||||
|| item.buckets.iter().any(|v| !v.is_finite() || *v <= 0.0)
|
||||
|| item.buckets.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
{
|
||||
return Err(SchemaError::InvalidBuckets);
|
||||
}
|
||||
} else if !item.buckets.is_empty() {
|
||||
return Err(SchemaError::InvalidBuckets);
|
||||
}
|
||||
let expected_rendered = if item.kind == MetricKind::Histogram {
|
||||
computed
|
||||
.checked_mul(item.buckets.len() + 3)
|
||||
.ok_or(SchemaError::BudgetExceeded)?
|
||||
} else {
|
||||
computed
|
||||
};
|
||||
if computed != item.max_logical_series || expected_rendered != item.max_rendered_series {
|
||||
return Err(SchemaError::InvalidCeiling);
|
||||
}
|
||||
logical = logical
|
||||
.checked_add(computed)
|
||||
.ok_or(SchemaError::BudgetExceeded)?;
|
||||
rendered = rendered
|
||||
.checked_add(expected_rendered)
|
||||
.ok_or(SchemaError::BudgetExceeded)?;
|
||||
}
|
||||
validate_budget(logical, rendered)?;
|
||||
Ok(SchemaBudget {
|
||||
logical_series: logical,
|
||||
rendered_series: rendered,
|
||||
})
|
||||
}
|
||||
|
||||
fn valid_static_text(value: &str) -> bool {
|
||||
!value.trim().is_empty()
|
||||
&& value.len() <= 512
|
||||
&& !value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\'))
|
||||
}
|
||||
|
||||
fn valid_metric_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
|
||||
&& name.starts_with("crank_")
|
||||
}
|
||||
|
||||
fn valid_label_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= 64
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
|
||||
}
|
||||
|
||||
fn valid_domain_value(value: &str) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 256
|
||||
&& !value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\'))
|
||||
}
|
||||
|
||||
pub fn render_metric_schema_json() -> Result<String, SchemaError> {
|
||||
let budget = schema_budget()?;
|
||||
let mut out = format!(
|
||||
"{{\n \"schema_version\": {METRIC_SCHEMA_VERSION},\n \"budget\": {{\"logical_series\": {}, \"rendered_series\": {}, \"max_logical_series\": {MAX_LOGICAL_SERIES_PER_PROCESS}, \"max_rendered_series\": {MAX_RENDERED_SERIES_PER_PROCESS}}},\n \"global_labels\": [\n",
|
||||
budget.logical_series, budget.rendered_series
|
||||
);
|
||||
for (index, label) in PROCESS_CONSTANT_LABELS.iter().enumerate() {
|
||||
out.push_str(&format!(
|
||||
" {{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}\n",
|
||||
label.name,
|
||||
label.class.as_str(),
|
||||
json_strings(label.values),
|
||||
comma(index, PROCESS_CONSTANT_LABELS.len())
|
||||
));
|
||||
}
|
||||
out.push_str(" ],\n \"metrics\": [\n");
|
||||
for (index, item) in METRIC_SCHEMA.iter().enumerate() {
|
||||
out.push_str(&format!(" {{\"name\": \"{}\", \"kind\": \"{}\", \"unit\": \"{}\", \"help\": \"{}\", \"processes\": [{}], \"labels\": [", item.name, item.kind.as_str(), item.unit.as_str(), item.description, json_processes(item.processes)));
|
||||
for (label_index, domain) in item.label_domains.iter().enumerate() {
|
||||
out.push_str(&format!(
|
||||
"{{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}",
|
||||
domain.name,
|
||||
domain.class.as_str(),
|
||||
json_strings(domain.values),
|
||||
comma(label_index, item.label_domains.len())
|
||||
));
|
||||
}
|
||||
out.push_str(&format!("], \"buckets_seconds\": [{}], \"max_logical_series\": {}, \"max_rendered_series\": {}}}{}\n", item.buckets.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(", "), item.max_logical_series, item.max_rendered_series, comma(index, METRIC_SCHEMA.len())));
|
||||
}
|
||||
out.push_str(" ]\n}\n");
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn json_strings(values: &[&str]) -> String {
|
||||
values
|
||||
.iter()
|
||||
.map(|value| format!("\"{value}\""))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
fn json_processes(values: &[MetricProcess]) -> String {
|
||||
values
|
||||
.iter()
|
||||
.map(|value| format!("\"{}\"", value.as_str()))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
const fn comma(index: usize, len: usize) -> &'static str {
|
||||
if index + 1 == len { "" } else { "," }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
|
||||
use crank_metrics::{
|
||||
HttpMethod, HttpRoute, HttpStatusClass, McpMethod, McpOutcome, McpResponseMode,
|
||||
record_http_request, record_mcp_request,
|
||||
};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
#[test]
|
||||
fn one_million_hostile_values_collapse_to_the_same_closed_series() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for index in 0..1_000_000_u32 {
|
||||
let canary = format!("workspace-{index}-request-trace-url-secret");
|
||||
record_http_request(
|
||||
HttpRoute::from_matched_path(&canary),
|
||||
HttpMethod::classify(&canary),
|
||||
HttpStatusClass::from_status(999),
|
||||
Duration::ZERO,
|
||||
);
|
||||
record_mcp_request(
|
||||
McpMethod::Invalid,
|
||||
McpResponseMode::Unknown,
|
||||
McpOutcome::Other,
|
||||
Duration::ZERO,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let series = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.map(|(key, _, _, _)| format!("{:?}", key.key()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert_eq!(series.len(), 4);
|
||||
assert!(series.iter().all(|value| !value.contains("workspace-")));
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use crank_metrics::{
|
||||
metric_schema, record_cache_outcome, record_confirmation_outcome, record_export_failure,
|
||||
record_http_request, record_idempotency_outcome, record_invocation_history_lost,
|
||||
record_limit_rejection, record_mcp_request, record_tool_invocation, record_upstream_request,
|
||||
set_catalog, set_db_pool_connections, set_mcp_active_sessions,
|
||||
set_catalog, set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
|
||||
};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
@@ -80,6 +80,26 @@ fn application_outcomes_are_distinct_from_transport_success() {
|
||||
assert_eq!(ConfirmationOutcome::Required.as_str(), "required");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impossible_tool_outcome_pairs_are_normalized() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
record_tool_invocation(
|
||||
InvocationSource::AgentToolCall,
|
||||
ToolOutcome::Success,
|
||||
ToolErrorKind::Mapping,
|
||||
Duration::ZERO,
|
||||
);
|
||||
});
|
||||
let snapshot = snapshotter.snapshot().into_vec();
|
||||
assert!(snapshot.iter().any(|(key, _, _, _)| {
|
||||
key.key().name() == "crank_tool_invocations_total"
|
||||
&& has_label(key.key(), "outcome", "success")
|
||||
&& has_label(key.key(), "error_kind", "none")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropped_product_guards_record_aborted_outcomes() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
@@ -128,6 +148,7 @@ fn diverse_calls_cannot_create_unbounded_series() {
|
||||
McpMethod::ToolsCall,
|
||||
McpResponseMode::Json,
|
||||
McpOutcome::ToolError,
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
record_tool_invocation(
|
||||
InvocationSource::AgentToolCall,
|
||||
@@ -147,7 +168,7 @@ fn diverse_calls_cannot_create_unbounded_series() {
|
||||
.map(|(key, _, _, _)| format!("{:?}", key.key()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(series.len(), 8);
|
||||
assert_eq!(series.len(), 9);
|
||||
assert!(
|
||||
series
|
||||
.iter()
|
||||
@@ -172,8 +193,10 @@ fn every_recording_api_matches_the_declared_schema() {
|
||||
McpMethod::ToolsCall,
|
||||
McpResponseMode::Json,
|
||||
McpOutcome::Success,
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
set_mcp_active_sessions(1);
|
||||
set_mcp_session_metrics_fresh(true);
|
||||
let _stream = InFlightGuard::mcp_stream();
|
||||
let _runtime = InFlightGuard::runtime();
|
||||
record_tool_invocation(
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use crank_metrics::{
|
||||
ExemplarTraceId, HttpMethod, HttpRoute, HttpStatusClass, exemplar_snapshot,
|
||||
record_http_request_with_exemplar, reset_exemplars_for_test,
|
||||
};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
#[test]
|
||||
fn trace_id_is_validated_and_never_becomes_an_ordinary_label() {
|
||||
assert!(ExemplarTraceId::parse("0123456789abcdef0123456789abcdef").is_some());
|
||||
assert!(ExemplarTraceId::parse("0123456789ABCDEF0123456789ABCDEF").is_none());
|
||||
assert!(ExemplarTraceId::parse("short").is_none());
|
||||
|
||||
reset_exemplars_for_test();
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
record_http_request_with_exemplar(
|
||||
HttpRoute::from_matched_path("/health"),
|
||||
HttpMethod::Get,
|
||||
HttpStatusClass::Success,
|
||||
Duration::from_millis(10),
|
||||
ExemplarTraceId::parse("0123456789abcdef0123456789abcdef"),
|
||||
);
|
||||
});
|
||||
|
||||
let aggregate = snapshotter.snapshot().into_vec();
|
||||
assert!(
|
||||
aggregate
|
||||
.iter()
|
||||
.all(|(key, _, _, _)| { key.key().labels().all(|label| label.key() != "trace_id") })
|
||||
);
|
||||
let exemplars = exemplar_snapshot();
|
||||
assert_eq!(exemplars.len(), 1);
|
||||
assert_eq!(
|
||||
exemplars[0].trace_id.as_str(),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use std::{
|
||||
hint::black_box,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use crank_metrics::{CacheOutcome, record_cache_outcome};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
const SAMPLES: usize = 40;
|
||||
const WORK_UNITS: u64 = 500_000;
|
||||
|
||||
#[test]
|
||||
fn reproducible_foundation_profile_stays_inside_latency_and_cpu_proxy_budgets() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let mut baseline = Vec::with_capacity(SAMPLES);
|
||||
let mut enabled = Vec::with_capacity(SAMPLES);
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for index in 0..SAMPLES {
|
||||
if index % 2 == 0 {
|
||||
baseline.push(sample(false));
|
||||
enabled.push(sample(true));
|
||||
} else {
|
||||
enabled.push(sample(true));
|
||||
baseline.push(sample(false));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
baseline.sort_unstable();
|
||||
enabled.sort_unstable();
|
||||
let baseline_p95 = baseline[SAMPLES * 95 / 100];
|
||||
let enabled_p95 = enabled[SAMPLES * 95 / 100];
|
||||
let baseline_cpu: Duration = baseline.iter().copied().sum();
|
||||
let enabled_cpu: Duration = enabled.iter().copied().sum();
|
||||
let series = snapshotter.snapshot().into_vec().len();
|
||||
|
||||
println!(
|
||||
"metrics_profile_v1 samples={SAMPLES} work_units={WORK_UNITS} baseline_p95_ns={} enabled_p95_ns={} baseline_cpu_ns={} enabled_cpu_ns={} logical_series={series}",
|
||||
baseline_p95.as_nanos(),
|
||||
enabled_p95.as_nanos(),
|
||||
baseline_cpu.as_nanos(),
|
||||
enabled_cpu.as_nanos()
|
||||
);
|
||||
assert!(enabled_p95.as_nanos() * 100 <= baseline_p95.as_nanos() * 105);
|
||||
assert!(enabled_cpu.as_nanos() * 100 <= baseline_cpu.as_nanos() * 110);
|
||||
assert_eq!(series, 1);
|
||||
}
|
||||
|
||||
fn sample(with_metrics: bool) -> Duration {
|
||||
let started = Instant::now();
|
||||
let mut value = 0x9e37_79b9_u64;
|
||||
for index in 0..WORK_UNITS {
|
||||
value = value.rotate_left(7) ^ index.wrapping_mul(0x100_0000_01b3);
|
||||
}
|
||||
black_box(value);
|
||||
if with_metrics {
|
||||
record_cache_outcome(CacheOutcome::Hit);
|
||||
}
|
||||
started.elapsed()
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crank_metrics::{
|
||||
HTTP_ROUTE_DOMAIN, HttpRoute, MAX_LOGICAL_SERIES_PER_PROCESS, MAX_PRODUCT_LABELS_PER_FAMILY,
|
||||
MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, metric_schema,
|
||||
render_metric_schema_json, schema_budget, validate_budget,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
|
||||
assert_eq!(METRIC_SCHEMA_VERSION, 1);
|
||||
assert_eq!(metric_schema().len(), 23);
|
||||
|
||||
let budget = schema_budget().expect("canonical schema must be valid");
|
||||
assert!(budget.logical_series <= MAX_LOGICAL_SERIES_PER_PROCESS);
|
||||
assert!(budget.rendered_series <= MAX_RENDERED_SERIES_PER_PROCESS);
|
||||
|
||||
for definition in metric_schema() {
|
||||
assert!(!definition.description.trim().is_empty());
|
||||
assert!(definition.labels.len() <= MAX_PRODUCT_LABELS_PER_FAMILY);
|
||||
assert!(!definition.processes.is_empty());
|
||||
assert_eq!(definition.labels.len(), definition.label_domains.len());
|
||||
assert!(definition.label_domains.iter().all(|domain| {
|
||||
!domain.values.is_empty()
|
||||
&& domain.values.iter().all(|value| !value.is_empty())
|
||||
&& domain.values.iter().copied().collect::<BTreeSet<_>>().len()
|
||||
== domain.values.len()
|
||||
}));
|
||||
if definition.kind.as_str() == "histogram" {
|
||||
assert!(!definition.buckets.is_empty());
|
||||
assert!(definition.buckets.windows(2).all(|pair| pair[0] < pair[1]));
|
||||
} else {
|
||||
assert!(definition.buckets.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
let first = render_metric_schema_json().expect("schema renders");
|
||||
let second = render_metric_schema_json().expect("schema renders deterministically");
|
||||
assert_eq!(first, second);
|
||||
assert!(first.len() < 65_536);
|
||||
assert!(!first.contains("trace_id"));
|
||||
assert!(!first.contains("request_id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frozen_budget_edges_are_exact() {
|
||||
assert!(validate_budget(4_999, 14_999).is_ok());
|
||||
assert!(validate_budget(5_000, 15_000).is_ok());
|
||||
assert!(validate_budget(5_001, 15_000).is_err());
|
||||
assert!(validate_budget(5_000, 15_001).is_err());
|
||||
assert_eq!(crank_metrics::max_exemplar_slots(), 9_170);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_registered_route_round_trips_without_dynamic_segments() {
|
||||
assert_eq!(HTTP_ROUTE_DOMAIN.len(), 57);
|
||||
for route in HTTP_ROUTE_DOMAIN {
|
||||
assert_eq!(HttpRoute::from_matched_path(route).as_str(), *route);
|
||||
assert!(!route.contains("trace_id"));
|
||||
assert!(!route.contains("request_id"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_schema_validation_reports_its_category() {
|
||||
assert_eq!(
|
||||
schema_budget(),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 4_513,
|
||||
rendered_series: 14_338
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_budget_math_counts_histogram_bucket_sum_and_count_series() {
|
||||
for definition in metric_schema() {
|
||||
let logical = definition
|
||||
.label_domains
|
||||
.iter()
|
||||
.map(|domain| domain.values.len())
|
||||
.product::<usize>();
|
||||
let logical = logical.max(1);
|
||||
let rendered = if definition.kind.as_str() == "histogram" {
|
||||
logical * (definition.buckets.len() + 3)
|
||||
} else {
|
||||
logical
|
||||
};
|
||||
assert_eq!(
|
||||
definition.max_logical_series, logical,
|
||||
"{}",
|
||||
definition.name
|
||||
);
|
||||
assert_eq!(
|
||||
definition.max_rendered_series, rendered,
|
||||
"{}",
|
||||
definition.name
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user