feat: freeze typed metrics registry and bounded exemplars

This commit is contained in:
2026-08-14 11:52:00 +03:00
parent 17c8e3a8f0
commit 8e709acea9
29 changed files with 1690 additions and 103 deletions
@@ -0,0 +1,36 @@
use std::{fs, process::ExitCode};
const SNAPSHOT: &str = "docs/schemas/metrics-registry-v1.json";
fn main() -> ExitCode {
let rendered = match crank_metrics::render_metric_schema_json() {
Ok(rendered) => rendered,
Err(_) => {
eprintln!("metrics_contract_invalid");
return ExitCode::FAILURE;
}
};
let args = std::env::args().skip(1).collect::<Vec<_>>();
if args.is_empty() {
print!("{rendered}");
return ExitCode::SUCCESS;
}
if args.as_slice() != ["--check"] {
eprintln!("metrics_contract_invalid_command");
return ExitCode::FAILURE;
}
match fs::read_to_string(SNAPSHOT) {
Ok(snapshot) if snapshot == rendered => {
println!("metrics_contract_ok schema_version=1");
ExitCode::SUCCESS
}
Ok(_) => {
eprintln!("metrics_contract_drift");
ExitCode::FAILURE
}
Err(_) => {
eprintln!("metrics_contract_missing");
ExitCode::FAILURE
}
}
}
+95
View File
@@ -0,0 +1,95 @@
use std::{
collections::BTreeMap,
sync::{Mutex, OnceLock},
};
use crate::{DURATION_BUCKETS_SECONDS, max_exemplar_slots};
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct ExemplarTraceId([u8; 32]);
impl ExemplarTraceId {
pub fn parse(value: &str) -> Option<Self> {
if value.len() != 32
|| value
.bytes()
.any(|byte| !byte.is_ascii_digit() && !(b'a'..=b'f').contains(&byte))
|| value.bytes().all(|byte| byte == b'0')
{
return None;
}
let mut bytes = [0; 32];
bytes.copy_from_slice(value.as_bytes());
Some(Self(bytes))
}
pub fn as_str(&self) -> &str {
std::str::from_utf8(&self.0).expect("validated ASCII trace id")
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct ExemplarObservation {
pub metric: &'static str,
pub labels: Vec<(&'static str, &'static str)>,
pub bucket_upper_bound: Option<f64>,
pub value: f64,
pub trace_id: ExemplarTraceId,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct ExemplarKey {
metric: &'static str,
labels: Vec<(&'static str, &'static str)>,
bucket_index: usize,
}
fn store() -> &'static Mutex<BTreeMap<ExemplarKey, ExemplarObservation>> {
static STORE: OnceLock<Mutex<BTreeMap<ExemplarKey, ExemplarObservation>>> = OnceLock::new();
STORE.get_or_init(|| Mutex::new(BTreeMap::new()))
}
pub(crate) fn record_exemplar(
metric: &'static str,
labels: Vec<(&'static str, &'static str)>,
value: f64,
trace_id: Option<ExemplarTraceId>,
) {
let Some(trace_id) = trace_id else { return };
let bucket_index = DURATION_BUCKETS_SECONDS
.iter()
.position(|bound| value <= *bound)
.unwrap_or(DURATION_BUCKETS_SECONDS.len());
let observation = ExemplarObservation {
metric,
labels: labels.clone(),
bucket_upper_bound: DURATION_BUCKETS_SECONDS.get(bucket_index).copied(),
value,
trace_id,
};
let Ok(mut observations) = store().lock() else {
return;
};
let key = ExemplarKey {
metric,
labels,
bucket_index,
};
if observations.contains_key(&key) || observations.len() < max_exemplar_slots() {
observations.insert(key, observation);
}
}
pub fn exemplar_snapshot() -> Vec<ExemplarObservation> {
store()
.lock()
.map(|observations| observations.values().cloned().collect())
.unwrap_or_default()
}
#[doc(hidden)]
pub fn reset_exemplars_for_test() {
if let Ok(mut observations) = store().lock() {
observations.clear();
}
}
+15 -4
View File
@@ -4,10 +4,14 @@
//! ключи и значения labels сосредоточены здесь, поэтому пользовательские
//! идентификаторы и тексты нельзя случайно превратить во временные ряды.
mod exemplar;
mod labels;
mod record;
mod schema;
pub use exemplar::{
ExemplarObservation, ExemplarTraceId, exemplar_snapshot, reset_exemplars_for_test,
};
pub use labels::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
@@ -17,10 +21,17 @@ pub use labels::{
pub use record::{
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, initialize_gauges,
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
set_db_pool_connections, set_mcp_active_sessions,
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
record_tool_invocation, record_tool_invocation_with_exemplar, record_upstream_request,
record_upstream_request_with_exemplar, set_catalog, set_db_pool_connections,
set_mcp_active_sessions, set_mcp_session_metrics_fresh,
};
pub use schema::{
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
DURATION_BUCKETS_SECONDS, HTTP_ROUTE_DOMAIN, LabelClass, LabelDomain, MAX_EXPOSITION_BYTES,
MAX_LOGICAL_SERIES_PER_PROCESS, MAX_METRIC_FAMILIES, MAX_PRODUCT_LABELS_PER_FAMILY,
MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, MetricDefinition, MetricKind,
MetricProcess, MetricService, MetricUnit, PROCESS_CONSTANT_LABELS, SchemaBudget, SchemaError,
max_exemplar_slots, metric_schema, render_metric_schema_json, schema_budget, validate_budget,
validate_schema,
};
+135 -6
View File
@@ -3,9 +3,9 @@ use std::time::{Duration, Instant};
use metrics::Gauge;
use crate::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
UpstreamOutcome,
};
@@ -14,6 +14,16 @@ pub fn record_http_request(
method: HttpMethod,
status: HttpStatusClass,
duration: Duration,
) {
record_http_request_with_exemplar(route, method, status, duration, None);
}
pub fn record_http_request_with_exemplar(
route: HttpRoute,
method: HttpMethod,
status: HttpStatusClass,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
metrics::counter!(
"crank_http_requests_total",
@@ -28,9 +38,30 @@ pub fn record_http_request(
"method" => method.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
"crank_http_request_duration_seconds",
vec![("route", route.as_str()), ("method", method.as_str())],
duration.as_secs_f64(),
exemplar,
);
}
pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, outcome: McpOutcome) {
pub fn record_mcp_request(
method: McpMethod,
response_mode: McpResponseMode,
outcome: McpOutcome,
duration: Duration,
) {
record_mcp_request_with_exemplar(method, response_mode, outcome, duration, None);
}
pub fn record_mcp_request_with_exemplar(
method: McpMethod,
response_mode: McpResponseMode,
outcome: McpOutcome,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
metrics::counter!(
"crank_mcp_requests_total",
"method" => method.as_str(),
@@ -38,18 +69,54 @@ pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, out
"outcome" => outcome.as_str()
)
.increment(1);
metrics::histogram!(
"crank_mcp_request_duration_seconds",
"method" => method.as_str(),
"outcome" => outcome.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
"crank_mcp_request_duration_seconds",
vec![("method", method.as_str()), ("outcome", outcome.as_str())],
duration.as_secs_f64(),
exemplar,
);
}
pub fn set_mcp_active_sessions(count: u64) {
metrics::gauge!("crank_mcp_active_sessions").set(count as f64);
}
pub fn set_mcp_session_metrics_fresh(fresh: bool) {
metrics::gauge!("crank_mcp_session_metrics_fresh").set(if fresh { 1.0 } else { 0.0 });
}
pub fn record_tool_invocation(
source: InvocationSource,
outcome: ToolOutcome,
error_kind: ToolErrorKind,
duration: Duration,
) {
record_tool_invocation_with_exemplar(source, outcome, error_kind, duration, None);
}
pub fn record_tool_invocation_with_exemplar(
source: InvocationSource,
outcome: ToolOutcome,
error_kind: ToolErrorKind,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
let error_kind = match outcome {
ToolOutcome::Success => ToolErrorKind::None,
ToolOutcome::Aborted => ToolErrorKind::Aborted,
ToolOutcome::Error
if matches!(error_kind, ToolErrorKind::None | ToolErrorKind::Aborted) =>
{
ToolErrorKind::ProtocolAdapter
}
ToolOutcome::Error => error_kind,
};
metrics::counter!(
"crank_tool_invocations_total",
"source" => source.as_str(),
@@ -63,12 +130,27 @@ pub fn record_tool_invocation(
"outcome" => outcome.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
"crank_tool_invocation_duration_seconds",
vec![("source", source.as_str()), ("outcome", outcome.as_str())],
duration.as_secs_f64(),
exemplar,
);
}
pub fn record_upstream_request(
operation_kind: UpstreamOperationKind,
outcome: UpstreamOutcome,
duration: Duration,
) {
record_upstream_request_with_exemplar(operation_kind, outcome, duration, None);
}
pub fn record_upstream_request_with_exemplar(
operation_kind: UpstreamOperationKind,
outcome: UpstreamOutcome,
duration: Duration,
exemplar: Option<ExemplarTraceId>,
) {
metrics::counter!(
"crank_upstream_requests_total",
@@ -82,11 +164,21 @@ pub fn record_upstream_request(
"outcome" => outcome.as_str()
)
.record(duration.as_secs_f64());
crate::exemplar::record_exemplar(
"crank_upstream_request_duration_seconds",
vec![
("operation_kind", operation_kind.as_str()),
("outcome", outcome.as_str()),
],
duration.as_secs_f64(),
exemplar,
);
}
pub struct ToolInvocationMetrics {
source: InvocationSource,
started_at: Option<Instant>,
exemplar: Option<ExemplarTraceId>,
}
impl ToolInvocationMetrics {
@@ -94,6 +186,18 @@ impl ToolInvocationMetrics {
Self {
source,
started_at: Some(Instant::now()),
exemplar: None,
}
}
pub fn start_with_exemplar(
source: InvocationSource,
exemplar: Option<ExemplarTraceId>,
) -> Self {
Self {
source,
started_at: Some(Instant::now()),
exemplar,
}
}
@@ -103,7 +207,13 @@ impl ToolInvocationMetrics {
fn finish(&mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) {
if let Some(started_at) = self.started_at.take() {
record_tool_invocation(self.source, outcome, error_kind, started_at.elapsed());
record_tool_invocation_with_exemplar(
self.source,
outcome,
error_kind,
started_at.elapsed(),
self.exemplar,
);
}
}
}
@@ -117,6 +227,7 @@ impl Drop for ToolInvocationMetrics {
pub struct UpstreamRequestMetrics {
operation_kind: UpstreamOperationKind,
started_at: Option<Instant>,
exemplar: Option<ExemplarTraceId>,
}
impl UpstreamRequestMetrics {
@@ -124,6 +235,18 @@ impl UpstreamRequestMetrics {
Self {
operation_kind,
started_at: Some(Instant::now()),
exemplar: None,
}
}
pub fn start_with_exemplar(
operation_kind: UpstreamOperationKind,
exemplar: Option<ExemplarTraceId>,
) -> Self {
Self {
operation_kind,
started_at: Some(Instant::now()),
exemplar,
}
}
@@ -133,7 +256,12 @@ impl UpstreamRequestMetrics {
fn finish(&mut self, outcome: UpstreamOutcome) {
if let Some(started_at) = self.started_at.take() {
record_upstream_request(self.operation_kind, outcome, started_at.elapsed());
record_upstream_request_with_exemplar(
self.operation_kind,
outcome,
started_at.elapsed(),
self.exemplar,
);
}
}
}
@@ -206,6 +334,7 @@ pub fn record_export_failure(signal: SignalType, exporter: Exporter) {
pub fn initialize_gauges() {
metrics::gauge!("crank_http_inflight").set(0.0);
metrics::gauge!("crank_mcp_active_sessions").set(0.0);
set_mcp_session_metrics_fresh(false);
metrics::gauge!("crank_mcp_active_streams").set(0.0);
metrics::gauge!("crank_runtime_inflight").set(0.0);
set_db_pool_connections(DbPoolState::Idle, 0);
+687 -56
View File
@@ -1,3 +1,12 @@
use std::collections::BTreeSet;
pub const METRIC_SCHEMA_VERSION: u32 = 1;
pub const MAX_METRIC_FAMILIES: usize = 128;
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 5_000;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 15_000;
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MetricKind {
Counter,
@@ -5,129 +14,619 @@ pub enum MetricKind {
Histogram,
}
impl MetricKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Counter => "counter",
Self::Gauge => "gauge",
Self::Histogram => "histogram",
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MetricUnit {
Count,
Seconds,
}
impl MetricUnit {
pub const fn as_str(self) -> &'static str {
match self {
Self::Count => "count",
Self::Seconds => "seconds",
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MetricProcess {
AdminApi,
McpServer,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MetricService {
AdminApi,
McpServer,
}
impl MetricService {
pub fn parse(value: &str) -> Option<Self> {
match value {
"admin-api" => Some(Self::AdminApi),
"mcp-server" => Some(Self::McpServer),
_ => None,
}
}
}
impl MetricProcess {
pub const fn as_str(self) -> &'static str {
match self {
Self::AdminApi => "admin_api",
Self::McpServer => "mcp_server",
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum LabelClass {
ProductClosed,
ProcessConstant,
}
impl LabelClass {
pub const fn as_str(self) -> &'static str {
match self {
Self::ProductClosed => "product_closed",
Self::ProcessConstant => "process_constant",
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct LabelDomain {
pub name: &'static str,
pub class: LabelClass,
pub values: &'static [&'static str],
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct MetricDefinition {
pub name: &'static str,
pub kind: MetricKind,
pub unit: MetricUnit,
pub labels: &'static [&'static str],
pub label_domains: &'static [LabelDomain],
pub buckets: &'static [f64],
pub processes: &'static [MetricProcess],
pub max_logical_series: usize,
pub max_rendered_series: usize,
pub description: &'static str,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct SchemaBudget {
pub logical_series: usize,
pub rendered_series: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum SchemaError {
TooManyFamilies,
DuplicateMetric,
InvalidMetric,
DuplicateLabel,
InvalidDomain,
InvalidBuckets,
InvalidCeiling,
BudgetExceeded,
}
pub const DURATION_BUCKETS_SECONDS: &[f64] = &[
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
];
pub const PROCESS_CONSTANT_LABELS: &[LabelDomain] = &[
LabelDomain {
name: "service",
class: LabelClass::ProcessConstant,
values: &["admin-api", "mcp-server"],
},
LabelDomain {
name: "version",
class: LabelClass::ProcessConstant,
values: &["validated_release_identity"],
},
LabelDomain {
name: "environment",
class: LabelClass::ProcessConstant,
values: &["validated_startup_value"],
},
];
const BOTH: &[MetricProcess] = &[MetricProcess::AdminApi, MetricProcess::McpServer];
const MCP: &[MetricProcess] = &[MetricProcess::McpServer];
pub const HTTP_ROUTE_DOMAIN: &[&str] = &[
"unmatched",
"/health",
"/ready",
"/api/auth/login",
"/api/auth/logout",
"/api/auth/session",
"/api/auth/profile",
"/api/auth/password",
"/api/admin/capabilities",
"/api/admin/workspaces",
"/api/admin/workspaces/{workspace_id}",
"/api/admin/workspaces/{workspace_id}/operations",
"/api/admin/workspaces/{workspace_id}/imports/openapi/preview",
"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create",
"/api/admin/workspaces/{workspace_id}/operations/analyze-quality",
"/api/admin/workspaces/{workspace_id}/operations/import",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate",
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export",
"/api/admin/workspaces/{workspace_id}/agents",
"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke",
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}",
"/api/admin/workspaces/{workspace_id}/auth-profiles",
"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}",
"/api/admin/workspaces/{workspace_id}/upstreams",
"/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}",
"/api/admin/workspaces/{workspace_id}/secrets",
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}",
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate",
"/api/admin/workspaces/{workspace_id}/export",
"/api/admin/workspaces/{workspace_id}/logs",
"/api/admin/workspaces/{workspace_id}/logs/{log_id}",
"/api/admin/workspaces/{workspace_id}/approvals",
"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}",
"/api/admin/workspaces/{workspace_id}/usage",
"/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}",
"/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}",
"/v1/{workspace_slug}/{agent_slug}",
"/v1/{workspace_slug}/{agent_slug}/approvals",
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve",
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}",
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny",
];
const METHODS: &[&str] = &[
"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER",
];
const STATUS: &[&str] = &["1xx", "2xx", "3xx", "4xx", "5xx", "other"];
const MCP_METHODS: &[&str] = &[
"initialize",
"initialized",
"ping",
"tools_list",
"tools_call",
"notification",
"unsupported",
"response",
"invalid",
];
const RESPONSE_MODES: &[&str] = &["json", "sse", "unknown"];
const MCP_OUTCOMES: &[&str] = &[
"success",
"client_error",
"server_error",
"jsonrpc_error",
"tool_error",
"aborted",
"other",
];
const SOURCES: &[&str] = &["internal", "admin_test_run", "agent_tool_call"];
const TOOL_OUTCOMES: &[&str] = &["success", "error", "aborted"];
const TOOL_ERRORS: &[&str] = &[
"none",
"schema",
"mapping",
"rest_adapter",
"protocol_adapter",
"unsupported_protocol",
"unsupported_execution_mode",
"concurrency_limit",
"invalid_prepared_request",
"confirmation_required",
"invalid_confirmation_token",
"confirmation_store",
"idempotency_store",
"idempotency_in_progress",
"idempotency_conflict",
"idempotency_outcome_unknown",
"missing_auth_profile",
"missing_secret",
"missing_secret_version",
"invalid_auth_secret",
"secret_crypto",
"aborted",
];
const OPERATION_KINDS: &[&str] = &["rest"];
const UPSTREAM_OUTCOMES: &[&str] = &[
"success",
"client_error",
"server_error",
"unexpected_status",
"timeout",
"transport_error",
"response_too_large",
"rejected",
"window_expired",
"invalid_response",
"invalid_request",
"configuration",
"aborted",
];
const LIMIT_STAGES: &[&str] = &["concurrency", "rate_limit", "mcp_stream"];
const CACHE_OUTCOMES: &[&str] = &[
"hit",
"miss",
"read_error",
"decode_error",
"evict_error",
"stored",
"write_error",
];
const IDEMPOTENCY_OUTCOMES: &[&str] = &[
"execute",
"replay",
"completed",
"conflict",
"in_progress",
"outcome_unknown",
"store_unavailable",
"error",
];
const CONFIRMATION_OUTCOMES: &[&str] = &[
"approved",
"required",
"invalid_token",
"store_unavailable",
"error",
];
const DB_STATES: &[&str] = &["idle", "used"];
const SIGNAL_TYPES: &[&str] = &["trace", "invocation_history"];
const EXPORTERS: &[&str] = &["otlp", "postgres"];
const fn domain(name: &'static str, values: &'static [&'static str]) -> LabelDomain {
LabelDomain {
name,
class: LabelClass::ProductClosed,
values,
}
}
#[allow(
clippy::too_many_arguments,
reason = "the canonical descriptor keeps every metric contract dimension explicit"
)]
const fn metric(
name: &'static str,
kind: MetricKind,
unit: MetricUnit,
labels: &'static [&'static str],
label_domains: &'static [LabelDomain],
buckets: &'static [f64],
processes: &'static [MetricProcess],
logical: usize,
description: &'static str,
) -> MetricDefinition {
let rendered = if matches!(kind, MetricKind::Histogram) {
logical * (buckets.len() + 3)
} else {
logical
};
MetricDefinition {
name,
kind,
unit,
labels,
label_domains,
buckets,
processes,
max_logical_series: logical,
max_rendered_series: rendered,
description,
}
}
const METRIC_SCHEMA: &[MetricDefinition] = &[
counter(
metric(
"crank_http_requests_total",
MetricKind::Counter,
MetricUnit::Count,
&["route", "method", "status_class"],
&[
domain("route", HTTP_ROUTE_DOMAIN),
domain("method", METHODS),
domain("status_class", STATUS),
],
&[],
BOTH,
57 * 10 * 6,
"Total HTTP requests.",
),
histogram(
metric(
"crank_http_request_duration_seconds",
MetricKind::Histogram,
MetricUnit::Seconds,
&["route", "method"],
&[
domain("route", HTTP_ROUTE_DOMAIN),
domain("method", METHODS),
],
DURATION_BUCKETS_SECONDS,
BOTH,
57 * 10,
"HTTP request duration in seconds.",
),
gauge(
metric(
"crank_http_inflight",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
BOTH,
1,
"HTTP requests currently being processed.",
),
counter(
metric(
"crank_mcp_requests_total",
MetricKind::Counter,
MetricUnit::Count,
&["method", "response_mode", "outcome"],
&[
domain("method", MCP_METHODS),
domain("response_mode", RESPONSE_MODES),
domain("outcome", MCP_OUTCOMES),
],
&[],
MCP,
9 * 3 * 7,
"Total MCP JSON-RPC requests.",
),
gauge(
metric(
"crank_mcp_request_duration_seconds",
MetricKind::Histogram,
MetricUnit::Seconds,
&["method", "outcome"],
&[
domain("method", MCP_METHODS),
domain("outcome", MCP_OUTCOMES),
],
DURATION_BUCKETS_SECONDS,
MCP,
9 * 7,
"MCP JSON-RPC request duration in seconds.",
),
metric(
"crank_mcp_active_sessions",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
MCP,
1,
"Active persisted MCP transport sessions.",
),
gauge(
"crank_mcp_active_streams",
metric(
"crank_mcp_session_metrics_fresh",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
MCP,
1,
"Whether the active-session gauge was refreshed successfully.",
),
metric(
"crank_mcp_active_streams",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
MCP,
1,
"MCP event streams currently being served.",
),
counter(
metric(
"crank_tool_invocations_total",
MetricKind::Counter,
MetricUnit::Count,
&["source", "outcome", "error_kind"],
&[
domain("source", SOURCES),
domain("outcome", TOOL_OUTCOMES),
domain("error_kind", TOOL_ERRORS),
],
&[],
BOTH,
3 * 3 * 22,
"Total tool invocations.",
),
histogram(
metric(
"crank_tool_invocation_duration_seconds",
MetricKind::Histogram,
MetricUnit::Seconds,
&["source", "outcome"],
&[domain("source", SOURCES), domain("outcome", TOOL_OUTCOMES)],
DURATION_BUCKETS_SECONDS,
BOTH,
3 * 3,
"Tool invocation duration in seconds.",
),
counter(
metric(
"crank_upstream_requests_total",
MetricKind::Counter,
MetricUnit::Count,
&["operation_kind", "outcome"],
&[
domain("operation_kind", OPERATION_KINDS),
domain("outcome", UPSTREAM_OUTCOMES),
],
&[],
BOTH,
13,
"Total upstream requests.",
),
histogram(
metric(
"crank_upstream_request_duration_seconds",
MetricKind::Histogram,
MetricUnit::Seconds,
&["operation_kind", "outcome"],
&[
domain("operation_kind", OPERATION_KINDS),
domain("outcome", UPSTREAM_OUTCOMES),
],
DURATION_BUCKETS_SECONDS,
BOTH,
13,
"Upstream request duration in seconds.",
),
gauge(
metric(
"crank_runtime_inflight",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
BOTH,
1,
"Runtime executions currently in progress.",
),
counter(
metric(
"crank_runtime_limit_rejections_total",
MetricKind::Counter,
MetricUnit::Count,
&["stage"],
&[domain("stage", LIMIT_STAGES)],
&[],
BOTH,
3,
"Runtime executions rejected by a bounded limit.",
),
counter(
metric(
"crank_runtime_cache_total",
MetricKind::Counter,
MetricUnit::Count,
&["outcome"],
&[domain("outcome", CACHE_OUTCOMES)],
&[],
BOTH,
7,
"Runtime response cache outcomes.",
),
counter(
metric(
"crank_idempotency_total",
MetricKind::Counter,
MetricUnit::Count,
&["outcome"],
&[domain("outcome", IDEMPOTENCY_OUTCOMES)],
&[],
BOTH,
8,
"Runtime idempotency outcomes.",
),
counter(
metric(
"crank_confirmation_total",
MetricKind::Counter,
MetricUnit::Count,
&["outcome"],
&[domain("outcome", CONFIRMATION_OUTCOMES)],
&[],
BOTH,
5,
"Runtime confirmation outcomes.",
),
gauge(
metric(
"crank_db_pool_connections",
MetricKind::Gauge,
MetricUnit::Count,
&["state"],
&[domain("state", DB_STATES)],
&[],
BOTH,
2,
"PostgreSQL pool connections by state.",
),
gauge(
metric(
"crank_catalog_tools",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
MCP,
1,
"Tools in the current published catalog.",
),
gauge(
metric(
"crank_catalog_estimated_context_tokens",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
MCP,
1,
"Estimated context tokens in the current published catalog.",
),
gauge(
metric(
"crank_catalog_warnings",
MetricKind::Gauge,
MetricUnit::Count,
&[],
&[],
&[],
MCP,
1,
"Warnings in the current published catalog.",
),
counter(
metric(
"crank_invocation_history_lost_total",
MetricKind::Counter,
MetricUnit::Count,
&[],
&[],
&[],
BOTH,
1,
"Invocation history records lost after an action completed.",
),
counter(
metric(
"crank_telemetry_export_failures_total",
MetricKind::Counter,
MetricUnit::Count,
&["signal_type", "exporter"],
&[
domain("signal_type", SIGNAL_TYPES),
domain("exporter", EXPORTERS),
],
&[],
BOTH,
4,
"Telemetry export failures.",
),
];
@@ -136,44 +635,176 @@ pub const fn metric_schema() -> &'static [MetricDefinition] {
METRIC_SCHEMA
}
const fn counter(
name: &'static str,
labels: &'static [&'static str],
description: &'static str,
) -> MetricDefinition {
MetricDefinition {
name,
kind: MetricKind::Counter,
unit: MetricUnit::Count,
labels,
description,
pub fn schema_budget() -> Result<SchemaBudget, SchemaError> {
validate_schema(METRIC_SCHEMA)
}
pub fn validate_budget(logical: usize, rendered: usize) -> Result<(), SchemaError> {
if logical > MAX_LOGICAL_SERIES_PER_PROCESS || rendered > MAX_RENDERED_SERIES_PER_PROCESS {
Err(SchemaError::BudgetExceeded)
} else {
Ok(())
}
}
const fn gauge(
name: &'static str,
labels: &'static [&'static str],
description: &'static str,
) -> MetricDefinition {
MetricDefinition {
name,
kind: MetricKind::Gauge,
unit: MetricUnit::Count,
labels,
description,
}
pub fn max_exemplar_slots() -> usize {
METRIC_SCHEMA
.iter()
.filter(|definition| definition.kind == MetricKind::Histogram)
.map(|definition| definition.max_logical_series * (definition.buckets.len() + 1))
.sum()
}
const fn histogram(
name: &'static str,
labels: &'static [&'static str],
description: &'static str,
) -> MetricDefinition {
MetricDefinition {
name,
kind: MetricKind::Histogram,
unit: MetricUnit::Seconds,
labels,
description,
pub fn validate_schema(schema: &[MetricDefinition]) -> Result<SchemaBudget, SchemaError> {
if schema.len() > MAX_METRIC_FAMILIES {
return Err(SchemaError::TooManyFamilies);
}
let mut names = BTreeSet::new();
let mut logical = 0usize;
let mut rendered = 0usize;
for item in schema {
if !names.insert(item.name) {
return Err(SchemaError::DuplicateMetric);
}
if !valid_metric_name(item.name)
|| !valid_static_text(item.description)
|| item.processes.is_empty()
|| item.labels.len() > MAX_PRODUCT_LABELS_PER_FAMILY
|| item.labels.len() != item.label_domains.len()
{
return Err(SchemaError::InvalidMetric);
}
let mut labels = BTreeSet::new();
let mut computed = 1usize;
for (label, domain) in item.labels.iter().zip(item.label_domains) {
if *label != domain.name || !valid_label_name(label) || !labels.insert(*label) {
return Err(SchemaError::DuplicateLabel);
}
if domain.class != LabelClass::ProductClosed
|| domain.values.is_empty()
|| domain.values.iter().any(|value| !valid_domain_value(value))
|| domain.values.iter().copied().collect::<BTreeSet<_>>().len()
!= domain.values.len()
{
return Err(SchemaError::InvalidDomain);
}
computed = computed
.checked_mul(domain.values.len())
.ok_or(SchemaError::BudgetExceeded)?;
}
if item.kind == MetricKind::Histogram {
if item.buckets.is_empty()
|| item.buckets.iter().any(|v| !v.is_finite() || *v <= 0.0)
|| item.buckets.windows(2).any(|pair| pair[0] >= pair[1])
{
return Err(SchemaError::InvalidBuckets);
}
} else if !item.buckets.is_empty() {
return Err(SchemaError::InvalidBuckets);
}
let expected_rendered = if item.kind == MetricKind::Histogram {
computed
.checked_mul(item.buckets.len() + 3)
.ok_or(SchemaError::BudgetExceeded)?
} else {
computed
};
if computed != item.max_logical_series || expected_rendered != item.max_rendered_series {
return Err(SchemaError::InvalidCeiling);
}
logical = logical
.checked_add(computed)
.ok_or(SchemaError::BudgetExceeded)?;
rendered = rendered
.checked_add(expected_rendered)
.ok_or(SchemaError::BudgetExceeded)?;
}
validate_budget(logical, rendered)?;
Ok(SchemaBudget {
logical_series: logical,
rendered_series: rendered,
})
}
fn valid_static_text(value: &str) -> bool {
!value.trim().is_empty()
&& value.len() <= 512
&& !value
.bytes()
.any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\'))
}
fn valid_metric_name(name: &str) -> bool {
!name.is_empty()
&& name
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
&& name.starts_with("crank_")
}
fn valid_label_name(name: &str) -> bool {
!name.is_empty()
&& name.len() <= 64
&& name
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
}
fn valid_domain_value(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 256
&& !value
.bytes()
.any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\'))
}
pub fn render_metric_schema_json() -> Result<String, SchemaError> {
let budget = schema_budget()?;
let mut out = format!(
"{{\n \"schema_version\": {METRIC_SCHEMA_VERSION},\n \"budget\": {{\"logical_series\": {}, \"rendered_series\": {}, \"max_logical_series\": {MAX_LOGICAL_SERIES_PER_PROCESS}, \"max_rendered_series\": {MAX_RENDERED_SERIES_PER_PROCESS}}},\n \"global_labels\": [\n",
budget.logical_series, budget.rendered_series
);
for (index, label) in PROCESS_CONSTANT_LABELS.iter().enumerate() {
out.push_str(&format!(
" {{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}\n",
label.name,
label.class.as_str(),
json_strings(label.values),
comma(index, PROCESS_CONSTANT_LABELS.len())
));
}
out.push_str(" ],\n \"metrics\": [\n");
for (index, item) in METRIC_SCHEMA.iter().enumerate() {
out.push_str(&format!(" {{\"name\": \"{}\", \"kind\": \"{}\", \"unit\": \"{}\", \"help\": \"{}\", \"processes\": [{}], \"labels\": [", item.name, item.kind.as_str(), item.unit.as_str(), item.description, json_processes(item.processes)));
for (label_index, domain) in item.label_domains.iter().enumerate() {
out.push_str(&format!(
"{{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}",
domain.name,
domain.class.as_str(),
json_strings(domain.values),
comma(label_index, item.label_domains.len())
));
}
out.push_str(&format!("], \"buckets_seconds\": [{}], \"max_logical_series\": {}, \"max_rendered_series\": {}}}{}\n", item.buckets.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(", "), item.max_logical_series, item.max_rendered_series, comma(index, METRIC_SCHEMA.len())));
}
out.push_str(" ]\n}\n");
Ok(out)
}
fn json_strings(values: &[&str]) -> String {
values
.iter()
.map(|value| format!("\"{value}\""))
.collect::<Vec<_>>()
.join(", ")
}
fn json_processes(values: &[MetricProcess]) -> String {
values
.iter()
.map(|value| format!("\"{}\"", value.as_str()))
.collect::<Vec<_>>()
.join(", ")
}
const fn comma(index: usize, len: usize) -> &'static str {
if index + 1 == len { "" } else { "," }
}