fix: harden typed metrics review findings

This commit is contained in:
2026-08-14 13:50:04 +03:00
parent 996a5461de
commit b7face0e94
30 changed files with 722 additions and 382 deletions
@@ -1,6 +1,6 @@
use std::{fs, process::ExitCode};
use std::process::ExitCode;
const SNAPSHOT: &str = "docs/schemas/metrics-registry-v1.json";
const SNAPSHOT: &str = include_str!("../../../../docs/schemas/metrics-registry-v1.json");
fn main() -> ExitCode {
let rendered = match crank_metrics::render_metric_schema_json() {
@@ -19,18 +19,14 @@ fn main() -> ExitCode {
eprintln!("metrics_contract_invalid_command");
return ExitCode::FAILURE;
}
match fs::read_to_string(SNAPSHOT) {
Ok(snapshot) if snapshot == rendered => {
match SNAPSHOT {
snapshot if snapshot == rendered => {
println!("metrics_contract_ok schema_version=1");
ExitCode::SUCCESS
}
Ok(_) => {
_ => {
eprintln!("metrics_contract_drift");
ExitCode::FAILURE
}
Err(_) => {
eprintln!("metrics_contract_missing");
ExitCode::FAILURE
}
}
}
+6 -12
View File
@@ -1,6 +1,6 @@
use std::{
collections::BTreeMap,
sync::{Mutex, OnceLock},
sync::{Arc, Mutex, OnceLock},
};
use crate::{DURATION_BUCKETS_SECONDS, max_exemplar_slots};
@@ -44,8 +44,9 @@ struct ExemplarKey {
bucket_index: usize,
}
fn store() -> &'static Mutex<BTreeMap<ExemplarKey, ExemplarObservation>> {
static STORE: OnceLock<Mutex<BTreeMap<ExemplarKey, ExemplarObservation>>> = OnceLock::new();
fn store() -> &'static Mutex<BTreeMap<ExemplarKey, Arc<ExemplarObservation>>> {
static STORE: OnceLock<Mutex<BTreeMap<ExemplarKey, Arc<ExemplarObservation>>>> =
OnceLock::new();
STORE.get_or_init(|| Mutex::new(BTreeMap::new()))
}
@@ -76,20 +77,13 @@ pub(crate) fn record_exemplar(
bucket_index,
};
if observations.contains_key(&key) || observations.len() < max_exemplar_slots() {
observations.insert(key, observation);
observations.insert(key, Arc::new(observation));
}
}
pub fn exemplar_snapshot() -> Vec<ExemplarObservation> {
pub fn exemplar_snapshot() -> Vec<Arc<ExemplarObservation>> {
store()
.lock()
.map(|observations| observations.values().cloned().collect())
.unwrap_or_default()
}
#[doc(hidden)]
pub fn reset_exemplars_for_test() {
if let Ok(mut observations) = store().lock() {
observations.clear();
}
}
+8 -162
View File
@@ -12,6 +12,9 @@ macro_rules! string_enum {
}
impl $name {
pub const ALL: &'static [Self] = &[$(Self::$variant),+];
pub const VALUES: &'static [&'static str] = &[$($value),+];
pub const fn as_str(self) -> &'static str {
match self {
$(Self::$variant => $value),+
@@ -30,169 +33,12 @@ impl HttpRoute {
}
pub fn from_matched_path(path: &str) -> Self {
match path {
"/health" => Self("/health"),
"/ready" => Self("/ready"),
"/api/auth/login" => Self("/api/auth/login"),
"/api/auth/logout" => Self("/api/auth/logout"),
"/api/auth/session" => Self("/api/auth/session"),
"/api/auth/profile" => Self("/api/auth/profile"),
"/api/auth/password" => Self("/api/auth/password"),
"/api/admin/capabilities" => Self("/api/admin/capabilities"),
"/api/admin/workspaces" => Self("/api/admin/workspaces"),
"/api/admin/workspaces/{workspace_id}" => Self("/api/admin/workspaces/{workspace_id}"),
"/api/admin/workspaces/{workspace_id}/operations" => {
Self("/api/admin/workspaces/{workspace_id}/operations")
}
"/api/admin/workspaces/{workspace_id}/imports/openapi/preview" => {
Self("/api/admin/workspaces/{workspace_id}/imports/openapi/preview")
}
"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create" => {
Self("/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create")
}
"/api/admin/workspaces/{workspace_id}/operations/analyze-quality" => {
Self("/api/admin/workspaces/{workspace_id}/operations/analyze-quality")
}
"/api/admin/workspaces/{workspace_id}/operations/import" => {
Self("/api/admin/workspaces/{workspace_id}/operations/import")
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}" => {
Self("/api/admin/workspaces/{workspace_id}/operations/{operation_id}")
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions" => {
Self("/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions")
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}" => {
Self(
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}",
)
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish" => {
Self("/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish")
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive" => {
Self("/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive")
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs" => {
Self("/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs")
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json" => {
Self(
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json",
)
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json" => {
Self(
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json",
)
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate" => {
Self(
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate",
)
}
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export" => {
Self("/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export")
}
"/api/admin/workspaces/{workspace_id}/agents" => {
Self("/api/admin/workspaces/{workspace_id}/agents")
}
"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview" => {
Self("/api/admin/workspaces/{workspace_id}/agents/tool-search/preview")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}" => {
Self("/api/admin/workspaces/{workspace_id}/agents/{agent_id}")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}" => {
Self("/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings" => {
Self("/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish" => {
Self("/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish" => {
Self("/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive" => {
Self("/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys" => {
Self("/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys")
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke" => {
Self(
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke",
)
}
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}" => {
Self(
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}",
)
}
"/api/admin/workspaces/{workspace_id}/auth-profiles" => {
Self("/api/admin/workspaces/{workspace_id}/auth-profiles")
}
"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}" => {
Self("/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}")
}
"/api/admin/workspaces/{workspace_id}/upstreams" => {
Self("/api/admin/workspaces/{workspace_id}/upstreams")
}
"/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}" => {
Self("/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}")
}
"/api/admin/workspaces/{workspace_id}/secrets" => {
Self("/api/admin/workspaces/{workspace_id}/secrets")
}
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}" => {
Self("/api/admin/workspaces/{workspace_id}/secrets/{secret_id}")
}
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate" => {
Self("/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate")
}
"/api/admin/workspaces/{workspace_id}/export" => {
Self("/api/admin/workspaces/{workspace_id}/export")
}
"/api/admin/workspaces/{workspace_id}/logs" => {
Self("/api/admin/workspaces/{workspace_id}/logs")
}
"/api/admin/workspaces/{workspace_id}/logs/{log_id}" => {
Self("/api/admin/workspaces/{workspace_id}/logs/{log_id}")
}
"/api/admin/workspaces/{workspace_id}/approvals" => {
Self("/api/admin/workspaces/{workspace_id}/approvals")
}
"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}" => {
Self("/api/admin/workspaces/{workspace_id}/approvals/{approval_id}")
}
"/api/admin/workspaces/{workspace_id}/usage" => {
Self("/api/admin/workspaces/{workspace_id}/usage")
}
"/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}" => {
Self("/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}")
}
"/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}" => {
Self("/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}")
}
"/v1/{workspace_slug}/{agent_slug}" => Self("/v1/{workspace_slug}/{agent_slug}"),
"/v1/{workspace_slug}/{agent_slug}/approvals" => {
Self("/v1/{workspace_slug}/{agent_slug}/approvals")
}
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve" => {
Self("/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve")
}
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}" => {
Self("/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}")
}
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny" => {
Self("/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny")
}
_ => Self::unmatched(),
}
crate::schema::HTTP_ROUTE_DOMAIN
.iter()
.copied()
.find(|candidate| *candidate != "unmatched" && *candidate == path)
.map_or_else(Self::unmatched, Self)
}
pub const fn as_str(self) -> &'static str {
self.0
}
+3 -5
View File
@@ -9,9 +9,7 @@ mod labels;
mod record;
mod schema;
pub use exemplar::{
ExemplarObservation, ExemplarTraceId, exemplar_snapshot, reset_exemplars_for_test,
};
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
pub use labels::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
@@ -32,6 +30,6 @@ pub use schema::{
MAX_LOGICAL_SERIES_PER_PROCESS, MAX_METRIC_FAMILIES, MAX_PRODUCT_LABELS_PER_FAMILY,
MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, MetricDefinition, MetricKind,
MetricProcess, MetricService, MetricUnit, PROCESS_CONSTANT_LABELS, SchemaBudget, SchemaError,
max_exemplar_slots, metric_schema, render_metric_schema_json, schema_budget, validate_budget,
validate_schema,
max_exemplar_slots, metric_schema, metric_schema_for_service, render_metric_schema_json,
schema_budget, validate_budget, validate_schema,
};
+7 -5
View File
@@ -331,15 +331,17 @@ pub fn record_export_failure(signal: SignalType, exporter: Exporter) {
.increment(1);
}
pub fn initialize_gauges() {
pub fn initialize_gauges(service: crate::MetricService) {
metrics::gauge!("crank_http_inflight").set(0.0);
metrics::gauge!("crank_mcp_active_sessions").set(0.0);
set_mcp_session_metrics_fresh(false);
metrics::gauge!("crank_mcp_active_streams").set(0.0);
metrics::gauge!("crank_runtime_inflight").set(0.0);
set_db_pool_connections(DbPoolState::Idle, 0);
set_db_pool_connections(DbPoolState::Used, 0);
set_catalog(0, 0, 0);
if service == crate::MetricService::McpServer {
metrics::gauge!("crank_mcp_active_sessions").set(0.0);
set_mcp_session_metrics_fresh(false);
metrics::gauge!("crank_mcp_active_streams").set(0.0);
set_catalog(0, 0, 0);
}
}
pub struct InFlightGuard {
+44 -97
View File
@@ -1,5 +1,11 @@
use std::collections::BTreeSet;
use crate::labels::{
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
pub const METRIC_SCHEMA_VERSION: u32 = 1;
pub const MAX_METRIC_FAMILIES: usize = 128;
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
@@ -59,6 +65,13 @@ impl MetricService {
_ => None,
}
}
pub const fn process(self) -> MetricProcess {
match self {
Self::AdminApi => MetricProcess::AdminApi,
Self::McpServer => MetricProcess::McpServer,
}
}
}
impl MetricProcess {
@@ -208,103 +221,23 @@ pub const HTTP_ROUTE_DOMAIN: &[&str] = &[
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}",
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny",
];
const METHODS: &[&str] = &[
"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER",
];
const STATUS: &[&str] = &["1xx", "2xx", "3xx", "4xx", "5xx", "other"];
const MCP_METHODS: &[&str] = &[
"initialize",
"initialized",
"ping",
"tools_list",
"tools_call",
"notification",
"unsupported",
"response",
"invalid",
];
const RESPONSE_MODES: &[&str] = &["json", "sse", "unknown"];
const MCP_OUTCOMES: &[&str] = &[
"success",
"client_error",
"server_error",
"jsonrpc_error",
"tool_error",
"aborted",
"other",
];
const SOURCES: &[&str] = &["internal", "admin_test_run", "agent_tool_call"];
const TOOL_OUTCOMES: &[&str] = &["success", "error", "aborted"];
const TOOL_ERRORS: &[&str] = &[
"none",
"schema",
"mapping",
"rest_adapter",
"protocol_adapter",
"unsupported_protocol",
"unsupported_execution_mode",
"concurrency_limit",
"invalid_prepared_request",
"confirmation_required",
"invalid_confirmation_token",
"confirmation_store",
"idempotency_store",
"idempotency_in_progress",
"idempotency_conflict",
"idempotency_outcome_unknown",
"missing_auth_profile",
"missing_secret",
"missing_secret_version",
"invalid_auth_secret",
"secret_crypto",
"aborted",
];
const OPERATION_KINDS: &[&str] = &["rest"];
const UPSTREAM_OUTCOMES: &[&str] = &[
"success",
"client_error",
"server_error",
"unexpected_status",
"timeout",
"transport_error",
"response_too_large",
"rejected",
"window_expired",
"invalid_response",
"invalid_request",
"configuration",
"aborted",
];
const LIMIT_STAGES: &[&str] = &["concurrency", "rate_limit", "mcp_stream"];
const CACHE_OUTCOMES: &[&str] = &[
"hit",
"miss",
"read_error",
"decode_error",
"evict_error",
"stored",
"write_error",
];
const IDEMPOTENCY_OUTCOMES: &[&str] = &[
"execute",
"replay",
"completed",
"conflict",
"in_progress",
"outcome_unknown",
"store_unavailable",
"error",
];
const CONFIRMATION_OUTCOMES: &[&str] = &[
"approved",
"required",
"invalid_token",
"store_unavailable",
"error",
];
const DB_STATES: &[&str] = &["idle", "used"];
const SIGNAL_TYPES: &[&str] = &["trace", "invocation_history"];
const EXPORTERS: &[&str] = &["otlp", "postgres"];
const METHODS: &[&str] = HttpMethod::VALUES;
const STATUS: &[&str] = HttpStatusClass::VALUES;
const MCP_METHODS: &[&str] = McpMethod::VALUES;
const RESPONSE_MODES: &[&str] = McpResponseMode::VALUES;
const MCP_OUTCOMES: &[&str] = McpOutcome::VALUES;
const SOURCES: &[&str] = InvocationSource::VALUES;
const TOOL_OUTCOMES: &[&str] = ToolOutcome::VALUES;
const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
const DB_STATES: &[&str] = DbPoolState::VALUES;
const SIGNAL_TYPES: &[&str] = SignalType::VALUES;
const EXPORTERS: &[&str] = Exporter::VALUES;
const fn domain(name: &'static str, values: &'static [&'static str]) -> LabelDomain {
LabelDomain {
@@ -635,6 +568,14 @@ pub const fn metric_schema() -> &'static [MetricDefinition] {
METRIC_SCHEMA
}
pub fn metric_schema_for_service(
service: MetricService,
) -> impl Iterator<Item = &'static MetricDefinition> {
METRIC_SCHEMA
.iter()
.filter(move |definition| definition.processes.contains(&service.process()))
}
pub fn schema_budget() -> Result<SchemaBudget, SchemaError> {
validate_schema(METRIC_SCHEMA)
}
@@ -745,8 +686,14 @@ fn valid_metric_name(name: &str) -> bool {
fn valid_label_name(name: &str) -> bool {
!name.is_empty()
&& name.len() <= 64
&& !name.starts_with("__")
&& name
.bytes()
.next()
.is_some_and(|byte| byte.is_ascii_lowercase() || byte == b'_')
&& name
.bytes()
.skip(1)
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
}