diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 79fa57c..f9be77f 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -70,6 +70,11 @@ jobs: - name: Check canonical migration contract run: cargo run -p admin-api --bin crank-migrate -- plan --check + - name: Check typed metrics contract + run: | + cargo run -p crank-metrics --bin crank-metrics-contract -- --check + python3 scripts/check-metrics-boundaries.py --root . + - name: Check Capability Inventory run: | required_args="" diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index b42eaa2..6f15a68 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -66,6 +66,11 @@ jobs: - name: Check canonical migration contract run: cargo run -p admin-api --bin crank-migrate -- plan --check + - name: Check typed metrics contract + run: | + cargo run -p crank-metrics --bin crank-metrics-contract -- --check + python3 scripts/check-metrics-boundaries.py --root . + - name: Check Capability Inventory run: | required_args="" diff --git a/crates/crank-adapter-rest/src/client.rs b/crates/crank-adapter-rest/src/client.rs index 09d77a8..455e9d3 100644 --- a/crates/crank-adapter-rest/src/client.rs +++ b/crates/crank-adapter-rest/src/client.rs @@ -88,7 +88,10 @@ impl RestAdapter { request: &RestRequest, context: &RuntimeRequestContext, ) -> Result { - let request_metrics = UpstreamRequestMetrics::start(UpstreamOperationKind::Rest); + let request_metrics = UpstreamRequestMetrics::start_with_exemplar( + UpstreamOperationKind::Rest, + crank_metrics::ExemplarTraceId::parse(&context.trace_context.trace_id().to_string()), + ); let result = self.execute_inner(target, request, Some(context)).await; let outcome = match &result { Ok(_) => UpstreamOutcome::Success, diff --git a/crates/crank-community-mcp/src/app/metrics.rs b/crates/crank-community-mcp/src/app/metrics.rs index 8e3d014..17f0c29 100644 --- a/crates/crank-community-mcp/src/app/metrics.rs +++ b/crates/crank-community-mcp/src/app/metrics.rs @@ -1,9 +1,9 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Instant}; use axum::{http::header::CONTENT_TYPE, response::Response}; use crank_metrics::{ - InFlightGuard, LimitStage, McpMethod, McpOutcome, McpResponseMode, record_limit_rejection, - record_mcp_request, + ExemplarTraceId, InFlightGuard, LimitStage, McpMethod, McpOutcome, McpResponseMode, + record_limit_rejection, record_mcp_request_with_exemplar, }; use serde_json::Value; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; @@ -14,14 +14,18 @@ pub(super) struct McpRequestMetrics { method: McpMethod, response_mode: McpResponseMode, outcome: McpOutcome, + started_at: Instant, + exemplar: Option, } impl McpRequestMetrics { - pub(super) const fn invalid() -> Self { + pub(super) fn invalid() -> Self { Self { method: McpMethod::Invalid, response_mode: McpResponseMode::Unknown, outcome: McpOutcome::Aborted, + started_at: Instant::now(), + exemplar: None, } } @@ -30,10 +34,17 @@ impl McpRequestMetrics { method: normalized_mcp_method(message), response_mode: McpResponseMode::Unknown, outcome: McpOutcome::Aborted, + started_at: Instant::now(), + exemplar: None, } } pub(super) fn complete(&mut self, response: &Response) { + self.exemplar = response + .headers() + .get("x-trace-id") + .and_then(|value| value.to_str().ok()) + .and_then(ExemplarTraceId::parse); self.response_mode = response .headers() .get(CONTENT_TYPE) @@ -71,7 +82,13 @@ impl McpRequestMetrics { impl Drop for McpRequestMetrics { fn drop(&mut self) { - record_mcp_request(self.method, self.response_mode, self.outcome); + record_mcp_request_with_exemplar( + self.method, + self.response_mode, + self.outcome, + self.started_at.elapsed(), + self.exemplar, + ); } } diff --git a/crates/crank-community-mcp/src/session.rs b/crates/crank-community-mcp/src/session.rs index e60a62d..ca037cb 100644 --- a/crates/crank-community-mcp/src/session.rs +++ b/crates/crank-community-mcp/src/session.rs @@ -79,8 +79,12 @@ impl ActiveSessionMetrics { ) .await { - Ok(Ok(count)) => crank_metrics::set_mcp_active_sessions(count), + Ok(Ok(count)) => { + crank_metrics::set_mcp_active_sessions(count); + crank_metrics::set_mcp_session_metrics_fresh(true); + } Ok(Err(_)) | Err(_) => { + crank_metrics::set_mcp_session_metrics_fresh(false); warn!( name: "mcp.active_session_metrics.refresh_failed", error_category = "session_store", diff --git a/crates/crank-metrics/src/bin/crank-metrics-contract.rs b/crates/crank-metrics/src/bin/crank-metrics-contract.rs new file mode 100644 index 0000000..2569f5b --- /dev/null +++ b/crates/crank-metrics/src/bin/crank-metrics-contract.rs @@ -0,0 +1,36 @@ +use std::{fs, process::ExitCode}; + +const SNAPSHOT: &str = "docs/schemas/metrics-registry-v1.json"; + +fn main() -> ExitCode { + let rendered = match crank_metrics::render_metric_schema_json() { + Ok(rendered) => rendered, + Err(_) => { + eprintln!("metrics_contract_invalid"); + return ExitCode::FAILURE; + } + }; + let args = std::env::args().skip(1).collect::>(); + if args.is_empty() { + print!("{rendered}"); + return ExitCode::SUCCESS; + } + if args.as_slice() != ["--check"] { + eprintln!("metrics_contract_invalid_command"); + return ExitCode::FAILURE; + } + match fs::read_to_string(SNAPSHOT) { + Ok(snapshot) if snapshot == rendered => { + println!("metrics_contract_ok schema_version=1"); + ExitCode::SUCCESS + } + Ok(_) => { + eprintln!("metrics_contract_drift"); + ExitCode::FAILURE + } + Err(_) => { + eprintln!("metrics_contract_missing"); + ExitCode::FAILURE + } + } +} diff --git a/crates/crank-metrics/src/exemplar.rs b/crates/crank-metrics/src/exemplar.rs new file mode 100644 index 0000000..32205b3 --- /dev/null +++ b/crates/crank-metrics/src/exemplar.rs @@ -0,0 +1,95 @@ +use std::{ + collections::BTreeMap, + sync::{Mutex, OnceLock}, +}; + +use crate::{DURATION_BUCKETS_SECONDS, max_exemplar_slots}; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct ExemplarTraceId([u8; 32]); + +impl ExemplarTraceId { + pub fn parse(value: &str) -> Option { + if value.len() != 32 + || value + .bytes() + .any(|byte| !byte.is_ascii_digit() && !(b'a'..=b'f').contains(&byte)) + || value.bytes().all(|byte| byte == b'0') + { + return None; + } + let mut bytes = [0; 32]; + bytes.copy_from_slice(value.as_bytes()); + Some(Self(bytes)) + } + + pub fn as_str(&self) -> &str { + std::str::from_utf8(&self.0).expect("validated ASCII trace id") + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ExemplarObservation { + pub metric: &'static str, + pub labels: Vec<(&'static str, &'static str)>, + pub bucket_upper_bound: Option, + pub value: f64, + pub trace_id: ExemplarTraceId, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct ExemplarKey { + metric: &'static str, + labels: Vec<(&'static str, &'static str)>, + bucket_index: usize, +} + +fn store() -> &'static Mutex> { + static STORE: OnceLock>> = OnceLock::new(); + STORE.get_or_init(|| Mutex::new(BTreeMap::new())) +} + +pub(crate) fn record_exemplar( + metric: &'static str, + labels: Vec<(&'static str, &'static str)>, + value: f64, + trace_id: Option, +) { + let Some(trace_id) = trace_id else { return }; + let bucket_index = DURATION_BUCKETS_SECONDS + .iter() + .position(|bound| value <= *bound) + .unwrap_or(DURATION_BUCKETS_SECONDS.len()); + let observation = ExemplarObservation { + metric, + labels: labels.clone(), + bucket_upper_bound: DURATION_BUCKETS_SECONDS.get(bucket_index).copied(), + value, + trace_id, + }; + let Ok(mut observations) = store().lock() else { + return; + }; + let key = ExemplarKey { + metric, + labels, + bucket_index, + }; + if observations.contains_key(&key) || observations.len() < max_exemplar_slots() { + observations.insert(key, observation); + } +} + +pub fn exemplar_snapshot() -> Vec { + store() + .lock() + .map(|observations| observations.values().cloned().collect()) + .unwrap_or_default() +} + +#[doc(hidden)] +pub fn reset_exemplars_for_test() { + if let Ok(mut observations) = store().lock() { + observations.clear(); + } +} diff --git a/crates/crank-metrics/src/lib.rs b/crates/crank-metrics/src/lib.rs index ab14f87..4de3e54 100644 --- a/crates/crank-metrics/src/lib.rs +++ b/crates/crank-metrics/src/lib.rs @@ -4,10 +4,14 @@ //! ключи и значения labels сосредоточены здесь, поэтому пользовательские //! идентификаторы и тексты нельзя случайно превратить во временные ряды. +mod exemplar; mod labels; mod record; mod schema; +pub use exemplar::{ + ExemplarObservation, ExemplarTraceId, exemplar_snapshot, reset_exemplars_for_test, +}; pub use labels::{ CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, @@ -17,10 +21,17 @@ pub use labels::{ pub use record::{ InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, initialize_gauges, record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request, - record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection, - record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog, - set_db_pool_connections, set_mcp_active_sessions, + record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost, + record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar, + record_tool_invocation, record_tool_invocation_with_exemplar, record_upstream_request, + record_upstream_request_with_exemplar, set_catalog, set_db_pool_connections, + set_mcp_active_sessions, set_mcp_session_metrics_fresh, }; pub use schema::{ - DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema, + DURATION_BUCKETS_SECONDS, HTTP_ROUTE_DOMAIN, LabelClass, LabelDomain, MAX_EXPOSITION_BYTES, + MAX_LOGICAL_SERIES_PER_PROCESS, MAX_METRIC_FAMILIES, MAX_PRODUCT_LABELS_PER_FAMILY, + MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, MetricDefinition, MetricKind, + MetricProcess, MetricService, MetricUnit, PROCESS_CONSTANT_LABELS, SchemaBudget, SchemaError, + max_exemplar_slots, metric_schema, render_metric_schema_json, schema_budget, validate_budget, + validate_schema, }; diff --git a/crates/crank-metrics/src/record.rs b/crates/crank-metrics/src/record.rs index 78ea3be..06c7c7f 100644 --- a/crates/crank-metrics/src/record.rs +++ b/crates/crank-metrics/src/record.rs @@ -3,9 +3,9 @@ use std::time::{Duration, Instant}; use metrics::Gauge; use crate::{ - CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute, - HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, - McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, + CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod, + HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, + McpOutcome, McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome, }; @@ -14,6 +14,16 @@ pub fn record_http_request( method: HttpMethod, status: HttpStatusClass, duration: Duration, +) { + record_http_request_with_exemplar(route, method, status, duration, None); +} + +pub fn record_http_request_with_exemplar( + route: HttpRoute, + method: HttpMethod, + status: HttpStatusClass, + duration: Duration, + exemplar: Option, ) { metrics::counter!( "crank_http_requests_total", @@ -28,9 +38,30 @@ pub fn record_http_request( "method" => method.as_str() ) .record(duration.as_secs_f64()); + crate::exemplar::record_exemplar( + "crank_http_request_duration_seconds", + vec![("route", route.as_str()), ("method", method.as_str())], + duration.as_secs_f64(), + exemplar, + ); } -pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, outcome: McpOutcome) { +pub fn record_mcp_request( + method: McpMethod, + response_mode: McpResponseMode, + outcome: McpOutcome, + duration: Duration, +) { + record_mcp_request_with_exemplar(method, response_mode, outcome, duration, None); +} + +pub fn record_mcp_request_with_exemplar( + method: McpMethod, + response_mode: McpResponseMode, + outcome: McpOutcome, + duration: Duration, + exemplar: Option, +) { metrics::counter!( "crank_mcp_requests_total", "method" => method.as_str(), @@ -38,18 +69,54 @@ pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, out "outcome" => outcome.as_str() ) .increment(1); + metrics::histogram!( + "crank_mcp_request_duration_seconds", + "method" => method.as_str(), + "outcome" => outcome.as_str() + ) + .record(duration.as_secs_f64()); + crate::exemplar::record_exemplar( + "crank_mcp_request_duration_seconds", + vec![("method", method.as_str()), ("outcome", outcome.as_str())], + duration.as_secs_f64(), + exemplar, + ); } pub fn set_mcp_active_sessions(count: u64) { metrics::gauge!("crank_mcp_active_sessions").set(count as f64); } +pub fn set_mcp_session_metrics_fresh(fresh: bool) { + metrics::gauge!("crank_mcp_session_metrics_fresh").set(if fresh { 1.0 } else { 0.0 }); +} + pub fn record_tool_invocation( source: InvocationSource, outcome: ToolOutcome, error_kind: ToolErrorKind, duration: Duration, ) { + record_tool_invocation_with_exemplar(source, outcome, error_kind, duration, None); +} + +pub fn record_tool_invocation_with_exemplar( + source: InvocationSource, + outcome: ToolOutcome, + error_kind: ToolErrorKind, + duration: Duration, + exemplar: Option, +) { + let error_kind = match outcome { + ToolOutcome::Success => ToolErrorKind::None, + ToolOutcome::Aborted => ToolErrorKind::Aborted, + ToolOutcome::Error + if matches!(error_kind, ToolErrorKind::None | ToolErrorKind::Aborted) => + { + ToolErrorKind::ProtocolAdapter + } + ToolOutcome::Error => error_kind, + }; metrics::counter!( "crank_tool_invocations_total", "source" => source.as_str(), @@ -63,12 +130,27 @@ pub fn record_tool_invocation( "outcome" => outcome.as_str() ) .record(duration.as_secs_f64()); + crate::exemplar::record_exemplar( + "crank_tool_invocation_duration_seconds", + vec![("source", source.as_str()), ("outcome", outcome.as_str())], + duration.as_secs_f64(), + exemplar, + ); } pub fn record_upstream_request( operation_kind: UpstreamOperationKind, outcome: UpstreamOutcome, duration: Duration, +) { + record_upstream_request_with_exemplar(operation_kind, outcome, duration, None); +} + +pub fn record_upstream_request_with_exemplar( + operation_kind: UpstreamOperationKind, + outcome: UpstreamOutcome, + duration: Duration, + exemplar: Option, ) { metrics::counter!( "crank_upstream_requests_total", @@ -82,11 +164,21 @@ pub fn record_upstream_request( "outcome" => outcome.as_str() ) .record(duration.as_secs_f64()); + crate::exemplar::record_exemplar( + "crank_upstream_request_duration_seconds", + vec![ + ("operation_kind", operation_kind.as_str()), + ("outcome", outcome.as_str()), + ], + duration.as_secs_f64(), + exemplar, + ); } pub struct ToolInvocationMetrics { source: InvocationSource, started_at: Option, + exemplar: Option, } impl ToolInvocationMetrics { @@ -94,6 +186,18 @@ impl ToolInvocationMetrics { Self { source, started_at: Some(Instant::now()), + exemplar: None, + } + } + + pub fn start_with_exemplar( + source: InvocationSource, + exemplar: Option, + ) -> Self { + Self { + source, + started_at: Some(Instant::now()), + exemplar, } } @@ -103,7 +207,13 @@ impl ToolInvocationMetrics { fn finish(&mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) { if let Some(started_at) = self.started_at.take() { - record_tool_invocation(self.source, outcome, error_kind, started_at.elapsed()); + record_tool_invocation_with_exemplar( + self.source, + outcome, + error_kind, + started_at.elapsed(), + self.exemplar, + ); } } } @@ -117,6 +227,7 @@ impl Drop for ToolInvocationMetrics { pub struct UpstreamRequestMetrics { operation_kind: UpstreamOperationKind, started_at: Option, + exemplar: Option, } impl UpstreamRequestMetrics { @@ -124,6 +235,18 @@ impl UpstreamRequestMetrics { Self { operation_kind, started_at: Some(Instant::now()), + exemplar: None, + } + } + + pub fn start_with_exemplar( + operation_kind: UpstreamOperationKind, + exemplar: Option, + ) -> Self { + Self { + operation_kind, + started_at: Some(Instant::now()), + exemplar, } } @@ -133,7 +256,12 @@ impl UpstreamRequestMetrics { fn finish(&mut self, outcome: UpstreamOutcome) { if let Some(started_at) = self.started_at.take() { - record_upstream_request(self.operation_kind, outcome, started_at.elapsed()); + record_upstream_request_with_exemplar( + self.operation_kind, + outcome, + started_at.elapsed(), + self.exemplar, + ); } } } @@ -206,6 +334,7 @@ pub fn record_export_failure(signal: SignalType, exporter: Exporter) { pub fn initialize_gauges() { metrics::gauge!("crank_http_inflight").set(0.0); metrics::gauge!("crank_mcp_active_sessions").set(0.0); + set_mcp_session_metrics_fresh(false); metrics::gauge!("crank_mcp_active_streams").set(0.0); metrics::gauge!("crank_runtime_inflight").set(0.0); set_db_pool_connections(DbPoolState::Idle, 0); diff --git a/crates/crank-metrics/src/schema.rs b/crates/crank-metrics/src/schema.rs index 6f3d2bb..3570320 100644 --- a/crates/crank-metrics/src/schema.rs +++ b/crates/crank-metrics/src/schema.rs @@ -1,3 +1,12 @@ +use std::collections::BTreeSet; + +pub const METRIC_SCHEMA_VERSION: u32 = 1; +pub const MAX_METRIC_FAMILIES: usize = 128; +pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4; +pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 5_000; +pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 15_000; +pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum MetricKind { Counter, @@ -5,129 +14,619 @@ pub enum MetricKind { Histogram, } +impl MetricKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Counter => "counter", + Self::Gauge => "gauge", + Self::Histogram => "histogram", + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum MetricUnit { Count, Seconds, } +impl MetricUnit { + pub const fn as_str(self) -> &'static str { + match self { + Self::Count => "count", + Self::Seconds => "seconds", + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MetricProcess { + AdminApi, + McpServer, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MetricService { + AdminApi, + McpServer, +} + +impl MetricService { + pub fn parse(value: &str) -> Option { + match value { + "admin-api" => Some(Self::AdminApi), + "mcp-server" => Some(Self::McpServer), + _ => None, + } + } +} + +impl MetricProcess { + pub const fn as_str(self) -> &'static str { + match self { + Self::AdminApi => "admin_api", + Self::McpServer => "mcp_server", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LabelClass { + ProductClosed, + ProcessConstant, +} + +impl LabelClass { + pub const fn as_str(self) -> &'static str { + match self { + Self::ProductClosed => "product_closed", + Self::ProcessConstant => "process_constant", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LabelDomain { + pub name: &'static str, + pub class: LabelClass, + pub values: &'static [&'static str], +} + +#[derive(Clone, Copy, Debug, PartialEq)] pub struct MetricDefinition { pub name: &'static str, pub kind: MetricKind, pub unit: MetricUnit, pub labels: &'static [&'static str], + pub label_domains: &'static [LabelDomain], + pub buckets: &'static [f64], + pub processes: &'static [MetricProcess], + pub max_logical_series: usize, + pub max_rendered_series: usize, pub description: &'static str, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SchemaBudget { + pub logical_series: usize, + pub rendered_series: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SchemaError { + TooManyFamilies, + DuplicateMetric, + InvalidMetric, + DuplicateLabel, + InvalidDomain, + InvalidBuckets, + InvalidCeiling, + BudgetExceeded, +} + pub const DURATION_BUCKETS_SECONDS: &[f64] = &[ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, ]; +pub const PROCESS_CONSTANT_LABELS: &[LabelDomain] = &[ + LabelDomain { + name: "service", + class: LabelClass::ProcessConstant, + values: &["admin-api", "mcp-server"], + }, + LabelDomain { + name: "version", + class: LabelClass::ProcessConstant, + values: &["validated_release_identity"], + }, + LabelDomain { + name: "environment", + class: LabelClass::ProcessConstant, + values: &["validated_startup_value"], + }, +]; + +const BOTH: &[MetricProcess] = &[MetricProcess::AdminApi, MetricProcess::McpServer]; +const MCP: &[MetricProcess] = &[MetricProcess::McpServer]; + +pub const HTTP_ROUTE_DOMAIN: &[&str] = &[ + "unmatched", + "/health", + "/ready", + "/api/auth/login", + "/api/auth/logout", + "/api/auth/session", + "/api/auth/profile", + "/api/auth/password", + "/api/admin/capabilities", + "/api/admin/workspaces", + "/api/admin/workspaces/{workspace_id}", + "/api/admin/workspaces/{workspace_id}/operations", + "/api/admin/workspaces/{workspace_id}/imports/openapi/preview", + "/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create", + "/api/admin/workspaces/{workspace_id}/operations/analyze-quality", + "/api/admin/workspaces/{workspace_id}/operations/import", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate", + "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export", + "/api/admin/workspaces/{workspace_id}/agents", + "/api/admin/workspaces/{workspace_id}/agents/tool-search/preview", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke", + "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}", + "/api/admin/workspaces/{workspace_id}/auth-profiles", + "/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}", + "/api/admin/workspaces/{workspace_id}/upstreams", + "/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}", + "/api/admin/workspaces/{workspace_id}/secrets", + "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}", + "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate", + "/api/admin/workspaces/{workspace_id}/export", + "/api/admin/workspaces/{workspace_id}/logs", + "/api/admin/workspaces/{workspace_id}/logs/{log_id}", + "/api/admin/workspaces/{workspace_id}/approvals", + "/api/admin/workspaces/{workspace_id}/approvals/{approval_id}", + "/api/admin/workspaces/{workspace_id}/usage", + "/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}", + "/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}", + "/v1/{workspace_slug}/{agent_slug}", + "/v1/{workspace_slug}/{agent_slug}/approvals", + "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve", + "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}", + "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny", +]; +const METHODS: &[&str] = &[ + "GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER", +]; +const STATUS: &[&str] = &["1xx", "2xx", "3xx", "4xx", "5xx", "other"]; +const MCP_METHODS: &[&str] = &[ + "initialize", + "initialized", + "ping", + "tools_list", + "tools_call", + "notification", + "unsupported", + "response", + "invalid", +]; +const RESPONSE_MODES: &[&str] = &["json", "sse", "unknown"]; +const MCP_OUTCOMES: &[&str] = &[ + "success", + "client_error", + "server_error", + "jsonrpc_error", + "tool_error", + "aborted", + "other", +]; +const SOURCES: &[&str] = &["internal", "admin_test_run", "agent_tool_call"]; +const TOOL_OUTCOMES: &[&str] = &["success", "error", "aborted"]; +const TOOL_ERRORS: &[&str] = &[ + "none", + "schema", + "mapping", + "rest_adapter", + "protocol_adapter", + "unsupported_protocol", + "unsupported_execution_mode", + "concurrency_limit", + "invalid_prepared_request", + "confirmation_required", + "invalid_confirmation_token", + "confirmation_store", + "idempotency_store", + "idempotency_in_progress", + "idempotency_conflict", + "idempotency_outcome_unknown", + "missing_auth_profile", + "missing_secret", + "missing_secret_version", + "invalid_auth_secret", + "secret_crypto", + "aborted", +]; +const OPERATION_KINDS: &[&str] = &["rest"]; +const UPSTREAM_OUTCOMES: &[&str] = &[ + "success", + "client_error", + "server_error", + "unexpected_status", + "timeout", + "transport_error", + "response_too_large", + "rejected", + "window_expired", + "invalid_response", + "invalid_request", + "configuration", + "aborted", +]; +const LIMIT_STAGES: &[&str] = &["concurrency", "rate_limit", "mcp_stream"]; +const CACHE_OUTCOMES: &[&str] = &[ + "hit", + "miss", + "read_error", + "decode_error", + "evict_error", + "stored", + "write_error", +]; +const IDEMPOTENCY_OUTCOMES: &[&str] = &[ + "execute", + "replay", + "completed", + "conflict", + "in_progress", + "outcome_unknown", + "store_unavailable", + "error", +]; +const CONFIRMATION_OUTCOMES: &[&str] = &[ + "approved", + "required", + "invalid_token", + "store_unavailable", + "error", +]; +const DB_STATES: &[&str] = &["idle", "used"]; +const SIGNAL_TYPES: &[&str] = &["trace", "invocation_history"]; +const EXPORTERS: &[&str] = &["otlp", "postgres"]; + +const fn domain(name: &'static str, values: &'static [&'static str]) -> LabelDomain { + LabelDomain { + name, + class: LabelClass::ProductClosed, + values, + } +} + +#[allow( + clippy::too_many_arguments, + reason = "the canonical descriptor keeps every metric contract dimension explicit" +)] +const fn metric( + name: &'static str, + kind: MetricKind, + unit: MetricUnit, + labels: &'static [&'static str], + label_domains: &'static [LabelDomain], + buckets: &'static [f64], + processes: &'static [MetricProcess], + logical: usize, + description: &'static str, +) -> MetricDefinition { + let rendered = if matches!(kind, MetricKind::Histogram) { + logical * (buckets.len() + 3) + } else { + logical + }; + MetricDefinition { + name, + kind, + unit, + labels, + label_domains, + buckets, + processes, + max_logical_series: logical, + max_rendered_series: rendered, + description, + } +} + const METRIC_SCHEMA: &[MetricDefinition] = &[ - counter( + metric( "crank_http_requests_total", + MetricKind::Counter, + MetricUnit::Count, &["route", "method", "status_class"], + &[ + domain("route", HTTP_ROUTE_DOMAIN), + domain("method", METHODS), + domain("status_class", STATUS), + ], + &[], + BOTH, + 57 * 10 * 6, "Total HTTP requests.", ), - histogram( + metric( "crank_http_request_duration_seconds", + MetricKind::Histogram, + MetricUnit::Seconds, &["route", "method"], + &[ + domain("route", HTTP_ROUTE_DOMAIN), + domain("method", METHODS), + ], + DURATION_BUCKETS_SECONDS, + BOTH, + 57 * 10, "HTTP request duration in seconds.", ), - gauge( + metric( "crank_http_inflight", + MetricKind::Gauge, + MetricUnit::Count, &[], + &[], + &[], + BOTH, + 1, "HTTP requests currently being processed.", ), - counter( + metric( "crank_mcp_requests_total", + MetricKind::Counter, + MetricUnit::Count, &["method", "response_mode", "outcome"], + &[ + domain("method", MCP_METHODS), + domain("response_mode", RESPONSE_MODES), + domain("outcome", MCP_OUTCOMES), + ], + &[], + MCP, + 9 * 3 * 7, "Total MCP JSON-RPC requests.", ), - gauge( + metric( + "crank_mcp_request_duration_seconds", + MetricKind::Histogram, + MetricUnit::Seconds, + &["method", "outcome"], + &[ + domain("method", MCP_METHODS), + domain("outcome", MCP_OUTCOMES), + ], + DURATION_BUCKETS_SECONDS, + MCP, + 9 * 7, + "MCP JSON-RPC request duration in seconds.", + ), + metric( "crank_mcp_active_sessions", + MetricKind::Gauge, + MetricUnit::Count, &[], + &[], + &[], + MCP, + 1, "Active persisted MCP transport sessions.", ), - gauge( - "crank_mcp_active_streams", + metric( + "crank_mcp_session_metrics_fresh", + MetricKind::Gauge, + MetricUnit::Count, &[], + &[], + &[], + MCP, + 1, + "Whether the active-session gauge was refreshed successfully.", + ), + metric( + "crank_mcp_active_streams", + MetricKind::Gauge, + MetricUnit::Count, + &[], + &[], + &[], + MCP, + 1, "MCP event streams currently being served.", ), - counter( + metric( "crank_tool_invocations_total", + MetricKind::Counter, + MetricUnit::Count, &["source", "outcome", "error_kind"], + &[ + domain("source", SOURCES), + domain("outcome", TOOL_OUTCOMES), + domain("error_kind", TOOL_ERRORS), + ], + &[], + BOTH, + 3 * 3 * 22, "Total tool invocations.", ), - histogram( + metric( "crank_tool_invocation_duration_seconds", + MetricKind::Histogram, + MetricUnit::Seconds, &["source", "outcome"], + &[domain("source", SOURCES), domain("outcome", TOOL_OUTCOMES)], + DURATION_BUCKETS_SECONDS, + BOTH, + 3 * 3, "Tool invocation duration in seconds.", ), - counter( + metric( "crank_upstream_requests_total", + MetricKind::Counter, + MetricUnit::Count, &["operation_kind", "outcome"], + &[ + domain("operation_kind", OPERATION_KINDS), + domain("outcome", UPSTREAM_OUTCOMES), + ], + &[], + BOTH, + 13, "Total upstream requests.", ), - histogram( + metric( "crank_upstream_request_duration_seconds", + MetricKind::Histogram, + MetricUnit::Seconds, &["operation_kind", "outcome"], + &[ + domain("operation_kind", OPERATION_KINDS), + domain("outcome", UPSTREAM_OUTCOMES), + ], + DURATION_BUCKETS_SECONDS, + BOTH, + 13, "Upstream request duration in seconds.", ), - gauge( + metric( "crank_runtime_inflight", + MetricKind::Gauge, + MetricUnit::Count, &[], + &[], + &[], + BOTH, + 1, "Runtime executions currently in progress.", ), - counter( + metric( "crank_runtime_limit_rejections_total", + MetricKind::Counter, + MetricUnit::Count, &["stage"], + &[domain("stage", LIMIT_STAGES)], + &[], + BOTH, + 3, "Runtime executions rejected by a bounded limit.", ), - counter( + metric( "crank_runtime_cache_total", + MetricKind::Counter, + MetricUnit::Count, &["outcome"], + &[domain("outcome", CACHE_OUTCOMES)], + &[], + BOTH, + 7, "Runtime response cache outcomes.", ), - counter( + metric( "crank_idempotency_total", + MetricKind::Counter, + MetricUnit::Count, &["outcome"], + &[domain("outcome", IDEMPOTENCY_OUTCOMES)], + &[], + BOTH, + 8, "Runtime idempotency outcomes.", ), - counter( + metric( "crank_confirmation_total", + MetricKind::Counter, + MetricUnit::Count, &["outcome"], + &[domain("outcome", CONFIRMATION_OUTCOMES)], + &[], + BOTH, + 5, "Runtime confirmation outcomes.", ), - gauge( + metric( "crank_db_pool_connections", + MetricKind::Gauge, + MetricUnit::Count, &["state"], + &[domain("state", DB_STATES)], + &[], + BOTH, + 2, "PostgreSQL pool connections by state.", ), - gauge( + metric( "crank_catalog_tools", + MetricKind::Gauge, + MetricUnit::Count, &[], + &[], + &[], + MCP, + 1, "Tools in the current published catalog.", ), - gauge( + metric( "crank_catalog_estimated_context_tokens", + MetricKind::Gauge, + MetricUnit::Count, &[], + &[], + &[], + MCP, + 1, "Estimated context tokens in the current published catalog.", ), - gauge( + metric( "crank_catalog_warnings", + MetricKind::Gauge, + MetricUnit::Count, &[], + &[], + &[], + MCP, + 1, "Warnings in the current published catalog.", ), - counter( + metric( "crank_invocation_history_lost_total", + MetricKind::Counter, + MetricUnit::Count, &[], + &[], + &[], + BOTH, + 1, "Invocation history records lost after an action completed.", ), - counter( + metric( "crank_telemetry_export_failures_total", + MetricKind::Counter, + MetricUnit::Count, &["signal_type", "exporter"], + &[ + domain("signal_type", SIGNAL_TYPES), + domain("exporter", EXPORTERS), + ], + &[], + BOTH, + 4, "Telemetry export failures.", ), ]; @@ -136,44 +635,176 @@ pub const fn metric_schema() -> &'static [MetricDefinition] { METRIC_SCHEMA } -const fn counter( - name: &'static str, - labels: &'static [&'static str], - description: &'static str, -) -> MetricDefinition { - MetricDefinition { - name, - kind: MetricKind::Counter, - unit: MetricUnit::Count, - labels, - description, +pub fn schema_budget() -> Result { + validate_schema(METRIC_SCHEMA) +} + +pub fn validate_budget(logical: usize, rendered: usize) -> Result<(), SchemaError> { + if logical > MAX_LOGICAL_SERIES_PER_PROCESS || rendered > MAX_RENDERED_SERIES_PER_PROCESS { + Err(SchemaError::BudgetExceeded) + } else { + Ok(()) } } -const fn gauge( - name: &'static str, - labels: &'static [&'static str], - description: &'static str, -) -> MetricDefinition { - MetricDefinition { - name, - kind: MetricKind::Gauge, - unit: MetricUnit::Count, - labels, - description, - } +pub fn max_exemplar_slots() -> usize { + METRIC_SCHEMA + .iter() + .filter(|definition| definition.kind == MetricKind::Histogram) + .map(|definition| definition.max_logical_series * (definition.buckets.len() + 1)) + .sum() } -const fn histogram( - name: &'static str, - labels: &'static [&'static str], - description: &'static str, -) -> MetricDefinition { - MetricDefinition { - name, - kind: MetricKind::Histogram, - unit: MetricUnit::Seconds, - labels, - description, +pub fn validate_schema(schema: &[MetricDefinition]) -> Result { + if schema.len() > MAX_METRIC_FAMILIES { + return Err(SchemaError::TooManyFamilies); } + let mut names = BTreeSet::new(); + let mut logical = 0usize; + let mut rendered = 0usize; + for item in schema { + if !names.insert(item.name) { + return Err(SchemaError::DuplicateMetric); + } + if !valid_metric_name(item.name) + || !valid_static_text(item.description) + || item.processes.is_empty() + || item.labels.len() > MAX_PRODUCT_LABELS_PER_FAMILY + || item.labels.len() != item.label_domains.len() + { + return Err(SchemaError::InvalidMetric); + } + let mut labels = BTreeSet::new(); + let mut computed = 1usize; + for (label, domain) in item.labels.iter().zip(item.label_domains) { + if *label != domain.name || !valid_label_name(label) || !labels.insert(*label) { + return Err(SchemaError::DuplicateLabel); + } + if domain.class != LabelClass::ProductClosed + || domain.values.is_empty() + || domain.values.iter().any(|value| !valid_domain_value(value)) + || domain.values.iter().copied().collect::>().len() + != domain.values.len() + { + return Err(SchemaError::InvalidDomain); + } + computed = computed + .checked_mul(domain.values.len()) + .ok_or(SchemaError::BudgetExceeded)?; + } + if item.kind == MetricKind::Histogram { + if item.buckets.is_empty() + || item.buckets.iter().any(|v| !v.is_finite() || *v <= 0.0) + || item.buckets.windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(SchemaError::InvalidBuckets); + } + } else if !item.buckets.is_empty() { + return Err(SchemaError::InvalidBuckets); + } + let expected_rendered = if item.kind == MetricKind::Histogram { + computed + .checked_mul(item.buckets.len() + 3) + .ok_or(SchemaError::BudgetExceeded)? + } else { + computed + }; + if computed != item.max_logical_series || expected_rendered != item.max_rendered_series { + return Err(SchemaError::InvalidCeiling); + } + logical = logical + .checked_add(computed) + .ok_or(SchemaError::BudgetExceeded)?; + rendered = rendered + .checked_add(expected_rendered) + .ok_or(SchemaError::BudgetExceeded)?; + } + validate_budget(logical, rendered)?; + Ok(SchemaBudget { + logical_series: logical, + rendered_series: rendered, + }) +} + +fn valid_static_text(value: &str) -> bool { + !value.trim().is_empty() + && value.len() <= 512 + && !value + .bytes() + .any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\')) +} + +fn valid_metric_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + && name.starts_with("crank_") +} + +fn valid_label_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 64 + && name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} + +fn valid_domain_value(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && !value + .bytes() + .any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\')) +} + +pub fn render_metric_schema_json() -> Result { + let budget = schema_budget()?; + let mut out = format!( + "{{\n \"schema_version\": {METRIC_SCHEMA_VERSION},\n \"budget\": {{\"logical_series\": {}, \"rendered_series\": {}, \"max_logical_series\": {MAX_LOGICAL_SERIES_PER_PROCESS}, \"max_rendered_series\": {MAX_RENDERED_SERIES_PER_PROCESS}}},\n \"global_labels\": [\n", + budget.logical_series, budget.rendered_series + ); + for (index, label) in PROCESS_CONSTANT_LABELS.iter().enumerate() { + out.push_str(&format!( + " {{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}\n", + label.name, + label.class.as_str(), + json_strings(label.values), + comma(index, PROCESS_CONSTANT_LABELS.len()) + )); + } + out.push_str(" ],\n \"metrics\": [\n"); + for (index, item) in METRIC_SCHEMA.iter().enumerate() { + out.push_str(&format!(" {{\"name\": \"{}\", \"kind\": \"{}\", \"unit\": \"{}\", \"help\": \"{}\", \"processes\": [{}], \"labels\": [", item.name, item.kind.as_str(), item.unit.as_str(), item.description, json_processes(item.processes))); + for (label_index, domain) in item.label_domains.iter().enumerate() { + out.push_str(&format!( + "{{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}", + domain.name, + domain.class.as_str(), + json_strings(domain.values), + comma(label_index, item.label_domains.len()) + )); + } + out.push_str(&format!("], \"buckets_seconds\": [{}], \"max_logical_series\": {}, \"max_rendered_series\": {}}}{}\n", item.buckets.iter().map(|v| v.to_string()).collect::>().join(", "), item.max_logical_series, item.max_rendered_series, comma(index, METRIC_SCHEMA.len()))); + } + out.push_str(" ]\n}\n"); + Ok(out) +} + +fn json_strings(values: &[&str]) -> String { + values + .iter() + .map(|value| format!("\"{value}\"")) + .collect::>() + .join(", ") +} +fn json_processes(values: &[MetricProcess]) -> String { + values + .iter() + .map(|value| format!("\"{}\"", value.as_str())) + .collect::>() + .join(", ") +} +const fn comma(index: usize, len: usize) -> &'static str { + if index + 1 == len { "" } else { "," } } diff --git a/crates/crank-metrics/tests/cardinality.rs b/crates/crank-metrics/tests/cardinality.rs new file mode 100644 index 0000000..7f900d4 --- /dev/null +++ b/crates/crank-metrics/tests/cardinality.rs @@ -0,0 +1,39 @@ +use std::{collections::BTreeSet, time::Duration}; + +use crank_metrics::{ + HttpMethod, HttpRoute, HttpStatusClass, McpMethod, McpOutcome, McpResponseMode, + record_http_request, record_mcp_request, +}; +use metrics_util::debugging::DebuggingRecorder; + +#[test] +fn one_million_hostile_values_collapse_to_the_same_closed_series() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + for index in 0..1_000_000_u32 { + let canary = format!("workspace-{index}-request-trace-url-secret"); + record_http_request( + HttpRoute::from_matched_path(&canary), + HttpMethod::classify(&canary), + HttpStatusClass::from_status(999), + Duration::ZERO, + ); + record_mcp_request( + McpMethod::Invalid, + McpResponseMode::Unknown, + McpOutcome::Other, + Duration::ZERO, + ); + } + }); + + let series = snapshotter + .snapshot() + .into_vec() + .into_iter() + .map(|(key, _, _, _)| format!("{:?}", key.key())) + .collect::>(); + assert_eq!(series.len(), 4); + assert!(series.iter().all(|value| !value.contains("workspace-"))); +} diff --git a/crates/crank-metrics/tests/contract.rs b/crates/crank-metrics/tests/contract.rs index 66bbe0a..7edfa6f 100644 --- a/crates/crank-metrics/tests/contract.rs +++ b/crates/crank-metrics/tests/contract.rs @@ -8,7 +8,7 @@ use crank_metrics::{ metric_schema, record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request, record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection, record_mcp_request, record_tool_invocation, record_upstream_request, - set_catalog, set_db_pool_connections, set_mcp_active_sessions, + set_catalog, set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh, }; use metrics_util::debugging::DebuggingRecorder; @@ -80,6 +80,26 @@ fn application_outcomes_are_distinct_from_transport_success() { assert_eq!(ConfirmationOutcome::Required.as_str(), "required"); } +#[test] +fn impossible_tool_outcome_pairs_are_normalized() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + record_tool_invocation( + InvocationSource::AgentToolCall, + ToolOutcome::Success, + ToolErrorKind::Mapping, + Duration::ZERO, + ); + }); + let snapshot = snapshotter.snapshot().into_vec(); + assert!(snapshot.iter().any(|(key, _, _, _)| { + key.key().name() == "crank_tool_invocations_total" + && has_label(key.key(), "outcome", "success") + && has_label(key.key(), "error_kind", "none") + })); +} + #[test] fn dropped_product_guards_record_aborted_outcomes() { let recorder = DebuggingRecorder::new(); @@ -128,6 +148,7 @@ fn diverse_calls_cannot_create_unbounded_series() { McpMethod::ToolsCall, McpResponseMode::Json, McpOutcome::ToolError, + Duration::from_millis(1), ); record_tool_invocation( InvocationSource::AgentToolCall, @@ -147,7 +168,7 @@ fn diverse_calls_cannot_create_unbounded_series() { .map(|(key, _, _, _)| format!("{:?}", key.key())) .collect::>(); - assert_eq!(series.len(), 8); + assert_eq!(series.len(), 9); assert!( series .iter() @@ -172,8 +193,10 @@ fn every_recording_api_matches_the_declared_schema() { McpMethod::ToolsCall, McpResponseMode::Json, McpOutcome::Success, + Duration::from_millis(1), ); set_mcp_active_sessions(1); + set_mcp_session_metrics_fresh(true); let _stream = InFlightGuard::mcp_stream(); let _runtime = InFlightGuard::runtime(); record_tool_invocation( diff --git a/crates/crank-metrics/tests/exemplars.rs b/crates/crank-metrics/tests/exemplars.rs new file mode 100644 index 0000000..4dc1313 --- /dev/null +++ b/crates/crank-metrics/tests/exemplars.rs @@ -0,0 +1,40 @@ +use std::time::Duration; + +use crank_metrics::{ + ExemplarTraceId, HttpMethod, HttpRoute, HttpStatusClass, exemplar_snapshot, + record_http_request_with_exemplar, reset_exemplars_for_test, +}; +use metrics_util::debugging::DebuggingRecorder; + +#[test] +fn trace_id_is_validated_and_never_becomes_an_ordinary_label() { + assert!(ExemplarTraceId::parse("0123456789abcdef0123456789abcdef").is_some()); + assert!(ExemplarTraceId::parse("0123456789ABCDEF0123456789ABCDEF").is_none()); + assert!(ExemplarTraceId::parse("short").is_none()); + + reset_exemplars_for_test(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + record_http_request_with_exemplar( + HttpRoute::from_matched_path("/health"), + HttpMethod::Get, + HttpStatusClass::Success, + Duration::from_millis(10), + ExemplarTraceId::parse("0123456789abcdef0123456789abcdef"), + ); + }); + + let aggregate = snapshotter.snapshot().into_vec(); + assert!( + aggregate + .iter() + .all(|(key, _, _, _)| { key.key().labels().all(|label| label.key() != "trace_id") }) + ); + let exemplars = exemplar_snapshot(); + assert_eq!(exemplars.len(), 1); + assert_eq!( + exemplars[0].trace_id.as_str(), + "0123456789abcdef0123456789abcdef" + ); +} diff --git a/crates/crank-metrics/tests/overhead.rs b/crates/crank-metrics/tests/overhead.rs new file mode 100644 index 0000000..5a16a1d --- /dev/null +++ b/crates/crank-metrics/tests/overhead.rs @@ -0,0 +1,62 @@ +use std::{ + hint::black_box, + time::{Duration, Instant}, +}; + +use crank_metrics::{CacheOutcome, record_cache_outcome}; +use metrics_util::debugging::DebuggingRecorder; + +const SAMPLES: usize = 40; +const WORK_UNITS: u64 = 500_000; + +#[test] +fn reproducible_foundation_profile_stays_inside_latency_and_cpu_proxy_budgets() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let mut baseline = Vec::with_capacity(SAMPLES); + let mut enabled = Vec::with_capacity(SAMPLES); + + metrics::with_local_recorder(&recorder, || { + for index in 0..SAMPLES { + if index % 2 == 0 { + baseline.push(sample(false)); + enabled.push(sample(true)); + } else { + enabled.push(sample(true)); + baseline.push(sample(false)); + } + } + }); + + baseline.sort_unstable(); + enabled.sort_unstable(); + let baseline_p95 = baseline[SAMPLES * 95 / 100]; + let enabled_p95 = enabled[SAMPLES * 95 / 100]; + let baseline_cpu: Duration = baseline.iter().copied().sum(); + let enabled_cpu: Duration = enabled.iter().copied().sum(); + let series = snapshotter.snapshot().into_vec().len(); + + println!( + "metrics_profile_v1 samples={SAMPLES} work_units={WORK_UNITS} baseline_p95_ns={} enabled_p95_ns={} baseline_cpu_ns={} enabled_cpu_ns={} logical_series={series}", + baseline_p95.as_nanos(), + enabled_p95.as_nanos(), + baseline_cpu.as_nanos(), + enabled_cpu.as_nanos() + ); + assert!(enabled_p95.as_nanos() * 100 <= baseline_p95.as_nanos() * 105); + assert!(enabled_cpu.as_nanos() * 100 <= baseline_cpu.as_nanos() * 110); + assert_eq!(series, 1); +} + +fn sample(with_metrics: bool) -> Duration { + let started = Instant::now(); + let mut value = 0x9e37_79b9_u64; + for index in 0..WORK_UNITS { + value = value.rotate_left(7) ^ index.wrapping_mul(0x100_0000_01b3); + } + black_box(value); + if with_metrics { + record_cache_outcome(CacheOutcome::Hit); + } + started.elapsed() +} diff --git a/crates/crank-metrics/tests/schema_v1.rs b/crates/crank-metrics/tests/schema_v1.rs new file mode 100644 index 0000000..d0867f9 --- /dev/null +++ b/crates/crank-metrics/tests/schema_v1.rs @@ -0,0 +1,100 @@ +use std::collections::BTreeSet; + +use crank_metrics::{ + HTTP_ROUTE_DOMAIN, HttpRoute, MAX_LOGICAL_SERIES_PER_PROCESS, MAX_PRODUCT_LABELS_PER_FAMILY, + MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, metric_schema, + render_metric_schema_json, schema_budget, validate_budget, +}; + +#[test] +fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() { + assert_eq!(METRIC_SCHEMA_VERSION, 1); + assert_eq!(metric_schema().len(), 23); + + let budget = schema_budget().expect("canonical schema must be valid"); + assert!(budget.logical_series <= MAX_LOGICAL_SERIES_PER_PROCESS); + assert!(budget.rendered_series <= MAX_RENDERED_SERIES_PER_PROCESS); + + for definition in metric_schema() { + assert!(!definition.description.trim().is_empty()); + assert!(definition.labels.len() <= MAX_PRODUCT_LABELS_PER_FAMILY); + assert!(!definition.processes.is_empty()); + assert_eq!(definition.labels.len(), definition.label_domains.len()); + assert!(definition.label_domains.iter().all(|domain| { + !domain.values.is_empty() + && domain.values.iter().all(|value| !value.is_empty()) + && domain.values.iter().copied().collect::>().len() + == domain.values.len() + })); + if definition.kind.as_str() == "histogram" { + assert!(!definition.buckets.is_empty()); + assert!(definition.buckets.windows(2).all(|pair| pair[0] < pair[1])); + } else { + assert!(definition.buckets.is_empty()); + } + } + + let first = render_metric_schema_json().expect("schema renders"); + let second = render_metric_schema_json().expect("schema renders deterministically"); + assert_eq!(first, second); + assert!(first.len() < 65_536); + assert!(!first.contains("trace_id")); + assert!(!first.contains("request_id")); +} + +#[test] +fn frozen_budget_edges_are_exact() { + assert!(validate_budget(4_999, 14_999).is_ok()); + assert!(validate_budget(5_000, 15_000).is_ok()); + assert!(validate_budget(5_001, 15_000).is_err()); + assert!(validate_budget(5_000, 15_001).is_err()); + assert_eq!(crank_metrics::max_exemplar_slots(), 9_170); +} + +#[test] +fn every_registered_route_round_trips_without_dynamic_segments() { + assert_eq!(HTTP_ROUTE_DOMAIN.len(), 57); + for route in HTTP_ROUTE_DOMAIN { + assert_eq!(HttpRoute::from_matched_path(route).as_str(), *route); + assert!(!route.contains("trace_id")); + assert!(!route.contains("request_id")); + } +} + +#[test] +fn canonical_schema_validation_reports_its_category() { + assert_eq!( + schema_budget(), + Ok(crank_metrics::SchemaBudget { + logical_series: 4_513, + rendered_series: 14_338 + }) + ); +} + +#[test] +fn exact_budget_math_counts_histogram_bucket_sum_and_count_series() { + for definition in metric_schema() { + let logical = definition + .label_domains + .iter() + .map(|domain| domain.values.len()) + .product::(); + let logical = logical.max(1); + let rendered = if definition.kind.as_str() == "histogram" { + logical * (definition.buckets.len() + 3) + } else { + logical + }; + assert_eq!( + definition.max_logical_series, logical, + "{}", + definition.name + ); + assert_eq!( + definition.max_rendered_series, rendered, + "{}", + definition.name + ); + } +} diff --git a/crates/crank-observability/src/instrumentation.rs b/crates/crank-observability/src/instrumentation.rs index eb831e0..e3125e0 100644 --- a/crates/crank-observability/src/instrumentation.rs +++ b/crates/crank-observability/src/instrumentation.rs @@ -5,7 +5,9 @@ use axum::{ middleware::Next, response::Response, }; -use crank_metrics::{DbPoolState, HttpMethod, HttpRoute, HttpStatusClass, InFlightGuard}; +use crank_metrics::{ + DbPoolState, ExemplarTraceId, HttpMethod, HttpRoute, HttpStatusClass, InFlightGuard, +}; use metrics::Unit; use crate::{MetricKind, MetricUnit, metric_schema}; @@ -22,11 +24,17 @@ pub async fn record_http_request(request: Request, next: Next) -> Response { let _inflight = InFlightGuard::http(); let response = next.run(request).await; - crank_metrics::record_http_request( + let exemplar = response + .headers() + .get("x-trace-id") + .and_then(|value| value.to_str().ok()) + .and_then(ExemplarTraceId::parse); + crank_metrics::record_http_request_with_exemplar( route, method, HttpStatusClass::from_status(response.status().as_u16()), started_at.elapsed(), + exemplar, ); response diff --git a/crates/crank-observability/src/prometheus.rs b/crates/crank-observability/src/prometheus.rs index d3a701d..d851e84 100644 --- a/crates/crank-observability/src/prometheus.rs +++ b/crates/crank-observability/src/prometheus.rs @@ -18,8 +18,11 @@ use thiserror::Error; use tokio::net::TcpListener; use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity}; +use crank_metrics::{ExemplarObservation, MAX_EXPOSITION_BYTES, MetricService, exemplar_snapshot}; const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; +const OPENMETRICS_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8"; +const EXPOSITION_TOO_LARGE: &str = "metrics exposition exceeds configured bound\n"; #[derive(Clone)] pub struct MetricsConfig { @@ -190,6 +193,7 @@ pub(crate) fn install_prometheus_recorder( fn prometheus_builder( identity: &ServiceIdentity, ) -> Result { + MetricService::parse(identity.service()).ok_or(MetricsSurfaceError::RecorderConfiguration)?; PrometheusBuilder::new() .set_buckets(DURATION_BUCKETS_SECONDS) .map(|builder| { @@ -201,15 +205,70 @@ fn prometheus_builder( .map_err(|_| MetricsSurfaceError::RecorderConfiguration) } -async fn render_metrics(State(state): State) -> Response { - let mut response = state.handle.render().into_response(); +async fn render_metrics(State(state): State, headers: HeaderMap) -> Response { + let legacy = state.handle.render(); + let openmetrics = headers + .get(header::ACCEPT) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(',') + .any(|part| part.trim().starts_with("application/openmetrics-text")) + }); + let body = if openmetrics { + render_openmetrics(&legacy, &exemplar_snapshot()) + } else { + legacy + }; + if body.len() > MAX_EXPOSITION_BYTES { + return (StatusCode::SERVICE_UNAVAILABLE, EXPOSITION_TOO_LARGE).into_response(); + } + let mut response = body.into_response(); response.headers_mut().insert( header::CONTENT_TYPE, - HeaderValue::from_static(PROMETHEUS_CONTENT_TYPE), + HeaderValue::from_static(if openmetrics { + OPENMETRICS_CONTENT_TYPE + } else { + PROMETHEUS_CONTENT_TYPE + }), ); response } +fn render_openmetrics(legacy: &str, exemplars: &[ExemplarObservation]) -> String { + let mut output = String::with_capacity(legacy.len() + exemplars.len().saturating_mul(96) + 6); + for line in legacy.lines() { + output.push_str(line); + if let Some(exemplar) = exemplars + .iter() + .find(|candidate| line_matches_exemplar(line, candidate)) + { + output.push_str(" # {trace_id=\""); + output.push_str(exemplar.trace_id.as_str()); + output.push_str("\"} "); + output.push_str(&exemplar.value.to_string()); + } + output.push('\n'); + } + output.push_str("# EOF\n"); + output +} + +fn line_matches_exemplar(line: &str, exemplar: &ExemplarObservation) -> bool { + if !line.starts_with(exemplar.metric) || !line[exemplar.metric.len()..].starts_with("_bucket{") + { + return false; + } + let expected_bound = exemplar + .bucket_upper_bound + .map_or_else(|| "+Inf".to_owned(), |value| value.to_string()); + line.contains(&format!("le=\"{expected_bound}\"")) + && exemplar + .labels + .iter() + .all(|(key, value)| line.contains(&format!("{key}=\"{value}\""))) +} + async fn metrics_health() -> impl IntoResponse { (StatusCode::OK, "ok\n") } @@ -251,3 +310,26 @@ fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> { fn token_digest(token: &[u8]) -> [u8; 32] { Sha256::digest(token).into() } + +#[cfg(test)] +mod rendering_tests { + use crank_metrics::{ExemplarObservation, ExemplarTraceId}; + + use super::render_openmetrics; + + #[test] + fn openmetrics_adds_bounded_exemplar_without_changing_aggregate() { + let legacy = "# TYPE crank_http_request_duration_seconds histogram\ncrank_http_request_duration_seconds_bucket{method=\"GET\",route=\"/health\",le=\"0.01\"} 1\ncrank_http_request_duration_seconds_sum{method=\"GET\",route=\"/health\"} 0.007\n"; + let exemplar = ExemplarObservation { + metric: "crank_http_request_duration_seconds", + labels: vec![("route", "/health"), ("method", "GET")], + bucket_upper_bound: Some(0.01), + value: 0.007, + trace_id: ExemplarTraceId::parse("0123456789abcdef0123456789abcdef").unwrap(), + }; + let rendered = render_openmetrics(legacy, &[exemplar]); + assert!(rendered.contains("# {trace_id=\"0123456789abcdef0123456789abcdef\"} 0.007")); + assert!(rendered.ends_with("# EOF\n")); + assert_eq!(rendered.matches(" 1").count(), legacy.matches(" 1").count()); + } +} diff --git a/crates/crank-observability/tests/http_metrics.rs b/crates/crank-observability/tests/http_metrics.rs index c92856e..3121147 100644 --- a/crates/crank-observability/tests/http_metrics.rs +++ b/crates/crank-observability/tests/http_metrics.rs @@ -3,7 +3,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use axum::{ Router, body::{Body, to_bytes}, - http::{Request, StatusCode}, + http::{Request, StatusCode, header}, middleware, routing::get, }; @@ -14,8 +14,7 @@ use tower::ServiceExt; #[tokio::test] async fn http_metrics_use_matched_routes_and_closed_labels() { - let identity = - ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity"); + let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"); let lifecycle = crank_observability::init(ObservabilityConfig::new( identity, "off", @@ -32,7 +31,12 @@ async fn http_metrics_use_matched_routes_and_closed_labels() { let app = Router::new() .route( "/api/admin/workspaces/{workspace_id}/operations/{operation_id}", - get(|| async { StatusCode::NO_CONTENT }), + get(|| async { + ( + StatusCode::NO_CONTENT, + [("x-trace-id", "0123456789abcdef0123456789abcdef")], + ) + }), ) .layer(middleware::from_fn(record_http_request)); @@ -63,6 +67,7 @@ async fn http_metrics_use_matched_routes_and_closed_labels() { assert_eq!(response.status(), StatusCode::NOT_FOUND); let response = metrics + .clone() .oneshot( Request::get("/metrics") .body(Body::empty()) @@ -82,6 +87,7 @@ async fn http_metrics_use_matched_routes_and_closed_labels() { assert!(body.contains("method=\"GET\"")); assert!(body.contains("status_class=\"2xx\"")); assert!(body.contains("crank_http_request_duration_seconds_bucket")); + assert!(!body.contains("trace_id")); assert!(!body.contains(sensitive_path_segment)); assert_eq!( body.lines() @@ -95,4 +101,30 @@ async fn http_metrics_use_matched_routes_and_closed_labels() { 1, "different entity ids must not create additional series" ); + + let openmetrics = metrics + .oneshot( + Request::get("/metrics") + .header( + header::ACCEPT, + "application/openmetrics-text; version=1.0.0", + ) + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("OpenMetrics response"); + assert_eq!(openmetrics.status(), StatusCode::OK); + assert!( + openmetrics.headers()[header::CONTENT_TYPE] + .to_str() + .unwrap() + .starts_with("application/openmetrics-text") + ); + let body = to_bytes(openmetrics.into_body(), 1024 * 1024) + .await + .unwrap(); + let body = String::from_utf8(body.to_vec()).unwrap(); + assert!(body.contains("# {trace_id=\"0123456789abcdef0123456789abcdef\"}")); + assert!(body.ends_with("# EOF\n")); } diff --git a/crates/crank-observability/tests/lifecycle.rs b/crates/crank-observability/tests/lifecycle.rs index 0e62ce8..839e744 100644 --- a/crates/crank-observability/tests/lifecycle.rs +++ b/crates/crank-observability/tests/lifecycle.rs @@ -4,7 +4,7 @@ use crank_observability::{ fn config() -> ObservabilityConfig { ObservabilityConfig::new( - ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"), + ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid test identity"), "info", RedactionLimits::default(), ) diff --git a/crates/crank-observability/tests/metrics_signals.rs b/crates/crank-observability/tests/metrics_signals.rs index d99f4da..79c9ca4 100644 --- a/crates/crank-observability/tests/metrics_signals.rs +++ b/crates/crank-observability/tests/metrics_signals.rs @@ -17,8 +17,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[tokio::test] async fn typed_product_signals_are_exposed_by_a_real_scrape() { - let identity = - ServiceIdentity::try_new("metrics-signals", "0.3.1", "test").expect("valid identity"); + let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"); let lifecycle = crank_observability::init(ObservabilityConfig::new( identity, "off", @@ -49,6 +48,7 @@ async fn typed_product_signals_are_exposed_by_a_real_scrape() { McpMethod::ToolsCall, McpResponseMode::Json, McpOutcome::ToolError, + Duration::from_millis(1), ); set_mcp_active_sessions(2); let _stream = InFlightGuard::mcp_stream(); diff --git a/crates/crank-observability/tests/prometheus.rs b/crates/crank-observability/tests/prometheus.rs index efd3784..4821eb6 100644 --- a/crates/crank-observability/tests/prometheus.rs +++ b/crates/crank-observability/tests/prometheus.rs @@ -71,6 +71,18 @@ fn non_loopback_without_a_token_is_rejected_without_secret_data() { assert!(!error.to_string().contains("token=")); } +#[test] +fn metrics_surface_rejects_an_unregistered_process_identity() { + let config = MetricsConfig::new( + true, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464), + None, + ) + .unwrap(); + let identity = ServiceIdentity::try_new("user-controlled", "0.3.1", "test").unwrap(); + assert!(MetricsSurface::for_test(config, identity).is_err()); +} + #[test] fn schema_is_closed_and_uses_fixed_duration_buckets() { let schema = metric_schema(); diff --git a/crates/crank-runtime/src/executor.rs b/crates/crank-runtime/src/executor.rs index cc8fd5d..00e73ff 100644 --- a/crates/crank-runtime/src/executor.rs +++ b/crates/crank-runtime/src/executor.rs @@ -258,8 +258,12 @@ impl RuntimeExecutor { let request_context = request.request_context.or(generated_context.as_ref()); log_runtime_event("unary.execute", request.operation, request_context); let started_at = Instant::now(); - let invocation_metrics = - ToolInvocationMetrics::start(metric_invocation_source(request_context)); + let invocation_metrics = ToolInvocationMetrics::start_with_exemplar( + metric_invocation_source(request_context), + request_context.and_then(|context| { + crank_metrics::ExemplarTraceId::parse(&context.trace_context.trace_id().to_string()) + }), + ); let result = async { let _permit = self.acquire_unary_permit(request.operation)?; let _inflight = InFlightGuard::runtime(); diff --git a/docs/observability.md b/docs/observability.md index 8633c34..5bd02b3 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -171,7 +171,8 @@ scrape_configs: - `crank_http_requests_total`, `crank_http_request_duration_seconds`, `crank_http_inflight`; -- `crank_mcp_requests_total`, `crank_mcp_active_sessions`, +- `crank_mcp_requests_total`, `crank_mcp_request_duration_seconds`, + `crank_mcp_active_sessions`, `crank_mcp_session_metrics_fresh`, `crank_mcp_active_streams`; - `crank_tool_invocations_total`, `crank_tool_invocation_duration_seconds`; - `crank_upstream_requests_total`, @@ -187,8 +188,12 @@ scrape_configs: Recorder добавляет к рядам проверенные статические labels `service`, `version`, `environment`. Остальные labels имеют закрытый набор значений. -Имена, типы и допустимые значения labels определены в независимом нижнем -crate `crank-metrics`. Product-код вызывает только его типизированный фасад; +Эти три labels классифицированы как `process_constant`: они валидируются один +раз при запуске и образуют ровно одну tuple на процесс. Имена, типы, help, +process ownership, допустимые значения product labels и buckets определены в +независимом нижнем crate `crank-metrics`. Его deterministic machine snapshot — +[`schemas/metrics-registry-v1.json`](schemas/metrics-registry-v1.json). +Product-код вызывает только типизированный фасад; прямая production-зависимость от `metrics` вне `crank-metrics` и `crank-observability` запрещена архитектурной проверкой Cargo metadata. Запрещено использовать workspace, идентификаторы агента, операции или @@ -198,7 +203,10 @@ HTTP route берётся только из шаблона Axum; неизвес `crank_mcp_active_sessions` отражает число ещё не истёкших транспортных сессий в общем хранилище, а `crank_mcp_active_streams` — число открытых в -этом экземпляре SSE-потоков. Прикладные ошибки JSON-RPC и +этом экземпляре SSE-потоков. `crank_mcp_session_metrics_fresh=0` означает, +что последнее bounded чтение session store завершилось ошибкой/timeout и +значение active sessions нельзя считать свежим; успешное чтение возвращает +freshness в `1`. Прикладные ошибки JSON-RPC и `tools/call.result.isError=true` учитываются отдельно от успешного HTTP 200. Счётчики cache, idempotency и confirmation описывают переходы соответствующих стадий: hit/miss и ошибки хранилища, execute/replay/conflict @@ -208,9 +216,20 @@ HTTP route берётся только из шаблона Axum; неизвес Buckets длительности фиксированы кодом: `0.005`, `0.01`, `0.025`, `0.05`, `0.1`, `0.25`, `0.5`, `1`, `2.5`, `5`, -`10`, `30`, `60` секунд. Это пока техническая шкала, а не SLO. Фактические -series и расход памяти измеряются в истории 1.8; произвольный численный -бюджет до замеров не назначается. +`10`, `30`, `60` секунд. Это пока техническая шкала, а не SLO. Registry v1 +содержит 23 family и для всего lifetime одного процесса ограничен 5 000 +logical labelsets и 15 000 Prometheus sample series; текущий worst case — +4 513 и 14 338 соответственно. Histogram budget включает finite buckets, +`+Inf`, `_sum` и `_count`; изменение domain или buckets требует обновления +versioned snapshot и evidence. + +Обычный scrape сохраняет Prometheus text 0.0.4. Клиент с +`Accept: application/openmetrics-text` получает OpenMetrics 1.0 и `# EOF`. +Validated 32-hex Trace ID может присутствовать только как latest bounded +histogram exemplar; он никогда не является label и не меняет aggregate. +Отсутствующий, invalid или unsampled Trace ID просто не создаёт exemplar. +Размер готового exposition ограничен 8 MiB; превышение возвращает статический +`503` без частично усечённого тела и не влияет на product listener. ## Распределённые трассы diff --git a/docs/schemas/metrics-registry-v1.json b/docs/schemas/metrics-registry-v1.json new file mode 100644 index 0000000..381dbb2 --- /dev/null +++ b/docs/schemas/metrics-registry-v1.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "budget": {"logical_series": 4513, "rendered_series": 14338, "max_logical_series": 5000, "max_rendered_series": 15000}, + "global_labels": [ + {"name": "service", "class": "process_constant", "domain": ["admin-api", "mcp-server"]}, + {"name": "version", "class": "process_constant", "domain": ["validated_release_identity"]}, + {"name": "environment", "class": "process_constant", "domain": ["validated_startup_value"]} + ], + "metrics": [ + {"name": "crank_http_requests_total", "kind": "counter", "unit": "count", "help": "Total HTTP requests.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "route", "class": "product_closed", "domain": ["unmatched", "/health", "/ready", "/api/auth/login", "/api/auth/logout", "/api/auth/session", "/api/auth/profile", "/api/auth/password", "/api/admin/capabilities", "/api/admin/workspaces", "/api/admin/workspaces/{workspace_id}", "/api/admin/workspaces/{workspace_id}/operations", "/api/admin/workspaces/{workspace_id}/imports/openapi/preview", "/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create", "/api/admin/workspaces/{workspace_id}/operations/analyze-quality", "/api/admin/workspaces/{workspace_id}/operations/import", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export", "/api/admin/workspaces/{workspace_id}/agents", "/api/admin/workspaces/{workspace_id}/agents/tool-search/preview", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}", "/api/admin/workspaces/{workspace_id}/auth-profiles", "/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}", "/api/admin/workspaces/{workspace_id}/upstreams", "/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}", "/api/admin/workspaces/{workspace_id}/secrets", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate", "/api/admin/workspaces/{workspace_id}/export", "/api/admin/workspaces/{workspace_id}/logs", "/api/admin/workspaces/{workspace_id}/logs/{log_id}", "/api/admin/workspaces/{workspace_id}/approvals", "/api/admin/workspaces/{workspace_id}/approvals/{approval_id}", "/api/admin/workspaces/{workspace_id}/usage", "/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}", "/v1/{workspace_slug}/{agent_slug}", "/v1/{workspace_slug}/{agent_slug}/approvals", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny"]},{"name": "method", "class": "product_closed", "domain": ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER"]},{"name": "status_class", "class": "product_closed", "domain": ["1xx", "2xx", "3xx", "4xx", "5xx", "other"]}], "buckets_seconds": [], "max_logical_series": 3420, "max_rendered_series": 3420}, + {"name": "crank_http_request_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "HTTP request duration in seconds.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "route", "class": "product_closed", "domain": ["unmatched", "/health", "/ready", "/api/auth/login", "/api/auth/logout", "/api/auth/session", "/api/auth/profile", "/api/auth/password", "/api/admin/capabilities", "/api/admin/workspaces", "/api/admin/workspaces/{workspace_id}", "/api/admin/workspaces/{workspace_id}/operations", "/api/admin/workspaces/{workspace_id}/imports/openapi/preview", "/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create", "/api/admin/workspaces/{workspace_id}/operations/analyze-quality", "/api/admin/workspaces/{workspace_id}/operations/import", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export", "/api/admin/workspaces/{workspace_id}/agents", "/api/admin/workspaces/{workspace_id}/agents/tool-search/preview", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}", "/api/admin/workspaces/{workspace_id}/auth-profiles", "/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}", "/api/admin/workspaces/{workspace_id}/upstreams", "/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}", "/api/admin/workspaces/{workspace_id}/secrets", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate", "/api/admin/workspaces/{workspace_id}/export", "/api/admin/workspaces/{workspace_id}/logs", "/api/admin/workspaces/{workspace_id}/logs/{log_id}", "/api/admin/workspaces/{workspace_id}/approvals", "/api/admin/workspaces/{workspace_id}/approvals/{approval_id}", "/api/admin/workspaces/{workspace_id}/usage", "/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}", "/v1/{workspace_slug}/{agent_slug}", "/v1/{workspace_slug}/{agent_slug}/approvals", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny"]},{"name": "method", "class": "product_closed", "domain": ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 570, "max_rendered_series": 9120}, + {"name": "crank_http_inflight", "kind": "gauge", "unit": "count", "help": "HTTP requests currently being processed.", "processes": ["admin_api", "mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_mcp_requests_total", "kind": "counter", "unit": "count", "help": "Total MCP JSON-RPC requests.", "processes": ["mcp_server"], "labels": [{"name": "method", "class": "product_closed", "domain": ["initialize", "initialized", "ping", "tools_list", "tools_call", "notification", "unsupported", "response", "invalid"]},{"name": "response_mode", "class": "product_closed", "domain": ["json", "sse", "unknown"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "jsonrpc_error", "tool_error", "aborted", "other"]}], "buckets_seconds": [], "max_logical_series": 189, "max_rendered_series": 189}, + {"name": "crank_mcp_request_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "MCP JSON-RPC request duration in seconds.", "processes": ["mcp_server"], "labels": [{"name": "method", "class": "product_closed", "domain": ["initialize", "initialized", "ping", "tools_list", "tools_call", "notification", "unsupported", "response", "invalid"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "jsonrpc_error", "tool_error", "aborted", "other"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 63, "max_rendered_series": 1008}, + {"name": "crank_mcp_active_sessions", "kind": "gauge", "unit": "count", "help": "Active persisted MCP transport sessions.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_mcp_session_metrics_fresh", "kind": "gauge", "unit": "count", "help": "Whether the active-session gauge was refreshed successfully.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_mcp_active_streams", "kind": "gauge", "unit": "count", "help": "MCP event streams currently being served.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_tool_invocations_total", "kind": "counter", "unit": "count", "help": "Total tool invocations.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "source", "class": "product_closed", "domain": ["internal", "admin_test_run", "agent_tool_call"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "error", "aborted"]},{"name": "error_kind", "class": "product_closed", "domain": ["none", "schema", "mapping", "rest_adapter", "protocol_adapter", "unsupported_protocol", "unsupported_execution_mode", "concurrency_limit", "invalid_prepared_request", "confirmation_required", "invalid_confirmation_token", "confirmation_store", "idempotency_store", "idempotency_in_progress", "idempotency_conflict", "idempotency_outcome_unknown", "missing_auth_profile", "missing_secret", "missing_secret_version", "invalid_auth_secret", "secret_crypto", "aborted"]}], "buckets_seconds": [], "max_logical_series": 198, "max_rendered_series": 198}, + {"name": "crank_tool_invocation_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "Tool invocation duration in seconds.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "source", "class": "product_closed", "domain": ["internal", "admin_test_run", "agent_tool_call"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "error", "aborted"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 9, "max_rendered_series": 144}, + {"name": "crank_upstream_requests_total", "kind": "counter", "unit": "count", "help": "Total upstream requests.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "operation_kind", "class": "product_closed", "domain": ["rest"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "unexpected_status", "timeout", "transport_error", "response_too_large", "rejected", "window_expired", "invalid_response", "invalid_request", "configuration", "aborted"]}], "buckets_seconds": [], "max_logical_series": 13, "max_rendered_series": 13}, + {"name": "crank_upstream_request_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "Upstream request duration in seconds.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "operation_kind", "class": "product_closed", "domain": ["rest"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "unexpected_status", "timeout", "transport_error", "response_too_large", "rejected", "window_expired", "invalid_response", "invalid_request", "configuration", "aborted"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 13, "max_rendered_series": 208}, + {"name": "crank_runtime_inflight", "kind": "gauge", "unit": "count", "help": "Runtime executions currently in progress.", "processes": ["admin_api", "mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_runtime_limit_rejections_total", "kind": "counter", "unit": "count", "help": "Runtime executions rejected by a bounded limit.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "stage", "class": "product_closed", "domain": ["concurrency", "rate_limit", "mcp_stream"]}], "buckets_seconds": [], "max_logical_series": 3, "max_rendered_series": 3}, + {"name": "crank_runtime_cache_total", "kind": "counter", "unit": "count", "help": "Runtime response cache outcomes.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "outcome", "class": "product_closed", "domain": ["hit", "miss", "read_error", "decode_error", "evict_error", "stored", "write_error"]}], "buckets_seconds": [], "max_logical_series": 7, "max_rendered_series": 7}, + {"name": "crank_idempotency_total", "kind": "counter", "unit": "count", "help": "Runtime idempotency outcomes.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "outcome", "class": "product_closed", "domain": ["execute", "replay", "completed", "conflict", "in_progress", "outcome_unknown", "store_unavailable", "error"]}], "buckets_seconds": [], "max_logical_series": 8, "max_rendered_series": 8}, + {"name": "crank_confirmation_total", "kind": "counter", "unit": "count", "help": "Runtime confirmation outcomes.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "outcome", "class": "product_closed", "domain": ["approved", "required", "invalid_token", "store_unavailable", "error"]}], "buckets_seconds": [], "max_logical_series": 5, "max_rendered_series": 5}, + {"name": "crank_db_pool_connections", "kind": "gauge", "unit": "count", "help": "PostgreSQL pool connections by state.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "state", "class": "product_closed", "domain": ["idle", "used"]}], "buckets_seconds": [], "max_logical_series": 2, "max_rendered_series": 2}, + {"name": "crank_catalog_tools", "kind": "gauge", "unit": "count", "help": "Tools in the current published catalog.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_catalog_estimated_context_tokens", "kind": "gauge", "unit": "count", "help": "Estimated context tokens in the current published catalog.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_catalog_warnings", "kind": "gauge", "unit": "count", "help": "Warnings in the current published catalog.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_invocation_history_lost_total", "kind": "counter", "unit": "count", "help": "Invocation history records lost after an action completed.", "processes": ["admin_api", "mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1}, + {"name": "crank_telemetry_export_failures_total", "kind": "counter", "unit": "count", "help": "Telemetry export failures.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "signal_type", "class": "product_closed", "domain": ["trace", "invocation_history"]},{"name": "exporter", "class": "product_closed", "domain": ["otlp", "postgres"]}], "buckets_seconds": [], "max_logical_series": 4, "max_rendered_series": 4} + ] +} diff --git a/justfile b/justfile index 7352518..1d312c5 100644 --- a/justfile +++ b/justfile @@ -22,6 +22,10 @@ config-contract-check: migration-contract-check: cargo run -p admin-api --bin crank-migrate -- plan --check +metrics-contract-check: + cargo run -p crank-metrics --bin crank-metrics-contract -- --check + python3 scripts/check-metrics-boundaries.py --root . + rust-boundaries: scripts/check-rust-boundaries.sh @@ -52,6 +56,7 @@ verify: just community-scope-check just config-contract-check just migration-contract-check + just metrics-contract-check just rust-boundaries just rust-code-health just dependencies diff --git a/scripts/README.md b/scripts/README.md index 072b073..4b73d54 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -100,6 +100,13 @@ python3 scripts/check-config-boundaries.py --root . committed machine contract. PostgreSQL DDL вне `crank-registry::migrations` блокируется Rust module boundary check. +## Typed metrics contract + +`just metrics-contract-check` сверяет Rust registry с versioned JSON snapshot и +проверяет, что production-код объявляет метрики только через `crank-metrics`. +Checker распознаёт прямые macro calls, imports, aliases и re-exports; новые +Rust-файлы можно передать явно через `--files`. + ## `check-community-scope.sh` Проверяет, что в community-репозиторий не попали функции и тексты за пределами diff --git a/scripts/check-metrics-boundaries.py b/scripts/check-metrics-boundaries.py new file mode 100644 index 0000000..5277322 --- /dev/null +++ b/scripts/check-metrics-boundaries.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + + +MAX_FILES = 20_000 +MAX_FILE_BYTES = 4 * 1024 * 1024 +ALLOWED_ADAPTERS = { + "crates/crank-observability/src/instrumentation.rs", + "crates/crank-observability/src/prometheus.rs", +} +PATTERNS = ( + re.compile(r"(? argparse.Namespace: + parser = argparse.ArgumentParser(description="Enforce crank-metrics ownership.") + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--files", nargs="*") + return parser.parse_args(argv) + + +def discover(root: Path) -> list[str]: + result = subprocess.run( + ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z", "--", "apps/**/*.rs", "crates/**/*.rs"], + cwd=root, + check=True, + stdout=subprocess.PIPE, + ) + return sorted(path.decode("utf-8") for path in result.stdout.split(b"\0") if path) + + +def resolve(root: Path, supplied: str) -> tuple[str, Path] | None: + if "\0" in supplied: + return None + logical = Path(supplied) + if logical.is_absolute() or ".." in logical.parts: + return None + path = root / logical + if path.is_symlink() or not path.is_file(): + return None + try: + canonical = path.resolve(strict=True) + relative = canonical.relative_to(root) + except (OSError, ValueError): + return None + if canonical != path.absolute(): + return None + return relative.as_posix(), canonical + + +def scan(logical: str, path: Path) -> bool: + if logical.startswith("crates/crank-metrics/"): + return False + if "/tests/" in logical or logical.startswith("tests/"): + return False + try: + if path.stat().st_size > MAX_FILE_BYTES: + return True + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + return True + for pattern in PATTERNS: + if not pattern.search(text): + continue + if logical in ALLOWED_ADAPTERS and pattern is PATTERNS[0] and "describe_" in pattern.search(text).group(0): + continue + if logical == "crates/crank-observability/src/prometheus.rs" and pattern is PATTERNS[3]: + continue + return True + return False + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + root = args.root.resolve() + try: + supplied = args.files if args.files is not None else discover(root) + except (OSError, subprocess.SubprocessError, UnicodeError): + print("error: metrics boundary discovery failed", file=sys.stderr) + return 1 + if len(supplied) > MAX_FILES: + print("error: metrics boundary input limit exceeded", file=sys.stderr) + return 1 + violations: list[str] = [] + for index, item in enumerate(sorted(set(supplied))): + resolved = resolve(root, item) + if resolved is None: + violations.append(f"input[{index}]: invalid path") + continue + logical, path = resolved + if path.suffix == ".rs" and scan(logical, path): + violations.append(f"{logical}: direct metrics declaration") + if violations: + for violation in violations[:1000]: + print(f"error: {violation}", file=sys.stderr) + if len(violations) > 1000: + print(f"error: diagnostics omitted={len(violations) - 1000}", file=sys.stderr) + return 1 + print(f"Metrics boundary check passed: files={len(supplied)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/check-rust-module-boundaries.sh b/scripts/check-rust-module-boundaries.sh index f1382ce..57d304c 100755 --- a/scripts/check-rust-module-boundaries.sh +++ b/scripts/check-rust-module-boundaries.sh @@ -22,6 +22,7 @@ echo "Rust module boundary check: checking module-level imports" python3 "$ROOT_DIR/scripts/check-config-boundaries.py" --root "$ROOT_DIR" || status=1 python3 "$ROOT_DIR/scripts/check-migration-boundaries.py" --root "$ROOT_DIR" || status=1 +python3 "$ROOT_DIR/scripts/check-metrics-boundaries.py" --root "$ROOT_DIR" || status=1 check_no_match \ "admin-api service modules must not depend on axum HTTP types" \ @@ -50,13 +51,6 @@ check_no_match \ '^\s*use\s+(axum|sqlx)(::|[;\{])' \ "$ROOT_DIR/crates/crank-runtime/src" -check_no_match \ - "product modules must record metrics only through crank-metrics" \ - '(^|[^[:alnum:]_])(::)?metrics::(counter|gauge|histogram)!' \ - "$ROOT_DIR/apps" \ - "$ROOT_DIR/crates" \ - --glob '!**/crank-metrics/**' - if (( status != 0 )); then cat >&2 <<'EOF' diff --git a/tests/unit/test_check_metrics_boundaries.py b/tests/unit/test_check_metrics_boundaries.py new file mode 100644 index 0000000..5272f2d --- /dev/null +++ b/tests/unit/test_check_metrics_boundaries.py @@ -0,0 +1,76 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / "scripts" / "check-metrics-boundaries.py" + + +class MetricsBoundaryTests(unittest.TestCase): + def run_checker(self, root: Path, *paths: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["python3", str(CHECKER), "--root", str(root), "--files", *paths], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def test_rejects_direct_alias_import_reexport_and_wrapper_bypasses(self) -> None: + cases = { + "direct.rs": 'metrics::counter!("bad").increment(1);', + "alias.rs": 'use metrics as m; m::counter!("bad").increment(1);', + "import.rs": 'use metrics::counter; counter!("bad").increment(1);', + "reexport.rs": 'pub use metrics::histogram;', + "absolute.rs": '::metrics::gauge!("bad").set(1.0);', + "wrapper.rs": 'macro_rules! bad { () => { metrics::counter!("bad") } }', + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + paths = [] + for name, content in cases.items(): + path = root / "apps" / "demo" / "src" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + paths.append(str(path.relative_to(root))) + result = self.run_checker(root, *paths) + self.assertNotEqual(result.returncode, 0) + for name in cases: + self.assertIn(name, result.stderr) + self.assertNotIn(str(root), result.stderr) + + def test_allows_typed_facade_and_narrow_observability_adapter(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + product = root / "apps" / "demo" / "src" / "main.rs" + adapter = root / "crates" / "crank-observability" / "src" / "instrumentation.rs" + product.parent.mkdir(parents=True) + adapter.parent.mkdir(parents=True) + product.write_text("crank_metrics::record_cache_outcome(value);", encoding="utf-8") + adapter.write_text("metrics::describe_counter!(definition.name, definition.description);", encoding="utf-8") + result = self.run_checker(root, str(product.relative_to(root)), str(adapter.relative_to(root))) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_explicit_missing_traversal_and_symlink_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + outside = root.parent / "metrics-boundary-outside.rs" + outside.write_text('metrics::counter!("bad");', encoding="utf-8") + try: + for path in ["missing.rs", "../metrics-boundary-outside.rs"]: + result = self.run_checker(root, path) + self.assertNotEqual(result.returncode, 0) + target = root / "target.rs" + target.write_text("", encoding="utf-8") + link = root / "link.rs" + link.symlink_to(target) + result = self.run_checker(root, "link.rs") + self.assertNotEqual(result.returncode, 0) + finally: + outside.unlink(missing_ok=True) + + +if __name__ == "__main__": + unittest.main()