feat: freeze typed metrics registry and bounded exemplars
This commit is contained in:
@@ -70,6 +70,11 @@ jobs:
|
||||
- name: Check canonical migration contract
|
||||
run: cargo run -p admin-api --bin crank-migrate -- plan --check
|
||||
|
||||
- name: Check typed metrics contract
|
||||
run: |
|
||||
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
|
||||
python3 scripts/check-metrics-boundaries.py --root .
|
||||
|
||||
- name: Check Capability Inventory
|
||||
run: |
|
||||
required_args=""
|
||||
|
||||
@@ -66,6 +66,11 @@ jobs:
|
||||
- name: Check canonical migration contract
|
||||
run: cargo run -p admin-api --bin crank-migrate -- plan --check
|
||||
|
||||
- name: Check typed metrics contract
|
||||
run: |
|
||||
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
|
||||
python3 scripts/check-metrics-boundaries.py --root .
|
||||
|
||||
- name: Check Capability Inventory
|
||||
run: |
|
||||
required_args=""
|
||||
|
||||
@@ -88,7 +88,10 @@ impl RestAdapter {
|
||||
request: &RestRequest,
|
||||
context: &RuntimeRequestContext,
|
||||
) -> Result<RestResponse, RestAdapterError> {
|
||||
let request_metrics = UpstreamRequestMetrics::start(UpstreamOperationKind::Rest);
|
||||
let request_metrics = UpstreamRequestMetrics::start_with_exemplar(
|
||||
UpstreamOperationKind::Rest,
|
||||
crank_metrics::ExemplarTraceId::parse(&context.trace_context.trace_id().to_string()),
|
||||
);
|
||||
let result = self.execute_inner(target, request, Some(context)).await;
|
||||
let outcome = match &result {
|
||||
Ok(_) => UpstreamOutcome::Success,
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use std::sync::Arc;
|
||||
use std::{sync::Arc, time::Instant};
|
||||
|
||||
use axum::{http::header::CONTENT_TYPE, response::Response};
|
||||
use crank_metrics::{
|
||||
InFlightGuard, LimitStage, McpMethod, McpOutcome, McpResponseMode, record_limit_rejection,
|
||||
record_mcp_request,
|
||||
ExemplarTraceId, InFlightGuard, LimitStage, McpMethod, McpOutcome, McpResponseMode,
|
||||
record_limit_rejection, record_mcp_request_with_exemplar,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
@@ -14,14 +14,18 @@ pub(super) struct McpRequestMetrics {
|
||||
method: McpMethod,
|
||||
response_mode: McpResponseMode,
|
||||
outcome: McpOutcome,
|
||||
started_at: Instant,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
}
|
||||
|
||||
impl McpRequestMetrics {
|
||||
pub(super) const fn invalid() -> Self {
|
||||
pub(super) fn invalid() -> Self {
|
||||
Self {
|
||||
method: McpMethod::Invalid,
|
||||
response_mode: McpResponseMode::Unknown,
|
||||
outcome: McpOutcome::Aborted,
|
||||
started_at: Instant::now(),
|
||||
exemplar: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,10 +34,17 @@ impl McpRequestMetrics {
|
||||
method: normalized_mcp_method(message),
|
||||
response_mode: McpResponseMode::Unknown,
|
||||
outcome: McpOutcome::Aborted,
|
||||
started_at: Instant::now(),
|
||||
exemplar: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn complete(&mut self, response: &Response) {
|
||||
self.exemplar = response
|
||||
.headers()
|
||||
.get("x-trace-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(ExemplarTraceId::parse);
|
||||
self.response_mode = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
@@ -71,7 +82,13 @@ impl McpRequestMetrics {
|
||||
|
||||
impl Drop for McpRequestMetrics {
|
||||
fn drop(&mut self) {
|
||||
record_mcp_request(self.method, self.response_mode, self.outcome);
|
||||
record_mcp_request_with_exemplar(
|
||||
self.method,
|
||||
self.response_mode,
|
||||
self.outcome,
|
||||
self.started_at.elapsed(),
|
||||
self.exemplar,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -79,8 +79,12 @@ impl ActiveSessionMetrics {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(count)) => crank_metrics::set_mcp_active_sessions(count),
|
||||
Ok(Ok(count)) => {
|
||||
crank_metrics::set_mcp_active_sessions(count);
|
||||
crank_metrics::set_mcp_session_metrics_fresh(true);
|
||||
}
|
||||
Ok(Err(_)) | Err(_) => {
|
||||
crank_metrics::set_mcp_session_metrics_fresh(false);
|
||||
warn!(
|
||||
name: "mcp.active_session_metrics.refresh_failed",
|
||||
error_category = "session_store",
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
use std::{fs, process::ExitCode};
|
||||
|
||||
const SNAPSHOT: &str = "docs/schemas/metrics-registry-v1.json";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let rendered = match crank_metrics::render_metric_schema_json() {
|
||||
Ok(rendered) => rendered,
|
||||
Err(_) => {
|
||||
eprintln!("metrics_contract_invalid");
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
};
|
||||
let args = std::env::args().skip(1).collect::<Vec<_>>();
|
||||
if args.is_empty() {
|
||||
print!("{rendered}");
|
||||
return ExitCode::SUCCESS;
|
||||
}
|
||||
if args.as_slice() != ["--check"] {
|
||||
eprintln!("metrics_contract_invalid_command");
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
match fs::read_to_string(SNAPSHOT) {
|
||||
Ok(snapshot) if snapshot == rendered => {
|
||||
println!("metrics_contract_ok schema_version=1");
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
Ok(_) => {
|
||||
eprintln!("metrics_contract_drift");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("metrics_contract_missing");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Mutex, OnceLock},
|
||||
};
|
||||
|
||||
use crate::{DURATION_BUCKETS_SECONDS, max_exemplar_slots};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
pub struct ExemplarTraceId([u8; 32]);
|
||||
|
||||
impl ExemplarTraceId {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
if value.len() != 32
|
||||
|| value
|
||||
.bytes()
|
||||
.any(|byte| !byte.is_ascii_digit() && !(b'a'..=b'f').contains(&byte))
|
||||
|| value.bytes().all(|byte| byte == b'0')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut bytes = [0; 32];
|
||||
bytes.copy_from_slice(value.as_bytes());
|
||||
Some(Self(bytes))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
std::str::from_utf8(&self.0).expect("validated ASCII trace id")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ExemplarObservation {
|
||||
pub metric: &'static str,
|
||||
pub labels: Vec<(&'static str, &'static str)>,
|
||||
pub bucket_upper_bound: Option<f64>,
|
||||
pub value: f64,
|
||||
pub trace_id: ExemplarTraceId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
struct ExemplarKey {
|
||||
metric: &'static str,
|
||||
labels: Vec<(&'static str, &'static str)>,
|
||||
bucket_index: usize,
|
||||
}
|
||||
|
||||
fn store() -> &'static Mutex<BTreeMap<ExemplarKey, ExemplarObservation>> {
|
||||
static STORE: OnceLock<Mutex<BTreeMap<ExemplarKey, ExemplarObservation>>> = OnceLock::new();
|
||||
STORE.get_or_init(|| Mutex::new(BTreeMap::new()))
|
||||
}
|
||||
|
||||
pub(crate) fn record_exemplar(
|
||||
metric: &'static str,
|
||||
labels: Vec<(&'static str, &'static str)>,
|
||||
value: f64,
|
||||
trace_id: Option<ExemplarTraceId>,
|
||||
) {
|
||||
let Some(trace_id) = trace_id else { return };
|
||||
let bucket_index = DURATION_BUCKETS_SECONDS
|
||||
.iter()
|
||||
.position(|bound| value <= *bound)
|
||||
.unwrap_or(DURATION_BUCKETS_SECONDS.len());
|
||||
let observation = ExemplarObservation {
|
||||
metric,
|
||||
labels: labels.clone(),
|
||||
bucket_upper_bound: DURATION_BUCKETS_SECONDS.get(bucket_index).copied(),
|
||||
value,
|
||||
trace_id,
|
||||
};
|
||||
let Ok(mut observations) = store().lock() else {
|
||||
return;
|
||||
};
|
||||
let key = ExemplarKey {
|
||||
metric,
|
||||
labels,
|
||||
bucket_index,
|
||||
};
|
||||
if observations.contains_key(&key) || observations.len() < max_exemplar_slots() {
|
||||
observations.insert(key, observation);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn exemplar_snapshot() -> Vec<ExemplarObservation> {
|
||||
store()
|
||||
.lock()
|
||||
.map(|observations| observations.values().cloned().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn reset_exemplars_for_test() {
|
||||
if let Ok(mut observations) = store().lock() {
|
||||
observations.clear();
|
||||
}
|
||||
}
|
||||
@@ -4,10 +4,14 @@
|
||||
//! ключи и значения labels сосредоточены здесь, поэтому пользовательские
|
||||
//! идентификаторы и тексты нельзя случайно превратить во временные ряды.
|
||||
|
||||
mod exemplar;
|
||||
mod labels;
|
||||
mod record;
|
||||
mod schema;
|
||||
|
||||
pub use exemplar::{
|
||||
ExemplarObservation, ExemplarTraceId, exemplar_snapshot, reset_exemplars_for_test,
|
||||
};
|
||||
pub use labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
@@ -17,10 +21,17 @@ pub use labels::{
|
||||
pub use record::{
|
||||
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, initialize_gauges,
|
||||
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
|
||||
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
|
||||
set_db_pool_connections, set_mcp_active_sessions,
|
||||
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
|
||||
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
|
||||
record_tool_invocation, record_tool_invocation_with_exemplar, record_upstream_request,
|
||||
record_upstream_request_with_exemplar, set_catalog, set_db_pool_connections,
|
||||
set_mcp_active_sessions, set_mcp_session_metrics_fresh,
|
||||
};
|
||||
pub use schema::{
|
||||
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
|
||||
DURATION_BUCKETS_SECONDS, HTTP_ROUTE_DOMAIN, LabelClass, LabelDomain, MAX_EXPOSITION_BYTES,
|
||||
MAX_LOGICAL_SERIES_PER_PROCESS, MAX_METRIC_FAMILIES, MAX_PRODUCT_LABELS_PER_FAMILY,
|
||||
MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, MetricDefinition, MetricKind,
|
||||
MetricProcess, MetricService, MetricUnit, PROCESS_CONSTANT_LABELS, SchemaBudget, SchemaError,
|
||||
max_exemplar_slots, metric_schema, render_metric_schema_json, schema_budget, validate_budget,
|
||||
validate_schema,
|
||||
};
|
||||
|
||||
@@ -3,9 +3,9 @@ use std::time::{Duration, Instant};
|
||||
use metrics::Gauge;
|
||||
|
||||
use crate::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
|
||||
UpstreamOutcome,
|
||||
};
|
||||
|
||||
@@ -14,6 +14,16 @@ pub fn record_http_request(
|
||||
method: HttpMethod,
|
||||
status: HttpStatusClass,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_http_request_with_exemplar(route, method, status, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_http_request_with_exemplar(
|
||||
route: HttpRoute,
|
||||
method: HttpMethod,
|
||||
status: HttpStatusClass,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
metrics::counter!(
|
||||
"crank_http_requests_total",
|
||||
@@ -28,9 +38,30 @@ pub fn record_http_request(
|
||||
"method" => method.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_http_request_duration_seconds",
|
||||
vec![("route", route.as_str()), ("method", method.as_str())],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, outcome: McpOutcome) {
|
||||
pub fn record_mcp_request(
|
||||
method: McpMethod,
|
||||
response_mode: McpResponseMode,
|
||||
outcome: McpOutcome,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_mcp_request_with_exemplar(method, response_mode, outcome, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_mcp_request_with_exemplar(
|
||||
method: McpMethod,
|
||||
response_mode: McpResponseMode,
|
||||
outcome: McpOutcome,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
metrics::counter!(
|
||||
"crank_mcp_requests_total",
|
||||
"method" => method.as_str(),
|
||||
@@ -38,18 +69,54 @@ pub fn record_mcp_request(method: McpMethod, response_mode: McpResponseMode, out
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
metrics::histogram!(
|
||||
"crank_mcp_request_duration_seconds",
|
||||
"method" => method.as_str(),
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_mcp_request_duration_seconds",
|
||||
vec![("method", method.as_str()), ("outcome", outcome.as_str())],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn set_mcp_active_sessions(count: u64) {
|
||||
metrics::gauge!("crank_mcp_active_sessions").set(count as f64);
|
||||
}
|
||||
|
||||
pub fn set_mcp_session_metrics_fresh(fresh: bool) {
|
||||
metrics::gauge!("crank_mcp_session_metrics_fresh").set(if fresh { 1.0 } else { 0.0 });
|
||||
}
|
||||
|
||||
pub fn record_tool_invocation(
|
||||
source: InvocationSource,
|
||||
outcome: ToolOutcome,
|
||||
error_kind: ToolErrorKind,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_tool_invocation_with_exemplar(source, outcome, error_kind, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_tool_invocation_with_exemplar(
|
||||
source: InvocationSource,
|
||||
outcome: ToolOutcome,
|
||||
error_kind: ToolErrorKind,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
let error_kind = match outcome {
|
||||
ToolOutcome::Success => ToolErrorKind::None,
|
||||
ToolOutcome::Aborted => ToolErrorKind::Aborted,
|
||||
ToolOutcome::Error
|
||||
if matches!(error_kind, ToolErrorKind::None | ToolErrorKind::Aborted) =>
|
||||
{
|
||||
ToolErrorKind::ProtocolAdapter
|
||||
}
|
||||
ToolOutcome::Error => error_kind,
|
||||
};
|
||||
metrics::counter!(
|
||||
"crank_tool_invocations_total",
|
||||
"source" => source.as_str(),
|
||||
@@ -63,12 +130,27 @@ pub fn record_tool_invocation(
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_tool_invocation_duration_seconds",
|
||||
vec![("source", source.as_str()), ("outcome", outcome.as_str())],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_upstream_request(
|
||||
operation_kind: UpstreamOperationKind,
|
||||
outcome: UpstreamOutcome,
|
||||
duration: Duration,
|
||||
) {
|
||||
record_upstream_request_with_exemplar(operation_kind, outcome, duration, None);
|
||||
}
|
||||
|
||||
pub fn record_upstream_request_with_exemplar(
|
||||
operation_kind: UpstreamOperationKind,
|
||||
outcome: UpstreamOutcome,
|
||||
duration: Duration,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) {
|
||||
metrics::counter!(
|
||||
"crank_upstream_requests_total",
|
||||
@@ -82,11 +164,21 @@ pub fn record_upstream_request(
|
||||
"outcome" => outcome.as_str()
|
||||
)
|
||||
.record(duration.as_secs_f64());
|
||||
crate::exemplar::record_exemplar(
|
||||
"crank_upstream_request_duration_seconds",
|
||||
vec![
|
||||
("operation_kind", operation_kind.as_str()),
|
||||
("outcome", outcome.as_str()),
|
||||
],
|
||||
duration.as_secs_f64(),
|
||||
exemplar,
|
||||
);
|
||||
}
|
||||
|
||||
pub struct ToolInvocationMetrics {
|
||||
source: InvocationSource,
|
||||
started_at: Option<Instant>,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
}
|
||||
|
||||
impl ToolInvocationMetrics {
|
||||
@@ -94,6 +186,18 @@ impl ToolInvocationMetrics {
|
||||
Self {
|
||||
source,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_with_exemplar(
|
||||
source: InvocationSource,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) -> Self {
|
||||
Self {
|
||||
source,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +207,13 @@ impl ToolInvocationMetrics {
|
||||
|
||||
fn finish(&mut self, outcome: ToolOutcome, error_kind: ToolErrorKind) {
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
record_tool_invocation(self.source, outcome, error_kind, started_at.elapsed());
|
||||
record_tool_invocation_with_exemplar(
|
||||
self.source,
|
||||
outcome,
|
||||
error_kind,
|
||||
started_at.elapsed(),
|
||||
self.exemplar,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,6 +227,7 @@ impl Drop for ToolInvocationMetrics {
|
||||
pub struct UpstreamRequestMetrics {
|
||||
operation_kind: UpstreamOperationKind,
|
||||
started_at: Option<Instant>,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
}
|
||||
|
||||
impl UpstreamRequestMetrics {
|
||||
@@ -124,6 +235,18 @@ impl UpstreamRequestMetrics {
|
||||
Self {
|
||||
operation_kind,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_with_exemplar(
|
||||
operation_kind: UpstreamOperationKind,
|
||||
exemplar: Option<ExemplarTraceId>,
|
||||
) -> Self {
|
||||
Self {
|
||||
operation_kind,
|
||||
started_at: Some(Instant::now()),
|
||||
exemplar,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +256,12 @@ impl UpstreamRequestMetrics {
|
||||
|
||||
fn finish(&mut self, outcome: UpstreamOutcome) {
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
record_upstream_request(self.operation_kind, outcome, started_at.elapsed());
|
||||
record_upstream_request_with_exemplar(
|
||||
self.operation_kind,
|
||||
outcome,
|
||||
started_at.elapsed(),
|
||||
self.exemplar,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -206,6 +334,7 @@ pub fn record_export_failure(signal: SignalType, exporter: Exporter) {
|
||||
pub fn initialize_gauges() {
|
||||
metrics::gauge!("crank_http_inflight").set(0.0);
|
||||
metrics::gauge!("crank_mcp_active_sessions").set(0.0);
|
||||
set_mcp_session_metrics_fresh(false);
|
||||
metrics::gauge!("crank_mcp_active_streams").set(0.0);
|
||||
metrics::gauge!("crank_runtime_inflight").set(0.0);
|
||||
set_db_pool_connections(DbPoolState::Idle, 0);
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
pub const METRIC_SCHEMA_VERSION: u32 = 1;
|
||||
pub const MAX_METRIC_FAMILIES: usize = 128;
|
||||
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 5_000;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 15_000;
|
||||
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricKind {
|
||||
Counter,
|
||||
@@ -5,129 +14,619 @@ pub enum MetricKind {
|
||||
Histogram,
|
||||
}
|
||||
|
||||
impl MetricKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Counter => "counter",
|
||||
Self::Gauge => "gauge",
|
||||
Self::Histogram => "histogram",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricUnit {
|
||||
Count,
|
||||
Seconds,
|
||||
}
|
||||
|
||||
impl MetricUnit {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Count => "count",
|
||||
Self::Seconds => "seconds",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricProcess {
|
||||
AdminApi,
|
||||
McpServer,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MetricService {
|
||||
AdminApi,
|
||||
McpServer,
|
||||
}
|
||||
|
||||
impl MetricService {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"admin-api" => Some(Self::AdminApi),
|
||||
"mcp-server" => Some(Self::McpServer),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MetricProcess {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::AdminApi => "admin_api",
|
||||
Self::McpServer => "mcp_server",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum LabelClass {
|
||||
ProductClosed,
|
||||
ProcessConstant,
|
||||
}
|
||||
|
||||
impl LabelClass {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ProductClosed => "product_closed",
|
||||
Self::ProcessConstant => "process_constant",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct LabelDomain {
|
||||
pub name: &'static str,
|
||||
pub class: LabelClass,
|
||||
pub values: &'static [&'static str],
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct MetricDefinition {
|
||||
pub name: &'static str,
|
||||
pub kind: MetricKind,
|
||||
pub unit: MetricUnit,
|
||||
pub labels: &'static [&'static str],
|
||||
pub label_domains: &'static [LabelDomain],
|
||||
pub buckets: &'static [f64],
|
||||
pub processes: &'static [MetricProcess],
|
||||
pub max_logical_series: usize,
|
||||
pub max_rendered_series: usize,
|
||||
pub description: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct SchemaBudget {
|
||||
pub logical_series: usize,
|
||||
pub rendered_series: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum SchemaError {
|
||||
TooManyFamilies,
|
||||
DuplicateMetric,
|
||||
InvalidMetric,
|
||||
DuplicateLabel,
|
||||
InvalidDomain,
|
||||
InvalidBuckets,
|
||||
InvalidCeiling,
|
||||
BudgetExceeded,
|
||||
}
|
||||
|
||||
pub const DURATION_BUCKETS_SECONDS: &[f64] = &[
|
||||
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
|
||||
];
|
||||
|
||||
pub const PROCESS_CONSTANT_LABELS: &[LabelDomain] = &[
|
||||
LabelDomain {
|
||||
name: "service",
|
||||
class: LabelClass::ProcessConstant,
|
||||
values: &["admin-api", "mcp-server"],
|
||||
},
|
||||
LabelDomain {
|
||||
name: "version",
|
||||
class: LabelClass::ProcessConstant,
|
||||
values: &["validated_release_identity"],
|
||||
},
|
||||
LabelDomain {
|
||||
name: "environment",
|
||||
class: LabelClass::ProcessConstant,
|
||||
values: &["validated_startup_value"],
|
||||
},
|
||||
];
|
||||
|
||||
const BOTH: &[MetricProcess] = &[MetricProcess::AdminApi, MetricProcess::McpServer];
|
||||
const MCP: &[MetricProcess] = &[MetricProcess::McpServer];
|
||||
|
||||
pub const HTTP_ROUTE_DOMAIN: &[&str] = &[
|
||||
"unmatched",
|
||||
"/health",
|
||||
"/ready",
|
||||
"/api/auth/login",
|
||||
"/api/auth/logout",
|
||||
"/api/auth/session",
|
||||
"/api/auth/profile",
|
||||
"/api/auth/password",
|
||||
"/api/admin/capabilities",
|
||||
"/api/admin/workspaces",
|
||||
"/api/admin/workspaces/{workspace_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/operations",
|
||||
"/api/admin/workspaces/{workspace_id}/imports/openapi/preview",
|
||||
"/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/analyze-quality",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/import",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate",
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export",
|
||||
"/api/admin/workspaces/{workspace_id}/agents",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/tool-search/preview",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke",
|
||||
"/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/auth-profiles",
|
||||
"/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/upstreams",
|
||||
"/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/secrets",
|
||||
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate",
|
||||
"/api/admin/workspaces/{workspace_id}/export",
|
||||
"/api/admin/workspaces/{workspace_id}/logs",
|
||||
"/api/admin/workspaces/{workspace_id}/logs/{log_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/approvals",
|
||||
"/api/admin/workspaces/{workspace_id}/approvals/{approval_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/usage",
|
||||
"/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}",
|
||||
"/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}",
|
||||
"/v1/{workspace_slug}/{agent_slug}",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}",
|
||||
"/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny",
|
||||
];
|
||||
const METHODS: &[&str] = &[
|
||||
"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER",
|
||||
];
|
||||
const STATUS: &[&str] = &["1xx", "2xx", "3xx", "4xx", "5xx", "other"];
|
||||
const MCP_METHODS: &[&str] = &[
|
||||
"initialize",
|
||||
"initialized",
|
||||
"ping",
|
||||
"tools_list",
|
||||
"tools_call",
|
||||
"notification",
|
||||
"unsupported",
|
||||
"response",
|
||||
"invalid",
|
||||
];
|
||||
const RESPONSE_MODES: &[&str] = &["json", "sse", "unknown"];
|
||||
const MCP_OUTCOMES: &[&str] = &[
|
||||
"success",
|
||||
"client_error",
|
||||
"server_error",
|
||||
"jsonrpc_error",
|
||||
"tool_error",
|
||||
"aborted",
|
||||
"other",
|
||||
];
|
||||
const SOURCES: &[&str] = &["internal", "admin_test_run", "agent_tool_call"];
|
||||
const TOOL_OUTCOMES: &[&str] = &["success", "error", "aborted"];
|
||||
const TOOL_ERRORS: &[&str] = &[
|
||||
"none",
|
||||
"schema",
|
||||
"mapping",
|
||||
"rest_adapter",
|
||||
"protocol_adapter",
|
||||
"unsupported_protocol",
|
||||
"unsupported_execution_mode",
|
||||
"concurrency_limit",
|
||||
"invalid_prepared_request",
|
||||
"confirmation_required",
|
||||
"invalid_confirmation_token",
|
||||
"confirmation_store",
|
||||
"idempotency_store",
|
||||
"idempotency_in_progress",
|
||||
"idempotency_conflict",
|
||||
"idempotency_outcome_unknown",
|
||||
"missing_auth_profile",
|
||||
"missing_secret",
|
||||
"missing_secret_version",
|
||||
"invalid_auth_secret",
|
||||
"secret_crypto",
|
||||
"aborted",
|
||||
];
|
||||
const OPERATION_KINDS: &[&str] = &["rest"];
|
||||
const UPSTREAM_OUTCOMES: &[&str] = &[
|
||||
"success",
|
||||
"client_error",
|
||||
"server_error",
|
||||
"unexpected_status",
|
||||
"timeout",
|
||||
"transport_error",
|
||||
"response_too_large",
|
||||
"rejected",
|
||||
"window_expired",
|
||||
"invalid_response",
|
||||
"invalid_request",
|
||||
"configuration",
|
||||
"aborted",
|
||||
];
|
||||
const LIMIT_STAGES: &[&str] = &["concurrency", "rate_limit", "mcp_stream"];
|
||||
const CACHE_OUTCOMES: &[&str] = &[
|
||||
"hit",
|
||||
"miss",
|
||||
"read_error",
|
||||
"decode_error",
|
||||
"evict_error",
|
||||
"stored",
|
||||
"write_error",
|
||||
];
|
||||
const IDEMPOTENCY_OUTCOMES: &[&str] = &[
|
||||
"execute",
|
||||
"replay",
|
||||
"completed",
|
||||
"conflict",
|
||||
"in_progress",
|
||||
"outcome_unknown",
|
||||
"store_unavailable",
|
||||
"error",
|
||||
];
|
||||
const CONFIRMATION_OUTCOMES: &[&str] = &[
|
||||
"approved",
|
||||
"required",
|
||||
"invalid_token",
|
||||
"store_unavailable",
|
||||
"error",
|
||||
];
|
||||
const DB_STATES: &[&str] = &["idle", "used"];
|
||||
const SIGNAL_TYPES: &[&str] = &["trace", "invocation_history"];
|
||||
const EXPORTERS: &[&str] = &["otlp", "postgres"];
|
||||
|
||||
const fn domain(name: &'static str, values: &'static [&'static str]) -> LabelDomain {
|
||||
LabelDomain {
|
||||
name,
|
||||
class: LabelClass::ProductClosed,
|
||||
values,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(
|
||||
clippy::too_many_arguments,
|
||||
reason = "the canonical descriptor keeps every metric contract dimension explicit"
|
||||
)]
|
||||
const fn metric(
|
||||
name: &'static str,
|
||||
kind: MetricKind,
|
||||
unit: MetricUnit,
|
||||
labels: &'static [&'static str],
|
||||
label_domains: &'static [LabelDomain],
|
||||
buckets: &'static [f64],
|
||||
processes: &'static [MetricProcess],
|
||||
logical: usize,
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
let rendered = if matches!(kind, MetricKind::Histogram) {
|
||||
logical * (buckets.len() + 3)
|
||||
} else {
|
||||
logical
|
||||
};
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind,
|
||||
unit,
|
||||
labels,
|
||||
label_domains,
|
||||
buckets,
|
||||
processes,
|
||||
max_logical_series: logical,
|
||||
max_rendered_series: rendered,
|
||||
description,
|
||||
}
|
||||
}
|
||||
|
||||
const METRIC_SCHEMA: &[MetricDefinition] = &[
|
||||
counter(
|
||||
metric(
|
||||
"crank_http_requests_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["route", "method", "status_class"],
|
||||
&[
|
||||
domain("route", HTTP_ROUTE_DOMAIN),
|
||||
domain("method", METHODS),
|
||||
domain("status_class", STATUS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
57 * 10 * 6,
|
||||
"Total HTTP requests.",
|
||||
),
|
||||
histogram(
|
||||
metric(
|
||||
"crank_http_request_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["route", "method"],
|
||||
&[
|
||||
domain("route", HTTP_ROUTE_DOMAIN),
|
||||
domain("method", METHODS),
|
||||
],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
BOTH,
|
||||
57 * 10,
|
||||
"HTTP request duration in seconds.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_http_inflight",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"HTTP requests currently being processed.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_mcp_requests_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["method", "response_mode", "outcome"],
|
||||
&[
|
||||
domain("method", MCP_METHODS),
|
||||
domain("response_mode", RESPONSE_MODES),
|
||||
domain("outcome", MCP_OUTCOMES),
|
||||
],
|
||||
&[],
|
||||
MCP,
|
||||
9 * 3 * 7,
|
||||
"Total MCP JSON-RPC requests.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_mcp_request_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["method", "outcome"],
|
||||
&[
|
||||
domain("method", MCP_METHODS),
|
||||
domain("outcome", MCP_OUTCOMES),
|
||||
],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
MCP,
|
||||
9 * 7,
|
||||
"MCP JSON-RPC request duration in seconds.",
|
||||
),
|
||||
metric(
|
||||
"crank_mcp_active_sessions",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Active persisted MCP transport sessions.",
|
||||
),
|
||||
gauge(
|
||||
"crank_mcp_active_streams",
|
||||
metric(
|
||||
"crank_mcp_session_metrics_fresh",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Whether the active-session gauge was refreshed successfully.",
|
||||
),
|
||||
metric(
|
||||
"crank_mcp_active_streams",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"MCP event streams currently being served.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_tool_invocations_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["source", "outcome", "error_kind"],
|
||||
&[
|
||||
domain("source", SOURCES),
|
||||
domain("outcome", TOOL_OUTCOMES),
|
||||
domain("error_kind", TOOL_ERRORS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
3 * 3 * 22,
|
||||
"Total tool invocations.",
|
||||
),
|
||||
histogram(
|
||||
metric(
|
||||
"crank_tool_invocation_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["source", "outcome"],
|
||||
&[domain("source", SOURCES), domain("outcome", TOOL_OUTCOMES)],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
BOTH,
|
||||
3 * 3,
|
||||
"Tool invocation duration in seconds.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_upstream_requests_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["operation_kind", "outcome"],
|
||||
&[
|
||||
domain("operation_kind", OPERATION_KINDS),
|
||||
domain("outcome", UPSTREAM_OUTCOMES),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
13,
|
||||
"Total upstream requests.",
|
||||
),
|
||||
histogram(
|
||||
metric(
|
||||
"crank_upstream_request_duration_seconds",
|
||||
MetricKind::Histogram,
|
||||
MetricUnit::Seconds,
|
||||
&["operation_kind", "outcome"],
|
||||
&[
|
||||
domain("operation_kind", OPERATION_KINDS),
|
||||
domain("outcome", UPSTREAM_OUTCOMES),
|
||||
],
|
||||
DURATION_BUCKETS_SECONDS,
|
||||
BOTH,
|
||||
13,
|
||||
"Upstream request duration in seconds.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_runtime_inflight",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"Runtime executions currently in progress.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["stage"],
|
||||
&[domain("stage", LIMIT_STAGES)],
|
||||
&[],
|
||||
BOTH,
|
||||
3,
|
||||
"Runtime executions rejected by a bounded limit.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_runtime_cache_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["outcome"],
|
||||
&[domain("outcome", CACHE_OUTCOMES)],
|
||||
&[],
|
||||
BOTH,
|
||||
7,
|
||||
"Runtime response cache outcomes.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_idempotency_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["outcome"],
|
||||
&[domain("outcome", IDEMPOTENCY_OUTCOMES)],
|
||||
&[],
|
||||
BOTH,
|
||||
8,
|
||||
"Runtime idempotency outcomes.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_confirmation_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["outcome"],
|
||||
&[domain("outcome", CONFIRMATION_OUTCOMES)],
|
||||
&[],
|
||||
BOTH,
|
||||
5,
|
||||
"Runtime confirmation outcomes.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_db_pool_connections",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&["state"],
|
||||
&[domain("state", DB_STATES)],
|
||||
&[],
|
||||
BOTH,
|
||||
2,
|
||||
"PostgreSQL pool connections by state.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_catalog_tools",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Tools in the current published catalog.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_catalog_estimated_context_tokens",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Estimated context tokens in the current published catalog.",
|
||||
),
|
||||
gauge(
|
||||
metric(
|
||||
"crank_catalog_warnings",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
MCP,
|
||||
1,
|
||||
"Warnings in the current published catalog.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_invocation_history_lost_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"Invocation history records lost after an action completed.",
|
||||
),
|
||||
counter(
|
||||
metric(
|
||||
"crank_telemetry_export_failures_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["signal_type", "exporter"],
|
||||
&[
|
||||
domain("signal_type", SIGNAL_TYPES),
|
||||
domain("exporter", EXPORTERS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
4,
|
||||
"Telemetry export failures.",
|
||||
),
|
||||
];
|
||||
@@ -136,44 +635,176 @@ pub const fn metric_schema() -> &'static [MetricDefinition] {
|
||||
METRIC_SCHEMA
|
||||
}
|
||||
|
||||
const fn counter(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Counter,
|
||||
unit: MetricUnit::Count,
|
||||
labels,
|
||||
description,
|
||||
pub fn schema_budget() -> Result<SchemaBudget, SchemaError> {
|
||||
validate_schema(METRIC_SCHEMA)
|
||||
}
|
||||
|
||||
pub fn validate_budget(logical: usize, rendered: usize) -> Result<(), SchemaError> {
|
||||
if logical > MAX_LOGICAL_SERIES_PER_PROCESS || rendered > MAX_RENDERED_SERIES_PER_PROCESS {
|
||||
Err(SchemaError::BudgetExceeded)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn gauge(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Gauge,
|
||||
unit: MetricUnit::Count,
|
||||
labels,
|
||||
description,
|
||||
}
|
||||
pub fn max_exemplar_slots() -> usize {
|
||||
METRIC_SCHEMA
|
||||
.iter()
|
||||
.filter(|definition| definition.kind == MetricKind::Histogram)
|
||||
.map(|definition| definition.max_logical_series * (definition.buckets.len() + 1))
|
||||
.sum()
|
||||
}
|
||||
|
||||
const fn histogram(
|
||||
name: &'static str,
|
||||
labels: &'static [&'static str],
|
||||
description: &'static str,
|
||||
) -> MetricDefinition {
|
||||
MetricDefinition {
|
||||
name,
|
||||
kind: MetricKind::Histogram,
|
||||
unit: MetricUnit::Seconds,
|
||||
labels,
|
||||
description,
|
||||
pub fn validate_schema(schema: &[MetricDefinition]) -> Result<SchemaBudget, SchemaError> {
|
||||
if schema.len() > MAX_METRIC_FAMILIES {
|
||||
return Err(SchemaError::TooManyFamilies);
|
||||
}
|
||||
let mut names = BTreeSet::new();
|
||||
let mut logical = 0usize;
|
||||
let mut rendered = 0usize;
|
||||
for item in schema {
|
||||
if !names.insert(item.name) {
|
||||
return Err(SchemaError::DuplicateMetric);
|
||||
}
|
||||
if !valid_metric_name(item.name)
|
||||
|| !valid_static_text(item.description)
|
||||
|| item.processes.is_empty()
|
||||
|| item.labels.len() > MAX_PRODUCT_LABELS_PER_FAMILY
|
||||
|| item.labels.len() != item.label_domains.len()
|
||||
{
|
||||
return Err(SchemaError::InvalidMetric);
|
||||
}
|
||||
let mut labels = BTreeSet::new();
|
||||
let mut computed = 1usize;
|
||||
for (label, domain) in item.labels.iter().zip(item.label_domains) {
|
||||
if *label != domain.name || !valid_label_name(label) || !labels.insert(*label) {
|
||||
return Err(SchemaError::DuplicateLabel);
|
||||
}
|
||||
if domain.class != LabelClass::ProductClosed
|
||||
|| domain.values.is_empty()
|
||||
|| domain.values.iter().any(|value| !valid_domain_value(value))
|
||||
|| domain.values.iter().copied().collect::<BTreeSet<_>>().len()
|
||||
!= domain.values.len()
|
||||
{
|
||||
return Err(SchemaError::InvalidDomain);
|
||||
}
|
||||
computed = computed
|
||||
.checked_mul(domain.values.len())
|
||||
.ok_or(SchemaError::BudgetExceeded)?;
|
||||
}
|
||||
if item.kind == MetricKind::Histogram {
|
||||
if item.buckets.is_empty()
|
||||
|| item.buckets.iter().any(|v| !v.is_finite() || *v <= 0.0)
|
||||
|| item.buckets.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
{
|
||||
return Err(SchemaError::InvalidBuckets);
|
||||
}
|
||||
} else if !item.buckets.is_empty() {
|
||||
return Err(SchemaError::InvalidBuckets);
|
||||
}
|
||||
let expected_rendered = if item.kind == MetricKind::Histogram {
|
||||
computed
|
||||
.checked_mul(item.buckets.len() + 3)
|
||||
.ok_or(SchemaError::BudgetExceeded)?
|
||||
} else {
|
||||
computed
|
||||
};
|
||||
if computed != item.max_logical_series || expected_rendered != item.max_rendered_series {
|
||||
return Err(SchemaError::InvalidCeiling);
|
||||
}
|
||||
logical = logical
|
||||
.checked_add(computed)
|
||||
.ok_or(SchemaError::BudgetExceeded)?;
|
||||
rendered = rendered
|
||||
.checked_add(expected_rendered)
|
||||
.ok_or(SchemaError::BudgetExceeded)?;
|
||||
}
|
||||
validate_budget(logical, rendered)?;
|
||||
Ok(SchemaBudget {
|
||||
logical_series: logical,
|
||||
rendered_series: rendered,
|
||||
})
|
||||
}
|
||||
|
||||
fn valid_static_text(value: &str) -> bool {
|
||||
!value.trim().is_empty()
|
||||
&& value.len() <= 512
|
||||
&& !value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\'))
|
||||
}
|
||||
|
||||
fn valid_metric_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
|
||||
&& name.starts_with("crank_")
|
||||
}
|
||||
|
||||
fn valid_label_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= 64
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
|
||||
}
|
||||
|
||||
fn valid_domain_value(value: &str) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 256
|
||||
&& !value
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_control() || matches!(byte, b'"' | b'\\'))
|
||||
}
|
||||
|
||||
pub fn render_metric_schema_json() -> Result<String, SchemaError> {
|
||||
let budget = schema_budget()?;
|
||||
let mut out = format!(
|
||||
"{{\n \"schema_version\": {METRIC_SCHEMA_VERSION},\n \"budget\": {{\"logical_series\": {}, \"rendered_series\": {}, \"max_logical_series\": {MAX_LOGICAL_SERIES_PER_PROCESS}, \"max_rendered_series\": {MAX_RENDERED_SERIES_PER_PROCESS}}},\n \"global_labels\": [\n",
|
||||
budget.logical_series, budget.rendered_series
|
||||
);
|
||||
for (index, label) in PROCESS_CONSTANT_LABELS.iter().enumerate() {
|
||||
out.push_str(&format!(
|
||||
" {{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}\n",
|
||||
label.name,
|
||||
label.class.as_str(),
|
||||
json_strings(label.values),
|
||||
comma(index, PROCESS_CONSTANT_LABELS.len())
|
||||
));
|
||||
}
|
||||
out.push_str(" ],\n \"metrics\": [\n");
|
||||
for (index, item) in METRIC_SCHEMA.iter().enumerate() {
|
||||
out.push_str(&format!(" {{\"name\": \"{}\", \"kind\": \"{}\", \"unit\": \"{}\", \"help\": \"{}\", \"processes\": [{}], \"labels\": [", item.name, item.kind.as_str(), item.unit.as_str(), item.description, json_processes(item.processes)));
|
||||
for (label_index, domain) in item.label_domains.iter().enumerate() {
|
||||
out.push_str(&format!(
|
||||
"{{\"name\": \"{}\", \"class\": \"{}\", \"domain\": [{}]}}{}",
|
||||
domain.name,
|
||||
domain.class.as_str(),
|
||||
json_strings(domain.values),
|
||||
comma(label_index, item.label_domains.len())
|
||||
));
|
||||
}
|
||||
out.push_str(&format!("], \"buckets_seconds\": [{}], \"max_logical_series\": {}, \"max_rendered_series\": {}}}{}\n", item.buckets.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(", "), item.max_logical_series, item.max_rendered_series, comma(index, METRIC_SCHEMA.len())));
|
||||
}
|
||||
out.push_str(" ]\n}\n");
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn json_strings(values: &[&str]) -> String {
|
||||
values
|
||||
.iter()
|
||||
.map(|value| format!("\"{value}\""))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
fn json_processes(values: &[MetricProcess]) -> String {
|
||||
values
|
||||
.iter()
|
||||
.map(|value| format!("\"{}\"", value.as_str()))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
const fn comma(index: usize, len: usize) -> &'static str {
|
||||
if index + 1 == len { "" } else { "," }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
|
||||
use crank_metrics::{
|
||||
HttpMethod, HttpRoute, HttpStatusClass, McpMethod, McpOutcome, McpResponseMode,
|
||||
record_http_request, record_mcp_request,
|
||||
};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
#[test]
|
||||
fn one_million_hostile_values_collapse_to_the_same_closed_series() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for index in 0..1_000_000_u32 {
|
||||
let canary = format!("workspace-{index}-request-trace-url-secret");
|
||||
record_http_request(
|
||||
HttpRoute::from_matched_path(&canary),
|
||||
HttpMethod::classify(&canary),
|
||||
HttpStatusClass::from_status(999),
|
||||
Duration::ZERO,
|
||||
);
|
||||
record_mcp_request(
|
||||
McpMethod::Invalid,
|
||||
McpResponseMode::Unknown,
|
||||
McpOutcome::Other,
|
||||
Duration::ZERO,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let series = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.map(|(key, _, _, _)| format!("{:?}", key.key()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert_eq!(series.len(), 4);
|
||||
assert!(series.iter().all(|value| !value.contains("workspace-")));
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use crank_metrics::{
|
||||
metric_schema, record_cache_outcome, record_confirmation_outcome, record_export_failure,
|
||||
record_http_request, record_idempotency_outcome, record_invocation_history_lost,
|
||||
record_limit_rejection, record_mcp_request, record_tool_invocation, record_upstream_request,
|
||||
set_catalog, set_db_pool_connections, set_mcp_active_sessions,
|
||||
set_catalog, set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
|
||||
};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
@@ -80,6 +80,26 @@ fn application_outcomes_are_distinct_from_transport_success() {
|
||||
assert_eq!(ConfirmationOutcome::Required.as_str(), "required");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn impossible_tool_outcome_pairs_are_normalized() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
record_tool_invocation(
|
||||
InvocationSource::AgentToolCall,
|
||||
ToolOutcome::Success,
|
||||
ToolErrorKind::Mapping,
|
||||
Duration::ZERO,
|
||||
);
|
||||
});
|
||||
let snapshot = snapshotter.snapshot().into_vec();
|
||||
assert!(snapshot.iter().any(|(key, _, _, _)| {
|
||||
key.key().name() == "crank_tool_invocations_total"
|
||||
&& has_label(key.key(), "outcome", "success")
|
||||
&& has_label(key.key(), "error_kind", "none")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropped_product_guards_record_aborted_outcomes() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
@@ -128,6 +148,7 @@ fn diverse_calls_cannot_create_unbounded_series() {
|
||||
McpMethod::ToolsCall,
|
||||
McpResponseMode::Json,
|
||||
McpOutcome::ToolError,
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
record_tool_invocation(
|
||||
InvocationSource::AgentToolCall,
|
||||
@@ -147,7 +168,7 @@ fn diverse_calls_cannot_create_unbounded_series() {
|
||||
.map(|(key, _, _, _)| format!("{:?}", key.key()))
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(series.len(), 8);
|
||||
assert_eq!(series.len(), 9);
|
||||
assert!(
|
||||
series
|
||||
.iter()
|
||||
@@ -172,8 +193,10 @@ fn every_recording_api_matches_the_declared_schema() {
|
||||
McpMethod::ToolsCall,
|
||||
McpResponseMode::Json,
|
||||
McpOutcome::Success,
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
set_mcp_active_sessions(1);
|
||||
set_mcp_session_metrics_fresh(true);
|
||||
let _stream = InFlightGuard::mcp_stream();
|
||||
let _runtime = InFlightGuard::runtime();
|
||||
record_tool_invocation(
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use crank_metrics::{
|
||||
ExemplarTraceId, HttpMethod, HttpRoute, HttpStatusClass, exemplar_snapshot,
|
||||
record_http_request_with_exemplar, reset_exemplars_for_test,
|
||||
};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
#[test]
|
||||
fn trace_id_is_validated_and_never_becomes_an_ordinary_label() {
|
||||
assert!(ExemplarTraceId::parse("0123456789abcdef0123456789abcdef").is_some());
|
||||
assert!(ExemplarTraceId::parse("0123456789ABCDEF0123456789ABCDEF").is_none());
|
||||
assert!(ExemplarTraceId::parse("short").is_none());
|
||||
|
||||
reset_exemplars_for_test();
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
record_http_request_with_exemplar(
|
||||
HttpRoute::from_matched_path("/health"),
|
||||
HttpMethod::Get,
|
||||
HttpStatusClass::Success,
|
||||
Duration::from_millis(10),
|
||||
ExemplarTraceId::parse("0123456789abcdef0123456789abcdef"),
|
||||
);
|
||||
});
|
||||
|
||||
let aggregate = snapshotter.snapshot().into_vec();
|
||||
assert!(
|
||||
aggregate
|
||||
.iter()
|
||||
.all(|(key, _, _, _)| { key.key().labels().all(|label| label.key() != "trace_id") })
|
||||
);
|
||||
let exemplars = exemplar_snapshot();
|
||||
assert_eq!(exemplars.len(), 1);
|
||||
assert_eq!(
|
||||
exemplars[0].trace_id.as_str(),
|
||||
"0123456789abcdef0123456789abcdef"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use std::{
|
||||
hint::black_box,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use crank_metrics::{CacheOutcome, record_cache_outcome};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
const SAMPLES: usize = 40;
|
||||
const WORK_UNITS: u64 = 500_000;
|
||||
|
||||
#[test]
|
||||
fn reproducible_foundation_profile_stays_inside_latency_and_cpu_proxy_budgets() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let mut baseline = Vec::with_capacity(SAMPLES);
|
||||
let mut enabled = Vec::with_capacity(SAMPLES);
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for index in 0..SAMPLES {
|
||||
if index % 2 == 0 {
|
||||
baseline.push(sample(false));
|
||||
enabled.push(sample(true));
|
||||
} else {
|
||||
enabled.push(sample(true));
|
||||
baseline.push(sample(false));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
baseline.sort_unstable();
|
||||
enabled.sort_unstable();
|
||||
let baseline_p95 = baseline[SAMPLES * 95 / 100];
|
||||
let enabled_p95 = enabled[SAMPLES * 95 / 100];
|
||||
let baseline_cpu: Duration = baseline.iter().copied().sum();
|
||||
let enabled_cpu: Duration = enabled.iter().copied().sum();
|
||||
let series = snapshotter.snapshot().into_vec().len();
|
||||
|
||||
println!(
|
||||
"metrics_profile_v1 samples={SAMPLES} work_units={WORK_UNITS} baseline_p95_ns={} enabled_p95_ns={} baseline_cpu_ns={} enabled_cpu_ns={} logical_series={series}",
|
||||
baseline_p95.as_nanos(),
|
||||
enabled_p95.as_nanos(),
|
||||
baseline_cpu.as_nanos(),
|
||||
enabled_cpu.as_nanos()
|
||||
);
|
||||
assert!(enabled_p95.as_nanos() * 100 <= baseline_p95.as_nanos() * 105);
|
||||
assert!(enabled_cpu.as_nanos() * 100 <= baseline_cpu.as_nanos() * 110);
|
||||
assert_eq!(series, 1);
|
||||
}
|
||||
|
||||
fn sample(with_metrics: bool) -> Duration {
|
||||
let started = Instant::now();
|
||||
let mut value = 0x9e37_79b9_u64;
|
||||
for index in 0..WORK_UNITS {
|
||||
value = value.rotate_left(7) ^ index.wrapping_mul(0x100_0000_01b3);
|
||||
}
|
||||
black_box(value);
|
||||
if with_metrics {
|
||||
record_cache_outcome(CacheOutcome::Hit);
|
||||
}
|
||||
started.elapsed()
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crank_metrics::{
|
||||
HTTP_ROUTE_DOMAIN, HttpRoute, MAX_LOGICAL_SERIES_PER_PROCESS, MAX_PRODUCT_LABELS_PER_FAMILY,
|
||||
MAX_RENDERED_SERIES_PER_PROCESS, METRIC_SCHEMA_VERSION, metric_schema,
|
||||
render_metric_schema_json, schema_budget, validate_budget,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
|
||||
assert_eq!(METRIC_SCHEMA_VERSION, 1);
|
||||
assert_eq!(metric_schema().len(), 23);
|
||||
|
||||
let budget = schema_budget().expect("canonical schema must be valid");
|
||||
assert!(budget.logical_series <= MAX_LOGICAL_SERIES_PER_PROCESS);
|
||||
assert!(budget.rendered_series <= MAX_RENDERED_SERIES_PER_PROCESS);
|
||||
|
||||
for definition in metric_schema() {
|
||||
assert!(!definition.description.trim().is_empty());
|
||||
assert!(definition.labels.len() <= MAX_PRODUCT_LABELS_PER_FAMILY);
|
||||
assert!(!definition.processes.is_empty());
|
||||
assert_eq!(definition.labels.len(), definition.label_domains.len());
|
||||
assert!(definition.label_domains.iter().all(|domain| {
|
||||
!domain.values.is_empty()
|
||||
&& domain.values.iter().all(|value| !value.is_empty())
|
||||
&& domain.values.iter().copied().collect::<BTreeSet<_>>().len()
|
||||
== domain.values.len()
|
||||
}));
|
||||
if definition.kind.as_str() == "histogram" {
|
||||
assert!(!definition.buckets.is_empty());
|
||||
assert!(definition.buckets.windows(2).all(|pair| pair[0] < pair[1]));
|
||||
} else {
|
||||
assert!(definition.buckets.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
let first = render_metric_schema_json().expect("schema renders");
|
||||
let second = render_metric_schema_json().expect("schema renders deterministically");
|
||||
assert_eq!(first, second);
|
||||
assert!(first.len() < 65_536);
|
||||
assert!(!first.contains("trace_id"));
|
||||
assert!(!first.contains("request_id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frozen_budget_edges_are_exact() {
|
||||
assert!(validate_budget(4_999, 14_999).is_ok());
|
||||
assert!(validate_budget(5_000, 15_000).is_ok());
|
||||
assert!(validate_budget(5_001, 15_000).is_err());
|
||||
assert!(validate_budget(5_000, 15_001).is_err());
|
||||
assert_eq!(crank_metrics::max_exemplar_slots(), 9_170);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_registered_route_round_trips_without_dynamic_segments() {
|
||||
assert_eq!(HTTP_ROUTE_DOMAIN.len(), 57);
|
||||
for route in HTTP_ROUTE_DOMAIN {
|
||||
assert_eq!(HttpRoute::from_matched_path(route).as_str(), *route);
|
||||
assert!(!route.contains("trace_id"));
|
||||
assert!(!route.contains("request_id"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_schema_validation_reports_its_category() {
|
||||
assert_eq!(
|
||||
schema_budget(),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 4_513,
|
||||
rendered_series: 14_338
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_budget_math_counts_histogram_bucket_sum_and_count_series() {
|
||||
for definition in metric_schema() {
|
||||
let logical = definition
|
||||
.label_domains
|
||||
.iter()
|
||||
.map(|domain| domain.values.len())
|
||||
.product::<usize>();
|
||||
let logical = logical.max(1);
|
||||
let rendered = if definition.kind.as_str() == "histogram" {
|
||||
logical * (definition.buckets.len() + 3)
|
||||
} else {
|
||||
logical
|
||||
};
|
||||
assert_eq!(
|
||||
definition.max_logical_series, logical,
|
||||
"{}",
|
||||
definition.name
|
||||
);
|
||||
assert_eq!(
|
||||
definition.max_rendered_series, rendered,
|
||||
"{}",
|
||||
definition.name
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,9 @@ use axum::{
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use crank_metrics::{DbPoolState, HttpMethod, HttpRoute, HttpStatusClass, InFlightGuard};
|
||||
use crank_metrics::{
|
||||
DbPoolState, ExemplarTraceId, HttpMethod, HttpRoute, HttpStatusClass, InFlightGuard,
|
||||
};
|
||||
use metrics::Unit;
|
||||
|
||||
use crate::{MetricKind, MetricUnit, metric_schema};
|
||||
@@ -22,11 +24,17 @@ pub async fn record_http_request(request: Request, next: Next) -> Response {
|
||||
let _inflight = InFlightGuard::http();
|
||||
|
||||
let response = next.run(request).await;
|
||||
crank_metrics::record_http_request(
|
||||
let exemplar = response
|
||||
.headers()
|
||||
.get("x-trace-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(ExemplarTraceId::parse);
|
||||
crank_metrics::record_http_request_with_exemplar(
|
||||
route,
|
||||
method,
|
||||
HttpStatusClass::from_status(response.status().as_u16()),
|
||||
started_at.elapsed(),
|
||||
exemplar,
|
||||
);
|
||||
|
||||
response
|
||||
|
||||
@@ -18,8 +18,11 @@ use thiserror::Error;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
|
||||
use crank_metrics::{ExemplarObservation, MAX_EXPOSITION_BYTES, MetricService, exemplar_snapshot};
|
||||
|
||||
const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
|
||||
const OPENMETRICS_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
|
||||
const EXPOSITION_TOO_LARGE: &str = "metrics exposition exceeds configured bound\n";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MetricsConfig {
|
||||
@@ -190,6 +193,7 @@ pub(crate) fn install_prometheus_recorder(
|
||||
fn prometheus_builder(
|
||||
identity: &ServiceIdentity,
|
||||
) -> Result<PrometheusBuilder, MetricsSurfaceError> {
|
||||
MetricService::parse(identity.service()).ok_or(MetricsSurfaceError::RecorderConfiguration)?;
|
||||
PrometheusBuilder::new()
|
||||
.set_buckets(DURATION_BUCKETS_SECONDS)
|
||||
.map(|builder| {
|
||||
@@ -201,15 +205,70 @@ fn prometheus_builder(
|
||||
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
|
||||
}
|
||||
|
||||
async fn render_metrics(State(state): State<MetricsState>) -> Response {
|
||||
let mut response = state.handle.render().into_response();
|
||||
async fn render_metrics(State(state): State<MetricsState>, headers: HeaderMap) -> Response {
|
||||
let legacy = state.handle.render();
|
||||
let openmetrics = headers
|
||||
.get(header::ACCEPT)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| {
|
||||
value
|
||||
.split(',')
|
||||
.any(|part| part.trim().starts_with("application/openmetrics-text"))
|
||||
});
|
||||
let body = if openmetrics {
|
||||
render_openmetrics(&legacy, &exemplar_snapshot())
|
||||
} else {
|
||||
legacy
|
||||
};
|
||||
if body.len() > MAX_EXPOSITION_BYTES {
|
||||
return (StatusCode::SERVICE_UNAVAILABLE, EXPOSITION_TOO_LARGE).into_response();
|
||||
}
|
||||
let mut response = body.into_response();
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static(PROMETHEUS_CONTENT_TYPE),
|
||||
HeaderValue::from_static(if openmetrics {
|
||||
OPENMETRICS_CONTENT_TYPE
|
||||
} else {
|
||||
PROMETHEUS_CONTENT_TYPE
|
||||
}),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn render_openmetrics(legacy: &str, exemplars: &[ExemplarObservation]) -> String {
|
||||
let mut output = String::with_capacity(legacy.len() + exemplars.len().saturating_mul(96) + 6);
|
||||
for line in legacy.lines() {
|
||||
output.push_str(line);
|
||||
if let Some(exemplar) = exemplars
|
||||
.iter()
|
||||
.find(|candidate| line_matches_exemplar(line, candidate))
|
||||
{
|
||||
output.push_str(" # {trace_id=\"");
|
||||
output.push_str(exemplar.trace_id.as_str());
|
||||
output.push_str("\"} ");
|
||||
output.push_str(&exemplar.value.to_string());
|
||||
}
|
||||
output.push('\n');
|
||||
}
|
||||
output.push_str("# EOF\n");
|
||||
output
|
||||
}
|
||||
|
||||
fn line_matches_exemplar(line: &str, exemplar: &ExemplarObservation) -> bool {
|
||||
if !line.starts_with(exemplar.metric) || !line[exemplar.metric.len()..].starts_with("_bucket{")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let expected_bound = exemplar
|
||||
.bucket_upper_bound
|
||||
.map_or_else(|| "+Inf".to_owned(), |value| value.to_string());
|
||||
line.contains(&format!("le=\"{expected_bound}\""))
|
||||
&& exemplar
|
||||
.labels
|
||||
.iter()
|
||||
.all(|(key, value)| line.contains(&format!("{key}=\"{value}\"")))
|
||||
}
|
||||
|
||||
async fn metrics_health() -> impl IntoResponse {
|
||||
(StatusCode::OK, "ok\n")
|
||||
}
|
||||
@@ -251,3 +310,26 @@ fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
|
||||
fn token_digest(token: &[u8]) -> [u8; 32] {
|
||||
Sha256::digest(token).into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod rendering_tests {
|
||||
use crank_metrics::{ExemplarObservation, ExemplarTraceId};
|
||||
|
||||
use super::render_openmetrics;
|
||||
|
||||
#[test]
|
||||
fn openmetrics_adds_bounded_exemplar_without_changing_aggregate() {
|
||||
let legacy = "# TYPE crank_http_request_duration_seconds histogram\ncrank_http_request_duration_seconds_bucket{method=\"GET\",route=\"/health\",le=\"0.01\"} 1\ncrank_http_request_duration_seconds_sum{method=\"GET\",route=\"/health\"} 0.007\n";
|
||||
let exemplar = ExemplarObservation {
|
||||
metric: "crank_http_request_duration_seconds",
|
||||
labels: vec![("route", "/health"), ("method", "GET")],
|
||||
bucket_upper_bound: Some(0.01),
|
||||
value: 0.007,
|
||||
trace_id: ExemplarTraceId::parse("0123456789abcdef0123456789abcdef").unwrap(),
|
||||
};
|
||||
let rendered = render_openmetrics(legacy, &[exemplar]);
|
||||
assert!(rendered.contains("# {trace_id=\"0123456789abcdef0123456789abcdef\"} 0.007"));
|
||||
assert!(rendered.ends_with("# EOF\n"));
|
||||
assert_eq!(rendered.matches(" 1").count(), legacy.matches(" 1").count());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
http::{Request, StatusCode, header},
|
||||
middleware,
|
||||
routing::get,
|
||||
};
|
||||
@@ -14,8 +14,7 @@ use tower::ServiceExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_metrics_use_matched_routes_and_closed_labels() {
|
||||
let identity =
|
||||
ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity");
|
||||
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity");
|
||||
let lifecycle = crank_observability::init(ObservabilityConfig::new(
|
||||
identity,
|
||||
"off",
|
||||
@@ -32,7 +31,12 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/api/admin/workspaces/{workspace_id}/operations/{operation_id}",
|
||||
get(|| async { StatusCode::NO_CONTENT }),
|
||||
get(|| async {
|
||||
(
|
||||
StatusCode::NO_CONTENT,
|
||||
[("x-trace-id", "0123456789abcdef0123456789abcdef")],
|
||||
)
|
||||
}),
|
||||
)
|
||||
.layer(middleware::from_fn(record_http_request));
|
||||
|
||||
@@ -63,6 +67,7 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
|
||||
let response = metrics
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::get("/metrics")
|
||||
.body(Body::empty())
|
||||
@@ -82,6 +87,7 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
|
||||
assert!(body.contains("method=\"GET\""));
|
||||
assert!(body.contains("status_class=\"2xx\""));
|
||||
assert!(body.contains("crank_http_request_duration_seconds_bucket"));
|
||||
assert!(!body.contains("trace_id"));
|
||||
assert!(!body.contains(sensitive_path_segment));
|
||||
assert_eq!(
|
||||
body.lines()
|
||||
@@ -95,4 +101,30 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
|
||||
1,
|
||||
"different entity ids must not create additional series"
|
||||
);
|
||||
|
||||
let openmetrics = metrics
|
||||
.oneshot(
|
||||
Request::get("/metrics")
|
||||
.header(
|
||||
header::ACCEPT,
|
||||
"application/openmetrics-text; version=1.0.0",
|
||||
)
|
||||
.body(Body::empty())
|
||||
.expect("request"),
|
||||
)
|
||||
.await
|
||||
.expect("OpenMetrics response");
|
||||
assert_eq!(openmetrics.status(), StatusCode::OK);
|
||||
assert!(
|
||||
openmetrics.headers()[header::CONTENT_TYPE]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("application/openmetrics-text")
|
||||
);
|
||||
let body = to_bytes(openmetrics.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = String::from_utf8(body.to_vec()).unwrap();
|
||||
assert!(body.contains("# {trace_id=\"0123456789abcdef0123456789abcdef\"}"));
|
||||
assert!(body.ends_with("# EOF\n"));
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use crank_observability::{
|
||||
|
||||
fn config() -> ObservabilityConfig {
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"),
|
||||
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid test identity"),
|
||||
"info",
|
||||
RedactionLimits::default(),
|
||||
)
|
||||
|
||||
@@ -17,8 +17,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
#[tokio::test]
|
||||
async fn typed_product_signals_are_exposed_by_a_real_scrape() {
|
||||
let identity =
|
||||
ServiceIdentity::try_new("metrics-signals", "0.3.1", "test").expect("valid identity");
|
||||
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity");
|
||||
let lifecycle = crank_observability::init(ObservabilityConfig::new(
|
||||
identity,
|
||||
"off",
|
||||
@@ -49,6 +48,7 @@ async fn typed_product_signals_are_exposed_by_a_real_scrape() {
|
||||
McpMethod::ToolsCall,
|
||||
McpResponseMode::Json,
|
||||
McpOutcome::ToolError,
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
set_mcp_active_sessions(2);
|
||||
let _stream = InFlightGuard::mcp_stream();
|
||||
|
||||
@@ -71,6 +71,18 @@ fn non_loopback_without_a_token_is_rejected_without_secret_data() {
|
||||
assert!(!error.to_string().contains("token="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metrics_surface_rejects_an_unregistered_process_identity() {
|
||||
let config = MetricsConfig::new(
|
||||
true,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let identity = ServiceIdentity::try_new("user-controlled", "0.3.1", "test").unwrap();
|
||||
assert!(MetricsSurface::for_test(config, identity).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_is_closed_and_uses_fixed_duration_buckets() {
|
||||
let schema = metric_schema();
|
||||
|
||||
@@ -258,8 +258,12 @@ impl RuntimeExecutor {
|
||||
let request_context = request.request_context.or(generated_context.as_ref());
|
||||
log_runtime_event("unary.execute", request.operation, request_context);
|
||||
let started_at = Instant::now();
|
||||
let invocation_metrics =
|
||||
ToolInvocationMetrics::start(metric_invocation_source(request_context));
|
||||
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
|
||||
metric_invocation_source(request_context),
|
||||
request_context.and_then(|context| {
|
||||
crank_metrics::ExemplarTraceId::parse(&context.trace_context.trace_id().to_string())
|
||||
}),
|
||||
);
|
||||
let result = async {
|
||||
let _permit = self.acquire_unary_permit(request.operation)?;
|
||||
let _inflight = InFlightGuard::runtime();
|
||||
|
||||
+26
-7
@@ -171,7 +171,8 @@ scrape_configs:
|
||||
|
||||
- `crank_http_requests_total`, `crank_http_request_duration_seconds`,
|
||||
`crank_http_inflight`;
|
||||
- `crank_mcp_requests_total`, `crank_mcp_active_sessions`,
|
||||
- `crank_mcp_requests_total`, `crank_mcp_request_duration_seconds`,
|
||||
`crank_mcp_active_sessions`, `crank_mcp_session_metrics_fresh`,
|
||||
`crank_mcp_active_streams`;
|
||||
- `crank_tool_invocations_total`, `crank_tool_invocation_duration_seconds`;
|
||||
- `crank_upstream_requests_total`,
|
||||
@@ -187,8 +188,12 @@ scrape_configs:
|
||||
|
||||
Recorder добавляет к рядам проверенные статические labels `service`,
|
||||
`version`, `environment`. Остальные labels имеют закрытый набор значений.
|
||||
Имена, типы и допустимые значения labels определены в независимом нижнем
|
||||
crate `crank-metrics`. Product-код вызывает только его типизированный фасад;
|
||||
Эти три labels классифицированы как `process_constant`: они валидируются один
|
||||
раз при запуске и образуют ровно одну tuple на процесс. Имена, типы, help,
|
||||
process ownership, допустимые значения product labels и buckets определены в
|
||||
независимом нижнем crate `crank-metrics`. Его deterministic machine snapshot —
|
||||
[`schemas/metrics-registry-v1.json`](schemas/metrics-registry-v1.json).
|
||||
Product-код вызывает только типизированный фасад;
|
||||
прямая production-зависимость от `metrics` вне `crank-metrics` и
|
||||
`crank-observability` запрещена архитектурной проверкой Cargo metadata.
|
||||
Запрещено использовать workspace, идентификаторы агента, операции или
|
||||
@@ -198,7 +203,10 @@ HTTP route берётся только из шаблона Axum; неизвес
|
||||
|
||||
`crank_mcp_active_sessions` отражает число ещё не истёкших транспортных
|
||||
сессий в общем хранилище, а `crank_mcp_active_streams` — число открытых в
|
||||
этом экземпляре SSE-потоков. Прикладные ошибки JSON-RPC и
|
||||
этом экземпляре SSE-потоков. `crank_mcp_session_metrics_fresh=0` означает,
|
||||
что последнее bounded чтение session store завершилось ошибкой/timeout и
|
||||
значение active sessions нельзя считать свежим; успешное чтение возвращает
|
||||
freshness в `1`. Прикладные ошибки JSON-RPC и
|
||||
`tools/call.result.isError=true` учитываются отдельно от успешного HTTP 200.
|
||||
Счётчики cache, idempotency и confirmation описывают переходы
|
||||
соответствующих стадий: hit/miss и ошибки хранилища, execute/replay/conflict
|
||||
@@ -208,9 +216,20 @@ HTTP route берётся только из шаблона Axum; неизвес
|
||||
|
||||
Buckets длительности фиксированы кодом:
|
||||
`0.005`, `0.01`, `0.025`, `0.05`, `0.1`, `0.25`, `0.5`, `1`, `2.5`, `5`,
|
||||
`10`, `30`, `60` секунд. Это пока техническая шкала, а не SLO. Фактические
|
||||
series и расход памяти измеряются в истории 1.8; произвольный численный
|
||||
бюджет до замеров не назначается.
|
||||
`10`, `30`, `60` секунд. Это пока техническая шкала, а не SLO. Registry v1
|
||||
содержит 23 family и для всего lifetime одного процесса ограничен 5 000
|
||||
logical labelsets и 15 000 Prometheus sample series; текущий worst case —
|
||||
4 513 и 14 338 соответственно. Histogram budget включает finite buckets,
|
||||
`+Inf`, `_sum` и `_count`; изменение domain или buckets требует обновления
|
||||
versioned snapshot и evidence.
|
||||
|
||||
Обычный scrape сохраняет Prometheus text 0.0.4. Клиент с
|
||||
`Accept: application/openmetrics-text` получает OpenMetrics 1.0 и `# EOF`.
|
||||
Validated 32-hex Trace ID может присутствовать только как latest bounded
|
||||
histogram exemplar; он никогда не является label и не меняет aggregate.
|
||||
Отсутствующий, invalid или unsampled Trace ID просто не создаёт exemplar.
|
||||
Размер готового exposition ограничен 8 MiB; превышение возвращает статический
|
||||
`503` без частично усечённого тела и не влияет на product listener.
|
||||
|
||||
## Распределённые трассы
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"budget": {"logical_series": 4513, "rendered_series": 14338, "max_logical_series": 5000, "max_rendered_series": 15000},
|
||||
"global_labels": [
|
||||
{"name": "service", "class": "process_constant", "domain": ["admin-api", "mcp-server"]},
|
||||
{"name": "version", "class": "process_constant", "domain": ["validated_release_identity"]},
|
||||
{"name": "environment", "class": "process_constant", "domain": ["validated_startup_value"]}
|
||||
],
|
||||
"metrics": [
|
||||
{"name": "crank_http_requests_total", "kind": "counter", "unit": "count", "help": "Total HTTP requests.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "route", "class": "product_closed", "domain": ["unmatched", "/health", "/ready", "/api/auth/login", "/api/auth/logout", "/api/auth/session", "/api/auth/profile", "/api/auth/password", "/api/admin/capabilities", "/api/admin/workspaces", "/api/admin/workspaces/{workspace_id}", "/api/admin/workspaces/{workspace_id}/operations", "/api/admin/workspaces/{workspace_id}/imports/openapi/preview", "/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create", "/api/admin/workspaces/{workspace_id}/operations/analyze-quality", "/api/admin/workspaces/{workspace_id}/operations/import", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export", "/api/admin/workspaces/{workspace_id}/agents", "/api/admin/workspaces/{workspace_id}/agents/tool-search/preview", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}", "/api/admin/workspaces/{workspace_id}/auth-profiles", "/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}", "/api/admin/workspaces/{workspace_id}/upstreams", "/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}", "/api/admin/workspaces/{workspace_id}/secrets", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate", "/api/admin/workspaces/{workspace_id}/export", "/api/admin/workspaces/{workspace_id}/logs", "/api/admin/workspaces/{workspace_id}/logs/{log_id}", "/api/admin/workspaces/{workspace_id}/approvals", "/api/admin/workspaces/{workspace_id}/approvals/{approval_id}", "/api/admin/workspaces/{workspace_id}/usage", "/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}", "/v1/{workspace_slug}/{agent_slug}", "/v1/{workspace_slug}/{agent_slug}/approvals", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny"]},{"name": "method", "class": "product_closed", "domain": ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER"]},{"name": "status_class", "class": "product_closed", "domain": ["1xx", "2xx", "3xx", "4xx", "5xx", "other"]}], "buckets_seconds": [], "max_logical_series": 3420, "max_rendered_series": 3420},
|
||||
{"name": "crank_http_request_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "HTTP request duration in seconds.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "route", "class": "product_closed", "domain": ["unmatched", "/health", "/ready", "/api/auth/login", "/api/auth/logout", "/api/auth/session", "/api/auth/profile", "/api/auth/password", "/api/admin/capabilities", "/api/admin/workspaces", "/api/admin/workspaces/{workspace_id}", "/api/admin/workspaces/{workspace_id}/operations", "/api/admin/workspaces/{workspace_id}/imports/openapi/preview", "/api/admin/workspaces/{workspace_id}/imports/openapi/{job_id}/create", "/api/admin/workspaces/{workspace_id}/operations/analyze-quality", "/api/admin/workspaces/{workspace_id}/operations/import", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/publish", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/input-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/samples/output-json", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/drafts/generate", "/api/admin/workspaces/{workspace_id}/operations/{operation_id}/export", "/api/admin/workspaces/{workspace_id}/agents", "/api/admin/workspaces/{workspace_id}/agents/tool-search/preview", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/versions/{version}", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/bindings", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/publish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/unpublish", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/archive", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}/revoke", "/api/admin/workspaces/{workspace_id}/agents/{agent_id}/platform-api-keys/{key_id}", "/api/admin/workspaces/{workspace_id}/auth-profiles", "/api/admin/workspaces/{workspace_id}/auth-profiles/{auth_profile_id}", "/api/admin/workspaces/{workspace_id}/upstreams", "/api/admin/workspaces/{workspace_id}/upstreams/{upstream_id}", "/api/admin/workspaces/{workspace_id}/secrets", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}", "/api/admin/workspaces/{workspace_id}/secrets/{secret_id}/rotate", "/api/admin/workspaces/{workspace_id}/export", "/api/admin/workspaces/{workspace_id}/logs", "/api/admin/workspaces/{workspace_id}/logs/{log_id}", "/api/admin/workspaces/{workspace_id}/approvals", "/api/admin/workspaces/{workspace_id}/approvals/{approval_id}", "/api/admin/workspaces/{workspace_id}/usage", "/api/admin/workspaces/{workspace_id}/usage/operations/{operation_id}", "/api/admin/workspaces/{workspace_id}/usage/agents/{agent_id}", "/v1/{workspace_slug}/{agent_slug}", "/v1/{workspace_slug}/{agent_slug}/approvals", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/approve", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}", "/v1/{workspace_slug}/{agent_slug}/approvals/{approval_id}/deny"]},{"name": "method", "class": "product_closed", "domain": ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD", "CONNECT", "TRACE", "OTHER"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 570, "max_rendered_series": 9120},
|
||||
{"name": "crank_http_inflight", "kind": "gauge", "unit": "count", "help": "HTTP requests currently being processed.", "processes": ["admin_api", "mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_mcp_requests_total", "kind": "counter", "unit": "count", "help": "Total MCP JSON-RPC requests.", "processes": ["mcp_server"], "labels": [{"name": "method", "class": "product_closed", "domain": ["initialize", "initialized", "ping", "tools_list", "tools_call", "notification", "unsupported", "response", "invalid"]},{"name": "response_mode", "class": "product_closed", "domain": ["json", "sse", "unknown"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "jsonrpc_error", "tool_error", "aborted", "other"]}], "buckets_seconds": [], "max_logical_series": 189, "max_rendered_series": 189},
|
||||
{"name": "crank_mcp_request_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "MCP JSON-RPC request duration in seconds.", "processes": ["mcp_server"], "labels": [{"name": "method", "class": "product_closed", "domain": ["initialize", "initialized", "ping", "tools_list", "tools_call", "notification", "unsupported", "response", "invalid"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "jsonrpc_error", "tool_error", "aborted", "other"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 63, "max_rendered_series": 1008},
|
||||
{"name": "crank_mcp_active_sessions", "kind": "gauge", "unit": "count", "help": "Active persisted MCP transport sessions.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_mcp_session_metrics_fresh", "kind": "gauge", "unit": "count", "help": "Whether the active-session gauge was refreshed successfully.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_mcp_active_streams", "kind": "gauge", "unit": "count", "help": "MCP event streams currently being served.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_tool_invocations_total", "kind": "counter", "unit": "count", "help": "Total tool invocations.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "source", "class": "product_closed", "domain": ["internal", "admin_test_run", "agent_tool_call"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "error", "aborted"]},{"name": "error_kind", "class": "product_closed", "domain": ["none", "schema", "mapping", "rest_adapter", "protocol_adapter", "unsupported_protocol", "unsupported_execution_mode", "concurrency_limit", "invalid_prepared_request", "confirmation_required", "invalid_confirmation_token", "confirmation_store", "idempotency_store", "idempotency_in_progress", "idempotency_conflict", "idempotency_outcome_unknown", "missing_auth_profile", "missing_secret", "missing_secret_version", "invalid_auth_secret", "secret_crypto", "aborted"]}], "buckets_seconds": [], "max_logical_series": 198, "max_rendered_series": 198},
|
||||
{"name": "crank_tool_invocation_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "Tool invocation duration in seconds.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "source", "class": "product_closed", "domain": ["internal", "admin_test_run", "agent_tool_call"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "error", "aborted"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 9, "max_rendered_series": 144},
|
||||
{"name": "crank_upstream_requests_total", "kind": "counter", "unit": "count", "help": "Total upstream requests.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "operation_kind", "class": "product_closed", "domain": ["rest"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "unexpected_status", "timeout", "transport_error", "response_too_large", "rejected", "window_expired", "invalid_response", "invalid_request", "configuration", "aborted"]}], "buckets_seconds": [], "max_logical_series": 13, "max_rendered_series": 13},
|
||||
{"name": "crank_upstream_request_duration_seconds", "kind": "histogram", "unit": "seconds", "help": "Upstream request duration in seconds.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "operation_kind", "class": "product_closed", "domain": ["rest"]},{"name": "outcome", "class": "product_closed", "domain": ["success", "client_error", "server_error", "unexpected_status", "timeout", "transport_error", "response_too_large", "rejected", "window_expired", "invalid_response", "invalid_request", "configuration", "aborted"]}], "buckets_seconds": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], "max_logical_series": 13, "max_rendered_series": 208},
|
||||
{"name": "crank_runtime_inflight", "kind": "gauge", "unit": "count", "help": "Runtime executions currently in progress.", "processes": ["admin_api", "mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_runtime_limit_rejections_total", "kind": "counter", "unit": "count", "help": "Runtime executions rejected by a bounded limit.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "stage", "class": "product_closed", "domain": ["concurrency", "rate_limit", "mcp_stream"]}], "buckets_seconds": [], "max_logical_series": 3, "max_rendered_series": 3},
|
||||
{"name": "crank_runtime_cache_total", "kind": "counter", "unit": "count", "help": "Runtime response cache outcomes.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "outcome", "class": "product_closed", "domain": ["hit", "miss", "read_error", "decode_error", "evict_error", "stored", "write_error"]}], "buckets_seconds": [], "max_logical_series": 7, "max_rendered_series": 7},
|
||||
{"name": "crank_idempotency_total", "kind": "counter", "unit": "count", "help": "Runtime idempotency outcomes.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "outcome", "class": "product_closed", "domain": ["execute", "replay", "completed", "conflict", "in_progress", "outcome_unknown", "store_unavailable", "error"]}], "buckets_seconds": [], "max_logical_series": 8, "max_rendered_series": 8},
|
||||
{"name": "crank_confirmation_total", "kind": "counter", "unit": "count", "help": "Runtime confirmation outcomes.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "outcome", "class": "product_closed", "domain": ["approved", "required", "invalid_token", "store_unavailable", "error"]}], "buckets_seconds": [], "max_logical_series": 5, "max_rendered_series": 5},
|
||||
{"name": "crank_db_pool_connections", "kind": "gauge", "unit": "count", "help": "PostgreSQL pool connections by state.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "state", "class": "product_closed", "domain": ["idle", "used"]}], "buckets_seconds": [], "max_logical_series": 2, "max_rendered_series": 2},
|
||||
{"name": "crank_catalog_tools", "kind": "gauge", "unit": "count", "help": "Tools in the current published catalog.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_catalog_estimated_context_tokens", "kind": "gauge", "unit": "count", "help": "Estimated context tokens in the current published catalog.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_catalog_warnings", "kind": "gauge", "unit": "count", "help": "Warnings in the current published catalog.", "processes": ["mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_invocation_history_lost_total", "kind": "counter", "unit": "count", "help": "Invocation history records lost after an action completed.", "processes": ["admin_api", "mcp_server"], "labels": [], "buckets_seconds": [], "max_logical_series": 1, "max_rendered_series": 1},
|
||||
{"name": "crank_telemetry_export_failures_total", "kind": "counter", "unit": "count", "help": "Telemetry export failures.", "processes": ["admin_api", "mcp_server"], "labels": [{"name": "signal_type", "class": "product_closed", "domain": ["trace", "invocation_history"]},{"name": "exporter", "class": "product_closed", "domain": ["otlp", "postgres"]}], "buckets_seconds": [], "max_logical_series": 4, "max_rendered_series": 4}
|
||||
]
|
||||
}
|
||||
@@ -22,6 +22,10 @@ config-contract-check:
|
||||
migration-contract-check:
|
||||
cargo run -p admin-api --bin crank-migrate -- plan --check
|
||||
|
||||
metrics-contract-check:
|
||||
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
|
||||
python3 scripts/check-metrics-boundaries.py --root .
|
||||
|
||||
rust-boundaries:
|
||||
scripts/check-rust-boundaries.sh
|
||||
|
||||
@@ -52,6 +56,7 @@ verify:
|
||||
just community-scope-check
|
||||
just config-contract-check
|
||||
just migration-contract-check
|
||||
just metrics-contract-check
|
||||
just rust-boundaries
|
||||
just rust-code-health
|
||||
just dependencies
|
||||
|
||||
@@ -100,6 +100,13 @@ python3 scripts/check-config-boundaries.py --root .
|
||||
committed machine contract. PostgreSQL DDL вне `crank-registry::migrations`
|
||||
блокируется Rust module boundary check.
|
||||
|
||||
## Typed metrics contract
|
||||
|
||||
`just metrics-contract-check` сверяет Rust registry с versioned JSON snapshot и
|
||||
проверяет, что production-код объявляет метрики только через `crank-metrics`.
|
||||
Checker распознаёт прямые macro calls, imports, aliases и re-exports; новые
|
||||
Rust-файлы можно передать явно через `--files`.
|
||||
|
||||
## `check-community-scope.sh`
|
||||
|
||||
Проверяет, что в community-репозиторий не попали функции и тексты за пределами
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_FILES = 20_000
|
||||
MAX_FILE_BYTES = 4 * 1024 * 1024
|
||||
ALLOWED_ADAPTERS = {
|
||||
"crates/crank-observability/src/instrumentation.rs",
|
||||
"crates/crank-observability/src/prometheus.rs",
|
||||
}
|
||||
PATTERNS = (
|
||||
re.compile(r"(?<![A-Za-z0-9_])(?:::)?metrics\s*::\s*(?:counter|gauge|histogram|describe_counter|describe_gauge|describe_histogram)\s*!"),
|
||||
re.compile(r"\b(?:pub\s+)?use\s+(?:::)?metrics\s*(?:::\s*(?:\{|counter|gauge|histogram)|\s+as\s+)"),
|
||||
re.compile(r"\bextern\s+crate\s+metrics(?:\s+as\s+\w+)?"),
|
||||
re.compile(r"\b(?:set_global_recorder|PrometheusRecorder|DebuggingRecorder)\b"),
|
||||
)
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Enforce crank-metrics ownership.")
|
||||
parser.add_argument("--root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--files", nargs="*")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def discover(root: Path) -> list[str]:
|
||||
result = subprocess.run(
|
||||
["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z", "--", "apps/**/*.rs", "crates/**/*.rs"],
|
||||
cwd=root,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
return sorted(path.decode("utf-8") for path in result.stdout.split(b"\0") if path)
|
||||
|
||||
|
||||
def resolve(root: Path, supplied: str) -> tuple[str, Path] | None:
|
||||
if "\0" in supplied:
|
||||
return None
|
||||
logical = Path(supplied)
|
||||
if logical.is_absolute() or ".." in logical.parts:
|
||||
return None
|
||||
path = root / logical
|
||||
if path.is_symlink() or not path.is_file():
|
||||
return None
|
||||
try:
|
||||
canonical = path.resolve(strict=True)
|
||||
relative = canonical.relative_to(root)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if canonical != path.absolute():
|
||||
return None
|
||||
return relative.as_posix(), canonical
|
||||
|
||||
|
||||
def scan(logical: str, path: Path) -> bool:
|
||||
if logical.startswith("crates/crank-metrics/"):
|
||||
return False
|
||||
if "/tests/" in logical or logical.startswith("tests/"):
|
||||
return False
|
||||
try:
|
||||
if path.stat().st_size > MAX_FILE_BYTES:
|
||||
return True
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeError):
|
||||
return True
|
||||
for pattern in PATTERNS:
|
||||
if not pattern.search(text):
|
||||
continue
|
||||
if logical in ALLOWED_ADAPTERS and pattern is PATTERNS[0] and "describe_" in pattern.search(text).group(0):
|
||||
continue
|
||||
if logical == "crates/crank-observability/src/prometheus.rs" and pattern is PATTERNS[3]:
|
||||
continue
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
root = args.root.resolve()
|
||||
try:
|
||||
supplied = args.files if args.files is not None else discover(root)
|
||||
except (OSError, subprocess.SubprocessError, UnicodeError):
|
||||
print("error: metrics boundary discovery failed", file=sys.stderr)
|
||||
return 1
|
||||
if len(supplied) > MAX_FILES:
|
||||
print("error: metrics boundary input limit exceeded", file=sys.stderr)
|
||||
return 1
|
||||
violations: list[str] = []
|
||||
for index, item in enumerate(sorted(set(supplied))):
|
||||
resolved = resolve(root, item)
|
||||
if resolved is None:
|
||||
violations.append(f"input[{index}]: invalid path")
|
||||
continue
|
||||
logical, path = resolved
|
||||
if path.suffix == ".rs" and scan(logical, path):
|
||||
violations.append(f"{logical}: direct metrics declaration")
|
||||
if violations:
|
||||
for violation in violations[:1000]:
|
||||
print(f"error: {violation}", file=sys.stderr)
|
||||
if len(violations) > 1000:
|
||||
print(f"error: diagnostics omitted={len(violations) - 1000}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"Metrics boundary check passed: files={len(supplied)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -22,6 +22,7 @@ echo "Rust module boundary check: checking module-level imports"
|
||||
|
||||
python3 "$ROOT_DIR/scripts/check-config-boundaries.py" --root "$ROOT_DIR" || status=1
|
||||
python3 "$ROOT_DIR/scripts/check-migration-boundaries.py" --root "$ROOT_DIR" || status=1
|
||||
python3 "$ROOT_DIR/scripts/check-metrics-boundaries.py" --root "$ROOT_DIR" || status=1
|
||||
|
||||
check_no_match \
|
||||
"admin-api service modules must not depend on axum HTTP types" \
|
||||
@@ -50,13 +51,6 @@ check_no_match \
|
||||
'^\s*use\s+(axum|sqlx)(::|[;\{])' \
|
||||
"$ROOT_DIR/crates/crank-runtime/src"
|
||||
|
||||
check_no_match \
|
||||
"product modules must record metrics only through crank-metrics" \
|
||||
'(^|[^[:alnum:]_])(::)?metrics::(counter|gauge|histogram)!' \
|
||||
"$ROOT_DIR/apps" \
|
||||
"$ROOT_DIR/crates" \
|
||||
--glob '!**/crank-metrics/**'
|
||||
|
||||
if (( status != 0 )); then
|
||||
cat >&2 <<'EOF'
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CHECKER = ROOT / "scripts" / "check-metrics-boundaries.py"
|
||||
|
||||
|
||||
class MetricsBoundaryTests(unittest.TestCase):
|
||||
def run_checker(self, root: Path, *paths: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["python3", str(CHECKER), "--root", str(root), "--files", *paths],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def test_rejects_direct_alias_import_reexport_and_wrapper_bypasses(self) -> None:
|
||||
cases = {
|
||||
"direct.rs": 'metrics::counter!("bad").increment(1);',
|
||||
"alias.rs": 'use metrics as m; m::counter!("bad").increment(1);',
|
||||
"import.rs": 'use metrics::counter; counter!("bad").increment(1);',
|
||||
"reexport.rs": 'pub use metrics::histogram;',
|
||||
"absolute.rs": '::metrics::gauge!("bad").set(1.0);',
|
||||
"wrapper.rs": 'macro_rules! bad { () => { metrics::counter!("bad") } }',
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
paths = []
|
||||
for name, content in cases.items():
|
||||
path = root / "apps" / "demo" / "src" / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
paths.append(str(path.relative_to(root)))
|
||||
result = self.run_checker(root, *paths)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
for name in cases:
|
||||
self.assertIn(name, result.stderr)
|
||||
self.assertNotIn(str(root), result.stderr)
|
||||
|
||||
def test_allows_typed_facade_and_narrow_observability_adapter(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
product = root / "apps" / "demo" / "src" / "main.rs"
|
||||
adapter = root / "crates" / "crank-observability" / "src" / "instrumentation.rs"
|
||||
product.parent.mkdir(parents=True)
|
||||
adapter.parent.mkdir(parents=True)
|
||||
product.write_text("crank_metrics::record_cache_outcome(value);", encoding="utf-8")
|
||||
adapter.write_text("metrics::describe_counter!(definition.name, definition.description);", encoding="utf-8")
|
||||
result = self.run_checker(root, str(product.relative_to(root)), str(adapter.relative_to(root)))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_explicit_missing_traversal_and_symlink_fail_closed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
outside = root.parent / "metrics-boundary-outside.rs"
|
||||
outside.write_text('metrics::counter!("bad");', encoding="utf-8")
|
||||
try:
|
||||
for path in ["missing.rs", "../metrics-boundary-outside.rs"]:
|
||||
result = self.run_checker(root, path)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
target = root / "target.rs"
|
||||
target.write_text("", encoding="utf-8")
|
||||
link = root / "link.rs"
|
||||
link.symlink_to(target)
|
||||
result = self.run_checker(root, "link.rs")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
finally:
|
||||
outside.unlink(missing_ok=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user