feat: freeze typed metrics registry and bounded exemplars

crates/crank-observability/src/instrumentation.rs

@@ -5,7 +5,9 @@ use axum::{
     middleware::Next,
     response::Response,
 };
-use crank_metrics::{DbPoolState, HttpMethod, HttpRoute, HttpStatusClass, InFlightGuard};
+use crank_metrics::{
+    DbPoolState, ExemplarTraceId, HttpMethod, HttpRoute, HttpStatusClass, InFlightGuard,
+};
 use metrics::Unit;
 
 use crate::{MetricKind, MetricUnit, metric_schema};
@@ -22,11 +24,17 @@ pub async fn record_http_request(request: Request, next: Next) -> Response {
     let _inflight = InFlightGuard::http();
 
     let response = next.run(request).await;
-    crank_metrics::record_http_request(
+    let exemplar = response
+        .headers()
+        .get("x-trace-id")
+        .and_then(|value| value.to_str().ok())
+        .and_then(ExemplarTraceId::parse);
+    crank_metrics::record_http_request_with_exemplar(
         route,
         method,
         HttpStatusClass::from_status(response.status().as_u16()),
         started_at.elapsed(),
+        exemplar,
     );
 
     response

crates/crank-observability/src/prometheus.rs

@@ -18,8 +18,11 @@ use thiserror::Error;
 use tokio::net::TcpListener;
 
 use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
+use crank_metrics::{ExemplarObservation, MAX_EXPOSITION_BYTES, MetricService, exemplar_snapshot};
 
 const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
+const OPENMETRICS_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
+const EXPOSITION_TOO_LARGE: &str = "metrics exposition exceeds configured bound\n";
 
 #[derive(Clone)]
 pub struct MetricsConfig {
@@ -190,6 +193,7 @@ pub(crate) fn install_prometheus_recorder(
 fn prometheus_builder(
     identity: &ServiceIdentity,
 ) -> Result<PrometheusBuilder, MetricsSurfaceError> {
+    MetricService::parse(identity.service()).ok_or(MetricsSurfaceError::RecorderConfiguration)?;
     PrometheusBuilder::new()
         .set_buckets(DURATION_BUCKETS_SECONDS)
         .map(|builder| {
@@ -201,15 +205,70 @@ fn prometheus_builder(
         .map_err(|_| MetricsSurfaceError::RecorderConfiguration)
 }
 
-async fn render_metrics(State(state): State<MetricsState>) -> Response {
-    let mut response = state.handle.render().into_response();
+async fn render_metrics(State(state): State<MetricsState>, headers: HeaderMap) -> Response {
+    let legacy = state.handle.render();
+    let openmetrics = headers
+        .get(header::ACCEPT)
+        .and_then(|value| value.to_str().ok())
+        .is_some_and(|value| {
+            value
+                .split(',')
+                .any(|part| part.trim().starts_with("application/openmetrics-text"))
+        });
+    let body = if openmetrics {
+        render_openmetrics(&legacy, &exemplar_snapshot())
+    } else {
+        legacy
+    };
+    if body.len() > MAX_EXPOSITION_BYTES {
+        return (StatusCode::SERVICE_UNAVAILABLE, EXPOSITION_TOO_LARGE).into_response();
+    }
+    let mut response = body.into_response();
     response.headers_mut().insert(
         header::CONTENT_TYPE,
-        HeaderValue::from_static(PROMETHEUS_CONTENT_TYPE),
+        HeaderValue::from_static(if openmetrics {
+            OPENMETRICS_CONTENT_TYPE
+        } else {
+            PROMETHEUS_CONTENT_TYPE
+        }),
     );
     response
 }
 
+fn render_openmetrics(legacy: &str, exemplars: &[ExemplarObservation]) -> String {
+    let mut output = String::with_capacity(legacy.len() + exemplars.len().saturating_mul(96) + 6);
+    for line in legacy.lines() {
+        output.push_str(line);
+        if let Some(exemplar) = exemplars
+            .iter()
+            .find(|candidate| line_matches_exemplar(line, candidate))
+        {
+            output.push_str(" # {trace_id=\"");
+            output.push_str(exemplar.trace_id.as_str());
+            output.push_str("\"} ");
+            output.push_str(&exemplar.value.to_string());
+        }
+        output.push('\n');
+    }
+    output.push_str("# EOF\n");
+    output
+}
+
+fn line_matches_exemplar(line: &str, exemplar: &ExemplarObservation) -> bool {
+    if !line.starts_with(exemplar.metric) || !line[exemplar.metric.len()..].starts_with("_bucket{")
+    {
+        return false;
+    }
+    let expected_bound = exemplar
+        .bucket_upper_bound
+        .map_or_else(|| "+Inf".to_owned(), |value| value.to_string());
+    line.contains(&format!("le=\"{expected_bound}\""))
+        && exemplar
+            .labels
+            .iter()
+            .all(|(key, value)| line.contains(&format!("{key}=\"{value}\"")))
+}
+
 async fn metrics_health() -> impl IntoResponse {
     (StatusCode::OK, "ok\n")
 }
@@ -251,3 +310,26 @@ fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
 fn token_digest(token: &[u8]) -> [u8; 32] {
     Sha256::digest(token).into()
 }
+
+#[cfg(test)]
+mod rendering_tests {
+    use crank_metrics::{ExemplarObservation, ExemplarTraceId};
+
+    use super::render_openmetrics;
+
+    #[test]
+    fn openmetrics_adds_bounded_exemplar_without_changing_aggregate() {
+        let legacy = "# TYPE crank_http_request_duration_seconds histogram\ncrank_http_request_duration_seconds_bucket{method=\"GET\",route=\"/health\",le=\"0.01\"} 1\ncrank_http_request_duration_seconds_sum{method=\"GET\",route=\"/health\"} 0.007\n";
+        let exemplar = ExemplarObservation {
+            metric: "crank_http_request_duration_seconds",
+            labels: vec![("route", "/health"), ("method", "GET")],
+            bucket_upper_bound: Some(0.01),
+            value: 0.007,
+            trace_id: ExemplarTraceId::parse("0123456789abcdef0123456789abcdef").unwrap(),
+        };
+        let rendered = render_openmetrics(legacy, &[exemplar]);
+        assert!(rendered.contains("# {trace_id=\"0123456789abcdef0123456789abcdef\"} 0.007"));
+        assert!(rendered.ends_with("# EOF\n"));
+        assert_eq!(rendered.matches(" 1").count(), legacy.matches(" 1").count());
+    }
+}

crates/crank-observability/tests/http_metrics.rs

@@ -3,7 +3,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use axum::{
     Router,
     body::{Body, to_bytes},
-    http::{Request, StatusCode},
+    http::{Request, StatusCode, header},
     middleware,
     routing::get,
 };
@@ -14,8 +14,7 @@ use tower::ServiceExt;
 
 #[tokio::test]
 async fn http_metrics_use_matched_routes_and_closed_labels() {
-    let identity =
-        ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity");
+    let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity");
     let lifecycle = crank_observability::init(ObservabilityConfig::new(
         identity,
         "off",
@@ -32,7 +31,12 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
     let app = Router::new()
         .route(
             "/api/admin/workspaces/{workspace_id}/operations/{operation_id}",
-            get(|| async { StatusCode::NO_CONTENT }),
+            get(|| async {
+                (
+                    StatusCode::NO_CONTENT,
+                    [("x-trace-id", "0123456789abcdef0123456789abcdef")],
+                )
+            }),
         )
         .layer(middleware::from_fn(record_http_request));
 
@@ -63,6 +67,7 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
     assert_eq!(response.status(), StatusCode::NOT_FOUND);
 
     let response = metrics
+        .clone()
         .oneshot(
             Request::get("/metrics")
                 .body(Body::empty())
@@ -82,6 +87,7 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
     assert!(body.contains("method=\"GET\""));
     assert!(body.contains("status_class=\"2xx\""));
     assert!(body.contains("crank_http_request_duration_seconds_bucket"));
+    assert!(!body.contains("trace_id"));
     assert!(!body.contains(sensitive_path_segment));
     assert_eq!(
         body.lines()
@@ -95,4 +101,30 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
         1,
         "different entity ids must not create additional series"
     );
+
+    let openmetrics = metrics
+        .oneshot(
+            Request::get("/metrics")
+                .header(
+                    header::ACCEPT,
+                    "application/openmetrics-text; version=1.0.0",
+                )
+                .body(Body::empty())
+                .expect("request"),
+        )
+        .await
+        .expect("OpenMetrics response");
+    assert_eq!(openmetrics.status(), StatusCode::OK);
+    assert!(
+        openmetrics.headers()[header::CONTENT_TYPE]
+            .to_str()
+            .unwrap()
+            .starts_with("application/openmetrics-text")
+    );
+    let body = to_bytes(openmetrics.into_body(), 1024 * 1024)
+        .await
+        .unwrap();
+    let body = String::from_utf8(body.to_vec()).unwrap();
+    assert!(body.contains("# {trace_id=\"0123456789abcdef0123456789abcdef\"}"));
+    assert!(body.ends_with("# EOF\n"));
 }

crates/crank-observability/tests/lifecycle.rs

@@ -4,7 +4,7 @@ use crank_observability::{
 
 fn config() -> ObservabilityConfig {
     ObservabilityConfig::new(
-        ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"),
+        ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid test identity"),
         "info",
         RedactionLimits::default(),
     )

crates/crank-observability/tests/metrics_signals.rs

@@ -17,8 +17,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
 #[tokio::test]
 async fn typed_product_signals_are_exposed_by_a_real_scrape() {
-    let identity =
-        ServiceIdentity::try_new("metrics-signals", "0.3.1", "test").expect("valid identity");
+    let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity");
     let lifecycle = crank_observability::init(ObservabilityConfig::new(
         identity,
         "off",
@@ -49,6 +48,7 @@ async fn typed_product_signals_are_exposed_by_a_real_scrape() {
         McpMethod::ToolsCall,
         McpResponseMode::Json,
         McpOutcome::ToolError,
+        Duration::from_millis(1),
     );
     set_mcp_active_sessions(2);
     let _stream = InFlightGuard::mcp_stream();

crates/crank-observability/tests/prometheus.rs

@@ -71,6 +71,18 @@ fn non_loopback_without_a_token_is_rejected_without_secret_data() {
     assert!(!error.to_string().contains("token="));
 }
 
+#[test]
+fn metrics_surface_rejects_an_unregistered_process_identity() {
+    let config = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
+        None,
+    )
+    .unwrap();
+    let identity = ServiceIdentity::try_new("user-controlled", "0.3.1", "test").unwrap();
+    assert!(MetricsSurface::for_test(config, identity).is_err());
+}
+
 #[test]
 fn schema_is_closed_and_uses_fixed_duration_buckets() {
     let schema = metric_schema();