наблюдаемость: ввести безопасный контракт метрик

crates/crank-observability/Cargo.toml

@@ -8,6 +8,7 @@ version.workspace = true
 
 [dependencies]
 axum.workspace = true
+crank-metrics = { path = "../crank-metrics" }
 metrics.workspace = true
 metrics-exporter-prometheus.workspace = true
 opentelemetry.workspace = true
@@ -21,7 +22,7 @@ sha2.workspace = true
 subtle.workspace = true
 thiserror.workspace = true
 time.workspace = true
-tokio = { workspace = true, features = ["net"] }
+tokio = { workspace = true, features = ["io-util", "net"] }
 tracing.workspace = true
 tracing-opentelemetry.workspace = true
 tracing-subscriber.workspace = true

crates/crank-observability/src/incidents.rs

@@ -14,13 +14,11 @@ pub fn record_operational_incident(incident: OperationalIncident) {
     });
     match incident {
         OperationalIncident::InvocationHistoryLost => {
-            metrics::counter!("crank_invocation_history_lost_total").increment(1);
-            metrics::counter!(
-                "crank_telemetry_export_failures_total",
-                "signal_type" => "invocation_history",
-                "exporter" => "postgres"
-            )
-            .increment(1);
+            crank_metrics::record_invocation_history_lost();
+            crank_metrics::record_export_failure(
+                crank_metrics::SignalType::InvocationHistory,
+                crank_metrics::Exporter::Postgres,
+            );
         }
     }
 }

crates/crank-observability/src/instrumentation.rs

@@ -5,7 +5,8 @@ use axum::{
     middleware::Next,
     response::Response,
 };
-use metrics::{Gauge, Unit};
+use crank_metrics::{DbPoolState, HttpMethod, HttpRoute, HttpStatusClass, InFlightGuard};
+use metrics::Unit;
 
 use crate::{MetricKind, MetricUnit, metric_schema};
 
@@ -13,36 +14,28 @@ pub async fn record_http_request(request: Request, next: Next) -> Response {
     let route = request
         .extensions()
         .get::<MatchedPath>()
-        .map_or("unmatched", MatchedPath::as_str)
-        .to_owned();
-    let method = normalized_http_method(request.method().as_str());
+        .map_or(HttpRoute::unmatched(), |path| {
+            HttpRoute::from_matched_path(path.as_str())
+        });
+    let method = HttpMethod::classify(request.method().as_str());
     let started_at = Instant::now();
-    let _inflight = GaugeGuard::increment("crank_http_inflight");
+    let _inflight = InFlightGuard::http();
 
     let response = next.run(request).await;
-    let status_class = status_class(response.status().as_u16());
-
-    metrics::counter!(
-        "crank_http_requests_total",
-        "route" => route.clone(),
-        "method" => method,
-        "status_class" => status_class
-    )
-    .increment(1);
-    metrics::histogram!(
-        "crank_http_request_duration_seconds",
-        "route" => route,
-        "method" => method
-    )
-    .record(started_at.elapsed().as_secs_f64());
+    crank_metrics::record_http_request(
+        route,
+        method,
+        HttpStatusClass::from_status(response.status().as_u16()),
+        started_at.elapsed(),
+    );
 
     response
 }
 
 pub fn record_db_pool_connections(total: u32, idle: usize) {
-    let idle = idle.min(total as usize) as f64;
-    metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(idle);
-    metrics::gauge!("crank_db_pool_connections", "state" => "used").set(f64::from(total) - idle);
+    let idle = u32::try_from(idle.min(total as usize)).unwrap_or(total);
+    crank_metrics::set_db_pool_connections(DbPoolState::Idle, idle);
+    crank_metrics::set_db_pool_connections(DbPoolState::Used, total.saturating_sub(idle));
 }
 
 pub(crate) fn register_metric_schema() {
@@ -64,70 +57,29 @@ pub(crate) fn register_metric_schema() {
         }
     }
 
-    metrics::gauge!("crank_http_inflight").set(0.0);
-    metrics::gauge!("crank_mcp_active_sessions").set(0.0);
-    metrics::gauge!("crank_runtime_inflight").set(0.0);
-    metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(0.0);
-    metrics::gauge!("crank_db_pool_connections", "state" => "used").set(0.0);
-    metrics::gauge!("crank_catalog_tools").set(0.0);
-    metrics::gauge!("crank_catalog_estimated_context_tokens").set(0.0);
-    metrics::gauge!("crank_catalog_warnings").set(0.0);
-}
-
-fn normalized_http_method(method: &str) -> &'static str {
-    match method {
-        "GET" => "GET",
-        "POST" => "POST",
-        "PUT" => "PUT",
-        "PATCH" => "PATCH",
-        "DELETE" => "DELETE",
-        "OPTIONS" => "OPTIONS",
-        "HEAD" => "HEAD",
-        "CONNECT" => "CONNECT",
-        "TRACE" => "TRACE",
-        _ => "OTHER",
-    }
-}
-
-fn status_class(status: u16) -> &'static str {
-    match status {
-        100..=199 => "1xx",
-        200..=299 => "2xx",
-        300..=399 => "3xx",
-        400..=499 => "4xx",
-        500..=599 => "5xx",
-        _ => "other",
-    }
-}
-
-struct GaugeGuard {
-    gauge: Gauge,
-}
-
-impl GaugeGuard {
-    fn increment(name: &'static str) -> Self {
-        let gauge = metrics::gauge!(name);
-        gauge.increment(1.0);
-        Self { gauge }
-    }
-}
-
-impl Drop for GaugeGuard {
-    fn drop(&mut self) {
-        self.gauge.decrement(1.0);
-    }
+    crank_metrics::initialize_gauges();
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{normalized_http_method, status_class};
+    use crank_metrics::{HttpMethod, HttpRoute, HttpStatusClass};
 
     #[test]
     fn normalizes_unbounded_http_values() {
-        assert_eq!(normalized_http_method("GET"), "GET");
-        assert_eq!(normalized_http_method("CUSTOM-user-controlled"), "OTHER");
-        assert_eq!(status_class(204), "2xx");
-        assert_eq!(status_class(429), "4xx");
-        assert_eq!(status_class(999), "other");
+        assert_eq!(HttpMethod::classify("GET"), HttpMethod::Get);
+        assert_eq!(
+            HttpMethod::classify("CUSTOM-user-controlled"),
+            HttpMethod::Other
+        );
+        assert_eq!(HttpStatusClass::from_status(204), HttpStatusClass::Success);
+        assert_eq!(
+            HttpStatusClass::from_status(429),
+            HttpStatusClass::ClientError
+        );
+        assert_eq!(HttpStatusClass::from_status(999), HttpStatusClass::Other);
+        assert_eq!(
+            HttpRoute::from_matched_path("/documents/{document_id}"),
+            HttpRoute::unmatched()
+        );
     }
 }

crates/crank-observability/src/lib.rs

@@ -5,7 +5,6 @@ mod incidents;
 mod instrumentation;
 mod lifecycle;
 mod logging;
-mod metrics_schema;
 mod otlp;
 mod prometheus;
 mod propagation;
@@ -14,6 +13,9 @@ mod schema;
 
 pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
 pub use correlation::RequestId;
+pub use crank_metrics::{
+    DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
+};
 pub use error_reporting::{
     CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
     with_request_correlation,
@@ -22,9 +24,6 @@ pub use incidents::{OperationalIncident, operational_incident_total, record_oper
 pub use instrumentation::{record_db_pool_connections, record_http_request};
 pub use lifecycle::{ObservabilityInitError, ObservabilityLifecycle, init};
 pub use logging::build_subscriber;
-pub use metrics_schema::{
-    DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
-};
 pub use otlp::{
     OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError, build_tracer_provider,
 };

crates/crank-observability/src/metrics_schema.rs

@@ -1,159 +0,0 @@
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum MetricKind {
-    Counter,
-    Gauge,
-    Histogram,
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum MetricUnit {
-    Count,
-    Seconds,
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub struct MetricDefinition {
-    pub name: &'static str,
-    pub kind: MetricKind,
-    pub unit: MetricUnit,
-    pub labels: &'static [&'static str],
-    pub description: &'static str,
-}
-
-pub const DURATION_BUCKETS_SECONDS: &[f64] = &[
-    0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
-];
-
-const METRIC_SCHEMA: &[MetricDefinition] = &[
-    counter(
-        "crank_http_requests_total",
-        &["route", "method", "status_class"],
-        "Total HTTP requests.",
-    ),
-    histogram(
-        "crank_http_request_duration_seconds",
-        &["route", "method"],
-        "HTTP request duration in seconds.",
-    ),
-    gauge(
-        "crank_http_inflight",
-        &[],
-        "HTTP requests currently being processed.",
-    ),
-    counter(
-        "crank_mcp_requests_total",
-        &["method", "response_mode", "outcome"],
-        "Total MCP JSON-RPC requests.",
-    ),
-    gauge(
-        "crank_mcp_active_sessions",
-        &[],
-        "Active MCP transport sessions.",
-    ),
-    counter(
-        "crank_tool_invocations_total",
-        &["source", "outcome", "error_kind"],
-        "Total tool invocations.",
-    ),
-    histogram(
-        "crank_tool_invocation_duration_seconds",
-        &["source", "outcome"],
-        "Tool invocation duration in seconds.",
-    ),
-    counter(
-        "crank_upstream_requests_total",
-        &["operation_kind", "outcome"],
-        "Total upstream requests.",
-    ),
-    histogram(
-        "crank_upstream_request_duration_seconds",
-        &["operation_kind", "outcome"],
-        "Upstream request duration in seconds.",
-    ),
-    gauge(
-        "crank_runtime_inflight",
-        &[],
-        "Runtime executions currently in progress.",
-    ),
-    counter(
-        "crank_runtime_limit_rejections_total",
-        &["stage"],
-        "Runtime executions rejected by a bounded limit.",
-    ),
-    gauge(
-        "crank_db_pool_connections",
-        &["state"],
-        "PostgreSQL pool connections by state.",
-    ),
-    gauge(
-        "crank_catalog_tools",
-        &[],
-        "Tools in the current published catalog.",
-    ),
-    gauge(
-        "crank_catalog_estimated_context_tokens",
-        &[],
-        "Estimated context tokens in the current published catalog.",
-    ),
-    gauge(
-        "crank_catalog_warnings",
-        &[],
-        "Warnings in the current published catalog.",
-    ),
-    counter(
-        "crank_invocation_history_lost_total",
-        &[],
-        "Invocation history records lost after an action completed.",
-    ),
-    counter(
-        "crank_telemetry_export_failures_total",
-        &["signal_type", "exporter"],
-        "Telemetry export failures.",
-    ),
-];
-
-pub const fn metric_schema() -> &'static [MetricDefinition] {
-    METRIC_SCHEMA
-}
-
-const fn counter(
-    name: &'static str,
-    labels: &'static [&'static str],
-    description: &'static str,
-) -> MetricDefinition {
-    MetricDefinition {
-        name,
-        kind: MetricKind::Counter,
-        unit: MetricUnit::Count,
-        labels,
-        description,
-    }
-}
-
-const fn gauge(
-    name: &'static str,
-    labels: &'static [&'static str],
-    description: &'static str,
-) -> MetricDefinition {
-    MetricDefinition {
-        name,
-        kind: MetricKind::Gauge,
-        unit: MetricUnit::Count,
-        labels,
-        description,
-    }
-}
-
-const fn histogram(
-    name: &'static str,
-    labels: &'static [&'static str],
-    description: &'static str,
-) -> MetricDefinition {
-    MetricDefinition {
-        name,
-        kind: MetricKind::Histogram,
-        unit: MetricUnit::Seconds,
-        labels,
-        description,
-    }
-}

crates/crank-observability/src/otlp.rs

@@ -340,12 +340,10 @@ impl SpanExporterTrait for ObservedSpanExporter {
         sanitize_trace_batch(&mut batch);
         let result = self.0.export(batch).await;
         if result.is_err() {
-            metrics::counter!(
-                "crank_telemetry_export_failures_total",
-                "signal_type" => "trace",
-                "exporter" => "otlp"
-            )
-            .increment(1);
+            crank_metrics::record_export_failure(
+                crank_metrics::SignalType::Trace,
+                crank_metrics::Exporter::Otlp,
+            );
         }
         result
     }

crates/crank-observability/src/prometheus.rs

@@ -155,7 +155,7 @@ impl MetricsSurface {
         Router::new()
             .route("/metrics", get(render_metrics))
             .route("/health", get(metrics_health))
-            .layer(middleware::from_fn_with_state(
+            .route_layer(middleware::from_fn_with_state(
                 self.state.clone(),
                 authorize_metrics,
             ))
@@ -179,6 +179,12 @@ pub struct MetricsServer {
 }
 
 impl MetricsServer {
+    pub fn local_addr(&self) -> Result<SocketAddr, MetricsServeError> {
+        self.listener
+            .local_addr()
+            .map_err(|_| MetricsServeError::LocalAddress)
+    }
+
     pub async fn serve(self) -> Result<(), MetricsServeError> {
         axum::serve(self.listener, self.router)
             .await
@@ -198,6 +204,8 @@ pub enum MetricsServeError {
     Bind,
     #[error("metrics listener stopped unexpectedly")]
     Serve,
+    #[error("failed to read metrics listener address")]
+    LocalAddress,
 }
 
 pub(crate) fn install_prometheus_recorder(
@@ -257,10 +265,15 @@ async fn authorize_metrics(
 }
 
 fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
-    headers
-        .get(header::AUTHORIZATION)?
-        .as_bytes()
-        .strip_prefix(b"Bearer ")
+    let value = headers.get(header::AUTHORIZATION)?.as_bytes();
+    let separator = value.iter().position(|byte| *byte == b' ')?;
+    let (scheme, token_with_spaces) = value.split_at(separator);
+    let token_start = token_with_spaces.iter().position(|byte| *byte != b' ')?;
+    let token = &token_with_spaces[token_start..];
+
+    scheme
+        .eq_ignore_ascii_case(b"bearer")
+        .then_some(token)
         .filter(|token| !token.is_empty())
 }
 

crates/crank-observability/tests/http_metrics.rs

@@ -31,19 +31,21 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
     let metrics = lifecycle.metrics_surface(config).router();
     let app = Router::new()
         .route(
-            "/documents/{document_id}",
+            "/api/admin/workspaces/{workspace_id}/operations/{operation_id}",
             get(|| async { StatusCode::NO_CONTENT }),
         )
         .layer(middleware::from_fn(record_http_request));
 
-    let sensitive_path_segment = "customer-secret-document-id";
+    let sensitive_path_segment = "customer-secret-operation-id";
     for index in 0..100 {
         let response = app
             .clone()
             .oneshot(
-                Request::get(format!("/documents/{sensitive_path_segment}-{index}"))
-                    .body(Body::empty())
-                    .expect("request"),
+                Request::get(format!(
+                    "/api/admin/workspaces/customer-secret-workspace-{index}/operations/{sensitive_path_segment}-{index}"
+                ))
+                .body(Body::empty())
+                .expect("request"),
             )
             .await
             .expect("response");
@@ -74,7 +76,9 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
     let body = String::from_utf8(body.to_vec()).expect("utf-8 metrics");
 
     assert!(body.contains("crank_http_requests_total"));
-    assert!(body.contains("route=\"/documents/{document_id}\""));
+    assert!(
+        body.contains("route=\"/api/admin/workspaces/{workspace_id}/operations/{operation_id}\"")
+    );
     assert!(body.contains("method=\"GET\""));
     assert!(body.contains("status_class=\"2xx\""));
     assert!(body.contains("crank_http_request_duration_seconds_bucket"));
@@ -83,7 +87,9 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
         body.lines()
             .filter(|line| {
                 line.starts_with("crank_http_requests_total{")
-                    && line.contains("route=\"/documents/{document_id}\"")
+                    && line.contains(
+                        "route=\"/api/admin/workspaces/{workspace_id}/operations/{operation_id}\"",
+                    )
             })
             .count(),
         1,

crates/crank-observability/tests/metrics_signals.rs

@@ -0,0 +1,134 @@
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    time::Duration,
+};
+
+use crank_metrics::{
+    CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
+    HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
+    McpOutcome, McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
+    UpstreamOutcome, record_cache_outcome, record_confirmation_outcome, record_export_failure,
+    record_http_request, record_idempotency_outcome, record_invocation_history_lost,
+    record_limit_rejection, record_mcp_request, record_tool_invocation, record_upstream_request,
+    set_catalog, set_db_pool_connections, set_mcp_active_sessions,
+};
+use crank_observability::{MetricsConfig, ObservabilityConfig, RedactionLimits, ServiceIdentity};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+#[tokio::test]
+async fn typed_product_signals_are_exposed_by_a_real_scrape() {
+    let identity =
+        ServiceIdentity::try_new("metrics-signals", "0.3.1", "test").expect("valid identity");
+    let lifecycle = crank_observability::init(ObservabilityConfig::new(
+        identity,
+        "off",
+        RedactionLimits::default(),
+    ))
+    .expect("observability lifecycle");
+    let config = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0),
+        None,
+    )
+    .expect("loopback metrics config");
+    let server = lifecycle
+        .metrics_surface(config)
+        .bind()
+        .await
+        .expect("metrics listener");
+    let address = server.local_addr().expect("listener address");
+    let task = tokio::spawn(server.serve());
+
+    record_http_request(
+        HttpRoute::from_matched_path("/api/auth/session"),
+        HttpMethod::Get,
+        HttpStatusClass::Success,
+        Duration::from_millis(7),
+    );
+    record_mcp_request(
+        McpMethod::ToolsCall,
+        McpResponseMode::Json,
+        McpOutcome::ToolError,
+    );
+    set_mcp_active_sessions(2);
+    let _stream = InFlightGuard::mcp_stream();
+    let _runtime = InFlightGuard::runtime();
+    record_tool_invocation(
+        InvocationSource::AgentToolCall,
+        ToolOutcome::Error,
+        ToolErrorKind::Mapping,
+        Duration::from_millis(11),
+    );
+    record_upstream_request(
+        UpstreamOperationKind::Rest,
+        UpstreamOutcome::Timeout,
+        Duration::from_millis(13),
+    );
+    record_limit_rejection(LimitStage::Concurrency);
+    record_cache_outcome(CacheOutcome::Hit);
+    record_idempotency_outcome(IdempotencyOutcome::Replay);
+    record_confirmation_outcome(ConfirmationOutcome::Required);
+    set_db_pool_connections(DbPoolState::Idle, 3);
+    set_db_pool_connections(DbPoolState::Used, 1);
+    set_catalog(5, 1200, 1);
+    record_invocation_history_lost();
+    record_export_failure(SignalType::Trace, Exporter::Otlp);
+
+    let response = raw_get(address, "/metrics").await;
+    task.abort();
+    let _ = task.await;
+
+    assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}");
+    for metric in [
+        "crank_http_requests_total",
+        "crank_mcp_requests_total",
+        "crank_mcp_active_sessions",
+        "crank_mcp_active_streams",
+        "crank_tool_invocations_total",
+        "crank_upstream_requests_total",
+        "crank_runtime_inflight",
+        "crank_runtime_limit_rejections_total",
+        "crank_runtime_cache_total",
+        "crank_idempotency_total",
+        "crank_confirmation_total",
+        "crank_db_pool_connections",
+        "crank_catalog_tools",
+        "crank_invocation_history_lost_total",
+        "crank_telemetry_export_failures_total",
+    ] {
+        assert!(response.contains(metric), "missing metric {metric}");
+    }
+    for expected_label in [
+        "outcome=\"tool_error\"",
+        "outcome=\"timeout\"",
+        "outcome=\"hit\"",
+        "outcome=\"replay\"",
+        "outcome=\"required\"",
+        "error_kind=\"mapping\"",
+    ] {
+        assert!(
+            response.contains(expected_label),
+            "missing label {expected_label}"
+        );
+    }
+    assert!(!response.contains("customer-secret"));
+}
+
+async fn raw_get(address: SocketAddr, path: &str) -> String {
+    let mut stream = tokio::net::TcpStream::connect(address)
+        .await
+        .expect("metrics listener connection");
+    stream
+        .write_all(
+            format!("GET {path} HTTP/1.1\r\nHost: {address}\r\nConnection: close\r\n\r\n")
+                .as_bytes(),
+        )
+        .await
+        .expect("metrics request");
+    let mut response = Vec::new();
+    stream
+        .read_to_end(&mut response)
+        .await
+        .expect("metrics response");
+    String::from_utf8(response).expect("utf-8 response")
+}

crates/crank-observability/tests/prometheus.rs

@@ -8,6 +8,7 @@ use crank_observability::{
     DURATION_BUCKETS_SECONDS, MetricsConfig, MetricsConfigError, MetricsSurface, ServiceIdentity,
     metric_schema,
 };
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tower::ServiceExt;
 
 fn identity() -> ServiceIdentity {
@@ -27,6 +28,33 @@ fn loopback_is_allowed_without_a_token() {
     assert!(!config.requires_authentication());
 }
 
+#[tokio::test]
+async fn default_service_ports_bind_and_serve_real_metrics_listeners() {
+    for (service, port) in [("admin-api", 9464), ("mcp-server", 9465)] {
+        let config = MetricsConfig::new(
+            true,
+            SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),
+            None,
+        )
+        .expect("default loopback metrics config");
+        let identity =
+            ServiceIdentity::try_new(service, "0.3.1", "test").expect("valid service identity");
+        let server = MetricsSurface::for_test(config, identity)
+            .expect("metrics surface")
+            .bind()
+            .await
+            .expect("default metrics port must bind");
+        assert_eq!(server.local_addr().expect("local address").port(), port);
+
+        let task = tokio::spawn(server.serve());
+        let response = raw_get(port, "/metrics").await;
+        assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}");
+        assert!(response.contains("content-type: text/plain"));
+        task.abort();
+        let _ = task.await;
+    }
+}
+
 #[test]
 fn non_loopback_without_a_token_is_rejected_without_secret_data() {
     let error = MetricsConfig::new(
@@ -51,8 +79,13 @@ fn schema_is_closed_and_uses_fixed_duration_buckets() {
     assert!(names.contains(&"crank_http_requests_total"));
     assert!(names.contains(&"crank_http_request_duration_seconds"));
     assert!(names.contains(&"crank_mcp_requests_total"));
+    assert!(names.contains(&"crank_mcp_active_sessions"));
+    assert!(names.contains(&"crank_mcp_active_streams"));
     assert!(names.contains(&"crank_tool_invocations_total"));
     assert!(names.contains(&"crank_runtime_inflight"));
+    assert!(names.contains(&"crank_runtime_cache_total"));
+    assert!(names.contains(&"crank_idempotency_total"));
+    assert!(names.contains(&"crank_confirmation_total"));
     assert!(names.contains(&"crank_db_pool_connections"));
     assert!(names.contains(&"crank_catalog_tools"));
     assert!(names.contains(&"crank_invocation_history_lost_total"));
@@ -108,7 +141,7 @@ async fn external_surface_protects_both_routes_and_exposes_nothing_else() {
             .clone()
             .oneshot(
                 Request::get(path)
-                    .header(header::AUTHORIZATION, format!("Bearer {token}"))
+                    .header(header::AUTHORIZATION, format!("bEaReR  {token}"))
                     .body(Body::empty())
                     .expect("request"),
             )
@@ -121,14 +154,35 @@ async fn external_surface_protects_both_routes_and_exposes_nothing_else() {
         assert!(!String::from_utf8_lossy(&body).contains(token));
     }
 
-    let absent = app
-        .oneshot(
-            Request::get("/api/operations")
-                .header(header::AUTHORIZATION, format!("Bearer {token}"))
-                .body(Body::empty())
-                .expect("request"),
+    for authorization in [None, Some(format!("Bearer {token}"))] {
+        let mut request = Request::get("/api/operations");
+        if let Some(authorization) = authorization {
+            request = request.header(header::AUTHORIZATION, authorization);
+        }
+        let absent = app
+            .clone()
+            .oneshot(request.body(Body::empty()).expect("request"))
+            .await
+            .expect("response");
+        assert_eq!(absent.status(), StatusCode::NOT_FOUND);
+    }
+}
+
+async fn raw_get(port: u16, path: &str) -> String {
+    let mut stream = tokio::net::TcpStream::connect((Ipv4Addr::LOCALHOST, port))
+        .await
+        .expect("metrics listener connection");
+    stream
+        .write_all(
+            format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n")
+                .as_bytes(),
         )
         .await
-        .expect("response");
-    assert_eq!(absent.status(), StatusCode::NOT_FOUND);
+        .expect("metrics request");
+    let mut response = Vec::new();
+    stream
+        .read_to_end(&mut response)
+        .await
+        .expect("metrics response");
+    String::from_utf8(response).expect("utf-8 response")
 }