наблюдаемость: ввести безопасный контракт метрик

crates/crank-observability/tests/http_metrics.rs

@@ -31,19 +31,21 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
     let metrics = lifecycle.metrics_surface(config).router();
     let app = Router::new()
         .route(
-            "/documents/{document_id}",
+            "/api/admin/workspaces/{workspace_id}/operations/{operation_id}",
             get(|| async { StatusCode::NO_CONTENT }),
         )
         .layer(middleware::from_fn(record_http_request));
 
-    let sensitive_path_segment = "customer-secret-document-id";
+    let sensitive_path_segment = "customer-secret-operation-id";
     for index in 0..100 {
         let response = app
             .clone()
             .oneshot(
-                Request::get(format!("/documents/{sensitive_path_segment}-{index}"))
-                    .body(Body::empty())
-                    .expect("request"),
+                Request::get(format!(
+                    "/api/admin/workspaces/customer-secret-workspace-{index}/operations/{sensitive_path_segment}-{index}"
+                ))
+                .body(Body::empty())
+                .expect("request"),
             )
             .await
             .expect("response");
@@ -74,7 +76,9 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
     let body = String::from_utf8(body.to_vec()).expect("utf-8 metrics");
 
     assert!(body.contains("crank_http_requests_total"));
-    assert!(body.contains("route=\"/documents/{document_id}\""));
+    assert!(
+        body.contains("route=\"/api/admin/workspaces/{workspace_id}/operations/{operation_id}\"")
+    );
     assert!(body.contains("method=\"GET\""));
     assert!(body.contains("status_class=\"2xx\""));
     assert!(body.contains("crank_http_request_duration_seconds_bucket"));
@@ -83,7 +87,9 @@ async fn http_metrics_use_matched_routes_and_closed_labels() {
         body.lines()
             .filter(|line| {
                 line.starts_with("crank_http_requests_total{")
-                    && line.contains("route=\"/documents/{document_id}\"")
+                    && line.contains(
+                        "route=\"/api/admin/workspaces/{workspace_id}/operations/{operation_id}\"",
+                    )
             })
             .count(),
         1,

crates/crank-observability/tests/metrics_signals.rs

@@ -0,0 +1,134 @@
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    time::Duration,
+};
+
+use crank_metrics::{
+    CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
+    HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
+    McpOutcome, McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
+    UpstreamOutcome, record_cache_outcome, record_confirmation_outcome, record_export_failure,
+    record_http_request, record_idempotency_outcome, record_invocation_history_lost,
+    record_limit_rejection, record_mcp_request, record_tool_invocation, record_upstream_request,
+    set_catalog, set_db_pool_connections, set_mcp_active_sessions,
+};
+use crank_observability::{MetricsConfig, ObservabilityConfig, RedactionLimits, ServiceIdentity};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+#[tokio::test]
+async fn typed_product_signals_are_exposed_by_a_real_scrape() {
+    let identity =
+        ServiceIdentity::try_new("metrics-signals", "0.3.1", "test").expect("valid identity");
+    let lifecycle = crank_observability::init(ObservabilityConfig::new(
+        identity,
+        "off",
+        RedactionLimits::default(),
+    ))
+    .expect("observability lifecycle");
+    let config = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0),
+        None,
+    )
+    .expect("loopback metrics config");
+    let server = lifecycle
+        .metrics_surface(config)
+        .bind()
+        .await
+        .expect("metrics listener");
+    let address = server.local_addr().expect("listener address");
+    let task = tokio::spawn(server.serve());
+
+    record_http_request(
+        HttpRoute::from_matched_path("/api/auth/session"),
+        HttpMethod::Get,
+        HttpStatusClass::Success,
+        Duration::from_millis(7),
+    );
+    record_mcp_request(
+        McpMethod::ToolsCall,
+        McpResponseMode::Json,
+        McpOutcome::ToolError,
+    );
+    set_mcp_active_sessions(2);
+    let _stream = InFlightGuard::mcp_stream();
+    let _runtime = InFlightGuard::runtime();
+    record_tool_invocation(
+        InvocationSource::AgentToolCall,
+        ToolOutcome::Error,
+        ToolErrorKind::Mapping,
+        Duration::from_millis(11),
+    );
+    record_upstream_request(
+        UpstreamOperationKind::Rest,
+        UpstreamOutcome::Timeout,
+        Duration::from_millis(13),
+    );
+    record_limit_rejection(LimitStage::Concurrency);
+    record_cache_outcome(CacheOutcome::Hit);
+    record_idempotency_outcome(IdempotencyOutcome::Replay);
+    record_confirmation_outcome(ConfirmationOutcome::Required);
+    set_db_pool_connections(DbPoolState::Idle, 3);
+    set_db_pool_connections(DbPoolState::Used, 1);
+    set_catalog(5, 1200, 1);
+    record_invocation_history_lost();
+    record_export_failure(SignalType::Trace, Exporter::Otlp);
+
+    let response = raw_get(address, "/metrics").await;
+    task.abort();
+    let _ = task.await;
+
+    assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}");
+    for metric in [
+        "crank_http_requests_total",
+        "crank_mcp_requests_total",
+        "crank_mcp_active_sessions",
+        "crank_mcp_active_streams",
+        "crank_tool_invocations_total",
+        "crank_upstream_requests_total",
+        "crank_runtime_inflight",
+        "crank_runtime_limit_rejections_total",
+        "crank_runtime_cache_total",
+        "crank_idempotency_total",
+        "crank_confirmation_total",
+        "crank_db_pool_connections",
+        "crank_catalog_tools",
+        "crank_invocation_history_lost_total",
+        "crank_telemetry_export_failures_total",
+    ] {
+        assert!(response.contains(metric), "missing metric {metric}");
+    }
+    for expected_label in [
+        "outcome=\"tool_error\"",
+        "outcome=\"timeout\"",
+        "outcome=\"hit\"",
+        "outcome=\"replay\"",
+        "outcome=\"required\"",
+        "error_kind=\"mapping\"",
+    ] {
+        assert!(
+            response.contains(expected_label),
+            "missing label {expected_label}"
+        );
+    }
+    assert!(!response.contains("customer-secret"));
+}
+
+async fn raw_get(address: SocketAddr, path: &str) -> String {
+    let mut stream = tokio::net::TcpStream::connect(address)
+        .await
+        .expect("metrics listener connection");
+    stream
+        .write_all(
+            format!("GET {path} HTTP/1.1\r\nHost: {address}\r\nConnection: close\r\n\r\n")
+                .as_bytes(),
+        )
+        .await
+        .expect("metrics request");
+    let mut response = Vec::new();
+    stream
+        .read_to_end(&mut response)
+        .await
+        .expect("metrics response");
+    String::from_utf8(response).expect("utf-8 response")
+}

crates/crank-observability/tests/prometheus.rs

@@ -8,6 +8,7 @@ use crank_observability::{
     DURATION_BUCKETS_SECONDS, MetricsConfig, MetricsConfigError, MetricsSurface, ServiceIdentity,
     metric_schema,
 };
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tower::ServiceExt;
 
 fn identity() -> ServiceIdentity {
@@ -27,6 +28,33 @@ fn loopback_is_allowed_without_a_token() {
     assert!(!config.requires_authentication());
 }
 
+#[tokio::test]
+async fn default_service_ports_bind_and_serve_real_metrics_listeners() {
+    for (service, port) in [("admin-api", 9464), ("mcp-server", 9465)] {
+        let config = MetricsConfig::new(
+            true,
+            SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),
+            None,
+        )
+        .expect("default loopback metrics config");
+        let identity =
+            ServiceIdentity::try_new(service, "0.3.1", "test").expect("valid service identity");
+        let server = MetricsSurface::for_test(config, identity)
+            .expect("metrics surface")
+            .bind()
+            .await
+            .expect("default metrics port must bind");
+        assert_eq!(server.local_addr().expect("local address").port(), port);
+
+        let task = tokio::spawn(server.serve());
+        let response = raw_get(port, "/metrics").await;
+        assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}");
+        assert!(response.contains("content-type: text/plain"));
+        task.abort();
+        let _ = task.await;
+    }
+}
+
 #[test]
 fn non_loopback_without_a_token_is_rejected_without_secret_data() {
     let error = MetricsConfig::new(
@@ -51,8 +79,13 @@ fn schema_is_closed_and_uses_fixed_duration_buckets() {
     assert!(names.contains(&"crank_http_requests_total"));
     assert!(names.contains(&"crank_http_request_duration_seconds"));
     assert!(names.contains(&"crank_mcp_requests_total"));
+    assert!(names.contains(&"crank_mcp_active_sessions"));
+    assert!(names.contains(&"crank_mcp_active_streams"));
     assert!(names.contains(&"crank_tool_invocations_total"));
     assert!(names.contains(&"crank_runtime_inflight"));
+    assert!(names.contains(&"crank_runtime_cache_total"));
+    assert!(names.contains(&"crank_idempotency_total"));
+    assert!(names.contains(&"crank_confirmation_total"));
     assert!(names.contains(&"crank_db_pool_connections"));
     assert!(names.contains(&"crank_catalog_tools"));
     assert!(names.contains(&"crank_invocation_history_lost_total"));
@@ -108,7 +141,7 @@ async fn external_surface_protects_both_routes_and_exposes_nothing_else() {
             .clone()
             .oneshot(
                 Request::get(path)
-                    .header(header::AUTHORIZATION, format!("Bearer {token}"))
+                    .header(header::AUTHORIZATION, format!("bEaReR  {token}"))
                     .body(Body::empty())
                     .expect("request"),
             )
@@ -121,14 +154,35 @@ async fn external_surface_protects_both_routes_and_exposes_nothing_else() {
         assert!(!String::from_utf8_lossy(&body).contains(token));
     }
 
-    let absent = app
-        .oneshot(
-            Request::get("/api/operations")
-                .header(header::AUTHORIZATION, format!("Bearer {token}"))
-                .body(Body::empty())
-                .expect("request"),
+    for authorization in [None, Some(format!("Bearer {token}"))] {
+        let mut request = Request::get("/api/operations");
+        if let Some(authorization) = authorization {
+            request = request.header(header::AUTHORIZATION, authorization);
+        }
+        let absent = app
+            .clone()
+            .oneshot(request.body(Body::empty()).expect("request"))
+            .await
+            .expect("response");
+        assert_eq!(absent.status(), StatusCode::NOT_FOUND);
+    }
+}
+
+async fn raw_get(port: u16, path: &str) -> String {
+    let mut stream = tokio::net::TcpStream::connect((Ipv4Addr::LOCALHOST, port))
+        .await
+        .expect("metrics listener connection");
+    stream
+        .write_all(
+            format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n")
+                .as_bytes(),
         )
         .await
-        .expect("response");
-    assert_eq!(absent.status(), StatusCode::NOT_FOUND);
+        .expect("metrics request");
+    let mut response = Vec::new();
+    stream
+        .read_to_end(&mut response)
+        .await
+        .expect("metrics response");
+    String::from_utf8(response).expect("utf-8 response")
 }