наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

crates/crank-observability/tests/correlation.rs

@@ -0,0 +1,42 @@
+use crank_observability::RequestId;
+use uuid::Version;
+
+#[test]
+fn preserves_valid_opaque_request_id() {
+    let request_id = RequestId::resolve(Some("req_test-123/abc"));
+
+    assert_eq!(request_id.as_str(), "req_test-123/abc");
+}
+
+#[test]
+fn replaces_missing_and_invalid_values_with_uuid_v7() {
+    for candidate in [
+        None,
+        Some(""),
+        Some("bad value"),
+        Some(" leading"),
+        Some("trailing "),
+        Some("bad,value"),
+        Some("bad;value"),
+        Some("я"),
+    ] {
+        let request_id = RequestId::resolve(candidate);
+        let parsed = uuid::Uuid::parse_str(request_id.as_str()).expect("generated UUID");
+
+        assert_eq!(parsed.get_version(), Some(Version::SortRand));
+    }
+}
+
+#[test]
+fn rejects_values_over_the_shared_limit() {
+    let oversized = "x".repeat(RequestId::MAX_LEN + 1);
+    let request_id = RequestId::resolve(Some(&oversized));
+
+    assert_ne!(request_id.as_str(), oversized);
+    assert_eq!(
+        uuid::Uuid::parse_str(request_id.as_str())
+            .expect("generated UUID")
+            .get_version(),
+        Some(Version::SortRand)
+    );
+}

crates/crank-observability/tests/critical_errors.rs

@@ -0,0 +1,33 @@
+use crank_observability::{CriticalErrorCategory, SentryConfig, SentryConfigError};
+
+#[test]
+fn missing_or_blank_dsn_disables_critical_error_channel() {
+    assert!(!SentryConfig::parse(None).expect("missing DSN").enabled());
+    assert!(!SentryConfig::parse(Some("")).expect("empty DSN").enabled());
+    assert!(
+        !SentryConfig::parse(Some("   "))
+            .expect("blank DSN")
+            .enabled()
+    );
+}
+
+#[test]
+fn invalid_explicit_dsn_is_rejected_without_echoing_the_value() {
+    let secret_value = "not-a-dsn?token=control-secret";
+    let error = SentryConfig::parse(Some(secret_value)).expect_err("invalid DSN must fail");
+
+    assert!(matches!(error, SentryConfigError::InvalidDsn));
+    assert!(!error.to_string().contains(secret_value));
+    assert!(!error.to_string().contains("control-secret"));
+}
+
+#[test]
+fn critical_error_categories_are_closed_and_stable() {
+    assert_eq!(CriticalErrorCategory::Panic.as_str(), "panic");
+    assert_eq!(CriticalErrorCategory::Startup.as_str(), "startup");
+    assert_eq!(CriticalErrorCategory::Internal.as_str(), "internal");
+    assert_eq!(
+        CriticalErrorCategory::DataIntegrity.as_str(),
+        "data_integrity"
+    );
+}

crates/crank-observability/tests/http_metrics.rs

@@ -0,0 +1,92 @@
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+use axum::{
+    Router,
+    body::{Body, to_bytes},
+    http::{Request, StatusCode},
+    middleware,
+    routing::get,
+};
+use crank_observability::{
+    MetricsConfig, ObservabilityConfig, RedactionLimits, ServiceIdentity, record_http_request,
+};
+use tower::ServiceExt;
+
+#[tokio::test]
+async fn http_metrics_use_matched_routes_and_closed_labels() {
+    let identity =
+        ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity");
+    let lifecycle = crank_observability::init(ObservabilityConfig::new(
+        identity,
+        "off",
+        RedactionLimits::default(),
+    ))
+    .expect("observability lifecycle");
+    let config = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
+        None,
+    )
+    .expect("loopback metrics");
+    let metrics = lifecycle.metrics_surface(config).router();
+    let app = Router::new()
+        .route(
+            "/documents/{document_id}",
+            get(|| async { StatusCode::NO_CONTENT }),
+        )
+        .layer(middleware::from_fn(record_http_request));
+
+    let sensitive_path_segment = "customer-secret-document-id";
+    for index in 0..100 {
+        let response = app
+            .clone()
+            .oneshot(
+                Request::get(format!("/documents/{sensitive_path_segment}-{index}"))
+                    .body(Body::empty())
+                    .expect("request"),
+            )
+            .await
+            .expect("response");
+        assert_eq!(response.status(), StatusCode::NO_CONTENT);
+    }
+
+    let response = app
+        .oneshot(
+            Request::get("/unknown/customer-controlled-path")
+                .body(Body::empty())
+                .expect("request"),
+        )
+        .await
+        .expect("response");
+    assert_eq!(response.status(), StatusCode::NOT_FOUND);
+
+    let response = metrics
+        .oneshot(
+            Request::get("/metrics")
+                .body(Body::empty())
+                .expect("request"),
+        )
+        .await
+        .expect("metrics response");
+    let body = to_bytes(response.into_body(), 1024 * 1024)
+        .await
+        .expect("bounded metrics body");
+    let body = String::from_utf8(body.to_vec()).expect("utf-8 metrics");
+
+    assert!(body.contains("crank_http_requests_total"));
+    assert!(body.contains("route=\"/documents/{document_id}\""));
+    assert!(body.contains("method=\"GET\""));
+    assert!(body.contains("status_class=\"2xx\""));
+    assert!(body.contains("crank_http_request_duration_seconds_bucket"));
+    assert!(!body.contains(sensitive_path_segment));
+    assert_eq!(
+        body.lines()
+            .filter(|line| {
+                line.starts_with("crank_http_requests_total{")
+                    && line.contains("route=\"/documents/{document_id}\"")
+            })
+            .count(),
+        1,
+        "different entity ids must not create additional series"
+    );
+}

crates/crank-observability/tests/incidents.rs

@@ -0,0 +1,15 @@
+use crank_observability::{
+    OperationalIncident, operational_incident_total, record_operational_incident,
+};
+
+#[test]
+fn history_loss_counter_has_no_dynamic_dimensions() {
+    let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+
+    record_operational_incident(OperationalIncident::InvocationHistoryLost);
+
+    assert!(
+        operational_incident_total(OperationalIncident::InvocationHistoryLost) > before,
+        "the closed incident counter must increase"
+    );
+}

crates/crank-observability/tests/json_logging.rs

@@ -0,0 +1,392 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber, safe_json,
+};
+use serde_json::Value;
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tracing_subscriber::fmt::MakeWriter;
+
+#[derive(Clone, Default)]
+struct SharedWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().expect("test writer lock").clone())
+            .expect("log output must be UTF-8")
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedWriter {
+    type Writer = SharedWriterGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedWriterGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedWriterGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedWriterGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer
+            .lock()
+            .map_err(|_| io::Error::other("test writer lock poisoned"))?
+            .extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+fn capture(service: &'static str, emit: impl FnOnce()) -> Vec<Value> {
+    capture_with_limits(service, RedactionLimits::default(), emit)
+}
+
+fn capture_with_limits(
+    service: &'static str,
+    limits: RedactionLimits,
+    emit: impl FnOnce(),
+) -> Vec<Value> {
+    let writer = SharedWriter::default();
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new(service, "0.3.1", "test").expect("valid test identity"),
+        "info",
+        limits,
+    );
+    let subscriber =
+        build_subscriber(config, writer.clone()).expect("test subscriber must be built");
+
+    tracing::subscriber::with_default(subscriber, emit);
+
+    writer
+        .output()
+        .lines()
+        .map(|line| {
+            assert!(!line.contains('\u{1b}'), "ANSI is forbidden: {line}");
+            serde_json::from_str(line).expect("every line must be one JSON object")
+        })
+        .collect()
+}
+
+#[test]
+fn schema_contract_is_identical_for_both_services() {
+    let outputs = ["admin-api", "mcp-server"].map(|service| {
+        capture(service, || {
+            tracing::info!(
+                name: "service.started",
+                target: "crank::startup",
+                port = 3101_u64,
+                "service started"
+            );
+        })
+    });
+
+    for (service, events) in ["admin-api", "mcp-server"].into_iter().zip(outputs.iter()) {
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(event["service"], service);
+        assert_eq!(event["version"], "0.3.1");
+        assert_eq!(event["environment"], "test");
+        assert_eq!(event["level"], "INFO");
+        assert_eq!(event["target"], "crank::startup");
+        assert_eq!(event["event"], "service.started");
+        assert!(event["fields"].is_object());
+        assert_eq!(event["fields"]["port"], 3101);
+        assert_eq!(event["fields"]["message"], "service started");
+        let timestamp = event["timestamp"].as_str().expect("timestamp string");
+        let parsed =
+            OffsetDateTime::parse(timestamp, &Rfc3339).expect("timestamp must be RFC 3339");
+        assert_eq!(parsed.offset(), time::UtcOffset::UTC);
+    }
+
+    let first_keys: Vec<_> = outputs[0][0]
+        .as_object()
+        .expect("object")
+        .keys()
+        .cloned()
+        .collect();
+    let second_keys: Vec<_> = outputs[1][0]
+        .as_object()
+        .expect("object")
+        .keys()
+        .cloned()
+        .collect();
+    assert_eq!(first_keys, second_keys);
+}
+
+#[test]
+fn correlation_fields_are_distinct_and_only_present_when_recorded() {
+    let present = capture("admin-api", || {
+        tracing::info!(
+            name: "admin.request.completed",
+            request_id = "req-123",
+            trace_id = "trace-456"
+        );
+    });
+    assert_eq!(present[0]["request_id"], "req-123");
+    assert_eq!(present[0]["trace_id"], "trace-456");
+    assert!(present[0]["fields"].get("request_id").is_none());
+    assert!(present[0]["fields"].get("trace_id").is_none());
+
+    let absent = capture("mcp-server", || {
+        tracing::info!(name: "mcp.request.completed", status = 200_u64);
+    });
+    assert!(absent[0].get("request_id").is_none());
+    assert!(absent[0].get("trace_id").is_none());
+}
+
+#[test]
+fn correlation_fields_preserve_scalar_display_values_before_field_limits() {
+    let limits = RedactionLimits {
+        max_object_fields: 1,
+        ..RedactionLimits::default()
+    };
+    let request_id = "123";
+    let events = capture_with_limits("admin-api", limits, || {
+        tracing::info!(
+            name: "admin.request.completed",
+            alpha = "field that consumes the object budget",
+            request_id = %request_id,
+            trace_id = true,
+        );
+    });
+
+    assert_eq!(events[0]["request_id"], "123");
+    assert_eq!(events[0]["trace_id"], "true");
+}
+
+#[test]
+fn empty_correlation_fields_are_omitted() {
+    let events = capture("admin-api", || {
+        tracing::info!(
+            name: "admin.request.completed",
+            request_id = "",
+            trace_id = ""
+        );
+    });
+
+    assert!(events[0].get("request_id").is_none());
+    assert!(events[0].get("trace_id").is_none());
+}
+
+#[test]
+fn formatter_redacts_fields_before_serialization() {
+    let context = safe_json(
+        &serde_json::json!({
+            "nested": {
+                "access_token": "nested-canary-secret",
+                "endpoint": "https://example.test/private?key=nested-canary-secret"
+            }
+        }),
+        RedactionLimits::default(),
+    )
+    .expect("safe nested context");
+    let events = capture("admin-api", || {
+        tracing::warn!(
+            name: "admin.request.rejected",
+            password = "canary-secret",
+            url = "https://example.test/path?token=canary-secret",
+            safe_fields = %context,
+            unsafe_context = ?serde_json::json!({"password": "debug-canary-secret"}),
+            error_code = "invalid_request"
+        );
+    });
+    let serialized = serde_json::to_string(&events[0]).expect("event JSON");
+
+    assert_eq!(events[0]["fields"]["password"], "[REDACTED]");
+    assert_eq!(events[0]["fields"]["url"], "https://example.test/path");
+    assert_eq!(
+        events[0]["fields"]["safe_fields"]["nested"]["access_token"],
+        "[REDACTED]"
+    );
+    assert_eq!(
+        events[0]["fields"]["safe_fields"]["nested"]["endpoint"],
+        "https://example.test/private"
+    );
+    assert_eq!(events[0]["fields"]["unsafe_context"], "[REDACTED]");
+    assert_eq!(events[0]["fields"]["error_code"], "invalid_request");
+    assert!(!serialized.contains("canary-secret"));
+}
+
+#[test]
+fn arbitrary_debug_text_is_never_written_verbatim() {
+    struct Credentials;
+
+    impl std::fmt::Debug for Credentials {
+        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            formatter.write_str("Credentials { password: \"debug-canary-secret\" }")
+        }
+    }
+
+    let events = capture("admin-api", || {
+        tracing::warn!(
+            name: "admin.debug.inspected",
+            details = ?Credentials
+        );
+    });
+    let serialized = serde_json::to_string(&events[0]).expect("event JSON");
+
+    assert_eq!(events[0]["fields"]["details"], "[REDACTED]");
+    assert!(!serialized.contains("debug-canary-secret"));
+}
+
+#[test]
+fn compound_sensitive_event_fields_are_redacted() {
+    let events = capture("admin-api", || {
+        tracing::warn!(
+            name: "admin.request.rejected",
+            client_api_key = "client-canary-secret",
+            authorization_header = "Bearer auth-canary-secret",
+            response_body = "response-canary-secret",
+            tool_arguments = "argument-canary-secret",
+            query_params = "query-canary-secret",
+        );
+    });
+    let serialized = serde_json::to_string(&events[0]).expect("event JSON");
+
+    for key in [
+        "client_api_key",
+        "authorization_header",
+        "response_body",
+        "tool_arguments",
+        "query_params",
+    ] {
+        assert_eq!(events[0]["fields"][key], "[REDACTED]");
+    }
+    assert!(!serialized.contains("canary-secret"));
+}
+
+#[test]
+fn oversized_event_falls_back_to_valid_bounded_json() {
+    let limits = RedactionLimits {
+        max_event_bytes: 512,
+        ..RedactionLimits::default()
+    };
+    let events = capture_with_limits("admin-api", limits, || {
+        tracing::info!(
+            name: "admin.payload.inspected",
+            description = %"x".repeat(1024)
+        );
+    });
+    let serialized = serde_json::to_vec(&events[0]).expect("bounded event JSON");
+
+    assert!(serialized.len() < limits.max_event_bytes);
+    assert_eq!(events[0]["fields"]["truncated"], true);
+}
+
+#[test]
+fn safe_json_honours_the_total_event_budget() {
+    let limits = RedactionLimits {
+        max_event_bytes: 512,
+        ..RedactionLimits::default()
+    };
+
+    let serialized = safe_json(
+        &serde_json::json!({"description": "x".repeat(4096)}),
+        limits,
+    )
+    .expect("safe JSON must remain serializable");
+
+    assert!(serialized.len() <= limits.max_event_bytes);
+    assert_eq!(
+        serde_json::from_str::<Value>(&serialized).expect("valid JSON")["truncated"],
+        true
+    );
+}
+
+#[test]
+fn subscriber_rejects_limits_that_cannot_hold_an_event() {
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
+        "info",
+        RedactionLimits {
+            max_event_bytes: 16,
+            ..RedactionLimits::default()
+        },
+    );
+
+    assert!(build_subscriber(config, SharedWriter::default()).is_err());
+}
+
+#[test]
+fn minimum_event_budget_handles_maximum_identity_labels() {
+    let writer = SharedWriter::default();
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new("s".repeat(64), "v".repeat(64), "e".repeat(64))
+            .expect("maximum identity labels are valid"),
+        "info",
+        RedactionLimits {
+            max_event_bytes: 512,
+            ..RedactionLimits::default()
+        },
+    );
+    let subscriber =
+        build_subscriber(config, writer.clone()).expect("minimum valid budget must be usable");
+
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::info!(
+            name: "event-name-that-is-intentionally-longer-than-the-fallback-limit",
+            description = %"x".repeat(4096),
+        );
+    });
+
+    let output = writer.output();
+    assert!(output.len() <= 512);
+    assert_eq!(output.lines().count(), 1);
+    serde_json::from_str::<Value>(output.trim_end()).expect("bounded line must remain valid JSON");
+}
+
+#[test]
+fn env_filter_is_applied_and_invalid_filter_is_safe() {
+    let writer = SharedWriter::default();
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
+        "warn",
+        RedactionLimits::default(),
+    );
+    let subscriber =
+        build_subscriber(config, writer.clone()).expect("test subscriber must be built");
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::info!(name: "filtered.info", "filtered");
+        tracing::warn!(name: "visible.warning", "visible");
+    });
+    let output = writer.output();
+
+    assert!(!output.contains("filtered.info"));
+    assert!(output.contains("visible.warning"));
+
+    let invalid = ObservabilityConfig::new(
+        ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
+        "[not a valid filter",
+        RedactionLimits::default(),
+    );
+    let error = build_subscriber(invalid, SharedWriter::default())
+        .err()
+        .expect("invalid filter must fail");
+    assert_eq!(error.to_string(), "invalid log filter");
+    assert!(!error.to_string().contains("not a valid filter"));
+}
+
+#[test]
+fn service_identity_rejects_empty_or_unsafe_labels() {
+    for (service, version, environment) in [
+        ("", "0.3.1", "test"),
+        ("admin api", "0.3.1", "test"),
+        ("admin-api", "", "test"),
+        ("admin-api", "0.3.1", "prod\nsecret"),
+    ] {
+        assert!(ServiceIdentity::try_new(service, version, environment).is_err());
+    }
+}

crates/crank-observability/tests/lifecycle.rs

@@ -0,0 +1,22 @@
+use crank_observability::{
+    ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity, init,
+};
+
+fn config() -> ObservabilityConfig {
+    ObservabilityConfig::new(
+        ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"),
+        "info",
+        RedactionLimits::default(),
+    )
+}
+
+#[test]
+fn repeated_global_initialization_returns_typed_error() {
+    let _lifecycle = init(config()).expect("first initialization must succeed");
+    let error = init(config()).expect_err("second initialization must fail");
+
+    assert!(matches!(
+        error,
+        ObservabilityInitError::SubscriberAlreadyInitialized
+    ));
+}

crates/crank-observability/tests/otlp.rs

@@ -0,0 +1,56 @@
+use std::time::Duration;
+
+use crank_observability::{OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError};
+
+#[test]
+fn absent_endpoint_disables_export_without_background_resources() {
+    let config = OtlpTraceConfig::try_new(
+        None,
+        None,
+        Duration::from_secs(10),
+        OtlpBatchConfig::default(),
+    )
+    .expect("missing endpoint must be valid");
+
+    assert!(!config.is_enabled());
+}
+
+#[test]
+fn explicit_config_accepts_only_bounded_http_protobuf() {
+    let config = OtlpTraceConfig::try_new(
+        Some("https://collector.example.test/v1/traces".to_owned()),
+        Some("http/protobuf".to_owned()),
+        Duration::from_secs(3),
+        OtlpBatchConfig::try_new(256, 64, Duration::from_millis(500), Duration::from_secs(3))
+            .unwrap(),
+    )
+    .expect("bounded HTTP protobuf config must be valid");
+
+    assert!(config.is_enabled());
+    assert_eq!(config.export_timeout(), Duration::from_secs(3));
+    assert_eq!(config.batch().max_queue_size(), 256);
+    assert_eq!(config.batch().max_export_batch_size(), 64);
+}
+
+#[test]
+fn invalid_values_return_safe_typed_errors() {
+    let secret_endpoint = "https://user:canary-secret@collector.example.test/v1/traces";
+    let error = OtlpTraceConfig::try_new(
+        Some(secret_endpoint.to_owned()),
+        Some("grpc".to_owned()),
+        Duration::ZERO,
+        OtlpBatchConfig::default(),
+    )
+    .expect_err("credentials in endpoint must be rejected");
+
+    assert!(matches!(
+        error,
+        OtlpTraceConfigError::InvalidEndpoint { .. }
+    ));
+    assert!(!error.to_string().contains(secret_endpoint));
+    assert!(!error.to_string().contains("canary-secret"));
+
+    let error = OtlpBatchConfig::try_new(8, 9, Duration::from_millis(1), Duration::from_secs(1))
+        .expect_err("batch cannot exceed queue");
+    assert!(matches!(error, OtlpTraceConfigError::InvalidBatchLimits));
+}

crates/crank-observability/tests/prometheus.rs

@@ -0,0 +1,134 @@
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+use axum::{
+    body::{Body, to_bytes},
+    http::{Request, StatusCode, header},
+};
+use crank_observability::{
+    DURATION_BUCKETS_SECONDS, MetricsConfig, MetricsConfigError, MetricsSurface, ServiceIdentity,
+    metric_schema,
+};
+use tower::ServiceExt;
+
+fn identity() -> ServiceIdentity {
+    ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity")
+}
+
+#[test]
+fn loopback_is_allowed_without_a_token() {
+    let config = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
+        None,
+    )
+    .expect("loopback metrics must be safe by default");
+
+    assert_eq!(config.bind_addr().to_string(), "127.0.0.1:9464");
+    assert!(!config.requires_authentication());
+}
+
+#[test]
+fn non_loopback_without_a_token_is_rejected_without_secret_data() {
+    let error = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9464),
+        None,
+    )
+    .expect_err("external metrics must require authentication");
+
+    assert!(matches!(
+        error,
+        MetricsConfigError::MissingTokenForExternalBind
+    ));
+    assert!(!error.to_string().contains("token="));
+}
+
+#[test]
+fn schema_is_closed_and_uses_fixed_duration_buckets() {
+    let schema = metric_schema();
+    let names: Vec<_> = schema.iter().map(|metric| metric.name).collect();
+
+    assert!(names.contains(&"crank_http_requests_total"));
+    assert!(names.contains(&"crank_http_request_duration_seconds"));
+    assert!(names.contains(&"crank_mcp_requests_total"));
+    assert!(names.contains(&"crank_tool_invocations_total"));
+    assert!(names.contains(&"crank_runtime_inflight"));
+    assert!(names.contains(&"crank_db_pool_connections"));
+    assert!(names.contains(&"crank_catalog_tools"));
+    assert!(names.contains(&"crank_invocation_history_lost_total"));
+    assert!(names.contains(&"crank_telemetry_export_failures_total"));
+
+    for metric in schema {
+        for forbidden in [
+            "workspace",
+            "agent_id",
+            "operation_id",
+            "request_id",
+            "url",
+            "error_message",
+            "text",
+        ] {
+            assert!(
+                !metric.labels.contains(&forbidden),
+                "{} exposes forbidden label {forbidden}",
+                metric.name
+            );
+        }
+    }
+
+    assert_eq!(
+        DURATION_BUCKETS_SECONDS,
+        &[
+            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0
+        ]
+    );
+}
+
+#[tokio::test]
+async fn external_surface_protects_both_routes_and_exposes_nothing_else() {
+    let token = "metrics-canary-secret";
+    let config = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9464),
+        Some(token.to_owned()),
+    )
+    .expect("external metrics with token");
+    let surface = MetricsSurface::for_test(config, identity()).expect("test metrics surface");
+    let app = surface.router();
+
+    for path in ["/metrics", "/health"] {
+        let unauthorized = app
+            .clone()
+            .oneshot(Request::get(path).body(Body::empty()).expect("request"))
+            .await
+            .expect("response");
+        assert_eq!(unauthorized.status(), StatusCode::UNAUTHORIZED);
+
+        let authorized = app
+            .clone()
+            .oneshot(
+                Request::get(path)
+                    .header(header::AUTHORIZATION, format!("Bearer {token}"))
+                    .body(Body::empty())
+                    .expect("request"),
+            )
+            .await
+            .expect("response");
+        assert_eq!(authorized.status(), StatusCode::OK);
+        let body = to_bytes(authorized.into_body(), 1024 * 1024)
+            .await
+            .expect("bounded body");
+        assert!(!String::from_utf8_lossy(&body).contains(token));
+    }
+
+    let absent = app
+        .oneshot(
+            Request::get("/api/operations")
+                .header(header::AUTHORIZATION, format!("Bearer {token}"))
+                .body(Body::empty())
+                .expect("request"),
+        )
+        .await
+        .expect("response");
+    assert_eq!(absent.status(), StatusCode::NOT_FOUND);
+}

crates/crank-observability/tests/propagation.rs

@@ -0,0 +1,65 @@
+use axum::http::{HeaderMap, HeaderValue};
+use crank_observability::{inject_current_trace_context, set_remote_trace_parent};
+use opentelemetry::{global, trace::TracerProvider as _};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
+use tracing::info_span;
+use tracing_subscriber::layer::SubscriberExt;
+
+const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
+
+#[test]
+fn valid_remote_parent_is_continued_and_request_id_is_unrelated() {
+    with_trace_dispatch(|| {
+        let mut incoming = HeaderMap::new();
+        incoming.insert(
+            "traceparent",
+            HeaderValue::from_static("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+        );
+        incoming.insert("x-request-id", HeaderValue::from_static("req-unrelated"));
+
+        let span = info_span!("http.request", request_id = "req-unrelated");
+        assert!(set_remote_trace_parent(&span, &incoming));
+        let _guard = span.enter();
+
+        let mut outgoing = HeaderMap::new();
+        assert!(inject_current_trace_context(&mut outgoing));
+        let propagated = outgoing["traceparent"].to_str().unwrap();
+        assert_eq!(&propagated[3..35], REMOTE_TRACE_ID);
+        assert!(!propagated.contains("req-unrelated"));
+        assert!(!outgoing.contains_key("baggage"));
+    });
+}
+
+#[test]
+fn invalid_parent_is_ignored_and_a_new_trace_is_created() {
+    with_trace_dispatch(|| {
+        let mut incoming = HeaderMap::new();
+        incoming.insert(
+            "traceparent",
+            HeaderValue::from_static("canary-invalid-traceparent"),
+        );
+
+        let span = info_span!("mcp.request");
+        assert!(!set_remote_trace_parent(&span, &incoming));
+        let _guard = span.enter();
+
+        let mut outgoing = HeaderMap::new();
+        assert!(inject_current_trace_context(&mut outgoing));
+        let propagated = outgoing["traceparent"].to_str().unwrap();
+        assert!(propagated.starts_with("00-"));
+        assert!(!propagated.contains("canary-invalid-traceparent"));
+    });
+}
+
+fn with_trace_dispatch(test: impl FnOnce()) {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("propagation-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _guard = tracing::dispatcher::set_default(&dispatch);
+
+    test();
+    provider.shutdown().unwrap();
+}

crates/crank-observability/tests/redaction.rs

@@ -0,0 +1,206 @@
+use crank_observability::{RedactionLimits, redact_value};
+use serde_json::{Value, json};
+
+const REDACTED: &str = "[REDACTED]";
+const TRUNCATED: &str = "[TRUNCATED]";
+
+#[test]
+fn sensitive_keys_are_redacted_case_insensitively() {
+    let keys = [
+        "password",
+        "PassWord",
+        "api_key",
+        "access-token",
+        "authorization",
+        "Proxy.Authorization",
+        "cookie",
+        "set_cookie",
+        "payload",
+        "request_body",
+        "arguments",
+        "result",
+        "response",
+    ];
+
+    for key in keys {
+        let cleaned = redact_value(&json!({ key: "canary-secret" }), RedactionLimits::default());
+        assert_eq!(cleaned[key], REDACTED, "key {key} was not redacted");
+        assert!(!cleaned.to_string().contains("canary-secret"));
+    }
+}
+
+#[test]
+fn generated_sensitive_key_variants_are_redacted() {
+    for canonical in [
+        "password",
+        "api_key",
+        "access_key",
+        "secret_key",
+        "proxy_authorization",
+        "set_cookie",
+        "query_string",
+        "request_body",
+    ] {
+        for separator in ["_", "-", "."] {
+            let variant = canonical
+                .split('_')
+                .collect::<Vec<_>>()
+                .join(separator)
+                .to_ascii_uppercase();
+            let cleaned = redact_value(
+                &json!({ variant.clone(): "canary-secret" }),
+                RedactionLimits::default(),
+            );
+
+            assert_eq!(
+                cleaned[&variant], REDACTED,
+                "key {variant} was not redacted"
+            );
+            assert!(!cleaned.to_string().contains("canary-secret"));
+        }
+    }
+}
+
+#[test]
+fn compound_sensitive_key_names_are_redacted() {
+    for key in [
+        "client_api_key",
+        "aws_access_key_id",
+        "http_authorization_header",
+        "response_body",
+        "tool_arguments",
+        "query_params",
+        "secret_value",
+    ] {
+        let cleaned = redact_value(&json!({ key: "canary-secret" }), RedactionLimits::default());
+
+        assert_eq!(cleaned[key], REDACTED, "key {key} was not redacted");
+        assert!(!cleaned.to_string().contains("canary-secret"));
+    }
+}
+
+#[test]
+fn url_query_and_fragment_are_removed_at_every_depth() {
+    let input = json!({
+        "url": "https://example.test/path?token=canary-secret#fragment",
+        "nested": [{
+            "endpoint_uri": "https://example.test/other?q=canary-secret"
+        }]
+    });
+
+    let cleaned = redact_value(&input, RedactionLimits::default());
+
+    assert_eq!(cleaned["url"], "https://example.test/path");
+    assert_eq!(
+        cleaned["nested"][0]["endpoint_uri"],
+        "https://example.test/other"
+    );
+    assert!(!cleaned.to_string().contains("canary-secret"));
+}
+
+#[test]
+fn url_credentials_and_compound_endpoint_fields_are_removed() {
+    let input = json!({
+        "upstream_endpoint": "https://user:canary-secret@example.test/path?token=canary-secret",
+    });
+
+    let cleaned = redact_value(&input, RedactionLimits::default());
+
+    assert_eq!(cleaned["upstream_endpoint"], "https://example.test/path");
+    assert!(!cleaned.to_string().contains("user"));
+    assert!(!cleaned.to_string().contains("canary-secret"));
+}
+
+#[test]
+fn nested_values_and_collections_respect_all_limits() {
+    let limits = RedactionLimits {
+        max_string_bytes: 16,
+        max_array_items: 3,
+        max_object_fields: 3,
+        max_depth: 2,
+        max_event_bytes: 256,
+    };
+    let input = json!({
+        "long": "абвгдежзийклмнопрсту",
+        "array": [1, 2, 3, 4, 5],
+        "object": {"a": 1, "b": 2, "c": 3, "d": 4},
+        "nested": {"level2": {"level3": "must not survive"}}
+    });
+
+    let cleaned = redact_value(&input, limits);
+    let object = cleaned
+        .as_object()
+        .expect("cleaned root must remain an object");
+
+    assert!(object.len() <= limits.max_object_fields);
+    assert!(
+        cleaned["long"]
+            .as_str()
+            .map(|value| value.len() <= limits.max_string_bytes)
+            .unwrap_or(true)
+    );
+    assert!(
+        cleaned["array"]
+            .as_array()
+            .map(|value| value.len() <= limits.max_array_items)
+            .unwrap_or(true)
+    );
+    assert!(!cleaned.to_string().contains("must not survive"));
+    assert!(cleaned.to_string().contains(TRUNCATED));
+}
+
+#[test]
+fn truncation_preserves_utf8_and_never_reveals_secret_fragments() {
+    let input = json!({
+        "secret_key": "секретное-значение",
+        "description": "я".repeat(2048),
+    });
+
+    let cleaned = redact_value(&input, RedactionLimits::default());
+    let serialized = serde_json::to_string(&cleaned).expect("cleaned value must be valid JSON");
+
+    assert_eq!(cleaned["secret_key"], REDACTED);
+    assert!(!serialized.contains("секретное"));
+    assert!(cleaned["description"].as_str().is_some());
+}
+
+#[test]
+fn redacted_value_does_not_mutate_input() {
+    let input = json!({"password": "canary-secret"});
+    let original = input.clone();
+
+    let _ = redact_value(&input, RedactionLimits::default());
+
+    assert_eq!(input, original);
+}
+
+#[test]
+fn object_keys_respect_the_string_limit() {
+    let limits = RedactionLimits {
+        max_string_bytes: 16,
+        ..RedactionLimits::default()
+    };
+    let long_key = format!("field-{}", "x".repeat(128));
+
+    let cleaned = redact_value(&json!({ long_key: "value" }), limits);
+
+    assert!(
+        cleaned
+            .as_object()
+            .expect("cleaned object")
+            .keys()
+            .all(|key| key.len() <= limits.max_string_bytes)
+    );
+}
+
+#[test]
+fn limits_have_finite_safe_defaults() {
+    let limits = RedactionLimits::default();
+
+    assert_eq!(limits.max_string_bytes, 1024);
+    assert_eq!(limits.max_array_items, 32);
+    assert_eq!(limits.max_object_fields, 64);
+    assert_eq!(limits.max_depth, 8);
+    assert_eq!(limits.max_event_bytes, 16 * 1024);
+    assert!(Value::Null.is_null());
+}