наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

crates/crank-adapter-rest/Cargo.toml

@@ -3,18 +3,26 @@ name = "crank-adapter-rest"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]
 async-trait = "0.1"
 crank-core = { path = "../crank-core" }
+crank-trace = { path = "../crank-trace" }
 futures-util = "0.3"
+metrics.workspace = true
+opentelemetry.workspace = true
 reqwest = { workspace = true, features = ["stream"] }
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
 
 [dev-dependencies]
 axum.workspace = true
+opentelemetry_sdk.workspace = true
 tokio.workspace = true
+tracing-subscriber.workspace = true

crates/crank-adapter-rest/src/client.rs

@@ -7,7 +7,9 @@ use std::{
 };
 
 use crank_core::{HttpMethod, RestTarget};
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
 use futures_util::StreamExt;
+use opentelemetry::{global, propagation::Injector, trace::TraceContextExt};
 use reqwest::{
     Client,
     dns::{Addrs, Name, Resolve, Resolving},
@@ -15,6 +17,8 @@ use reqwest::{
     redirect,
 };
 use serde_json::Value;
+use tracing::{Instrument, Span};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 use crate::{RestAdapterError, RestRequest, RestResponse};
 
@@ -66,10 +70,37 @@ impl RestAdapter {
         &self,
         target: &RestTarget,
         request: &RestRequest,
+    ) -> Result<RestResponse, RestAdapterError> {
+        let started_at = std::time::Instant::now();
+        let result = self.execute_inner(target, request).await;
+        let outcome = match &result {
+            Ok(_) => "success",
+            Err(error) => upstream_outcome(error),
+        };
+        metrics::counter!(
+            "crank_upstream_requests_total",
+            "operation_kind" => "rest",
+            "outcome" => outcome
+        )
+        .increment(1);
+        metrics::histogram!(
+            "crank_upstream_request_duration_seconds",
+            "operation_kind" => "rest",
+            "outcome" => outcome
+        )
+        .record(started_at.elapsed().as_secs_f64());
+        result
+    }
+
+    async fn execute_inner(
+        &self,
+        target: &RestTarget,
+        request: &RestRequest,
     ) -> Result<RestResponse, RestAdapterError> {
         let url = build_url(target, request)?;
         self.policy.validate_url(&url)?;
-        let headers = build_headers(target, request)?;
+        let mut headers = build_headers(target, request)?;
+        apply_current_trace_context(&mut headers);
         let client =
             self.client
                 .as_ref()
@@ -85,23 +116,60 @@ impl RestAdapter {
             builder = builder.json(body);
         }
 
-        let response = builder.send().await?;
-        let status = response.status();
-        let headers = normalize_headers(response.headers());
-        let body = decode_body(response, self.policy.max_response_bytes).await?;
+        let upstream_span = Stage::UpstreamHttp.span();
+        let result = async {
+            let response = builder.send().await?;
+            let status = response.status();
+            let headers = normalize_headers(response.headers());
+            let body = decode_body(response, self.policy.max_response_bytes).await?;
 
-        if !status.is_success() {
-            return Err(RestAdapterError::UnexpectedStatus {
-                status: status.as_u16(),
+            if !status.is_success() {
+                return Err(RestAdapterError::UnexpectedStatus {
+                    status: status.as_u16(),
+                    body,
+                });
+            }
+
+            Ok(RestResponse {
+                status_code: status.as_u16(),
+                headers,
                 body,
-            });
+            })
         }
+        .instrument(upstream_span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&upstream_span),
+            Err(_) => {
+                StageOutcome::Error.record(&upstream_span);
+                ErrorCategory::Upstream.record(&upstream_span);
+            }
+        }
+        result
+    }
+}
 
-        Ok(RestResponse {
-            status_code: status.as_u16(),
-            headers,
-            body,
-        })
+fn upstream_outcome(error: &RestAdapterError) -> &'static str {
+    match error {
+        RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
+            "client_error"
+        }
+        RestAdapterError::UnexpectedStatus { status, .. } if (500..600).contains(status) => {
+            "server_error"
+        }
+        RestAdapterError::UnexpectedStatus { .. } => "unexpected_status",
+        RestAdapterError::Transport(error) if error.is_timeout() => "timeout",
+        RestAdapterError::Transport(_) => "transport_error",
+        RestAdapterError::ResponseTooLarge { .. } => "response_too_large",
+        RestAdapterError::TargetNotAllowed { .. } => "rejected",
+        RestAdapterError::WindowExpired => "window_expired",
+        RestAdapterError::InvalidSseEvent => "invalid_response",
+        RestAdapterError::InvalidBaseUrl { .. }
+        | RestAdapterError::InvalidPathParameter { .. }
+        | RestAdapterError::InvalidQueryParameter { .. }
+        | RestAdapterError::InvalidHeaderName { .. }
+        | RestAdapterError::InvalidHeaderValue { .. } => "invalid_request",
+        RestAdapterError::InvalidConfiguration { .. } => "configuration",
     }
 }
 
@@ -407,6 +475,9 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
         HeaderName::try_from(name).map_err(|_| RestAdapterError::InvalidHeaderName {
             header: name.to_owned(),
         })?;
+    if is_trace_propagation_header(&header_name) {
+        return Ok(());
+    }
     let header_value =
         HeaderValue::try_from(value).map_err(|_| RestAdapterError::InvalidHeaderValue {
             header: name.to_owned(),
@@ -416,6 +487,38 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
     Ok(())
 }
 
+fn is_trace_propagation_header(name: &HeaderName) -> bool {
+    matches!(name.as_str(), "traceparent" | "tracestate" | "baggage")
+}
+
+fn apply_current_trace_context(headers: &mut HeaderMap) {
+    for header in ["traceparent", "tracestate", "baggage"] {
+        headers.remove(header);
+    }
+
+    let context = Span::current().context();
+    if !context.span().span_context().is_valid() {
+        return;
+    }
+    global::get_text_map_propagator(|propagator| {
+        propagator.inject_context(&context, &mut ReqwestHeaderInjector(headers));
+    });
+}
+
+struct ReqwestHeaderInjector<'a>(&'a mut HeaderMap);
+
+impl Injector for ReqwestHeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(name) = HeaderName::try_from(key) else {
+            return;
+        };
+        let Ok(value) = HeaderValue::try_from(value) else {
+            return;
+        };
+        self.0.insert(name, value);
+    }
+}
+
 async fn decode_body(
     response: reqwest::Response,
     max_response_bytes: usize,

crates/crank-adapter-rest/src/lib.rs

@@ -26,13 +26,15 @@ impl ProtocolAdapter for RestAdapter {
         &self,
         target: &Target,
         prepared: &PreparedRequest,
-        _context: &RuntimeRequestContext,
+        context: &RuntimeRequestContext,
     ) -> Result<AdapterResponse, ProtocolAdapterError> {
         let target = rest_target(target)?;
+        let mut headers = prepared.headers.clone();
+        headers.extend(context.outbound_headers());
         let request = RestRequest {
             path_params: prepared.path_params.clone(),
             query_params: prepared.query_params.clone(),
-            headers: prepared.headers.clone(),
+            headers,
             body: prepared.body.clone(),
             timeout_ms: prepared.timeout_ms,
         };

crates/crank-adapter-rest/tests/integration/client.rs

@@ -8,9 +8,19 @@ use axum::{
     routing::{get, post},
 };
 use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
-use crank_core::{HttpMethod, RestTarget};
+use crank_core::{
+    HttpMethod, PreparedRequest, ProtocolAdapter, RestTarget, RuntimeRequestContext, Target,
+};
+use opentelemetry::{
+    global,
+    trace::{TraceContextExt, TracerProvider as _},
+};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
 use serde_json::{Value, json};
 use tokio::net::TcpListener;
+use tracing::Instrument;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+use tracing_subscriber::layer::SubscriberExt;
 
 #[tokio::test]
 async fn executes_rest_request_and_normalizes_json_response() {
@@ -45,6 +55,120 @@ async fn executes_rest_request_and_normalizes_json_response() {
     );
 }
 
+#[tokio::test]
+async fn protocol_context_overrides_mapped_correlation_headers() {
+    let base_url = spawn_test_server().await;
+    let adapter = test_adapter();
+    let target = Target::Rest(RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::from([
+            ("x-request-id".to_owned(), "static-request".to_owned()),
+            (
+                "x-correlation-id".to_owned(),
+                "static-correlation".to_owned(),
+            ),
+        ]),
+    });
+    let prepared = PreparedRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        headers: BTreeMap::from([
+            ("x-request-id".to_owned(), "mapped-request".to_owned()),
+            (
+                "x-correlation-id".to_owned(),
+                "mapped-correlation".to_owned(),
+            ),
+        ]),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+        ..PreparedRequest::default()
+    };
+    let context = RuntimeRequestContext::new("req-runtime", "corr-runtime");
+
+    let response = adapter
+        .invoke_unary(&target, &prepared, &context)
+        .await
+        .unwrap();
+
+    assert_eq!(response.body["request_id"], "req-runtime");
+    assert_eq!(response.body["correlation_id"], "corr-runtime");
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn current_trace_context_overrides_mapped_traceparent() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("rest-propagation-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let span = tracing::info_span!("runtime.execute");
+    let context = span.context();
+    let expected_trace_id = context.span().span_context().trace_id().to_string();
+    let base_url = spawn_test_server().await;
+    let target = RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::new(),
+    };
+    let request = RestRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        query_params: BTreeMap::new(),
+        headers: BTreeMap::from([(
+            "traceparent".to_owned(),
+            "00-11111111111111111111111111111111-2222222222222222-01".to_owned(),
+        )]),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+    };
+
+    let response = test_adapter()
+        .execute(&target, &request)
+        .instrument(span)
+        .await
+        .unwrap();
+
+    assert_eq!(
+        &response.body["traceparent"].as_str().unwrap()[3..35],
+        expected_trace_id
+    );
+    provider.shutdown().unwrap();
+}
+
+#[tokio::test]
+async fn user_configured_propagation_headers_are_removed_without_trusted_context() {
+    let base_url = spawn_test_server().await;
+    let target = RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::from([
+            (
+                "traceparent".to_owned(),
+                "untrusted\ninvalid-value".to_owned(),
+            ),
+            ("tracestate".to_owned(), "vendor=value".to_owned()),
+            ("baggage".to_owned(), "secret=must-not-leave".to_owned()),
+        ]),
+    };
+    let request = RestRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        query_params: BTreeMap::new(),
+        headers: BTreeMap::new(),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+    };
+
+    let response = test_adapter().execute(&target, &request).await.unwrap();
+
+    assert!(response.body.get("traceparent").is_none());
+    assert!(response.body.get("tracestate").is_none());
+    assert!(response.body.get("baggage").is_none());
+}
+
 #[tokio::test]
 async fn returns_unexpected_status_with_normalized_body() {
     let base_url = spawn_test_server().await;
@@ -186,14 +310,57 @@ async fn create_user(
         .get("x-static")
         .and_then(|value| value.to_str().ok())
         .unwrap_or_default();
+    let request_id = headers
+        .get("x-request-id")
+        .and_then(|value| value.to_str().ok());
+    let correlation_id = headers
+        .get("x-correlation-id")
+        .and_then(|value| value.to_str().ok());
+    let traceparent = headers
+        .get("traceparent")
+        .and_then(|value| value.to_str().ok());
+    let tracestate = headers
+        .get("tracestate")
+        .and_then(|value| value.to_str().ok());
+    let baggage = headers.get("baggage").and_then(|value| value.to_str().ok());
 
-    Json(json!({
+    let mut response = json!({
         "id": user_id,
         "query": query.get("expand").cloned().unwrap_or_default(),
         "trace": trace,
         "static": static_header,
         "payload": payload
-    }))
+    });
+    let response = response.as_object_mut().unwrap();
+    if let Some(request_id) = request_id {
+        response.insert(
+            "request_id".to_owned(),
+            Value::String(request_id.to_owned()),
+        );
+    }
+    if let Some(correlation_id) = correlation_id {
+        response.insert(
+            "correlation_id".to_owned(),
+            Value::String(correlation_id.to_owned()),
+        );
+    }
+    if let Some(traceparent) = traceparent {
+        response.insert(
+            "traceparent".to_owned(),
+            Value::String(traceparent.to_owned()),
+        );
+    }
+    if let Some(tracestate) = tracestate {
+        response.insert(
+            "tracestate".to_owned(),
+            Value::String(tracestate.to_owned()),
+        );
+    }
+    if let Some(baggage) = baggage {
+        response.insert("baggage".to_owned(), Value::String(baggage.to_owned()));
+    }
+
+    Json(Value::Object(response.clone()))
 }
 
 async fn fail() -> (axum::http::StatusCode, Json<Value>) {