feat: harden community production foundation through story 1.5

2026-08-14 00:21:59 +03:00
parent c30461cc92
commit f6fc2e5c9b
161 changed files with 16758 additions and 2515 deletions
@@ -3,10 +3,10 @@ use std::{
    sync::{Arc, Mutex},
 };

-use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
+use admin_api::request_context::{REQUEST_ID_HEADER, TRACE_ID_HEADER, apply_request_context};
 use axum::{
    Router,
-    body::Body,
+    body::{Body, to_bytes},
    http::{HeaderMap, HeaderValue, Request, StatusCode},
    routing::get,
 };
@@ -64,6 +64,11 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
        .find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
        .unwrap();
    assert_eq!(event["request_id"], "req_admin_trace_123");
+    let trace_id = response.headers()[TRACE_ID_HEADER.as_str()]
+        .to_str()
+        .unwrap();
+    assert_eq!(trace_id.len(), 32);
+    assert_eq!(event["trace_id"], trace_id);
    assert_eq!(event["fields"]["status"], 200);
    assert_eq!(event["fields"]["route"], "/probe");

@@ -85,9 +90,144 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
        uuid::Uuid::parse_str(generated).unwrap().get_version(),
        Some(Version::SortRand)
    );
+    let generated_trace = invalid_response.headers()[TRACE_ID_HEADER.as_str()]
+        .to_str()
+        .unwrap();
+    assert_eq!(generated_trace.len(), 32);
+    assert_ne!(generated_trace, "canary-invalid-traceparent");
    assert!(!writer.output().contains("canary-invalid-traceparent"));
 }

+#[tokio::test(flavor = "current_thread")]
+async fn structured_boundary_error_carries_the_same_safe_ids() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    let dispatch = tracing::Dispatch::new(tracing_subscriber::registry());
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let response = error_probe_app()
+        .oneshot(
+            Request::builder()
+                .uri("/error")
+                .header("x-request-id", "request-boundary-error")
+                .header(
+                    "traceparent",
+                    "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+                )
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+    assert_eq!(
+        response.headers()[REQUEST_ID_HEADER],
+        "request-boundary-error"
+    );
+    assert_eq!(
+        response.headers()[TRACE_ID_HEADER],
+        "0af7651916cd43dd8448eb211c80319c"
+    );
+    let body = to_bytes(response.into_body(), 16 * 1024).await.unwrap();
+    let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
+    assert_eq!(payload["error"]["request_id"], "request-boundary-error");
+    assert_eq!(
+        payload["error"]["trace_id"],
+        "0af7651916cd43dd8448eb211c80319c"
+    );
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn boundary_status_matrix_keeps_ids_and_redacts_internal_causes() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let app = Router::new()
+        .route(
+            "/bad-request",
+            get(|| async { Err::<(), _>(admin_api::error::ApiError::validation("invalid")) }),
+        )
+        .route(
+            "/unauthorized",
+            get(|| async {
+                Err::<(), _>(admin_api::error::ApiError::unauthorized("unauthorized"))
+            }),
+        )
+        .route(
+            "/forbidden",
+            get(|| async { Err::<(), _>(admin_api::error::ApiError::forbidden("forbidden")) }),
+        )
+        .route(
+            "/internal",
+            get(|| async {
+                Err::<(), _>(admin_api::error::ApiError::internal(
+                    "postgres://canary-user:canary-password@private-host/database",
+                ))
+            }),
+        )
+        .route(
+            "/rate-limited",
+            get(|| async { StatusCode::TOO_MANY_REQUESTS }),
+        )
+        .layer(axum::middleware::from_fn(apply_request_context));
+
+    let responses = async {
+        let mut responses = Vec::new();
+        for (path, status) in [
+            ("/bad-request", StatusCode::BAD_REQUEST),
+            ("/unauthorized", StatusCode::UNAUTHORIZED),
+            ("/forbidden", StatusCode::FORBIDDEN),
+            ("/missing", StatusCode::NOT_FOUND),
+            ("/rate-limited", StatusCode::TOO_MANY_REQUESTS),
+            ("/internal", StatusCode::INTERNAL_SERVER_ERROR),
+        ] {
+            let response = app
+                .clone()
+                .oneshot(
+                    Request::builder()
+                        .uri(path)
+                        .header(REQUEST_ID_HEADER.as_str(), "matrix-request-id")
+                        .body(Body::empty())
+                        .unwrap(),
+                )
+                .await
+                .unwrap();
+            assert_eq!(response.status(), status);
+            assert_eq!(response.headers()[REQUEST_ID_HEADER], "matrix-request-id");
+            assert_eq!(response.headers()[TRACE_ID_HEADER].as_bytes().len(), 32);
+            responses.push(to_bytes(response.into_body(), 16 * 1024).await.unwrap());
+        }
+        responses
+    };
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let responses = responses.await;
+    let combined = responses
+        .iter()
+        .flat_map(|body| body.iter().copied())
+        .collect::<Vec<_>>();
+    assert!(
+        !combined
+            .windows("canary-password".len())
+            .any(|value| value == b"canary-password")
+    );
+    assert!(!writer.output().contains("canary-password"));
+    let internal_event: serde_json::Value = writer
+        .output()
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &serde_json::Value| event["event"] == "admin.response.internal_error")
+        .unwrap();
+    assert_eq!(internal_event["request_id"], "matrix-request-id");
+    assert_eq!(internal_event["trace_id"].as_str().unwrap().len(), 32);
+}
+
 #[tokio::test(flavor = "current_thread")]
 async fn covers_valid_invalid_and_absent_traceparent() {
    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
@@ -163,6 +303,14 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
        REQUEST_ID_HEADER,
        HeaderValue::from_static("second-request-id"),
    );
+    request.headers_mut().append(
+        "traceparent",
+        HeaderValue::from_static("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+    );
+    request.headers_mut().append(
+        "traceparent",
+        HeaderValue::from_static("00-1af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+    );

    let response = probe_app().oneshot(request).await.unwrap();
    let generated = response.headers()[REQUEST_ID_HEADER].to_str().unwrap();
@@ -173,6 +321,9 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
        uuid::Uuid::parse_str(generated).unwrap().get_version(),
        Some(Version::SortRand)
    );
+    let trace_id = response.headers()[TRACE_ID_HEADER].to_str().unwrap();
+    assert_ne!(trace_id, "0af7651916cd43dd8448eb211c80319c");
+    assert_ne!(trace_id, "1af7651916cd43dd8448eb211c80319c");
 }

 fn probe_app() -> Router {
@@ -181,6 +332,15 @@ fn probe_app() -> Router {
        .layer(axum::middleware::from_fn(apply_request_context))
 }

+fn error_probe_app() -> Router {
+    Router::new()
+        .route(
+            "/error",
+            get(|| async { Err::<(), _>(admin_api::error::ApiError::validation("invalid")) }),
+        )
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
 fn trace_probe_app() -> Router {
    Router::new()
        .route("/trace", get(observed_traceparent))