302 lines
11 KiB
Rust
302 lines
11 KiB
Rust
use super::*;
|
|
use std::{collections::BTreeSet, time::Duration};
|
|
|
|
use axum::{
|
|
Router,
|
|
body::Bytes,
|
|
extract::Request,
|
|
http::StatusCode,
|
|
middleware::{self, Next},
|
|
routing::post,
|
|
};
|
|
use crank_core::{ProductEventKind, WorkspaceId};
|
|
use crank_observability::{
|
|
OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider,
|
|
};
|
|
use opentelemetry::trace::TracerProvider as _;
|
|
use opentelemetry_proto::tonic::{
|
|
collector::trace::v1::ExportTraceServiceRequest,
|
|
common::v1::{KeyValue, any_value},
|
|
};
|
|
use opentelemetry_sdk::trace::SdkTracerProvider;
|
|
use prost::Message;
|
|
use tokio::{net::TcpListener, sync::mpsc};
|
|
use tracing::instrument::WithSubscriber;
|
|
use tracing_subscriber::layer::SubscriberExt;
|
|
|
|
pub(super) const RAW_DOCUMENT_CANARY: &str = "openapi: 3.1.0 raw-document-canary";
|
|
pub(super) const USER_ID_CANARY: &str = "user-id-canary-should-not-export";
|
|
const REQUEST_ID: &str = "req_onboarding_optin_0001";
|
|
|
|
#[tokio::test(flavor = "multi_thread")]
|
|
#[serial]
|
|
async fn onboarding_http_event_keeps_default_trace_export_disabled() {
|
|
let registry = test_registry().await;
|
|
let workspace_id = WorkspaceId::new("ws_default");
|
|
let config = OtlpTraceConfig::default();
|
|
let identity = service_identity();
|
|
assert!(!config.is_enabled());
|
|
assert!(
|
|
build_tracer_provider(&identity, &config)
|
|
.expect("disabled OTLP config is valid")
|
|
.is_none(),
|
|
"default config must not construct an exporter or batch queue"
|
|
);
|
|
|
|
let (collector_listener, _) = loopback_listener().await;
|
|
let (dispatch, provider) = local_trace_dispatch();
|
|
let app = app_with_request_dispatch(
|
|
build_test_app(
|
|
registry.clone(),
|
|
test_storage_root("onboarding_telemetry_default"),
|
|
),
|
|
dispatch,
|
|
);
|
|
let base_url = spawn_admin_api(app).await;
|
|
let client = authorized_client(&base_url).await;
|
|
let revision = onboarding_revision(&client, &base_url).await;
|
|
|
|
reject_canary_payload(&client, &base_url, revision).await;
|
|
let event = post_started_event(&client, &base_url, revision, false).await;
|
|
assert_eq!(event.status(), reqwest::StatusCode::OK);
|
|
assert_eq!(event.json::<Value>().await.unwrap()["accepted"], true);
|
|
assert_event_is_local(®istry, &workspace_id, "ui:telemetry-default-event").await;
|
|
|
|
assert!(
|
|
tokio::time::timeout(Duration::from_millis(300), collector_listener.accept())
|
|
.await
|
|
.is_err(),
|
|
"default telemetry config must make no connection to the external collector"
|
|
);
|
|
provider
|
|
.shutdown()
|
|
.expect("local trace provider shuts down");
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread")]
|
|
#[serial]
|
|
async fn onboarding_http_event_exports_only_sanitized_trace_when_operator_opts_in() {
|
|
let registry = test_registry().await;
|
|
let workspace_id = WorkspaceId::new("ws_default");
|
|
let (endpoint, mut exported_requests, collector_task) = spawn_otlp_collector().await;
|
|
let config = OtlpTraceConfig::try_new(
|
|
Some(endpoint),
|
|
Some("http/protobuf".to_owned()),
|
|
Duration::from_secs(2),
|
|
OtlpBatchConfig::try_new(128, 128, Duration::from_secs(60), Duration::from_secs(2))
|
|
.unwrap(),
|
|
)
|
|
.unwrap();
|
|
let identity = service_identity();
|
|
assert!(config.is_enabled());
|
|
let (provider, tracer) = build_tracer_provider(&identity, &config)
|
|
.expect("explicit OTLP config is valid")
|
|
.expect("operator endpoint enables the real exporter");
|
|
let subscriber =
|
|
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
|
let dispatch = tracing::Dispatch::new(subscriber);
|
|
let app = app_with_request_dispatch(
|
|
build_test_app(
|
|
registry.clone(),
|
|
test_storage_root("onboarding_telemetry_optin"),
|
|
),
|
|
dispatch,
|
|
);
|
|
let base_url = spawn_admin_api(app).await;
|
|
let client = authorized_client(&base_url).await;
|
|
let revision = onboarding_revision(&client, &base_url).await;
|
|
|
|
reject_canary_payload(&client, &base_url, revision).await;
|
|
let event = post_started_event(&client, &base_url, revision, true).await;
|
|
assert_eq!(event.status(), reqwest::StatusCode::OK);
|
|
assert_eq!(event.json::<Value>().await.unwrap()["accepted"], true);
|
|
assert_event_is_local(®istry, &workspace_id, "ui:telemetry-optin-event").await;
|
|
|
|
provider
|
|
.force_flush()
|
|
.expect("flush trace spans to the operator collector");
|
|
provider.shutdown().expect("exporter shuts down cleanly");
|
|
let mut request_count = 0;
|
|
let mut onboarding_request_id_exported = false;
|
|
while let Ok(body) = exported_requests.try_recv() {
|
|
request_count += 1;
|
|
let export = ExportTraceServiceRequest::decode(body.as_ref())
|
|
.expect("collector receives valid OTLP protobuf");
|
|
for resource_spans in export.resource_spans {
|
|
for scope_spans in resource_spans.scope_spans {
|
|
for span in scope_spans.spans {
|
|
assert_eq!(span.name, "http.request");
|
|
assert!(span.events.is_empty());
|
|
assert!(span.links.is_empty());
|
|
let attribute_keys: BTreeSet<&str> = span
|
|
.attributes
|
|
.iter()
|
|
.map(|attribute| attribute.key.as_str())
|
|
.collect();
|
|
assert!(
|
|
attribute_keys.iter().all(|key| *key == "request_id"),
|
|
"request spans may export only the request_id attribute"
|
|
);
|
|
if string_attribute(&span.attributes, "request_id") == Some(REQUEST_ID) {
|
|
onboarding_request_id_exported = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
assert!(!body_contains(&body, RAW_DOCUMENT_CANARY));
|
|
assert!(!body_contains(&body, USER_ID_CANARY));
|
|
}
|
|
assert!(
|
|
request_count > 0,
|
|
"explicit opt-in must reach the collector"
|
|
);
|
|
assert!(
|
|
onboarding_request_id_exported,
|
|
"the collected trace must include the real onboarding HTTP request"
|
|
);
|
|
collector_task.abort();
|
|
}
|
|
|
|
fn service_identity() -> ServiceIdentity {
|
|
ServiceIdentity::try_new("admin-api", "integration-test", "test")
|
|
.expect("static service identity is valid")
|
|
}
|
|
|
|
fn local_trace_dispatch() -> (tracing::Dispatch, SdkTracerProvider) {
|
|
let provider = SdkTracerProvider::builder().build();
|
|
let tracer = provider.tracer("crank");
|
|
let subscriber =
|
|
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
|
(tracing::Dispatch::new(subscriber), provider)
|
|
}
|
|
|
|
fn app_with_request_dispatch(app: Router, dispatch: tracing::Dispatch) -> Router {
|
|
app.layer(middleware::from_fn(move |request: Request, next: Next| {
|
|
let dispatch = dispatch.clone();
|
|
async move { next.run(request).with_subscriber(dispatch).await }
|
|
}))
|
|
}
|
|
|
|
pub(super) async fn loopback_listener() -> (TcpListener, String) {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let endpoint = format!("http://{}/v1/traces", listener.local_addr().unwrap());
|
|
(listener, endpoint)
|
|
}
|
|
|
|
pub(super) async fn spawn_otlp_collector() -> (
|
|
String,
|
|
mpsc::UnboundedReceiver<Bytes>,
|
|
tokio::task::JoinHandle<()>,
|
|
) {
|
|
let (listener, endpoint) = loopback_listener().await;
|
|
let (sender, receiver) = mpsc::unbounded_channel();
|
|
let app = Router::new().route(
|
|
"/v1/traces",
|
|
post(move |body: Bytes| {
|
|
let sender = sender.clone();
|
|
async move {
|
|
let _ = sender.send(body);
|
|
StatusCode::OK
|
|
}
|
|
}),
|
|
);
|
|
let handle = tokio::spawn(async move {
|
|
axum::serve(listener, app).await.unwrap();
|
|
});
|
|
(endpoint, receiver, handle)
|
|
}
|
|
|
|
pub(super) async fn onboarding_revision(
|
|
client: &reqwest::Client,
|
|
base_url: &(impl AsRef<str> + ?Sized),
|
|
) -> i64 {
|
|
let base_url = base_url.as_ref();
|
|
let response = client
|
|
.get(format!("{base_url}/onboarding"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
|
response.json::<Value>().await.unwrap()["revision"]
|
|
.as_i64()
|
|
.expect("onboarding returns a server revision")
|
|
}
|
|
|
|
pub(super) async fn reject_canary_payload(
|
|
client: &reqwest::Client,
|
|
base_url: &(impl AsRef<str> + ?Sized),
|
|
revision: i64,
|
|
) {
|
|
let base_url = base_url.as_ref();
|
|
let response = client
|
|
.post(format!("{base_url}/onboarding/events"))
|
|
.json(&json!({
|
|
"event": "started",
|
|
"idempotency_key": "ui:telemetry-canary-rejection",
|
|
"expected_revision": revision,
|
|
"user_id": USER_ID_CANARY,
|
|
"openapi_document": RAW_DOCUMENT_CANARY,
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
|
|
let body = response.text().await.unwrap();
|
|
assert!(!body.contains(USER_ID_CANARY));
|
|
assert!(!body.contains(RAW_DOCUMENT_CANARY));
|
|
}
|
|
|
|
async fn post_started_event(
|
|
client: &reqwest::Client,
|
|
base_url: &(impl AsRef<str> + ?Sized),
|
|
revision: i64,
|
|
with_request_id: bool,
|
|
) -> reqwest::Response {
|
|
let base_url = base_url.as_ref();
|
|
let mut request = client
|
|
.post(format!("{base_url}/onboarding/events"))
|
|
.json(&json!({
|
|
"event": "started",
|
|
"idempotency_key": if with_request_id {
|
|
"ui:telemetry-optin-event"
|
|
} else {
|
|
"ui:telemetry-default-event"
|
|
},
|
|
"expected_revision": revision,
|
|
}));
|
|
if with_request_id {
|
|
request = request.header("x-request-id", REQUEST_ID);
|
|
}
|
|
request.send().await.unwrap()
|
|
}
|
|
|
|
async fn assert_event_is_local(
|
|
registry: &crank_registry::PostgresRegistry,
|
|
workspace_id: &WorkspaceId,
|
|
idempotency_key: &str,
|
|
) {
|
|
let record = registry
|
|
.get_product_event_by_idempotency_key(workspace_id, idempotency_key)
|
|
.await
|
|
.unwrap()
|
|
.expect("successful HTTP event is persisted in the local product event store");
|
|
assert_eq!(record.event.kind, ProductEventKind::OnboardingStarted);
|
|
assert_eq!(record.event.idempotency_key, idempotency_key);
|
|
}
|
|
|
|
pub(super) fn string_attribute<'a>(attributes: &'a [KeyValue], key: &str) -> Option<&'a str> {
|
|
attributes.iter().find_map(|attribute| {
|
|
(attribute.key == key)
|
|
.then_some(attribute.value.as_ref()?.value.as_ref()?)
|
|
.and_then(|value| match value {
|
|
any_value::Value::StringValue(value) => Some(value.as_str()),
|
|
_ => None,
|
|
})
|
|
})
|
|
}
|
|
|
|
pub(super) fn body_contains(body: &[u8], canary: &str) -> bool {
|
|
body.windows(canary.len())
|
|
.any(|window| window == canary.as_bytes())
|
|
}
|