291 lines
11 KiB
Rust
291 lines
11 KiB
Rust
use super::telemetry::{
|
|
RAW_DOCUMENT_CANARY, USER_ID_CANARY, body_contains, loopback_listener, onboarding_revision,
|
|
reject_canary_payload, string_attribute,
|
|
};
|
|
use super::*;
|
|
use axum::{
|
|
Router,
|
|
body::{self, Bytes},
|
|
extract::Request,
|
|
http::StatusCode,
|
|
routing::any,
|
|
};
|
|
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
|
|
use prost::Message;
|
|
use std::{
|
|
collections::BTreeSet,
|
|
fs::{self, OpenOptions},
|
|
net::TcpListener as StdTcpListener,
|
|
os::unix::fs::{OpenOptionsExt, PermissionsExt},
|
|
path::PathBuf,
|
|
process::{Child, Command, Stdio},
|
|
time::{Duration, Instant},
|
|
};
|
|
|
|
const MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
|
const SESSION_SECRET: &str = "test-session-secret";
|
|
const PASSWORD_PEPPER: &str = "test-password-pepper";
|
|
const ADMIN_EMAIL: &str = "owner@crank.local";
|
|
const ADMIN_PASSWORD: &str = "test-password";
|
|
const REQUEST_ID: &str = "req_admin_process_optin_0001";
|
|
|
|
struct AdminProcess {
|
|
child: Child,
|
|
log_path: PathBuf,
|
|
secrets: Vec<String>,
|
|
}
|
|
|
|
impl AdminProcess {
|
|
fn stop_and_assert_logs_are_safe(&mut self) {
|
|
let _ = self.child.kill();
|
|
self.child.wait().expect("admin-api child must be reaped");
|
|
let logs = fs::read_to_string(&self.log_path).expect("child log must be readable");
|
|
for secret in &self.secrets {
|
|
assert!(
|
|
!logs.contains(secret),
|
|
"admin-api process log must not contain configured secrets or request canaries"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Drop for AdminProcess {
|
|
fn drop(&mut self) {
|
|
let _ = self.child.kill();
|
|
let _ = self.child.wait();
|
|
}
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread")]
|
|
#[serial]
|
|
async fn production_admin_process_keeps_default_otlp_export_disabled() {
|
|
let (registry, database_url) = test_registry_with_single_connection().await;
|
|
let _registry = registry;
|
|
let storage_root = test_storage_root("onboarding_telemetry_startup_default");
|
|
let (collector_listener, _) = loopback_listener().await;
|
|
let (mut process, base_url) = start_admin_process(&database_url, &storage_root, None);
|
|
wait_until_ready(&mut process, &base_url).await;
|
|
|
|
let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default");
|
|
let client = authorized_client(&workspace_url).await;
|
|
let revision = onboarding_revision(&client, &workspace_url).await;
|
|
reject_canary_payload(&client, &workspace_url, revision).await;
|
|
let response = post_startup_event(&client, &workspace_url, revision, false).await;
|
|
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
|
assert_eq!(response.json::<Value>().await.unwrap()["accepted"], true);
|
|
|
|
assert!(
|
|
tokio::time::timeout(Duration::from_secs(2), collector_listener.accept())
|
|
.await
|
|
.is_err(),
|
|
"default production startup must not connect to an OTLP collector"
|
|
);
|
|
process.stop_and_assert_logs_are_safe();
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread")]
|
|
#[serial]
|
|
async fn production_admin_process_exports_only_sanitized_request_trace_when_opted_in() {
|
|
let (registry, database_url) = test_registry_with_single_connection().await;
|
|
let _registry = registry;
|
|
let storage_root = test_storage_root("onboarding_telemetry_startup_optin");
|
|
let (endpoint, mut exported_requests, collector_task) = spawn_startup_collector().await;
|
|
let (mut process, base_url) =
|
|
start_admin_process(&database_url, &storage_root, Some(&endpoint));
|
|
wait_until_ready(&mut process, &base_url).await;
|
|
|
|
let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default");
|
|
let client = authorized_client(&workspace_url).await;
|
|
let revision = onboarding_revision(&client, &workspace_url).await;
|
|
reject_canary_payload(&client, &workspace_url, revision).await;
|
|
let response = post_startup_event(&client, &workspace_url, revision, true).await;
|
|
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
|
assert_eq!(response.json::<Value>().await.unwrap()["accepted"], true);
|
|
|
|
let mut request_id_exported = false;
|
|
tokio::time::timeout(Duration::from_secs(12), async {
|
|
while !request_id_exported {
|
|
let (path, body) = exported_requests
|
|
.recv()
|
|
.await
|
|
.expect("collector task must remain active");
|
|
assert_eq!(path, "/v1/traces", "trace-specific endpoint path is exact");
|
|
let export = ExportTraceServiceRequest::decode(body.as_ref())
|
|
.expect("production process must send valid OTLP protobuf");
|
|
for resource_spans in export.resource_spans {
|
|
for scope_spans in resource_spans.scope_spans {
|
|
for span in scope_spans.spans {
|
|
assert_eq!(span.name, "http.request");
|
|
assert!(span.events.is_empty());
|
|
assert!(span.links.is_empty());
|
|
let attribute_keys: BTreeSet<&str> = span
|
|
.attributes
|
|
.iter()
|
|
.map(|attribute| attribute.key.as_str())
|
|
.collect();
|
|
assert!(
|
|
attribute_keys.iter().all(|key| *key == "request_id"),
|
|
"production request spans may export only the request_id attribute"
|
|
);
|
|
if string_attribute(&span.attributes, "request_id") == Some(REQUEST_ID) {
|
|
request_id_exported = true;
|
|
}
|
|
let encoded = format!("{} {:?}", span.name, span.attributes);
|
|
assert!(!encoded.contains(USER_ID_CANARY));
|
|
assert!(!encoded.contains(RAW_DOCUMENT_CANARY));
|
|
}
|
|
}
|
|
}
|
|
assert!(!body_contains(&body, USER_ID_CANARY));
|
|
assert!(!body_contains(&body, RAW_DOCUMENT_CANARY));
|
|
}
|
|
})
|
|
.await
|
|
.expect("production exporter must send the onboarding request in a bounded OTLP batch");
|
|
assert!(
|
|
request_id_exported,
|
|
"collector must receive the trace for the onboarding event request"
|
|
);
|
|
process.stop_and_assert_logs_are_safe();
|
|
collector_task.abort();
|
|
}
|
|
|
|
fn start_admin_process(
|
|
database_url: &str,
|
|
storage_root: &std::path::Path,
|
|
otlp_endpoint: Option<&str>,
|
|
) -> (AdminProcess, String) {
|
|
fs::create_dir_all(storage_root).expect("isolated storage root must be created");
|
|
#[cfg(unix)]
|
|
fs::set_permissions(storage_root, fs::Permissions::from_mode(0o700))
|
|
.expect("isolated storage root must be private");
|
|
|
|
let listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve an ephemeral API port");
|
|
let address = listener.local_addr().unwrap();
|
|
drop(listener);
|
|
let base_url = format!("http://{address}");
|
|
let log_path = storage_root.join("admin-api.log");
|
|
let stdout = OpenOptions::new()
|
|
.write(true)
|
|
.create_new(true)
|
|
.mode(0o600)
|
|
.open(&log_path)
|
|
.expect("private child log must be created");
|
|
let stderr = stdout.try_clone().expect("child stderr log handle");
|
|
|
|
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
|
|
command
|
|
.env_clear()
|
|
.env("CRANK_DATABASE_URL", database_url)
|
|
.env("CRANK_ADMIN_BIND", address.to_string())
|
|
.env("CRANK_STORAGE_ROOT", storage_root)
|
|
.env("CRANK_MASTER_KEY", MASTER_KEY)
|
|
.env("CRANK_SESSION_SECRET", SESSION_SECRET)
|
|
.env("CRANK_PASSWORD_PEPPER", PASSWORD_PEPPER)
|
|
.env("CRANK_SESSION_TTL_HOURS", "24")
|
|
.env("CRANK_BOOTSTRAP_ADMIN_EMAIL", ADMIN_EMAIL)
|
|
.env("CRANK_BOOTSTRAP_ADMIN_PASSWORD", ADMIN_PASSWORD)
|
|
.env("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", "Integration Test")
|
|
.env("CRANK_METRICS_ENABLED", "false")
|
|
.env("CRANK_BASE_URL", &base_url)
|
|
.env("CRANK_LOG_LEVEL", "info")
|
|
.stdout(Stdio::from(stdout))
|
|
.stderr(Stdio::from(stderr));
|
|
if let Some(endpoint) = otlp_endpoint {
|
|
command
|
|
.env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", endpoint)
|
|
.env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "http/protobuf")
|
|
.env("OTEL_BSP_MAX_QUEUE_SIZE", "16")
|
|
.env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "1")
|
|
.env("OTEL_BSP_SCHEDULE_DELAY", "100")
|
|
.env("OTEL_BSP_EXPORT_TIMEOUT", "2000");
|
|
}
|
|
let child = command.spawn().expect("production admin-api binary starts");
|
|
(
|
|
AdminProcess {
|
|
child,
|
|
log_path,
|
|
secrets: vec![
|
|
database_url.to_owned(),
|
|
MASTER_KEY.to_owned(),
|
|
SESSION_SECRET.to_owned(),
|
|
PASSWORD_PEPPER.to_owned(),
|
|
ADMIN_PASSWORD.to_owned(),
|
|
USER_ID_CANARY.to_owned(),
|
|
RAW_DOCUMENT_CANARY.to_owned(),
|
|
],
|
|
},
|
|
base_url,
|
|
)
|
|
}
|
|
|
|
async fn spawn_startup_collector() -> (
|
|
String,
|
|
tokio::sync::mpsc::UnboundedReceiver<(String, Bytes)>,
|
|
tokio::task::JoinHandle<()>,
|
|
) {
|
|
let (listener, endpoint) = loopback_listener().await;
|
|
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
|
|
let app = Router::new().fallback(any(move |request: Request| {
|
|
let sender = sender.clone();
|
|
async move {
|
|
let path = request.uri().path().to_owned();
|
|
let body = body::to_bytes(request.into_body(), usize::MAX)
|
|
.await
|
|
.expect("collector request body must be readable");
|
|
let _ = sender.send((path, body));
|
|
StatusCode::OK
|
|
}
|
|
}));
|
|
let handle = tokio::spawn(async move {
|
|
axum::serve(listener, app).await.unwrap();
|
|
});
|
|
(endpoint, receiver, handle)
|
|
}
|
|
|
|
async fn wait_until_ready(process: &mut AdminProcess, base_url: &str) {
|
|
let deadline = Instant::now() + Duration::from_secs(45);
|
|
loop {
|
|
if let Some(status) = process.child.try_wait().expect("child status is readable") {
|
|
panic!("production admin-api exited before readiness: {status}");
|
|
}
|
|
if reqwest::Client::new()
|
|
.get(format!("{base_url}/health"))
|
|
.timeout(Duration::from_millis(500))
|
|
.send()
|
|
.await
|
|
.is_ok_and(|response| response.status().is_success())
|
|
{
|
|
return;
|
|
}
|
|
assert!(
|
|
Instant::now() < deadline,
|
|
"production admin-api readiness deadline"
|
|
);
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
}
|
|
}
|
|
|
|
async fn post_startup_event(
|
|
client: &reqwest::Client,
|
|
workspace_url: &str,
|
|
revision: i64,
|
|
with_request_id: bool,
|
|
) -> reqwest::Response {
|
|
let mut request = client
|
|
.post(format!("{workspace_url}/onboarding/events"))
|
|
.json(&json!({
|
|
"event": "started",
|
|
"idempotency_key": if with_request_id {
|
|
"ui:telemetry-process-optin"
|
|
} else {
|
|
"ui:telemetry-process-default"
|
|
},
|
|
"expected_revision": revision,
|
|
}));
|
|
if with_request_id {
|
|
request = request.header("x-request-id", REQUEST_ID);
|
|
}
|
|
request.send().await.unwrap()
|
|
}
|