Files
bsodfather ad23cd9a46
CI / Rust Checks (pull_request) Successful in 24m43s
CI / UI Checks (pull_request) Successful in 6s
CI / Community Image Smoke (pull_request) Successful in 9m5s
CI / Frontend E2E (pull_request) Successful in 12m43s
CI / Deploy (pull_request) Skipped
fix(import): подтвердить обязательные гарантии второго эпика
2026-10-10 13:25:05 +03:00

291 lines
11 KiB
Rust

use super::telemetry::{
RAW_DOCUMENT_CANARY, USER_ID_CANARY, body_contains, loopback_listener, onboarding_revision,
reject_canary_payload, string_attribute,
};
use super::*;
use axum::{
Router,
body::{self, Bytes},
extract::Request,
http::StatusCode,
routing::any,
};
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
use prost::Message;
use std::{
collections::BTreeSet,
fs::{self, OpenOptions},
net::TcpListener as StdTcpListener,
os::unix::fs::{OpenOptionsExt, PermissionsExt},
path::PathBuf,
process::{Child, Command, Stdio},
time::{Duration, Instant},
};
const MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
const SESSION_SECRET: &str = "test-session-secret";
const PASSWORD_PEPPER: &str = "test-password-pepper";
const ADMIN_EMAIL: &str = "owner@crank.local";
const ADMIN_PASSWORD: &str = "test-password";
const REQUEST_ID: &str = "req_admin_process_optin_0001";
struct AdminProcess {
child: Child,
log_path: PathBuf,
secrets: Vec<String>,
}
impl AdminProcess {
fn stop_and_assert_logs_are_safe(&mut self) {
let _ = self.child.kill();
self.child.wait().expect("admin-api child must be reaped");
let logs = fs::read_to_string(&self.log_path).expect("child log must be readable");
for secret in &self.secrets {
assert!(
!logs.contains(secret),
"admin-api process log must not contain configured secrets or request canaries"
);
}
}
}
impl Drop for AdminProcess {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn production_admin_process_keeps_default_otlp_export_disabled() {
let (registry, database_url) = test_registry_with_single_connection().await;
let _registry = registry;
let storage_root = test_storage_root("onboarding_telemetry_startup_default");
let (collector_listener, _) = loopback_listener().await;
let (mut process, base_url) = start_admin_process(&database_url, &storage_root, None);
wait_until_ready(&mut process, &base_url).await;
let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default");
let client = authorized_client(&workspace_url).await;
let revision = onboarding_revision(&client, &workspace_url).await;
reject_canary_payload(&client, &workspace_url, revision).await;
let response = post_startup_event(&client, &workspace_url, revision, false).await;
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(response.json::<Value>().await.unwrap()["accepted"], true);
assert!(
tokio::time::timeout(Duration::from_secs(2), collector_listener.accept())
.await
.is_err(),
"default production startup must not connect to an OTLP collector"
);
process.stop_and_assert_logs_are_safe();
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn production_admin_process_exports_only_sanitized_request_trace_when_opted_in() {
let (registry, database_url) = test_registry_with_single_connection().await;
let _registry = registry;
let storage_root = test_storage_root("onboarding_telemetry_startup_optin");
let (endpoint, mut exported_requests, collector_task) = spawn_startup_collector().await;
let (mut process, base_url) =
start_admin_process(&database_url, &storage_root, Some(&endpoint));
wait_until_ready(&mut process, &base_url).await;
let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default");
let client = authorized_client(&workspace_url).await;
let revision = onboarding_revision(&client, &workspace_url).await;
reject_canary_payload(&client, &workspace_url, revision).await;
let response = post_startup_event(&client, &workspace_url, revision, true).await;
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(response.json::<Value>().await.unwrap()["accepted"], true);
let mut request_id_exported = false;
tokio::time::timeout(Duration::from_secs(12), async {
while !request_id_exported {
let (path, body) = exported_requests
.recv()
.await
.expect("collector task must remain active");
assert_eq!(path, "/v1/traces", "trace-specific endpoint path is exact");
let export = ExportTraceServiceRequest::decode(body.as_ref())
.expect("production process must send valid OTLP protobuf");
for resource_spans in export.resource_spans {
for scope_spans in resource_spans.scope_spans {
for span in scope_spans.spans {
assert_eq!(span.name, "http.request");
assert!(span.events.is_empty());
assert!(span.links.is_empty());
let attribute_keys: BTreeSet<&str> = span
.attributes
.iter()
.map(|attribute| attribute.key.as_str())
.collect();
assert!(
attribute_keys.iter().all(|key| *key == "request_id"),
"production request spans may export only the request_id attribute"
);
if string_attribute(&span.attributes, "request_id") == Some(REQUEST_ID) {
request_id_exported = true;
}
let encoded = format!("{} {:?}", span.name, span.attributes);
assert!(!encoded.contains(USER_ID_CANARY));
assert!(!encoded.contains(RAW_DOCUMENT_CANARY));
}
}
}
assert!(!body_contains(&body, USER_ID_CANARY));
assert!(!body_contains(&body, RAW_DOCUMENT_CANARY));
}
})
.await
.expect("production exporter must send the onboarding request in a bounded OTLP batch");
assert!(
request_id_exported,
"collector must receive the trace for the onboarding event request"
);
process.stop_and_assert_logs_are_safe();
collector_task.abort();
}
fn start_admin_process(
database_url: &str,
storage_root: &std::path::Path,
otlp_endpoint: Option<&str>,
) -> (AdminProcess, String) {
fs::create_dir_all(storage_root).expect("isolated storage root must be created");
#[cfg(unix)]
fs::set_permissions(storage_root, fs::Permissions::from_mode(0o700))
.expect("isolated storage root must be private");
let listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve an ephemeral API port");
let address = listener.local_addr().unwrap();
drop(listener);
let base_url = format!("http://{address}");
let log_path = storage_root.join("admin-api.log");
let stdout = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&log_path)
.expect("private child log must be created");
let stderr = stdout.try_clone().expect("child stderr log handle");
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
command
.env_clear()
.env("CRANK_DATABASE_URL", database_url)
.env("CRANK_ADMIN_BIND", address.to_string())
.env("CRANK_STORAGE_ROOT", storage_root)
.env("CRANK_MASTER_KEY", MASTER_KEY)
.env("CRANK_SESSION_SECRET", SESSION_SECRET)
.env("CRANK_PASSWORD_PEPPER", PASSWORD_PEPPER)
.env("CRANK_SESSION_TTL_HOURS", "24")
.env("CRANK_BOOTSTRAP_ADMIN_EMAIL", ADMIN_EMAIL)
.env("CRANK_BOOTSTRAP_ADMIN_PASSWORD", ADMIN_PASSWORD)
.env("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", "Integration Test")
.env("CRANK_METRICS_ENABLED", "false")
.env("CRANK_BASE_URL", &base_url)
.env("CRANK_LOG_LEVEL", "info")
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr));
if let Some(endpoint) = otlp_endpoint {
command
.env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", endpoint)
.env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "http/protobuf")
.env("OTEL_BSP_MAX_QUEUE_SIZE", "16")
.env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "1")
.env("OTEL_BSP_SCHEDULE_DELAY", "100")
.env("OTEL_BSP_EXPORT_TIMEOUT", "2000");
}
let child = command.spawn().expect("production admin-api binary starts");
(
AdminProcess {
child,
log_path,
secrets: vec![
database_url.to_owned(),
MASTER_KEY.to_owned(),
SESSION_SECRET.to_owned(),
PASSWORD_PEPPER.to_owned(),
ADMIN_PASSWORD.to_owned(),
USER_ID_CANARY.to_owned(),
RAW_DOCUMENT_CANARY.to_owned(),
],
},
base_url,
)
}
async fn spawn_startup_collector() -> (
String,
tokio::sync::mpsc::UnboundedReceiver<(String, Bytes)>,
tokio::task::JoinHandle<()>,
) {
let (listener, endpoint) = loopback_listener().await;
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
let app = Router::new().fallback(any(move |request: Request| {
let sender = sender.clone();
async move {
let path = request.uri().path().to_owned();
let body = body::to_bytes(request.into_body(), usize::MAX)
.await
.expect("collector request body must be readable");
let _ = sender.send((path, body));
StatusCode::OK
}
}));
let handle = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(endpoint, receiver, handle)
}
async fn wait_until_ready(process: &mut AdminProcess, base_url: &str) {
let deadline = Instant::now() + Duration::from_secs(45);
loop {
if let Some(status) = process.child.try_wait().expect("child status is readable") {
panic!("production admin-api exited before readiness: {status}");
}
if reqwest::Client::new()
.get(format!("{base_url}/health"))
.timeout(Duration::from_millis(500))
.send()
.await
.is_ok_and(|response| response.status().is_success())
{
return;
}
assert!(
Instant::now() < deadline,
"production admin-api readiness deadline"
);
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
async fn post_startup_event(
client: &reqwest::Client,
workspace_url: &str,
revision: i64,
with_request_id: bool,
) -> reqwest::Response {
let mut request = client
.post(format!("{workspace_url}/onboarding/events"))
.json(&json!({
"event": "started",
"idempotency_key": if with_request_id {
"ui:telemetry-process-optin"
} else {
"ui:telemetry-process-default"
},
"expected_revision": revision,
}));
if with_request_id {
request = request.header("x-request-id", REQUEST_ID);
}
request.send().await.unwrap()
}