use std::{ fs, io::Read, os::unix::fs::PermissionsExt, process::{Child, Command, Output, Stdio}, sync::atomic::{AtomicU64, Ordering}, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use axum::{body::Body, http::Request}; use crank_observability::{ MetricService, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle, OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, bound_metric_service, }; use crank_registry::{MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresRegistry}; use crank_runtime::SecretCrypto; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tower::ServiceExt; const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000"; const REGISTERED_MASTER_KEY: &str = "registered-master-key-CANARY_SECRET_VALUE-00000000"; const WRONG_MASTER_KEY: &str = "wrong-master-key-CANARY_SECRET_VALUE-0000000000000"; static NEXT_STORAGE_ROOT: AtomicU64 = AtomicU64::new(0); fn isolated_metrics_bind() -> String { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("ephemeral metrics port"); let addr = listener.local_addr().unwrap().to_string(); drop(listener); addr } fn run_with(entries: &[(&str, &str)]) -> String { let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api")); for field in crank_config::field_registry() { command.env_remove(field.env_name); } command.envs([ ("CRANK_MASTER_KEY", TEST_MASTER_KEY), ("CRANK_SESSION_SECRET", "session"), ("CRANK_PASSWORD_PEPPER", "pepper"), ("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"), ("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"), ("CRANK_ADMIN_METRICS_BIND", isolated_metrics_bind().as_str()), ]); for (name, value) in entries { command.env(name, value); } let output = command.output().expect("admin binary executes"); assert!(!output.status.success()); let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8"); assert!(!stderr.contains("CANARY_SECRET_VALUE")); assert!(!stderr.contains("connection refused")); assert!(stderr.len() <= 65_536); stderr } #[test] fn invalid_config_fails_before_database_or_listener_side_effects() { for (entries, code) in [ ( vec![("CRANK_CONFIG_CANARY_UNKNOWN", "CANARY_SECRET_VALUE")], "config.unknown_field", ), (vec![("POSTGRES_PORT", "bad")], "config.invalid_type"), ( vec![ ("CRANK_DATABASE_URL", "postgres://db/crank"), ("POSTGRES_HOST", "other"), ], "config.conflict", ), (vec![("CRANK_LOG_LEVEL", "[")], "config.invalid_type"), ] { let stderr = run_with(&entries); assert!(stderr.contains(code), "{stderr}"); } } #[test] fn database_driver_failures_are_normalized_and_redacted() { let stderr = run_with(&[( "CRANK_DATABASE_URL", "postgres://CANARY_SECRET_VALUE:CANARY_SECRET_VALUE@127.0.0.1:1/crank", )]); assert!(stderr.contains("startup_failed"), "{stderr}"); } #[tokio::test] async fn fresh_database_startup_is_read_only() { let database_url = crank_test_support::postgres_schema_url("admin_startup_read_only").await; let stderr = run_with(&[("CRANK_DATABASE_URL", &database_url)]); assert!(stderr.contains("schema_missing"), "{stderr}"); let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); let present: bool = sqlx::query_scalar( "select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null", ) .fetch_one(&pool) .await .unwrap(); assert!(!present); } #[tokio::test] async fn real_admin_startup_runs_immediate_import_maintenance() { let unique = NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed); let epoch = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); let database_url = crank_test_support::postgres_schema_url(&format!("startup_maint_{epoch}_{unique}")).await; let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate")) .arg("apply") .env("CRANK_DATABASE_URL", &database_url) .output() .expect("migration command executes"); assert!( applied.status.success(), "{}", String::from_utf8_lossy(&applied.stderr) ); let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); assert_eq!( sqlx::query_scalar::<_, Option>("select max(version) from __crank_migrations") .fetch_one(&pool) .await .unwrap(), Some(23), "migration binary must install the current canonical ledger", ); sqlx::query( "insert into import_jobs ( id, workspace_id, kind, source_format, source_version, status, preview_payload, created_operation_ids, error_text, created_at, expires_at, finished_at ) values ( 'imp_startup_expired', 'ws_default', 'openapi', 'openapi', null, 'pending', '{\"source\": {\"source_id\": 1}}'::jsonb, '[]'::jsonb, null, now() - interval '2 hours', now() - interval '1 hour', null )", ) .execute(&pool) .await .unwrap(); let storage_root = std::env::temp_dir().join(format!( "crank-admin-startup-maintenance-{}-{epoch}-{unique}", std::process::id() )); fs::create_dir(&storage_root).unwrap(); fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap(); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let bind_addr = listener.local_addr().unwrap().to_string(); drop(listener); let metrics_bind = isolated_metrics_bind(); let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api")); for field in crank_config::field_registry() { command.env_remove(field.env_name); } let mut child = command .envs([ ("CRANK_DATABASE_URL", database_url.as_str()), ("CRANK_MASTER_KEY", TEST_MASTER_KEY), ("CRANK_SESSION_SECRET", "session"), ("CRANK_PASSWORD_PEPPER", "pepper"), ("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"), ("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"), ("CRANK_ADMIN_BIND", bind_addr.as_str()), ("CRANK_ADMIN_METRICS_BIND", metrics_bind.as_str()), ("CRANK_STORAGE_ROOT", storage_root.to_str().unwrap()), ]) .stdout(Stdio::null()) .stderr(Stdio::piped()) .spawn() .expect("admin binary starts"); let mut cleaned = false; for _ in 0..200 { if let Some(status) = child.try_wait().unwrap() { let mut stderr = String::new(); if let Some(mut stream) = child.stderr.take() { stream.read_to_string(&mut stderr).unwrap(); } panic!("admin binary exited before startup maintenance: {status}: {stderr}"); } let remaining: i64 = sqlx::query_scalar("select count(*) from import_jobs where id = 'imp_startup_expired'") .fetch_one(&pool) .await .unwrap(); if remaining == 0 { cleaned = true; break; } tokio::time::sleep(std::time::Duration::from_millis(25)).await; } child.kill().expect("admin binary can be stopped"); child.wait().expect("admin binary can be reaped"); fs::remove_dir_all(&storage_root).unwrap(); assert!(cleaned, "real startup did not clean the expired import job"); } #[tokio::test] async fn master_key_mismatch_blocks_startup_with_safe_diagnostic() { let database_url = crank_test_support::postgres_schema_url("admin_master_key_mismatch").await; let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate")) .arg("apply") .env("CRANK_DATABASE_URL", &database_url) .output() .expect("migration command executes"); assert!( applied.status.success(), "{}", String::from_utf8_lossy(&applied.stderr) ); let registry = PostgresRegistry::connect(&database_url).await.unwrap(); let crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap(); registry .verify_or_register_master_key_identity(MasterKeyIdentityCandidate { epoch: 1, fingerprint: crypto.master_key_fingerprint(), cipher_contract: MASTER_KEY_CIPHER_CONTRACT, observed_at: &OffsetDateTime::parse("2026-08-21T00:00:00Z", &Rfc3339).unwrap(), }) .await .unwrap(); let stderr = run_with(&[ ("CRANK_DATABASE_URL", &database_url), ("CRANK_MASTER_KEY", WRONG_MASTER_KEY), ]); assert!(stderr.contains("master_key_identity_mismatch"), "{stderr}"); assert!(!stderr.contains("registered-master-key")); assert!(!stderr.contains("wrong-master-key")); } #[tokio::test] async fn populated_database_without_identity_rejects_wrong_first_key() { let database_url = crank_test_support::postgres_schema_url("admin_master_key_first_registration_wrong").await; let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate")) .arg("apply") .env("CRANK_DATABASE_URL", &database_url) .output() .expect("migration command executes"); assert!( applied.status.success(), "{}", String::from_utf8_lossy(&applied.stderr) ); let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); let old_crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap(); let ciphertext = old_crypto .encrypt(&serde_json::json!({"token": "CANARY_SECRET_VALUE"})) .unwrap(); sqlx::query( "insert into secrets ( id, workspace_id, name, kind, status, current_version, created_at, updated_at ) values ( 'legacy_secret', 'ws_default', 'legacy secret', 'token', 'active', 1, now(), now() )", ) .execute(&pool) .await .unwrap(); sqlx::query( "insert into secret_versions ( secret_id, version, ciphertext, key_version, master_key_epoch, created_at, created_by ) values ( 'legacy_secret', 1, $1, $2, 1, now(), null )", ) .bind(ciphertext) .bind(old_crypto.key_version()) .execute(&pool) .await .unwrap(); let stderr = run_with(&[ ("CRANK_DATABASE_URL", &database_url), ("CRANK_MASTER_KEY", WRONG_MASTER_KEY), ]); assert!(stderr.contains("startup_failed"), "{stderr}"); assert!(!stderr.contains("CANARY_SECRET_VALUE")); let identities: i64 = sqlx::query_scalar("select count(*) from master_key_identities") .fetch_one(&pool) .await .unwrap(); assert_eq!(identities, 0); } // --- TD-B6/TD-E6/TD-E8: реальный startup-порядок binding → recorder → listeners --- struct AdminProcess { child: Child, base_url: String, storage_root: std::path::PathBuf, } impl AdminProcess { fn stop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); let _ = fs::remove_dir_all(&self.storage_root); } } impl Drop for AdminProcess { fn drop(&mut self) { self.stop(); } } async fn migrated_database_url(prefix: &str) -> String { let database_url = crank_test_support::postgres_schema_url(prefix).await; let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate")) .arg("apply") .env("CRANK_DATABASE_URL", &database_url) .output() .expect("migration command executes"); assert!( applied.status.success(), "migration must apply: {}", String::from_utf8_lossy(&applied.stderr) ); database_url } fn admin_command( database_url: &str, product_addr: &str, metrics_addr: &str, metrics_enabled: bool, storage_root: &std::path::Path, ) -> Command { let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api")); for field in crank_config::field_registry() { command.env_remove(field.env_name); } command.envs([ ("CRANK_DATABASE_URL", database_url), ("CRANK_MASTER_KEY", TEST_MASTER_KEY), ("CRANK_SESSION_SECRET", "session"), ("CRANK_PASSWORD_PEPPER", "pepper"), ("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"), ("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"), ("CRANK_ADMIN_BIND", product_addr), ("CRANK_ADMIN_METRICS_BIND", metrics_addr), ( "CRANK_METRICS_ENABLED", if metrics_enabled { "true" } else { "false" }, ), ("CRANK_LOG_LEVEL", "info"), ]); command.env("CRANK_STORAGE_ROOT", storage_root); command } fn spawn_admin( database_url: &str, product_addr: &str, metrics_addr: &str, metrics_enabled: bool, ) -> AdminProcess { let storage_root = std::env::temp_dir().join(format!( "crank-admin-f4-{}-{}-{metrics_enabled}", std::process::id(), NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed) )); fs::create_dir(&storage_root).unwrap(); fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap(); let child = admin_command( database_url, product_addr, metrics_addr, metrics_enabled, &storage_root, ) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() .expect("admin binary starts"); AdminProcess { child, base_url: format!("http://{product_addr}"), storage_root, } } fn run_admin_once( database_url: &str, product_addr: &str, metrics_addr: &str, metrics_enabled: bool, ) -> Output { let storage_root = std::env::temp_dir().join(format!( "crank-admin-f4-terminal-{}-{}", std::process::id(), NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed) )); fs::create_dir(&storage_root).unwrap(); fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap(); let output = admin_command( database_url, product_addr, metrics_addr, metrics_enabled, &storage_root, ) .output() .expect("admin binary executes"); let _ = fs::remove_dir_all(&storage_root); output } async fn wait_until_healthy(client: &reqwest::Client, process: &mut AdminProcess) { let deadline = Instant::now() + Duration::from_secs(60); loop { if let Some(status) = process.child.try_wait().expect("child status is readable") { panic!("admin binary exited before readiness: {status}"); } if client .get(format!("{}/health", process.base_url)) .timeout(Duration::from_millis(500)) .send() .await .is_ok_and(|response| response.status().is_success()) { return; } assert!(Instant::now() < deadline, "admin readiness deadline"); tokio::time::sleep(Duration::from_millis(100)).await; } } /// TD-B6 — реальный бинарник: service binding и recorder публикуются **до** /// product- и metrics-listener'ов. Product-запрос обязан быть записан (при /// неопубликованном binding'е recording — no-op), а metrics-listener обязан /// отдавать exposition с process-constant identity. #[tokio::test] async fn binding_precedes_recorder_and_listeners() { let database_url = migrated_database_url("admin_startup_binding_precedes").await; let product_addr = isolated_metrics_bind(); let metrics_addr = isolated_metrics_bind(); let mut process = spawn_admin(&database_url, &product_addr, &metrics_addr, true); let client = reqwest::Client::new(); wait_until_healthy(&client, &mut process).await; let health = client .get(format!("{}/health", process.base_url)) .send() .await .expect("product health request"); assert_eq!(health.status(), reqwest::StatusCode::OK); let response = client .get(format!("http://{metrics_addr}/metrics")) .send() .await .expect("metrics scrape"); assert_eq!(response.status(), reqwest::StatusCode::OK); let content_type = response .headers() .get(reqwest::header::CONTENT_TYPE) .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_owned(); assert_eq!(content_type, "text/plain; version=0.0.4; charset=utf-8"); let exposition = response.text().await.expect("metrics body"); assert!( exposition.contains("service=\"admin-api\""), "process-constant identity обязана присутствовать в exposition" ); assert!( exposition.contains("crank_http_requests_total{"), "recorder обязан отдавать записанные series: {}", &exposition[..exposition.len().min(300)] ); assert!( exposition.contains("route=\"/health\""), "запись product-запроса обязана быть под normalized route: {}", &exposition[..exposition.len().min(300)] ); process.stop(); } /// TD-E6 — `enabled=false` выключает metrics-listener, но не binding и не /// локальную запись: реальный бинарник стартует даже с уже занятым /// metrics-адресом, а in-process lifecycle сохраняет опубликованный binding и /// отдаёт записанные series без открытия listener'а. #[tokio::test] async fn disabled_listener_still_binds_service() { // (1) Реальный бинарник: metrics disabled, metrics-адрес занят тестом. let occupied = std::net::TcpListener::bind("127.0.0.1:0").expect("occupied metrics port"); let metrics_addr = occupied.local_addr().unwrap().to_string(); let database_url = migrated_database_url("admin_startup_metrics_disabled").await; let product_addr = isolated_metrics_bind(); let mut process = spawn_admin(&database_url, &product_addr, &metrics_addr, false); let client = reqwest::Client::new(); wait_until_healthy(&client, &mut process).await; assert_eq!( client .get(format!("{}/health", process.base_url)) .send() .await .expect("product health request") .status(), reqwest::StatusCode::OK, "disabled metrics listener не делает startup terminal" ); process.stop(); drop(occupied); // (2) In-process: disabled MetricsConfig сохраняет binding и локальную запись, // но не создаёт listener. let bind_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); let identity = ServiceIdentity::try_new("admin-api", env!("CARGO_PKG_VERSION"), "test").expect("identity"); let config = ObservabilityConfig::try_new(identity, "off", RedactionLimits::default()) .expect("observability config"); let lifecycle = ObservabilityLifecycle::init_with_exporters( config, SentryConfig::parse(None).expect("sentry config"), OtlpTraceConfig::default(), ) .expect("observability lifecycle"); assert_eq!( bound_metric_service(), Some(MetricService::AdminApi), "binding обязателен и при disabled listener" ); let metrics_config = MetricsConfig::new(false, bind_addr, None).expect("metrics config"); assert!(!metrics_config.enabled(), "listener обязан быть выключен"); let surface = lifecycle.metrics_surface(metrics_config); let probe = std::net::TcpListener::bind(bind_addr).expect("disabled listener не занимает адрес"); drop(probe); crank_metrics::record_http_request( crank_metrics::HttpRoute::from_matched_path("/health"), crank_metrics::HttpMethod::Get, crank_metrics::HttpStatusClass::Success, Duration::from_millis(1), ); let response = surface .router() .oneshot( Request::builder() .uri("/metrics") .body(Body::empty()) .unwrap(), ) .await .expect("in-process metrics request"); assert_eq!(response.status(), axum::http::StatusCode::OK); let body = axum::body::to_bytes(response.into_body(), usize::MAX) .await .expect("metrics body"); let exposition = String::from_utf8(body.to_vec()).expect("utf-8 exposition"); assert!( exposition.contains("crank_http_requests_total") && exposition.contains("service=\"admin-api\""), "локальная запись при disabled listener обязана сохраняться: {}", &exposition[..exposition.len().min(200)] ); } /// TD-E8 — ошибка binding'а metrics-listener'а terminal: тот же конфиг со /// свободным портом стартует, а с занятым — падает до открытия product /// listener'а (product-адрес остаётся свободным, listening-лог не появляется). #[tokio::test] async fn startup_binding_failures_are_terminal() { let database_url = migrated_database_url("admin_startup_binding_terminal").await; // Positive control: конфигурация со свободным metrics-портом стартует. let free_metrics = isolated_metrics_bind(); let product_addr = isolated_metrics_bind(); let mut process = spawn_admin(&database_url, &product_addr, &free_metrics, true); let client = reqwest::Client::new(); wait_until_healthy(&client, &mut process).await; process.stop(); // Тот же конфиг с занятым metrics-адресом обязан упасть ДО product listener. let occupied = std::net::TcpListener::bind("127.0.0.1:0").expect("occupied metrics port"); let occupied_addr = occupied.local_addr().unwrap().to_string(); let output = run_admin_once(&database_url, &product_addr, &occupied_addr, true); assert!( !output.status.success(), "bind-ошибка metrics-listener'а обязана быть terminal" ); let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8"); assert!(stderr.contains("startup_failed"), "{stderr}"); assert!( !stderr.contains("admin.server.listening"), "product listener не мог быть открыт до terminal-ошибки: {stderr}" ); assert!( std::net::TcpListener::bind(&product_addr).is_ok(), "product listener процесса не открывался" ); drop(occupied); }