Files
crank/apps/admin-api/tests/config_startup.rs
Hermes DevOps 75fe5a123c [skip ci] wip: preserve Hermes crank development checkpoint
Owner-authorized recovery of accumulated source changes.
Includes accepted work and in-progress changes; NOT a release or acceptance.
Parent baseline 1b6db09bd1. Runtime data, build caches and local credentials excluded.
2026-10-25 10:00:22 +00:00

604 lines
22 KiB
Rust
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use std::{
fs,
io::Read,
os::unix::fs::PermissionsExt,
process::{Child, Command, Output, Stdio},
sync::atomic::{AtomicU64, Ordering},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use axum::{body::Body, http::Request};
use crank_observability::{
MetricService, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle, OtlpTraceConfig,
RedactionLimits, SentryConfig, ServiceIdentity, bound_metric_service,
};
use crank_registry::{MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresRegistry};
use crank_runtime::SecretCrypto;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tower::ServiceExt;
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
const REGISTERED_MASTER_KEY: &str = "registered-master-key-CANARY_SECRET_VALUE-00000000";
const WRONG_MASTER_KEY: &str = "wrong-master-key-CANARY_SECRET_VALUE-0000000000000";
static NEXT_STORAGE_ROOT: AtomicU64 = AtomicU64::new(0);
fn isolated_metrics_bind() -> String {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("ephemeral metrics port");
let addr = listener.local_addr().unwrap().to_string();
drop(listener);
addr
}
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.envs([
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_METRICS_BIND", isolated_metrics_bind().as_str()),
]);
for (name, value) in entries {
command.env(name, value);
}
let output = command.output().expect("admin binary executes");
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8");
assert!(!stderr.contains("CANARY_SECRET_VALUE"));
assert!(!stderr.contains("connection refused"));
assert!(stderr.len() <= 65_536);
stderr
}
#[test]
fn invalid_config_fails_before_database_or_listener_side_effects() {
for (entries, code) in [
(
vec![("CRANK_CONFIG_CANARY_UNKNOWN", "CANARY_SECRET_VALUE")],
"config.unknown_field",
),
(vec![("POSTGRES_PORT", "bad")], "config.invalid_type"),
(
vec![
("CRANK_DATABASE_URL", "postgres://db/crank"),
("POSTGRES_HOST", "other"),
],
"config.conflict",
),
(vec![("CRANK_LOG_LEVEL", "[")], "config.invalid_type"),
] {
let stderr = run_with(&entries);
assert!(stderr.contains(code), "{stderr}");
}
}
#[test]
fn database_driver_failures_are_normalized_and_redacted() {
let stderr = run_with(&[(
"CRANK_DATABASE_URL",
"postgres://CANARY_SECRET_VALUE:CANARY_SECRET_VALUE@127.0.0.1:1/crank",
)]);
assert!(stderr.contains("startup_failed"), "{stderr}");
}
#[tokio::test]
async fn fresh_database_startup_is_read_only() {
let database_url = crank_test_support::postgres_schema_url("admin_startup_read_only").await;
let stderr = run_with(&[("CRANK_DATABASE_URL", &database_url)]);
assert!(stderr.contains("schema_missing"), "{stderr}");
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present);
}
#[tokio::test]
async fn real_admin_startup_runs_immediate_import_maintenance() {
let unique = NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed);
let epoch = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let database_url =
crank_test_support::postgres_schema_url(&format!("startup_maint_{epoch}_{unique}")).await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
assert_eq!(
sqlx::query_scalar::<_, Option<i64>>("select max(version) from __crank_migrations")
.fetch_one(&pool)
.await
.unwrap(),
Some(23),
"migration binary must install the current canonical ledger",
);
sqlx::query(
"insert into import_jobs (
id, workspace_id, kind, source_format, source_version, status,
preview_payload, created_operation_ids, error_text, created_at, expires_at, finished_at
) values (
'imp_startup_expired', 'ws_default', 'openapi', 'openapi', null, 'pending',
'{\"source\": {\"source_id\": 1}}'::jsonb, '[]'::jsonb, null,
now() - interval '2 hours', now() - interval '1 hour', null
)",
)
.execute(&pool)
.await
.unwrap();
let storage_root = std::env::temp_dir().join(format!(
"crank-admin-startup-maintenance-{}-{epoch}-{unique}",
std::process::id()
));
fs::create_dir(&storage_root).unwrap();
fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let bind_addr = listener.local_addr().unwrap().to_string();
drop(listener);
let metrics_bind = isolated_metrics_bind();
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
let mut child = command
.envs([
("CRANK_DATABASE_URL", database_url.as_str()),
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_BIND", bind_addr.as_str()),
("CRANK_ADMIN_METRICS_BIND", metrics_bind.as_str()),
("CRANK_STORAGE_ROOT", storage_root.to_str().unwrap()),
])
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("admin binary starts");
let mut cleaned = false;
for _ in 0..200 {
if let Some(status) = child.try_wait().unwrap() {
let mut stderr = String::new();
if let Some(mut stream) = child.stderr.take() {
stream.read_to_string(&mut stderr).unwrap();
}
panic!("admin binary exited before startup maintenance: {status}: {stderr}");
}
let remaining: i64 =
sqlx::query_scalar("select count(*) from import_jobs where id = 'imp_startup_expired'")
.fetch_one(&pool)
.await
.unwrap();
if remaining == 0 {
cleaned = true;
break;
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
child.kill().expect("admin binary can be stopped");
child.wait().expect("admin binary can be reaped");
fs::remove_dir_all(&storage_root).unwrap();
assert!(cleaned, "real startup did not clean the expired import job");
}
#[tokio::test]
async fn master_key_mismatch_blocks_startup_with_safe_diagnostic() {
let database_url = crank_test_support::postgres_schema_url("admin_master_key_mismatch").await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::parse("2026-08-21T00:00:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
let stderr = run_with(&[
("CRANK_DATABASE_URL", &database_url),
("CRANK_MASTER_KEY", WRONG_MASTER_KEY),
]);
assert!(stderr.contains("master_key_identity_mismatch"), "{stderr}");
assert!(!stderr.contains("registered-master-key"));
assert!(!stderr.contains("wrong-master-key"));
}
#[tokio::test]
async fn populated_database_without_identity_rejects_wrong_first_key() {
let database_url =
crank_test_support::postgres_schema_url("admin_master_key_first_registration_wrong").await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let old_crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap();
let ciphertext = old_crypto
.encrypt(&serde_json::json!({"token": "CANARY_SECRET_VALUE"}))
.unwrap();
sqlx::query(
"insert into secrets (
id, workspace_id, name, kind, status, current_version, created_at, updated_at
) values (
'legacy_secret', 'ws_default', 'legacy secret', 'token', 'active', 1, now(), now()
)",
)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"insert into secret_versions (
secret_id, version, ciphertext, key_version, master_key_epoch, created_at, created_by
) values (
'legacy_secret', 1, $1, $2, 1, now(), null
)",
)
.bind(ciphertext)
.bind(old_crypto.key_version())
.execute(&pool)
.await
.unwrap();
let stderr = run_with(&[
("CRANK_DATABASE_URL", &database_url),
("CRANK_MASTER_KEY", WRONG_MASTER_KEY),
]);
assert!(stderr.contains("startup_failed"), "{stderr}");
assert!(!stderr.contains("CANARY_SECRET_VALUE"));
let identities: i64 = sqlx::query_scalar("select count(*) from master_key_identities")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(identities, 0);
}
// --- TD-B6/TD-E6/TD-E8: реальный startup-порядок binding → recorder → listeners ---
struct AdminProcess {
child: Child,
base_url: String,
storage_root: std::path::PathBuf,
}
impl AdminProcess {
fn stop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
let _ = fs::remove_dir_all(&self.storage_root);
}
}
impl Drop for AdminProcess {
fn drop(&mut self) {
self.stop();
}
}
async fn migrated_database_url(prefix: &str) -> String {
let database_url = crank_test_support::postgres_schema_url(prefix).await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"migration must apply: {}",
String::from_utf8_lossy(&applied.stderr)
);
database_url
}
fn admin_command(
database_url: &str,
product_addr: &str,
metrics_addr: &str,
metrics_enabled: bool,
storage_root: &std::path::Path,
) -> Command {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.envs([
("CRANK_DATABASE_URL", database_url),
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_BIND", product_addr),
("CRANK_ADMIN_METRICS_BIND", metrics_addr),
(
"CRANK_METRICS_ENABLED",
if metrics_enabled { "true" } else { "false" },
),
("CRANK_LOG_LEVEL", "info"),
]);
command.env("CRANK_STORAGE_ROOT", storage_root);
command
}
fn spawn_admin(
database_url: &str,
product_addr: &str,
metrics_addr: &str,
metrics_enabled: bool,
) -> AdminProcess {
let storage_root = std::env::temp_dir().join(format!(
"crank-admin-f4-{}-{}-{metrics_enabled}",
std::process::id(),
NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&storage_root).unwrap();
fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap();
let child = admin_command(
database_url,
product_addr,
metrics_addr,
metrics_enabled,
&storage_root,
)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("admin binary starts");
AdminProcess {
child,
base_url: format!("http://{product_addr}"),
storage_root,
}
}
fn run_admin_once(
database_url: &str,
product_addr: &str,
metrics_addr: &str,
metrics_enabled: bool,
) -> Output {
let storage_root = std::env::temp_dir().join(format!(
"crank-admin-f4-terminal-{}-{}",
std::process::id(),
NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&storage_root).unwrap();
fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap();
let output = admin_command(
database_url,
product_addr,
metrics_addr,
metrics_enabled,
&storage_root,
)
.output()
.expect("admin binary executes");
let _ = fs::remove_dir_all(&storage_root);
output
}
async fn wait_until_healthy(client: &reqwest::Client, process: &mut AdminProcess) {
let deadline = Instant::now() + Duration::from_secs(60);
loop {
if let Some(status) = process.child.try_wait().expect("child status is readable") {
panic!("admin binary exited before readiness: {status}");
}
if client
.get(format!("{}/health", process.base_url))
.timeout(Duration::from_millis(500))
.send()
.await
.is_ok_and(|response| response.status().is_success())
{
return;
}
assert!(Instant::now() < deadline, "admin readiness deadline");
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
/// TD-B6 — реальный бинарник: service binding и recorder публикуются **до**
/// product- и metrics-listener'ов. Product-запрос обязан быть записан (при
/// неопубликованном binding'е recording — no-op), а metrics-listener обязан
/// отдавать exposition с process-constant identity.
#[tokio::test]
async fn binding_precedes_recorder_and_listeners() {
let database_url = migrated_database_url("admin_startup_binding_precedes").await;
let product_addr = isolated_metrics_bind();
let metrics_addr = isolated_metrics_bind();
let mut process = spawn_admin(&database_url, &product_addr, &metrics_addr, true);
let client = reqwest::Client::new();
wait_until_healthy(&client, &mut process).await;
let health = client
.get(format!("{}/health", process.base_url))
.send()
.await
.expect("product health request");
assert_eq!(health.status(), reqwest::StatusCode::OK);
let response = client
.get(format!("http://{metrics_addr}/metrics"))
.send()
.await
.expect("metrics scrape");
assert_eq!(response.status(), reqwest::StatusCode::OK);
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
assert_eq!(content_type, "text/plain; version=0.0.4; charset=utf-8");
let exposition = response.text().await.expect("metrics body");
assert!(
exposition.contains("service=\"admin-api\""),
"process-constant identity обязана присутствовать в exposition"
);
assert!(
exposition.contains("crank_http_requests_total{"),
"recorder обязан отдавать записанные series: {}",
&exposition[..exposition.len().min(300)]
);
assert!(
exposition.contains("route=\"/health\""),
"запись product-запроса обязана быть под normalized route: {}",
&exposition[..exposition.len().min(300)]
);
process.stop();
}
/// TD-E6 — `enabled=false` выключает metrics-listener, но не binding и не
/// локальную запись: реальный бинарник стартует даже с уже занятым
/// metrics-адресом, а in-process lifecycle сохраняет опубликованный binding и
/// отдаёт записанные series без открытия listener'а.
#[tokio::test]
async fn disabled_listener_still_binds_service() {
// (1) Реальный бинарник: metrics disabled, metrics-адрес занят тестом.
let occupied = std::net::TcpListener::bind("127.0.0.1:0").expect("occupied metrics port");
let metrics_addr = occupied.local_addr().unwrap().to_string();
let database_url = migrated_database_url("admin_startup_metrics_disabled").await;
let product_addr = isolated_metrics_bind();
let mut process = spawn_admin(&database_url, &product_addr, &metrics_addr, false);
let client = reqwest::Client::new();
wait_until_healthy(&client, &mut process).await;
assert_eq!(
client
.get(format!("{}/health", process.base_url))
.send()
.await
.expect("product health request")
.status(),
reqwest::StatusCode::OK,
"disabled metrics listener не делает startup terminal"
);
process.stop();
drop(occupied);
// (2) In-process: disabled MetricsConfig сохраняет binding и локальную запись,
// но не создаёт listener.
let bind_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let identity =
ServiceIdentity::try_new("admin-api", env!("CARGO_PKG_VERSION"), "test").expect("identity");
let config = ObservabilityConfig::try_new(identity, "off", RedactionLimits::default())
.expect("observability config");
let lifecycle = ObservabilityLifecycle::init_with_exporters(
config,
SentryConfig::parse(None).expect("sentry config"),
OtlpTraceConfig::default(),
)
.expect("observability lifecycle");
assert_eq!(
bound_metric_service(),
Some(MetricService::AdminApi),
"binding обязателен и при disabled listener"
);
let metrics_config = MetricsConfig::new(false, bind_addr, None).expect("metrics config");
assert!(!metrics_config.enabled(), "listener обязан быть выключен");
let surface = lifecycle.metrics_surface(metrics_config);
let probe =
std::net::TcpListener::bind(bind_addr).expect("disabled listener не занимает адрес");
drop(probe);
crank_metrics::record_http_request(
crank_metrics::HttpRoute::from_matched_path("/health"),
crank_metrics::HttpMethod::Get,
crank_metrics::HttpStatusClass::Success,
Duration::from_millis(1),
);
let response = surface
.router()
.oneshot(
Request::builder()
.uri("/metrics")
.body(Body::empty())
.unwrap(),
)
.await
.expect("in-process metrics request");
assert_eq!(response.status(), axum::http::StatusCode::OK);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.expect("metrics body");
let exposition = String::from_utf8(body.to_vec()).expect("utf-8 exposition");
assert!(
exposition.contains("crank_http_requests_total")
&& exposition.contains("service=\"admin-api\""),
"локальная запись при disabled listener обязана сохраняться: {}",
&exposition[..exposition.len().min(200)]
);
}
/// TD-E8 — ошибка binding'а metrics-listener'а terminal: тот же конфиг со
/// свободным портом стартует, а с занятым — падает до открытия product
/// listener'а (product-адрес остаётся свободным, listening-лог не появляется).
#[tokio::test]
async fn startup_binding_failures_are_terminal() {
let database_url = migrated_database_url("admin_startup_binding_terminal").await;
// Positive control: конфигурация со свободным metrics-портом стартует.
let free_metrics = isolated_metrics_bind();
let product_addr = isolated_metrics_bind();
let mut process = spawn_admin(&database_url, &product_addr, &free_metrics, true);
let client = reqwest::Client::new();
wait_until_healthy(&client, &mut process).await;
process.stop();
// Тот же конфиг с занятым metrics-адресом обязан упасть ДО product listener.
let occupied = std::net::TcpListener::bind("127.0.0.1:0").expect("occupied metrics port");
let occupied_addr = occupied.local_addr().unwrap().to_string();
let output = run_admin_once(&database_url, &product_addr, &occupied_addr, true);
assert!(
!output.status.success(),
"bind-ошибка metrics-listener'а обязана быть terminal"
);
let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8");
assert!(stderr.contains("startup_failed"), "{stderr}");
assert!(
!stderr.contains("admin.server.listening"),
"product listener не мог быть открыт до terminal-ошибки: {stderr}"
);
assert!(
std::net::TcpListener::bind(&product_addr).is_ok(),
"product listener процесса не открывался"
);
drop(occupied);
}