use std::{io, net::SocketAddr, process::ExitCode, time::Duration}; use admin_api::{ app::build_app, auth::{AuthSettings, BootstrapAdminConfig}, pool_metrics::spawn_postgres_pool_metrics, reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation}, service::AdminServiceBuilder, state::AppState, }; use crank_community_auth::PasswordIdentityProvider; use crank_config::{ AdminProcessConfig, CacheBackend as ConfigCacheBackend, ConfigSource, DatabaseSettings, DiagnosticCode, ObservabilitySettings, ProcessKind, parse_process, }; use crank_core::CacheBackend; use crank_observability::{ CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle, OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error, }; use crank_registry::{ MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry, }; use crank_runtime::{ RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores, RuntimeLimits, SecretCrypto, }; use sqlx::postgres::PgConnectOptions; use tokio::net::TcpListener; use tracing::{info, warn}; #[tokio::main] async fn main() -> ExitCode { match main_result().await { Ok(()) => ExitCode::SUCCESS, Err(error) => { eprintln!("{}", safe_startup_diagnostic(error.as_ref())); ExitCode::FAILURE } } } fn safe_startup_diagnostic(error: &(dyn std::error::Error + 'static)) -> String { let mut current = Some(error); while let Some(cause) = current { if let Some(config) = cause.downcast_ref::() { return config.to_json(); } if let Some(migration) = cause.downcast_ref::() { return serde_json::json!({ "status": "error", "code": migration.code(), "stage": migration.stage(), "version": migration.version(), "recovery": migration.recovery(), }) .to_string(); } if let Some(crank_registry::RegistryError::Migration(migration)) = cause.downcast_ref::() { return serde_json::json!({ "status": "error", "code": migration.code(), "stage": migration.stage(), "version": migration.version(), "recovery": migration.recovery(), }) .to_string(); } if let Some(crank_registry::RegistryError::MasterKeyIdentityMismatch { epoch }) = cause.downcast_ref::() { return serde_json::json!({ "status": "error", "code": "master_key_identity_mismatch", "stage": "startup.master_key_identity", "version": epoch, "recovery": "configure_same_master_key", }) .to_string(); } if cause .downcast_ref::() .is_some_and(|error| { matches!( error, crank_registry::RegistryError::InvalidMasterKeyIdentity ) }) { return serde_json::json!({ "status": "error", "code": "master_key_identity_invalid", "stage": "startup.master_key_identity", "version": null, "recovery": "contact_operator", }) .to_string(); } current = cause.source(); } serde_json::json!({ "status": "error", "code": "startup_failed", "stage": "startup", "version": null, "recovery": "contact_operator", }) .to_string() } async fn main_result() -> Result<(), Box> { let effective = parse_process(ProcessKind::AdminApi, ConfigSource::from_os()?)?; let config = effective .admin() .cloned() .ok_or_else(|| io::Error::other("admin configuration projection is unavailable"))?; preflight_config(&config)?; let observability = init_observability(&config.observability)?; for deprecation in effective.deprecations() { warn!( name: "config.deprecated", field = deprecation.field, source_class = deprecation.source_class, replacement = deprecation.replacement, removal_window = deprecation.removal_window, "deprecated configuration accepted" ); } let mut startup_completed = false; let result = run(config, &observability, &mut startup_completed).await; if result.is_err() { capture_critical_error(if startup_completed { CriticalErrorCategory::Internal } else { CriticalErrorCategory::Startup }); } result } async fn run( config: AdminProcessConfig, observability: &ObservabilityLifecycle, startup_completed: &mut bool, ) -> Result<(), Box> { let metrics_config = MetricsConfig::new( config.observability.metrics.enabled, config.observability.metrics.bind_addr, config .observability .metrics .bearer_token .as_ref() .map(|token| token.expose_secret().to_owned()), )?; let metrics_enabled = metrics_config.enabled(); let pool_config = postgres_pool_config(&config.database)?; let registry = PostgresRegistry::connect_with_options_and_pool_config( database_options(&config.database)?, pool_config, ) .await?; let metrics_server = if metrics_enabled { Some(observability.metrics_surface(metrics_config).bind().await?) } else { None }; if metrics_enabled { spawn_postgres_pool_metrics(registry.pool().clone()); } let base_url = config .runtime .base_url .clone() .unwrap_or_else(|| "http://localhost:3000".to_owned()); let auth_settings = AuthSettings { session_secret: config.session_secret.expose_secret().to_owned(), password_pepper: config.password_pepper.expose_secret().to_owned(), session_ttl_hours: config.session_ttl_hours, cookie_secure: base_url.starts_with("https://"), bootstrap_admin: BootstrapAdminConfig { email: config.bootstrap_email.clone(), password: config .bootstrap_password .as_ref() .map(|password| password.expose_secret().to_owned()) .unwrap_or_default(), display_name: config.bootstrap_display_name.clone(), }, }; let runtime_limits = RuntimeLimits::try_new( config.runtime.max_concurrent_unary, config.runtime.max_concurrent_sessions, )?; let cache_config = runtime_cache_config(&config)?; let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?; let api_rate_limit = RequestRateLimitConfig::new( config.rate_limit.requests_per_second, config.rate_limit.burst, )?; let secret_crypto = verified_startup_secret_crypto(®istry, config.runtime.master_key.expose_secret()) .await?; let artifact_store = open_reconciliation_store(config.storage_root.clone()).await?; let outbound_http_policy = crank_runtime::OutboundHttpPolicy::try_new_with_limits( config.runtime.outbound.allowed_hosts.clone(), config.runtime.outbound.denied_hosts.clone(), config.runtime.outbound.max_request_bytes, config.runtime.outbound.max_response_bytes, )?; let runtime = crank_runtime::community_with_outbound_policy(outbound_http_policy.clone()) .with_limits(runtime_limits) .with_response_cache(cache_stores.response.clone()) .with_coordination_store(cache_stores.coordination.clone()) .build(); let identity_provider = PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone()); let service = AdminServiceBuilder::new( registry.clone(), config.storage_root.clone(), auth_settings, secret_crypto, runtime, ) .with_artifact_store(artifact_store.clone()) .with_public_base_url(base_url) .with_outbound_http_policy(outbound_http_policy) .with_identity_provider(std::sync::Arc::new(identity_provider)) .build(); if config.demo_seed { service.seed_demo_assets().await?; } spawn_artifact_reconciliation(registry, artifact_store).await; spawn_invocation_log_cleanup(service.clone(), config.invocation_log_retention_days); let state = AppState { service, api_rate_limiter: if cache_config.backend.is_external() { RequestRateLimiter::new_shared(api_rate_limit, cache_stores.rate_limit.clone()) } else { RequestRateLimiter::new(api_rate_limit) }, trusted_proxy_ips: config.trusted_proxy_ips.clone(), }; let app = build_app(state); let listener = TcpListener::bind(config.bind_addr).await?; let make_service = app.into_make_service_with_connect_info::(); info!( name: "admin.postgres_pool.configured", runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary, admin_rate_limit_rps = api_rate_limit.requests_per_second, admin_rate_limit_burst = api_rate_limit.burst, cache_backend = %cache_config.backend, max_connections = pool_config.max_connections, min_connections = pool_config.min_connections, acquire_timeout_ms = pool_config.acquire_timeout_ms, idle_timeout_ms = pool_config.idle_timeout_ms, max_lifetime_ms = pool_config.max_lifetime_ms, invocation_log_retention_days = config.invocation_log_retention_days, "postgres pool configured" ); info!(name: "admin.server.listening", bind_address = %config.bind_addr, "admin-api listening"); *startup_completed = true; if let Some(metrics_server) = metrics_server { tokio::select! { result = axum::serve(listener, make_service) => result?, result = metrics_server.serve() => result?, } } else { axum::serve(listener, make_service).await?; } Ok(()) } fn init_observability( config: &ObservabilitySettings, ) -> Result> { let identity = ServiceIdentity::try_new( "admin-api", env!("CARGO_PKG_VERSION"), config.environment.clone(), )?; let base = ObservabilityConfig::try_new( identity, config.log_filter.clone(), RedactionLimits::default(), )?; let sentry = SentryConfig::parse( config .sentry_dsn .as_ref() .map(|value| value.expose_secret()), )?; let otlp = otlp_config(config)?; Ok(ObservabilityLifecycle::init_with_exporters( base, sentry, otlp, )?) } fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::ConfigError> { let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field); database_options(&config.database).map_err(|_| invalid("database.source"))?; postgres_pool_config(&config.database).map_err(|_| invalid("database.pool"))?; MetricsConfig::new( config.observability.metrics.enabled, config.observability.metrics.bind_addr, config .observability .metrics .bearer_token .as_ref() .map(|v| v.expose_secret().to_owned()), ) .map_err(|_| invalid("observability.metrics"))?; RuntimeLimits::try_new( config.runtime.max_concurrent_unary, config.runtime.max_concurrent_sessions, ) .map_err(|_| invalid("runtime.limits"))?; runtime_cache_config(config).map_err(|_| invalid("cache"))?; RequestRateLimitConfig::new( config.rate_limit.requests_per_second, config.rate_limit.burst, ) .map_err(|_| invalid("admin.rate_limit"))?; SecretCrypto::new(config.runtime.master_key.expose_secret()) .map_err(|_| invalid("runtime.master_key"))?; crank_runtime::OutboundHttpPolicy::try_new_with_limits( config.runtime.outbound.allowed_hosts.clone(), config.runtime.outbound.denied_hosts.clone(), config.runtime.outbound.max_request_bytes, config.runtime.outbound.max_response_bytes, ) .map_err(|_| invalid("runtime.outbound"))?; let identity = ServiceIdentity::try_new( "admin-api", env!("CARGO_PKG_VERSION"), config.observability.environment.clone(), ) .map_err(|_| invalid("observability.environment"))?; ObservabilityConfig::try_new( identity, config.observability.log_filter.clone(), RedactionLimits::default(), ) .map_err(|_| invalid("observability.log_filter"))?; SentryConfig::parse( config .observability .sentry_dsn .as_ref() .map(|v| v.expose_secret()), ) .map_err(|_| invalid("observability.sentry_dsn"))?; otlp_config(&config.observability).map_err(|_| invalid("observability.otlp"))?; Ok(()) } async fn verified_startup_secret_crypto( registry: &PostgresRegistry, master_key: &str, ) -> Result> { let active = registry.active_master_key_identity().await?; let secret_crypto = if let Some(identity) = active { SecretCrypto::with_epoch(master_key, identity.epoch)? } else { let crypto = SecretCrypto::new(master_key)?; let mut after_secret_id: Option = None; let mut after_version: Option = None; loop { let versions = registry .list_secret_versions_for_master_key_epoch_page( 1, after_secret_id.as_deref(), after_version, 1_000, ) .await?; if versions.is_empty() { break; } for version in versions { after_secret_id = Some(version.secret_version.secret_id.as_str().to_owned()); after_version = Some(version.secret_version.version); crypto.decrypt_for_epoch( &version.secret_version.key_version, version.master_key_epoch, &version.secret_version.ciphertext, )?; } } crypto }; let master_key_observed_at = time::OffsetDateTime::now_utc(); registry .verify_or_register_master_key_identity(MasterKeyIdentityCandidate { epoch: secret_crypto.master_key_epoch(), fingerprint: secret_crypto.master_key_fingerprint(), cipher_contract: MASTER_KEY_CIPHER_CONTRACT, observed_at: &master_key_observed_at, }) .await?; Ok(secret_crypto) } fn otlp_config( config: &ObservabilitySettings, ) -> Result { let values = &config.otlp; OtlpTraceConfig::from_values( values.endpoint.clone(), values.traces_endpoint.clone(), values.protocol.clone(), values.traces_protocol.clone(), values.timeout.clone(), values.traces_timeout.clone(), values .headers .as_ref() .map(|value| value.expose_secret().to_owned()), values .traces_headers .as_ref() .map(|value| value.expose_secret().to_owned()), values.max_queue_size, values.max_export_batch_size, values.schedule_delay.clone(), values.export_timeout.clone(), ) } fn postgres_pool_config( config: &DatabaseSettings, ) -> Result { PostgresPoolConfig::try_new( config.pool.max_connections, config.pool.min_connections, config.pool.acquire_timeout_ms, config.pool.idle_timeout_ms, config.pool.max_lifetime_ms, ) } fn database_options( config: &DatabaseSettings, ) -> Result> { if let Some(url) = &config.url { return url .expose_secret() .parse::() .map_err(|_| io::Error::other("database URL is invalid").into()); } Ok(PgConnectOptions::new() .host(&config.host) .port(config.port) .database(&config.database) .username(&config.username) .password(config.password.expose_secret())) } fn runtime_cache_config( config: &AdminProcessConfig, ) -> Result { RuntimeCacheConfig::try_new( match config.runtime.cache.backend { ConfigCacheBackend::Memory => CacheBackend::Memory, ConfigCacheBackend::Valkey => CacheBackend::Valkey, ConfigCacheBackend::Redis => CacheBackend::Redis, }, config .runtime .cache .url .as_ref() .map(|value| value.expose_secret().to_owned()), ) } fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) { tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(60 * 60)); loop { interval.tick().await; let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days); match service.cleanup_invocation_logs_before(cutoff).await { Ok(outcome) if outcome.deleted_records > 0 => info!( name: "admin.invocation_log_cleanup.completed", status = ?outcome.status, removed = outcome.deleted_records, requested_cutoff = %outcome.policy.requested_cutoff, effective_cutoff = %outcome.policy.effective_cutoff, preserved_usage_window_days = outcome.policy.preserved_usage_window_days, "expired invocation logs removed" ), Ok(outcome) => info!( name: "admin.invocation_log_cleanup.noop", status = ?outcome.status, requested_cutoff = %outcome.policy.requested_cutoff, effective_cutoff = %outcome.policy.effective_cutoff, preserved_usage_window_days = outcome.policy.preserved_usage_window_days, "no expired invocation logs removed" ), Err(_) => warn!( name: "admin.invocation_log_cleanup.failed", error_category = "registry_cleanup", "failed to remove expired invocation logs" ), } } }); }