516 lines
19 KiB
Rust
516 lines
19 KiB
Rust
use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
|
|
|
|
use admin_api::{
|
|
app::build_app,
|
|
auth::{AuthSettings, BootstrapAdminConfig},
|
|
pool_metrics::spawn_postgres_pool_metrics,
|
|
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
|
|
service::AdminServiceBuilder,
|
|
state::AppState,
|
|
};
|
|
use crank_community_auth::PasswordIdentityProvider;
|
|
use crank_config::{
|
|
AdminProcessConfig, CacheBackend as ConfigCacheBackend, ConfigSource, DatabaseSettings,
|
|
DiagnosticCode, ObservabilitySettings, ProcessKind, parse_process,
|
|
};
|
|
use crank_core::CacheBackend;
|
|
use crank_observability::{
|
|
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
|
|
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
|
|
};
|
|
use crank_registry::{
|
|
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
|
|
};
|
|
use crank_runtime::{
|
|
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
|
|
RuntimeLimits, SecretCrypto,
|
|
};
|
|
use sqlx::postgres::PgConnectOptions;
|
|
use tokio::net::TcpListener;
|
|
use tracing::{info, warn};
|
|
|
|
#[tokio::main]
|
|
async fn main() -> ExitCode {
|
|
match main_result().await {
|
|
Ok(()) => ExitCode::SUCCESS,
|
|
Err(error) => {
|
|
eprintln!("{}", safe_startup_diagnostic(error.as_ref()));
|
|
ExitCode::FAILURE
|
|
}
|
|
}
|
|
}
|
|
|
|
fn safe_startup_diagnostic(error: &(dyn std::error::Error + 'static)) -> String {
|
|
let mut current = Some(error);
|
|
while let Some(cause) = current {
|
|
if let Some(config) = cause.downcast_ref::<crank_config::ConfigError>() {
|
|
return config.to_json();
|
|
}
|
|
if let Some(migration) = cause.downcast_ref::<crank_registry::MigrationError>() {
|
|
return serde_json::json!({
|
|
"status": "error",
|
|
"code": migration.code(),
|
|
"stage": migration.stage(),
|
|
"version": migration.version(),
|
|
"recovery": migration.recovery(),
|
|
})
|
|
.to_string();
|
|
}
|
|
if let Some(crank_registry::RegistryError::Migration(migration)) =
|
|
cause.downcast_ref::<crank_registry::RegistryError>()
|
|
{
|
|
return serde_json::json!({
|
|
"status": "error",
|
|
"code": migration.code(),
|
|
"stage": migration.stage(),
|
|
"version": migration.version(),
|
|
"recovery": migration.recovery(),
|
|
})
|
|
.to_string();
|
|
}
|
|
if let Some(crank_registry::RegistryError::MasterKeyIdentityMismatch { epoch }) =
|
|
cause.downcast_ref::<crank_registry::RegistryError>()
|
|
{
|
|
return serde_json::json!({
|
|
"status": "error",
|
|
"code": "master_key_identity_mismatch",
|
|
"stage": "startup.master_key_identity",
|
|
"version": epoch,
|
|
"recovery": "configure_same_master_key",
|
|
})
|
|
.to_string();
|
|
}
|
|
if cause
|
|
.downcast_ref::<crank_registry::RegistryError>()
|
|
.is_some_and(|error| {
|
|
matches!(
|
|
error,
|
|
crank_registry::RegistryError::InvalidMasterKeyIdentity
|
|
)
|
|
})
|
|
{
|
|
return serde_json::json!({
|
|
"status": "error",
|
|
"code": "master_key_identity_invalid",
|
|
"stage": "startup.master_key_identity",
|
|
"version": null,
|
|
"recovery": "contact_operator",
|
|
})
|
|
.to_string();
|
|
}
|
|
current = cause.source();
|
|
}
|
|
serde_json::json!({
|
|
"status": "error",
|
|
"code": "startup_failed",
|
|
"stage": "startup",
|
|
"version": null,
|
|
"recovery": "contact_operator",
|
|
})
|
|
.to_string()
|
|
}
|
|
|
|
async fn main_result() -> Result<(), Box<dyn std::error::Error>> {
|
|
let effective = parse_process(ProcessKind::AdminApi, ConfigSource::from_os()?)?;
|
|
let config = effective
|
|
.admin()
|
|
.cloned()
|
|
.ok_or_else(|| io::Error::other("admin configuration projection is unavailable"))?;
|
|
preflight_config(&config)?;
|
|
let observability = init_observability(&config.observability)?;
|
|
for deprecation in effective.deprecations() {
|
|
warn!(
|
|
name: "config.deprecated",
|
|
field = deprecation.field,
|
|
source_class = deprecation.source_class,
|
|
replacement = deprecation.replacement,
|
|
removal_window = deprecation.removal_window,
|
|
"deprecated configuration accepted"
|
|
);
|
|
}
|
|
let mut startup_completed = false;
|
|
let result = run(config, &observability, &mut startup_completed).await;
|
|
if result.is_err() {
|
|
capture_critical_error(if startup_completed {
|
|
CriticalErrorCategory::Internal
|
|
} else {
|
|
CriticalErrorCategory::Startup
|
|
});
|
|
}
|
|
result
|
|
}
|
|
|
|
async fn run(
|
|
config: AdminProcessConfig,
|
|
observability: &ObservabilityLifecycle,
|
|
startup_completed: &mut bool,
|
|
) -> Result<(), Box<dyn std::error::Error>> {
|
|
let metrics_config = MetricsConfig::new(
|
|
config.observability.metrics.enabled,
|
|
config.observability.metrics.bind_addr,
|
|
config
|
|
.observability
|
|
.metrics
|
|
.bearer_token
|
|
.as_ref()
|
|
.map(|token| token.expose_secret().to_owned()),
|
|
)?;
|
|
let metrics_enabled = metrics_config.enabled();
|
|
let pool_config = postgres_pool_config(&config.database)?;
|
|
let registry = PostgresRegistry::connect_with_options_and_pool_config(
|
|
database_options(&config.database)?,
|
|
pool_config,
|
|
)
|
|
.await?;
|
|
let metrics_server = if metrics_enabled {
|
|
Some(observability.metrics_surface(metrics_config).bind().await?)
|
|
} else {
|
|
None
|
|
};
|
|
if metrics_enabled {
|
|
spawn_postgres_pool_metrics(registry.pool().clone());
|
|
}
|
|
let base_url = config
|
|
.runtime
|
|
.base_url
|
|
.clone()
|
|
.unwrap_or_else(|| "http://localhost:3000".to_owned());
|
|
let auth_settings = AuthSettings {
|
|
session_secret: config.session_secret.expose_secret().to_owned(),
|
|
password_pepper: config.password_pepper.expose_secret().to_owned(),
|
|
session_ttl_hours: config.session_ttl_hours,
|
|
cookie_secure: base_url.starts_with("https://"),
|
|
bootstrap_admin: BootstrapAdminConfig {
|
|
email: config.bootstrap_email.clone(),
|
|
password: config
|
|
.bootstrap_password
|
|
.as_ref()
|
|
.map(|password| password.expose_secret().to_owned())
|
|
.unwrap_or_default(),
|
|
display_name: config.bootstrap_display_name.clone(),
|
|
},
|
|
};
|
|
let runtime_limits = RuntimeLimits::try_new(
|
|
config.runtime.max_concurrent_unary,
|
|
config.runtime.max_concurrent_sessions,
|
|
)?;
|
|
let cache_config = runtime_cache_config(&config)?;
|
|
let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
|
|
let api_rate_limit = RequestRateLimitConfig::new(
|
|
config.rate_limit.requests_per_second,
|
|
config.rate_limit.burst,
|
|
)?;
|
|
let secret_crypto =
|
|
verified_startup_secret_crypto(®istry, config.runtime.master_key.expose_secret())
|
|
.await?;
|
|
let artifact_store = open_reconciliation_store(config.storage_root.clone()).await?;
|
|
let outbound_http_policy = crank_runtime::OutboundHttpPolicy::try_new_with_limits(
|
|
config.runtime.outbound.allowed_hosts.clone(),
|
|
config.runtime.outbound.denied_hosts.clone(),
|
|
config.runtime.outbound.max_request_bytes,
|
|
config.runtime.outbound.max_response_bytes,
|
|
)?;
|
|
let runtime = crank_runtime::community_with_outbound_policy(outbound_http_policy.clone())
|
|
.with_limits(runtime_limits)
|
|
.with_response_cache(cache_stores.response.clone())
|
|
.with_coordination_store(cache_stores.coordination.clone())
|
|
.build();
|
|
let identity_provider =
|
|
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
|
|
let service = AdminServiceBuilder::new(
|
|
registry.clone(),
|
|
config.storage_root.clone(),
|
|
auth_settings,
|
|
secret_crypto,
|
|
runtime,
|
|
)
|
|
.with_artifact_store(artifact_store.clone())
|
|
.with_public_base_url(base_url)
|
|
.with_outbound_http_policy(outbound_http_policy)
|
|
.with_external_reference_import(&config.external_references)?
|
|
.with_identity_provider(std::sync::Arc::new(identity_provider))
|
|
.build();
|
|
if config.demo_seed {
|
|
service.seed_demo_assets().await?;
|
|
}
|
|
spawn_artifact_reconciliation(registry, artifact_store).await;
|
|
spawn_invocation_log_cleanup(service.clone(), config.invocation_log_retention_days);
|
|
let state = AppState {
|
|
service,
|
|
api_rate_limiter: if cache_config.backend.is_external() {
|
|
RequestRateLimiter::new_shared(api_rate_limit, cache_stores.rate_limit.clone())
|
|
} else {
|
|
RequestRateLimiter::new(api_rate_limit)
|
|
},
|
|
trusted_proxy_ips: config.trusted_proxy_ips.clone(),
|
|
};
|
|
let app = build_app(state);
|
|
let listener = TcpListener::bind(config.bind_addr).await?;
|
|
let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
|
|
|
|
info!(
|
|
name: "admin.postgres_pool.configured",
|
|
runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
|
|
admin_rate_limit_rps = api_rate_limit.requests_per_second,
|
|
admin_rate_limit_burst = api_rate_limit.burst,
|
|
cache_backend = %cache_config.backend,
|
|
max_connections = pool_config.max_connections,
|
|
min_connections = pool_config.min_connections,
|
|
acquire_timeout_ms = pool_config.acquire_timeout_ms,
|
|
idle_timeout_ms = pool_config.idle_timeout_ms,
|
|
max_lifetime_ms = pool_config.max_lifetime_ms,
|
|
invocation_log_retention_days = config.invocation_log_retention_days,
|
|
"postgres pool configured"
|
|
);
|
|
info!(name: "admin.server.listening", bind_address = %config.bind_addr, "admin-api listening");
|
|
*startup_completed = true;
|
|
|
|
if let Some(metrics_server) = metrics_server {
|
|
tokio::select! {
|
|
result = axum::serve(listener, make_service) => result?,
|
|
result = metrics_server.serve() => result?,
|
|
}
|
|
} else {
|
|
axum::serve(listener, make_service).await?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn init_observability(
|
|
config: &ObservabilitySettings,
|
|
) -> Result<ObservabilityLifecycle, Box<dyn std::error::Error>> {
|
|
let identity = ServiceIdentity::try_new(
|
|
"admin-api",
|
|
env!("CARGO_PKG_VERSION"),
|
|
config.environment.clone(),
|
|
)?;
|
|
let base = ObservabilityConfig::try_new(
|
|
identity,
|
|
config.log_filter.clone(),
|
|
RedactionLimits::default(),
|
|
)?;
|
|
let sentry = SentryConfig::parse(
|
|
config
|
|
.sentry_dsn
|
|
.as_ref()
|
|
.map(|value| value.expose_secret()),
|
|
)?;
|
|
let otlp = otlp_config(config)?;
|
|
Ok(ObservabilityLifecycle::init_with_exporters(
|
|
base, sentry, otlp,
|
|
)?)
|
|
}
|
|
|
|
fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::ConfigError> {
|
|
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
|
|
database_options(&config.database).map_err(|_| invalid("database.source"))?;
|
|
postgres_pool_config(&config.database).map_err(|_| invalid("database.pool"))?;
|
|
MetricsConfig::new(
|
|
config.observability.metrics.enabled,
|
|
config.observability.metrics.bind_addr,
|
|
config
|
|
.observability
|
|
.metrics
|
|
.bearer_token
|
|
.as_ref()
|
|
.map(|v| v.expose_secret().to_owned()),
|
|
)
|
|
.map_err(|_| invalid("observability.metrics"))?;
|
|
RuntimeLimits::try_new(
|
|
config.runtime.max_concurrent_unary,
|
|
config.runtime.max_concurrent_sessions,
|
|
)
|
|
.map_err(|_| invalid("runtime.limits"))?;
|
|
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
|
|
RequestRateLimitConfig::new(
|
|
config.rate_limit.requests_per_second,
|
|
config.rate_limit.burst,
|
|
)
|
|
.map_err(|_| invalid("admin.rate_limit"))?;
|
|
SecretCrypto::new(config.runtime.master_key.expose_secret())
|
|
.map_err(|_| invalid("runtime.master_key"))?;
|
|
crank_runtime::OutboundHttpPolicy::try_new_with_limits(
|
|
config.runtime.outbound.allowed_hosts.clone(),
|
|
config.runtime.outbound.denied_hosts.clone(),
|
|
config.runtime.outbound.max_request_bytes,
|
|
config.runtime.outbound.max_response_bytes,
|
|
)
|
|
.map_err(|_| invalid("runtime.outbound"))?;
|
|
let identity = ServiceIdentity::try_new(
|
|
"admin-api",
|
|
env!("CARGO_PKG_VERSION"),
|
|
config.observability.environment.clone(),
|
|
)
|
|
.map_err(|_| invalid("observability.environment"))?;
|
|
ObservabilityConfig::try_new(
|
|
identity,
|
|
config.observability.log_filter.clone(),
|
|
RedactionLimits::default(),
|
|
)
|
|
.map_err(|_| invalid("observability.log_filter"))?;
|
|
SentryConfig::parse(
|
|
config
|
|
.observability
|
|
.sentry_dsn
|
|
.as_ref()
|
|
.map(|v| v.expose_secret()),
|
|
)
|
|
.map_err(|_| invalid("observability.sentry_dsn"))?;
|
|
otlp_config(&config.observability).map_err(|_| invalid("observability.otlp"))?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn verified_startup_secret_crypto(
|
|
registry: &PostgresRegistry,
|
|
master_key: &str,
|
|
) -> Result<SecretCrypto, Box<dyn std::error::Error>> {
|
|
let active = registry.active_master_key_identity().await?;
|
|
let secret_crypto = if let Some(identity) = active {
|
|
SecretCrypto::with_epoch(master_key, identity.epoch)?
|
|
} else {
|
|
let crypto = SecretCrypto::new(master_key)?;
|
|
let mut after_secret_id: Option<String> = None;
|
|
let mut after_version: Option<u32> = None;
|
|
loop {
|
|
let versions = registry
|
|
.list_secret_versions_for_master_key_epoch_page(
|
|
1,
|
|
after_secret_id.as_deref(),
|
|
after_version,
|
|
1_000,
|
|
)
|
|
.await?;
|
|
if versions.is_empty() {
|
|
break;
|
|
}
|
|
for version in versions {
|
|
after_secret_id = Some(version.secret_version.secret_id.as_str().to_owned());
|
|
after_version = Some(version.secret_version.version);
|
|
crypto.decrypt_for_epoch(
|
|
&version.secret_version.key_version,
|
|
version.master_key_epoch,
|
|
&version.secret_version.ciphertext,
|
|
)?;
|
|
}
|
|
}
|
|
crypto
|
|
};
|
|
let master_key_observed_at = time::OffsetDateTime::now_utc();
|
|
registry
|
|
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
|
|
epoch: secret_crypto.master_key_epoch(),
|
|
fingerprint: secret_crypto.master_key_fingerprint(),
|
|
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
|
|
observed_at: &master_key_observed_at,
|
|
})
|
|
.await?;
|
|
Ok(secret_crypto)
|
|
}
|
|
|
|
fn otlp_config(
|
|
config: &ObservabilitySettings,
|
|
) -> Result<OtlpTraceConfig, crank_observability::OtlpTraceConfigError> {
|
|
let values = &config.otlp;
|
|
OtlpTraceConfig::from_values(
|
|
values.endpoint.clone(),
|
|
values.traces_endpoint.clone(),
|
|
values.protocol.clone(),
|
|
values.traces_protocol.clone(),
|
|
values.timeout.clone(),
|
|
values.traces_timeout.clone(),
|
|
values
|
|
.headers
|
|
.as_ref()
|
|
.map(|value| value.expose_secret().to_owned()),
|
|
values
|
|
.traces_headers
|
|
.as_ref()
|
|
.map(|value| value.expose_secret().to_owned()),
|
|
values.max_queue_size,
|
|
values.max_export_batch_size,
|
|
values.schedule_delay.clone(),
|
|
values.export_timeout.clone(),
|
|
)
|
|
}
|
|
|
|
fn postgres_pool_config(
|
|
config: &DatabaseSettings,
|
|
) -> Result<PostgresPoolConfig, crank_registry::PostgresPoolConfigError> {
|
|
PostgresPoolConfig::try_new(
|
|
config.pool.max_connections,
|
|
config.pool.min_connections,
|
|
config.pool.acquire_timeout_ms,
|
|
config.pool.idle_timeout_ms,
|
|
config.pool.max_lifetime_ms,
|
|
)
|
|
}
|
|
|
|
fn database_options(
|
|
config: &DatabaseSettings,
|
|
) -> Result<PgConnectOptions, Box<dyn std::error::Error>> {
|
|
if let Some(url) = &config.url {
|
|
return url
|
|
.expose_secret()
|
|
.parse::<PgConnectOptions>()
|
|
.map_err(|_| io::Error::other("database URL is invalid").into());
|
|
}
|
|
Ok(PgConnectOptions::new()
|
|
.host(&config.host)
|
|
.port(config.port)
|
|
.database(&config.database)
|
|
.username(&config.username)
|
|
.password(config.password.expose_secret()))
|
|
}
|
|
|
|
fn runtime_cache_config(
|
|
config: &AdminProcessConfig,
|
|
) -> Result<RuntimeCacheConfig, crank_runtime::RuntimeCacheConfigError> {
|
|
RuntimeCacheConfig::try_new(
|
|
match config.runtime.cache.backend {
|
|
ConfigCacheBackend::Memory => CacheBackend::Memory,
|
|
ConfigCacheBackend::Valkey => CacheBackend::Valkey,
|
|
ConfigCacheBackend::Redis => CacheBackend::Redis,
|
|
},
|
|
config
|
|
.runtime
|
|
.cache
|
|
.url
|
|
.as_ref()
|
|
.map(|value| value.expose_secret().to_owned()),
|
|
)
|
|
}
|
|
|
|
fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
|
|
tokio::spawn(async move {
|
|
let mut interval = tokio::time::interval(Duration::from_secs(60 * 60));
|
|
loop {
|
|
interval.tick().await;
|
|
let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days);
|
|
match service.cleanup_invocation_logs_before(cutoff).await {
|
|
Ok(outcome) if outcome.deleted_records > 0 => info!(
|
|
name: "admin.invocation_log_cleanup.completed",
|
|
status = ?outcome.status,
|
|
removed = outcome.deleted_records,
|
|
requested_cutoff = %outcome.policy.requested_cutoff,
|
|
effective_cutoff = %outcome.policy.effective_cutoff,
|
|
preserved_usage_window_days = outcome.policy.preserved_usage_window_days,
|
|
"expired invocation logs removed"
|
|
),
|
|
Ok(outcome) => info!(
|
|
name: "admin.invocation_log_cleanup.noop",
|
|
status = ?outcome.status,
|
|
requested_cutoff = %outcome.policy.requested_cutoff,
|
|
effective_cutoff = %outcome.policy.effective_cutoff,
|
|
preserved_usage_window_days = outcome.policy.preserved_usage_window_days,
|
|
"no expired invocation logs removed"
|
|
),
|
|
Err(_) => warn!(
|
|
name: "admin.invocation_log_cleanup.failed",
|
|
error_category = "registry_cleanup",
|
|
"failed to remove expired invocation logs"
|
|
),
|
|
}
|
|
}
|
|
});
|
|
}
|