fix(openapi): harden story 2.1 production lifecycle
CI / Rust Checks (push) Failing after 4m6s
CI / UI Checks (push) Has been skipped
CI / Frontend E2E (push) Has been skipped
CI / Community Image Smoke (push) Has been skipped
CI / Deploy (push) Has been skipped

This commit is contained in:
2026-08-29 00:48:22 +03:00
parent 2c94af6791
commit bc03c33387
46 changed files with 2198 additions and 276 deletions
+5
View File
@@ -24,9 +24,14 @@ WORKDIR /app
COPY --from=builder /tmp/admin-api /usr/local/bin/admin-api
COPY --from=builder /tmp/crank-migrate /usr/local/bin/crank-migrate
COPY apps/admin-api/docker-entrypoint.sh /usr/local/bin/crank-admin-entrypoint
RUN chmod 0755 /usr/local/bin/crank-admin-entrypoint
ENV CRANK_ADMIN_BIND=0.0.0.0:3001
ENV CRANK_STORAGE_ROOT=/var/lib/crank/storage
EXPOSE 3001
ENTRYPOINT ["/usr/local/bin/crank-admin-entrypoint"]
CMD ["admin-api"]
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
set -eu
# Docker creates a fresh named volume as root:root 0755. The artifact store
# deliberately rejects that mode: before starting the API, provision exactly
# the private root its pinned-directory checks require.
if [ "$#" -gt 0 ] && [ "$1" = "admin-api" ]; then
storage_root="${CRANK_STORAGE_ROOT:-/var/lib/crank/storage}"
case "$storage_root" in
/*) ;;
*)
echo "CRANK_STORAGE_ROOT must be an absolute path" >&2
exit 64
;;
esac
if [ -L "$storage_root" ] || { [ -e "$storage_root" ] && [ ! -d "$storage_root" ]; }; then
echo "CRANK_STORAGE_ROOT must be a directory, not a symlink or file" >&2
exit 64
fi
umask 077
mkdir -p -- "$storage_root"
chmod 0700 -- "$storage_root"
fi
exec "$@"
+3 -2
View File
@@ -560,14 +560,15 @@ pub struct OpenApiImportCreateResponse {
pub findings: Vec<crank_import::rest::ImportFinding>,
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct OpenApiImportCreatedOperation {
pub operation_key: String,
pub operation_id: String,
pub name: String,
pub version: u32,
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct OpenApiImportSkippedOperation {
pub operation_key: String,
pub name: String,
-8
View File
@@ -195,14 +195,6 @@ impl ApiError {
}
}
pub(crate) fn source_unavailable() -> Self {
Self::openapi_upload(OpenApiUploadLocale::En, "source_unavailable")
}
pub(crate) fn source_integrity() -> Self {
Self::openapi_upload(OpenApiUploadLocale::En, "source_integrity")
}
pub(crate) fn rate_limited_with_context(message: impl Into<String>, context: Value) -> Self {
Self::RateLimited {
message: message.into(),
-1
View File
@@ -204,7 +204,6 @@ async fn run(
verified_startup_secret_crypto(&registry, config.runtime.master_key.expose_secret())
.await?;
let artifact_store = open_reconciliation_store(config.storage_root.clone()).await?;
registry.delete_expired_import_jobs().await?;
let outbound_http_policy = crank_runtime::OutboundHttpPolicy::try_new_with_limits(
config.runtime.outbound.allowed_hosts.clone(),
config.runtime.outbound.denied_hosts.clone(),
+174 -13
View File
@@ -1,5 +1,6 @@
use std::{
path::PathBuf,
sync::Arc,
time::{Duration, SystemTime},
};
@@ -7,7 +8,7 @@ use async_trait::async_trait;
use crank_artifacts::{
ArtifactError, ArtifactStore, ReconciliationCursor, ReconciliationMutation,
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
ReconciliationScanStop,
ReconciliationScanStop, TempScanCursor,
};
use crank_registry::{
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
@@ -25,6 +26,11 @@ pub const RECONCILIATION_LEASE: Duration = Duration::from_secs(5 * 60);
pub const RECONCILIATION_TRAVERSAL_LIMIT: usize = 4_096;
pub const RECONCILIATION_CANDIDATE_LIMIT: usize = 32;
pub const RECONCILIATION_MUTATION_LIMIT: usize = 32;
pub const RECONCILIATION_TICK_DEADLINE: Duration = Duration::from_secs(30);
pub const TEMP_CLEANUP_GRACE: Duration = Duration::from_secs(60 * 60);
pub const TEMP_CLEANUP_SCAN_LIMIT: usize = 512;
pub const TEMP_CLEANUP_DELETE_LIMIT: usize = 16;
pub const IMPORT_JOB_CLEANUP_LIMIT: u32 = 128;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Phase {
@@ -191,6 +197,8 @@ pub enum ReconciliationCoordinatorError {
Backend,
#[error("artifact reconciliation backend exceeded its bounded contract")]
Bounds,
#[error("artifact reconciliation backend exceeded its deadline")]
Deadline,
}
#[async_trait]
@@ -215,7 +223,7 @@ struct ReconciliationCoordinator<B: ReconciliationBackend> {
#[derive(Clone)]
struct PostgresReconciliationBackend {
registry: PostgresRegistry,
store: ArtifactStore,
store: Arc<ArtifactStore>,
}
enum PostgresReconciliationCursor {
@@ -230,7 +238,7 @@ impl std::fmt::Debug for PostgresReconciliationCursor {
}
impl PostgresReconciliationBackend {
fn new(registry: PostgresRegistry, store: ArtifactStore) -> Self {
fn new(registry: PostgresRegistry, store: Arc<ArtifactStore>) -> Self {
Self { registry, store }
}
@@ -245,7 +253,7 @@ impl PostgresReconciliationBackend {
&self,
artifact_ref: crank_artifacts::ArtifactRef,
) -> Result<ReconciliationPresence, ReconciliationCoordinatorError> {
let store = self.store.clone();
let store = Arc::clone(&self.store);
let result =
tokio::task::spawn_blocking(move || store.reconciliation_presence(&artifact_ref))
.await
@@ -518,7 +526,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
Phase::Recovery => ReconciliationNamespace::Quarantine,
Phase::Sweep => ReconciliationNamespace::Final,
};
let store = self.store.clone();
let store = Arc::clone(&self.store);
let (mut scan, candidates) = tokio::task::spawn_blocking(move || {
store.scan_reconciliation_namespace(
namespace,
@@ -560,7 +568,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
Phase::Recovery => Duration::ZERO,
Phase::Sweep => RECONCILIATION_GRACE,
};
let store = self.store.clone();
let store = Arc::clone(&self.store);
let registration = tokio::task::spawn_blocking(move || {
let registration = store.register_reconciliation(
&candidate,
@@ -621,7 +629,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
step.mutation_attempts += 1;
step.physical_mutation_possible = true;
let store = self.store.clone();
let store = Arc::clone(&self.store);
let mutation = match tokio::task::spawn_blocking(move || match phase {
Phase::Recovery => store.delete_quarantined_reconciliation(candidate),
Phase::Sweep => store.quarantine_reconciliation(candidate),
@@ -717,8 +725,8 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
/// async executor.
pub async fn open_reconciliation_store(
root: PathBuf,
) -> Result<ArtifactStore, ReconciliationCoordinatorError> {
tokio::task::spawn_blocking(move || ArtifactStore::open(root))
) -> Result<Arc<ArtifactStore>, ReconciliationCoordinatorError> {
tokio::task::spawn_blocking(move || ArtifactStore::open(root).map(Arc::new))
.await
.map_err(|_| ReconciliationCoordinatorError::Backend)?
.map_err(|_| ReconciliationCoordinatorError::Backend)
@@ -726,10 +734,25 @@ pub async fn open_reconciliation_store(
/// Runs the immediate bounded startup cycle, then schedules 15-minute ticks.
/// Recovery remains ahead of the final sweep even when it spans several ticks.
pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: ArtifactStore) {
let backend = PostgresReconciliationBackend::new(registry, store);
pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: Arc<ArtifactStore>) {
let cleanup_registry = registry.clone();
let backend = PostgresReconciliationBackend::new(registry, Arc::clone(&store));
let mut coordinator = ReconciliationCoordinator::new(backend);
observe_tick(coordinator.tick().await);
let mut temp_cursor = None;
let startup_maintenance = async {
observe_import_job_cleanup(&cleanup_registry).await;
observe_tick(run_bounded_reconciliation_tick(&mut coordinator).await);
temp_cursor = observe_stale_temp_cleanup(Arc::clone(&store), None).await;
};
if tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, startup_maintenance)
.await
.is_err()
{
warn!(
name: "admin.artifact_maintenance.startup_deadline",
"startup maintenance exceeded its shared deadline; remaining work will resume in background"
);
}
tokio::spawn(async move {
let mut interval = tokio::time::interval(RECONCILIATION_INTERVAL);
@@ -738,11 +761,148 @@ pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: Ar
interval.tick().await;
loop {
interval.tick().await;
observe_tick(coordinator.tick().await);
observe_import_job_cleanup(&cleanup_registry).await;
observe_tick(run_bounded_reconciliation_tick(&mut coordinator).await);
temp_cursor = observe_stale_temp_cleanup(Arc::clone(&store), temp_cursor).await;
}
});
}
async fn observe_import_job_cleanup(registry: &PostgresRegistry) {
let cleanup = async {
let mut deleted_jobs = 0_u64;
let mut detached_sources = 0_u64;
loop {
let report = registry
.cleanup_expired_import_jobs(IMPORT_JOB_CLEANUP_LIMIT)
.await?;
deleted_jobs = deleted_jobs.saturating_add(report.deleted_jobs);
detached_sources = detached_sources.saturating_add(report.detached_sources);
if !report.more_work {
return Ok::<_, crank_registry::RegistryError>((deleted_jobs, detached_sources));
}
// Each pass commits independently. Yield before requesting the
// next lock batch so normal import traffic can make progress.
tokio::task::yield_now().await;
}
};
match tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, cleanup).await {
Ok(Ok((deleted_jobs, detached_sources))) => info!(
name: "admin.openapi_import_cleanup.tick",
deleted_jobs,
detached_sources,
more_work = false,
"OpenAPI import cleanup tick completed"
),
Ok(Err(_)) => warn!(
name: "admin.openapi_import_cleanup.failed",
error_category = "registry",
"OpenAPI import cleanup tick will be retried"
),
Err(_) => warn!(
name: "admin.openapi_import_cleanup.failed",
error_category = "deadline",
"OpenAPI import cleanup tick exceeded its deadline"
),
}
}
async fn run_bounded_reconciliation_tick<B: ReconciliationBackend>(
coordinator: &mut ReconciliationCoordinator<B>,
) -> Result<ReconciliationTickReport, ReconciliationCoordinatorError> {
tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, coordinator.tick())
.await
.map_err(|_| ReconciliationCoordinatorError::Deadline)?
}
async fn observe_stale_temp_cleanup(
store: Arc<ArtifactStore>,
cursor: Option<TempScanCursor>,
) -> Option<TempScanCursor> {
let progress = Arc::new(std::sync::Mutex::new(cursor));
let cleanup_progress = Arc::clone(&progress);
let cleanup = async move {
let mut scanned = 0_usize;
let mut omitted = 0_usize;
let mut deleted = 0_usize;
let mut retryable = 0_usize;
loop {
let scanner = Arc::clone(&store);
let cursor = cleanup_progress
.lock()
.map_err(|_| ArtifactError::Storage)?
.clone();
let (scan, candidates, next_cursor) = tokio::task::spawn_blocking(move || {
scanner.scan_stale_temps_after(
TEMP_CLEANUP_GRACE,
cursor,
TEMP_CLEANUP_SCAN_LIMIT,
TEMP_CLEANUP_DELETE_LIMIT,
)
})
.await
.map_err(|_| ArtifactError::Storage)??;
scanned = scanned.saturating_add(scan.scanned);
omitted = omitted.saturating_add(scan.omitted);
*cleanup_progress
.lock()
.map_err(|_| ArtifactError::Storage)? = Some(next_cursor);
for candidate in candidates {
let store = Arc::clone(&store);
match tokio::task::spawn_blocking(move || store.delete_stale_temp(candidate)).await
{
Ok(Ok(())) => deleted += 1,
Ok(Err(_)) | Err(_) => retryable += 1,
}
}
if scan.complete {
return Ok::<_, ArtifactError>((
scanned,
omitted,
deleted,
retryable,
cleanup_progress
.lock()
.map_err(|_| ArtifactError::Storage)?
.clone(),
));
}
tokio::task::yield_now().await;
}
};
match tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, cleanup).await {
Ok(Ok((scanned, omitted, deleted, retryable, next_cursor))) => {
info!(
name: "admin.artifact_temp_cleanup.tick",
scanned,
omitted,
deleted,
retryable,
"artifact temporary-file cleanup tick completed"
);
next_cursor
}
Ok(Err(_)) => {
warn!(
name: "admin.artifact_temp_cleanup.failed",
error_category = "backend",
"artifact temporary-file cleanup tick will be retried"
);
progress.lock().ok().and_then(|cursor| cursor.clone())
}
Err(_) => {
warn!(
name: "admin.artifact_temp_cleanup.failed",
error_category = "deadline",
"artifact temporary-file cleanup tick exceeded its deadline"
);
progress.lock().ok().and_then(|cursor| cursor.clone())
}
}
}
fn observe_tick(result: Result<ReconciliationTickReport, ReconciliationCoordinatorError>) {
match result {
Ok(report) => info!(
@@ -772,6 +932,7 @@ fn observe_tick(result: Result<ReconciliationTickReport, ReconciliationCoordinat
error_category = match error {
ReconciliationCoordinatorError::Backend => "backend",
ReconciliationCoordinatorError::Bounds => "bounds",
ReconciliationCoordinatorError::Deadline => "deadline",
},
"artifact reconciliation tick will be retried"
),
+2 -2
View File
@@ -518,7 +518,7 @@ async fn production_backend_recovers_absence_before_sweep_and_restarts_after_mut
assert_ne!(unsafe_name, later.artifact_ref().digest_hex());
symlink("not-a-digest", shard.join(unsafe_name)).unwrap();
let backend = PostgresReconciliationBackend::new(registry.clone(), store.clone());
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone()));
let mut coordinator = ReconciliationCoordinator::with_tick_limits(
backend,
StepLimits {
@@ -660,7 +660,7 @@ async fn expired_claim_cursor_prevents_unsafe_prefix_starvation_across_ticks() {
.unwrap();
remove_registered_artifact(&store, &absent);
let backend = PostgresReconciliationBackend::new(registry.clone(), store);
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store));
let mut coordinator = ReconciliationCoordinator::new(backend);
let first = coordinator.tick().await.unwrap();
assert_eq!(first.candidates, RECONCILIATION_CANDIDATE_LIMIT);
+74 -8
View File
@@ -24,23 +24,89 @@ pub async fn health() -> Json<serde_json::Value> {
}
pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
match state.service.readiness().await {
Ok(()) => (
let checks = state.service.readiness().await;
let postgres = if checks.postgres {
"ready"
} else {
"not_ready"
};
let artifact_storage = if checks.artifact_storage {
"ready"
} else {
"not_ready"
};
if checks.is_ready() {
(
StatusCode::OK,
Json(json!({
"service": "admin-api",
"status": "ready",
"checks": { "postgres": "ready" }
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
})),
),
Err(error) => (
)
} else {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "admin-api",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": error.to_string()
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
})),
),
)
}
}
#[cfg(test)]
mod tests {
use crate::service::ReadinessChecks;
#[test]
fn readiness_checks_identify_a_postgres_failure() {
let checks = ReadinessChecks {
postgres: false,
artifact_storage: true,
};
assert!(!checks.is_ready());
assert_eq!(
if checks.postgres {
"ready"
} else {
"not_ready"
},
"not_ready"
);
assert_eq!(
if checks.artifact_storage {
"ready"
} else {
"not_ready"
},
"ready"
);
}
#[test]
fn readiness_checks_identify_an_artifact_storage_failure() {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: false,
};
assert!(!checks.is_ready());
assert_eq!(
if checks.postgres {
"ready"
} else {
"not_ready"
},
"ready"
);
assert_eq!(
if checks.artifact_storage {
"ready"
} else {
"not_ready"
},
"not_ready"
);
}
}
+77 -13
View File
@@ -70,7 +70,16 @@ async fn parse_openapi_upload(
.ok_or_else(|| ApiError::openapi_upload(locale, "invalid_filename"))?;
let mime_type = field
.content_type()
.map(ToString::to_string)
// Axum exposes the raw header value here. Persist only the MIME
// essence so parameters and case cannot make route and service
// validation disagree.
.map(|mime| {
mime.split(';')
.next()
.unwrap_or_default()
.trim()
.to_ascii_lowercase()
})
.ok_or_else(|| ApiError::openapi_upload(locale, "invalid_media_type"))?;
if !valid_upload_type(filename, &mime_type) {
return Err(ApiError::openapi_upload(locale, "invalid_media_type"));
@@ -123,35 +132,90 @@ fn valid_upload_type(filename: &str, mime_type: &str) -> bool {
}
fn openapi_upload_locale(headers: &HeaderMap) -> OpenApiUploadLocale {
let prefers_russian = headers
let Some(value) = headers
.get("accept-language")
.and_then(|value| value.to_str().ok())
.is_some_and(|value| {
value.split(',').any(|range| {
let language = range.split(';').next().unwrap_or_default().trim();
language.eq_ignore_ascii_case("ru")
|| language.to_ascii_lowercase().starts_with("ru-")
else {
return OpenApiUploadLocale::En;
};
// RFC 9110: highest q wins; ties preserve the header's order. Only the
// locales served by this endpoint participate in negotiation.
let mut preferred = (0_u16, usize::MAX, OpenApiUploadLocale::En);
for (index, range) in value.split(',').enumerate() {
let mut parts = range.split(';');
let language = parts.next().unwrap_or_default().trim();
let locale = if language.eq_ignore_ascii_case("ru")
|| language.to_ascii_lowercase().starts_with("ru-")
{
Some(OpenApiUploadLocale::Ru)
} else if language.eq_ignore_ascii_case("en")
|| language.to_ascii_lowercase().starts_with("en-")
|| language == "*"
{
Some(OpenApiUploadLocale::En)
} else {
None
};
let Some(locale) = locale else { continue };
let quality = match parts
.filter_map(|parameter| {
let (name, value) = parameter.trim().split_once('=')?;
name.eq_ignore_ascii_case("q").then_some(value.trim())
})
});
if prefers_russian {
OpenApiUploadLocale::Ru
} else {
OpenApiUploadLocale::En
.next()
{
None => Some(1_000_u16),
Some(value) => value
.parse::<f32>()
.ok()
.filter(|quality| (0.0..=1.0).contains(quality) && *quality > 0.0)
.map(|quality| (quality * 1_000.0).round() as u16),
};
let Some(quality) = quality else { continue };
if quality > preferred.0 || (quality == preferred.0 && index < preferred.1) {
preferred = (quality, index, locale);
}
}
preferred.2
}
pub async fn create_openapi_import(
Path(path): Path<WorkspaceImportPath>,
State(state): State<AppState>,
headers: HeaderMap,
Json(payload): Json<OpenApiImportCreatePayload>,
) -> Result<Json<Value>, ApiError> {
let imported = state
.service
.create_openapi_import(
.create_openapi_import_with_locale(
&path.workspace_id.as_str().into(),
&path.job_id.as_str().into(),
payload,
openapi_upload_locale(&headers),
)
.await?;
Ok(Json(json!(imported)))
}
#[cfg(test)]
mod tests {
use axum::http::{HeaderMap, HeaderValue};
use super::*;
fn locale(value: &str) -> OpenApiUploadLocale {
let mut headers = HeaderMap::new();
headers.insert("accept-language", HeaderValue::from_str(value).unwrap());
openapi_upload_locale(&headers)
}
#[test]
fn accept_language_honours_quality_zero_and_header_order() {
assert_eq!(locale("ru;q=0, en;q=0.5"), OpenApiUploadLocale::En);
assert_eq!(locale("en;q=0.5, ru;q=0.5"), OpenApiUploadLocale::En);
assert_eq!(locale("ru-RU;q=0.9, en;q=1"), OpenApiUploadLocale::En);
assert_eq!(locale("ru, en;q=0.5"), OpenApiUploadLocale::Ru);
assert_eq!(locale("ru;q=bad, en;q=0.5"), OpenApiUploadLocale::En);
}
}
+27 -9
View File
@@ -74,7 +74,7 @@ pub struct AdminService {
pub struct AdminServiceBuilder {
registry: PostgresRegistry,
storage_root: PathBuf,
artifact_store: Option<ArtifactStore>,
artifact_store: Option<Arc<ArtifactStore>>,
auth_settings: AuthSettings,
secret_crypto: SecretCrypto,
runtime: RuntimeExecutor,
@@ -88,6 +88,18 @@ pub struct AdminServiceBuilder {
pub use crate::dto::*;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ReadinessChecks {
pub postgres: bool,
pub artifact_storage: bool,
}
impl ReadinessChecks {
pub const fn is_ready(self) -> bool {
self.postgres && self.artifact_storage
}
}
#[derive(Clone, Debug)]
pub struct AdminAuditContext {
actor: AuditActor,
@@ -124,9 +136,16 @@ pub(super) struct CredentialAuditRecord<'a> {
}
impl AdminService {
pub async fn readiness(&self) -> Result<(), ApiError> {
self.registry.ping().await?;
Ok(())
pub async fn readiness(&self) -> ReadinessChecks {
let postgres = self.registry.ping().await.is_ok();
let store = Arc::clone(&self.artifact_store);
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
.await
.is_ok_and(|result| result.is_ok());
ReadinessChecks {
postgres,
artifact_storage,
}
}
#[cfg(test)]
@@ -230,7 +249,7 @@ impl AdminServiceBuilder {
/// Reuses the process-wide immutable artifact authority for OpenAPI
/// ingress and reconciliation. Tests may omit this and use their private
/// storage root instead.
pub fn with_artifact_store(mut self, artifact_store: ArtifactStore) -> Self {
pub fn with_artifact_store(mut self, artifact_store: Arc<ArtifactStore>) -> Self {
self.artifact_store = Some(artifact_store);
self
}
@@ -264,10 +283,9 @@ impl AdminServiceBuilder {
pub fn build(self) -> AdminService {
AdminService {
registry: self.registry,
artifact_store: Arc::new(
self.artifact_store
.unwrap_or_else(|| ArtifactStore::new(&self.storage_root)),
),
artifact_store: self
.artifact_store
.unwrap_or_else(|| Arc::new(ArtifactStore::new(&self.storage_root))),
runtime: self.runtime,
storage: LocalArtifactStorage::new(self.storage_root),
auth_settings: self.auth_settings,
+198 -48
View File
@@ -29,6 +29,10 @@ use crate::{
};
const IMPORT_JOB_TTL_HOURS: i64 = 24;
const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30);
const OPENAPI_PARSE_CONCURRENCY: usize = 4;
static OPENAPI_PARSE_SLOTS: tokio::sync::Semaphore =
tokio::sync::Semaphore::const_new(OPENAPI_PARSE_CONCURRENCY);
impl AdminService {
#[instrument(skip(self, upload), fields(workspace_id = %workspace_id.as_str()))]
@@ -38,7 +42,6 @@ impl AdminService {
upload: OpenApiUpload,
) -> Result<OpenApiImportPreviewResponse, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
self.registry.delete_expired_import_jobs().await?;
validate_openapi_upload(&upload)?;
let OpenApiUpload {
@@ -72,19 +75,38 @@ impl AdminService {
source_id.clone(),
source.updated_at,
);
let verified = self
let verified = match self
.registry
.read_artifact_source(&self.artifact_store, workspace_id, &source_id)
.await?;
.read_artifact_source(
std::sync::Arc::clone(&self.artifact_store),
workspace_id,
&source_id,
)
.await
{
Ok(verified) => verified,
Err(error) => {
detach_guard.detach_now().await;
return Err(ApiError::from(error));
}
};
if verified.source.blob.artifact_ref != *artifact.artifact_ref() {
detach_guard.detach_now().await;
return Err(ApiError::openapi_upload(locale, "source_integrity"));
}
let preview = parse_verified_preview(verified.bytes, locale).await?;
let preview = match parse_verified_preview(verified.bytes, locale).await {
Ok(preview) => preview,
Err(error) => {
detach_guard.detach_now().await;
return Err(error);
}
};
if preview
.groups
.iter()
.all(|group| group.operations.is_empty())
{
detach_guard.detach_now().await;
return Err(ApiError::openapi_upload(locale, "no_methods"));
}
let source_envelope = ImportJobSourceEnvelope {
@@ -93,15 +115,18 @@ impl AdminService {
};
let preview_value = serde_json::to_value(&preview)
.map_err(|error| ApiError::internal(error.to_string()))?;
let preview_digest = preview_digest(&preview_value)?;
let preview_payload = json!({
"source": {
"source_id": source_envelope.source_id.as_str(),
"digest": source_envelope.digest.as_str(),
},
"preview": preview_value,
"preview_digest": preview_digest,
});
self.registry
if let Err(error) = self
.registry
.create_import_job(CreateImportJobRequest {
id: &job_id,
workspace_id,
@@ -114,7 +139,11 @@ impl AdminService {
created_at: &now,
expires_at: &expires_at,
})
.await?;
.await
{
detach_guard.detach_now().await;
return Err(ApiError::from(error));
}
detach_guard.disarm();
Ok(OpenApiImportPreviewResponse {
@@ -132,9 +161,24 @@ impl AdminService {
workspace_id: &WorkspaceId,
job_id: &ImportJobId,
payload: OpenApiImportCreatePayload,
) -> Result<OpenApiImportCreateResponse, ApiError> {
self.create_openapi_import_with_locale(
workspace_id,
job_id,
payload,
OpenApiUploadLocale::En,
)
.await
}
pub async fn create_openapi_import_with_locale(
&self,
workspace_id: &WorkspaceId,
job_id: &ImportJobId,
payload: OpenApiImportCreatePayload,
locale: OpenApiUploadLocale,
) -> Result<OpenApiImportCreateResponse, ApiError> {
self.ensure_workspace_exists(workspace_id).await?;
self.registry.delete_expired_import_jobs().await?;
if !matches!(payload.conflict_mode.as_str(), "skip" | "rename") {
return Err(ApiError::validation(
@@ -185,22 +229,56 @@ impl AdminService {
application_key: &application_key,
conflict_mode,
operations: &[],
pre_skipped: &[],
finished_at: &finished_at,
})
.await?;
return Ok(openapi_import_response(applied, Vec::new()));
return Ok(openapi_import_response(applied));
}
let source = import_job_source(&job.preview_payload)?;
let verified = self
let source = import_job_source(&job.preview_payload, locale)?;
let verified = match self
.registry
.read_artifact_source(&self.artifact_store, workspace_id, &source.source_id)
.read_artifact_source(
std::sync::Arc::clone(&self.artifact_store),
workspace_id,
&source.source_id,
)
.await
.map_err(openapi_source_error)?;
{
Ok(verified) => verified,
Err(
error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable),
) => {
// Another request may have finished and detached the source
// after our unlocked Pending read. Re-read the job and ask
// the registry for its canonical, locked replay instead of
// exposing a false `source_unavailable` outcome.
let latest = self.registry.get_import_job(workspace_id, job_id).await?;
if latest.is_some_and(|latest| latest.status == ImportJobStatus::Completed) {
let applied = self
.registry
.apply_import_job(ApplyImportJobRequest {
id: job_id,
workspace_id,
application_key: &application_key,
conflict_mode,
operations: &[],
pre_skipped: &[],
finished_at: &finished_at,
})
.await?;
return Ok(openapi_import_response(applied));
}
return Err(openapi_source_error(locale, error));
}
Err(error) => return Err(openapi_source_error(locale, error)),
};
if verified.source.blob.artifact_ref != source.digest {
return Err(ApiError::source_integrity());
return Err(ApiError::openapi_upload(locale, "source_integrity"));
}
let preview = parse_verified_preview(verified.bytes, OpenApiUploadLocale::En).await?;
let preview = parse_verified_preview(verified.bytes, locale).await?;
verify_preview_contract(&job.preview_payload, &preview, locale)?;
let mut candidates = BTreeMap::new();
for group in &preview.groups {
for operation in &group.operations {
@@ -213,7 +291,7 @@ impl AdminService {
for operation_key in selected {
let Some(candidate) = candidates.get(&operation_key) else {
skipped.push(OpenApiImportSkippedOperation {
skipped.push(crank_registry::SkippedImportOperation {
operation_key,
name: String::new(),
reason: "operation was not found in import preview".to_owned(),
@@ -261,22 +339,21 @@ impl AdminService {
application_key: &application_key,
conflict_mode,
operations: &operations,
pre_skipped: &skipped,
finished_at: &finished_at,
})
.await?;
Ok(openapi_import_response(applied, skipped))
Ok(openapi_import_response(applied))
}
}
fn openapi_import_response(
applied: ImportJobApplyResult,
mut skipped: Vec<OpenApiImportSkippedOperation>,
) -> OpenApiImportCreateResponse {
fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse {
let created = applied
.created
.iter()
.map(|operation| OpenApiImportCreatedOperation {
operation_key: operation.operation_key.clone(),
operation_id: operation.operation_id.as_str().to_owned(),
name: operation.name.clone(),
version: operation.version,
@@ -300,22 +377,29 @@ fn openapi_import_response(
})
})
.collect::<Vec<_>>();
for operation in applied.skipped {
skipped.push(OpenApiImportSkippedOperation {
operation_key: operation.operation_key.clone(),
name: operation.name.clone(),
reason: "operation with this name already exists".to_owned(),
});
findings.push(ImportFinding {
code: operation.reason,
severity: ImportFindingSeverity::Warning,
message: format!(
"Операция {} уже существует и была пропущена.",
operation.name
),
operation_key: Some(operation.operation_key),
});
}
let skipped = applied
.skipped
.into_iter()
.map(|operation| {
let reason = operation.reason.clone();
let name = operation.name.clone();
findings.push(ImportFinding {
code: reason.clone(),
severity: ImportFindingSeverity::Warning,
message: if name.is_empty() {
"Выбранная операция отсутствует в исходном preview и была пропущена.".to_owned()
} else {
format!("Операция {name} уже существует и была пропущена.")
},
operation_key: Some(operation.operation_key.clone()),
});
OpenApiImportSkippedOperation {
operation_key: operation.operation_key.clone(),
name: operation.name,
reason,
}
})
.collect::<Vec<_>>();
info!(
name: "admin.openapi_import.completed",
created = created.len(),
@@ -361,14 +445,28 @@ async fn parse_verified_preview(
bytes: Vec<u8>,
locale: OpenApiUploadLocale,
) -> Result<crank_import::rest::ImportPreview, ApiError> {
tokio::task::spawn_blocking(move || {
let started = tokio::time::Instant::now();
let permit = tokio::time::timeout(OPENAPI_PARSE_DEADLINE, OPENAPI_PARSE_SLOTS.acquire())
.await
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?;
let parsing = tokio::task::spawn_blocking(move || {
// Keep the permit inside the blocking task. Timing out the caller
// cannot cancel CPU work already running, but abandoned parsers remain
// globally bounded and release capacity when they actually finish.
let _permit = permit;
let document = std::str::from_utf8(&bytes)
.map_err(|_| ApiError::openapi_upload(locale, "invalid_utf8"))?;
crank_import::rest::preview_document(document)
.map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))
})
.await
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
});
let remaining = OPENAPI_PARSE_DEADLINE
.checked_sub(started.elapsed())
.unwrap_or(std::time::Duration::ZERO);
tokio::time::timeout(remaining, parsing)
.await
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
}
fn artifact_error(locale: OpenApiUploadLocale, error: ArtifactError) -> ApiError {
@@ -383,30 +481,63 @@ fn artifact_error(locale: OpenApiUploadLocale, error: ArtifactError) -> ApiError
}
}
fn openapi_source_error(error: RegistryError) -> ApiError {
fn openapi_source_error(locale: OpenApiUploadLocale, error: RegistryError) -> ApiError {
match error {
RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable => {
ApiError::source_unavailable()
ApiError::openapi_upload(locale, "source_unavailable")
}
RegistryError::SourceIntegrity => ApiError::source_integrity(),
RegistryError::SourceIntegrity => ApiError::openapi_upload(locale, "source_integrity"),
other => ApiError::from(other),
}
}
fn import_job_source(payload: &serde_json::Value) -> Result<ImportJobSourceEnvelope, ApiError> {
fn preview_digest(preview: &serde_json::Value) -> Result<String, ApiError> {
let canonical =
serde_json::to_vec(preview).map_err(|error| ApiError::internal(error.to_string()))?;
Ok(format!("{:x}", Sha256::digest(canonical)))
}
fn verify_preview_contract(
payload: &serde_json::Value,
preview: &crank_import::rest::ImportPreview,
locale: OpenApiUploadLocale,
) -> Result<(), ApiError> {
// Pre-fingerprint jobs are legacy rolling-upgrade records. They retain the
// old reparse behavior; new jobs fail closed if parser output drifts or a
// persisted preview has been changed.
let Some(expected) = payload
.get("preview_digest")
.and_then(serde_json::Value::as_str)
else {
return Ok(());
};
let actual = preview_digest(
&serde_json::to_value(preview).map_err(|error| ApiError::internal(error.to_string()))?,
)?;
if actual == expected {
Ok(())
} else {
Err(ApiError::openapi_upload(locale, "source_integrity"))
}
}
fn import_job_source(
payload: &serde_json::Value,
locale: OpenApiUploadLocale,
) -> Result<ImportJobSourceEnvelope, ApiError> {
let source = payload
.get("source")
.ok_or_else(ApiError::source_unavailable)?;
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
let source_id = source
.get("source_id")
.and_then(serde_json::Value::as_str)
.filter(|value| value.len() <= 132)
.ok_or_else(ApiError::source_unavailable)?;
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
let digest = source
.get("digest")
.and_then(serde_json::Value::as_str)
.and_then(|value| value.parse().ok())
.ok_or_else(ApiError::source_integrity)?;
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
Ok(ImportJobSourceEnvelope {
source_id: ArtifactSourceId::new(source_id),
digest,
@@ -440,6 +571,25 @@ impl SourceDetachGuard {
fn disarm(&mut self) {
self.armed = false;
}
async fn detach_now(&mut self) {
if !self.armed {
return;
}
self.armed = false;
// This is the normal error path: await the database transition so a
// caller receives a completed cleanup before it can retry. `Drop`
// remains only the cancellation/shutdown safety net.
let _ = self
.registry
.detach_artifact_source(DetachArtifactSourceRequest {
workspace_id: &self.workspace_id,
source_id: &self.source_id,
expected_updated_at: Some(self.expected_updated_at),
detached_at: OffsetDateTime::now_utc(),
})
.await;
}
}
impl Drop for SourceDetachGuard {
+108 -1
View File
@@ -1,4 +1,11 @@
use std::process::Command;
use std::{
fs,
io::Read,
os::unix::fs::PermissionsExt,
process::{Command, Stdio},
sync::atomic::{AtomicU64, Ordering},
time::{SystemTime, UNIX_EPOCH},
};
use crank_registry::{MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresRegistry};
use crank_runtime::SecretCrypto;
@@ -7,6 +14,7 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339};
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
const REGISTERED_MASTER_KEY: &str = "registered-master-key-CANARY_SECRET_VALUE-00000000";
const WRONG_MASTER_KEY: &str = "wrong-master-key-CANARY_SECRET_VALUE-0000000000000";
static NEXT_STORAGE_ROOT: AtomicU64 = AtomicU64::new(0);
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
@@ -78,6 +86,105 @@ async fn fresh_database_startup_is_read_only() {
assert!(!present);
}
#[tokio::test]
async fn real_admin_startup_runs_immediate_import_maintenance() {
let unique = NEXT_STORAGE_ROOT.fetch_add(1, Ordering::Relaxed);
let epoch = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let database_url =
crank_test_support::postgres_schema_url(&format!("startup_maint_{epoch}_{unique}")).await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
assert_eq!(
sqlx::query_scalar::<_, Option<i64>>("select max(version) from __crank_migrations")
.fetch_one(&pool)
.await
.unwrap(),
Some(13),
"migration binary must install the current canonical ledger",
);
sqlx::query(
"insert into import_jobs (
id, workspace_id, kind, source_format, source_version, status,
preview_payload, created_operation_ids, error_text, created_at, expires_at, finished_at
) values (
'imp_startup_expired', 'ws_default', 'openapi', 'openapi', null, 'pending',
'{\"source\": {\"source_id\": 1}}'::jsonb, '[]'::jsonb, null,
now() - interval '2 hours', now() - interval '1 hour', null
)",
)
.execute(&pool)
.await
.unwrap();
let storage_root = std::env::temp_dir().join(format!(
"crank-admin-startup-maintenance-{}-{epoch}-{unique}",
std::process::id()
));
fs::create_dir(&storage_root).unwrap();
fs::set_permissions(&storage_root, fs::Permissions::from_mode(0o700)).unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let bind_addr = listener.local_addr().unwrap().to_string();
drop(listener);
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
let mut child = command
.envs([
("CRANK_DATABASE_URL", database_url.as_str()),
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_BIND", bind_addr.as_str()),
("CRANK_STORAGE_ROOT", storage_root.to_str().unwrap()),
])
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.expect("admin binary starts");
let mut cleaned = false;
for _ in 0..200 {
if let Some(status) = child.try_wait().unwrap() {
let mut stderr = String::new();
if let Some(mut stream) = child.stderr.take() {
stream.read_to_string(&mut stderr).unwrap();
}
panic!("admin binary exited before startup maintenance: {status}: {stderr}");
}
let remaining: i64 =
sqlx::query_scalar("select count(*) from import_jobs where id = 'imp_startup_expired'")
.fetch_one(&pool)
.await
.unwrap();
if remaining == 0 {
cleaned = true;
break;
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
child.kill().expect("admin binary can be stopped");
child.wait().expect("admin binary can be reaped");
fs::remove_dir_all(&storage_root).unwrap();
assert!(cleaned, "real startup did not clean the expired import job");
}
#[tokio::test]
async fn master_key_mismatch_blocks_startup_with_safe_diagnostic() {
let database_url = crank_test_support::postgres_schema_url("admin_master_key_mismatch").await;
@@ -81,6 +81,7 @@ async fn previews_openapi_and_creates_draft_operations() {
.unwrap();
assert_eq!(created.created.len(), 1);
assert_eq!(created.created[0].operation_key, "GET /v2/latest");
assert_eq!(created.created[0].name, "latest_rates");
assert!(created.skipped.is_empty());
@@ -176,6 +177,46 @@ async fn previews_openapi_and_creates_draft_operations() {
assert_eq!(renamed.findings[0].code, "operation_name_renamed");
}
#[tokio::test]
#[serial]
async fn unknown_selected_operations_are_persisted_in_the_canonical_replay() {
let registry = test_registry().await;
let service = test_service(
registry,
test_storage_root("openapi_import_unknown_replay"),
test_auth_settings(),
test_secret_crypto(),
);
let workspace_id = WorkspaceId::new("ws_default");
let preview = service
.preview_openapi_import(&workspace_id, openapi_upload())
.await
.unwrap();
let payload = OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned(), "GET /missing".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
conflict_mode: "skip".to_owned(),
};
let first = service
.create_openapi_import(
&workspace_id,
&preview.job_id.as_str().into(),
payload.clone(),
)
.await
.unwrap();
let replayed = service
.create_openapi_import(&workspace_id, &preview.job_id.as_str().into(), payload)
.await
.unwrap();
assert_eq!(first.created, replayed.created);
assert_eq!(first.skipped, replayed.skipped);
assert_eq!(first.skipped.len(), 1);
assert_eq!(first.skipped[0].operation_key, "GET /missing");
}
#[tokio::test]
#[serial]
async fn concurrent_openapi_import_replays_the_same_atomic_result() {
@@ -231,6 +272,55 @@ async fn concurrent_openapi_import_replays_the_same_atomic_result() {
assert!(conflicting_replay.is_err());
}
#[tokio::test]
#[serial]
async fn apply_fails_closed_when_the_preview_parser_contract_drifts() {
let registry = test_registry().await;
let service = test_service(
registry.clone(),
test_storage_root("openapi_import_parser_drift"),
test_auth_settings(),
test_secret_crypto(),
);
let workspace_id = WorkspaceId::new("ws_default");
let preview = service
.preview_openapi_import(&workspace_id, openapi_upload())
.await
.unwrap();
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
sqlx::query(
"update import_jobs
set preview_payload = jsonb_set(preview_payload, '{preview_digest}', to_jsonb($1::text))
where id = $2",
)
.bind("0".repeat(64))
.bind(job_id.as_str())
.execute(registry.pool())
.await
.unwrap();
let result = service
.create_openapi_import(
&workspace_id,
&job_id,
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
conflict_mode: "rename".to_owned(),
},
)
.await;
assert!(result.is_err());
assert!(
service
.list_operations(&workspace_id)
.await
.unwrap()
.is_empty()
);
}
fn openapi_upload() -> OpenApiUpload {
OpenApiUpload {
bytes: OPENAPI3.as_bytes().to_vec(),
@@ -385,6 +385,16 @@ async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() {
Some("no supported methods"),
)
.await;
assert_eq!(
sqlx::query_scalar::<_, i64>(
"select count(*) from artifact_sources where lifecycle = 'active'",
)
.fetch_one(registry.pool())
.await
.unwrap(),
0,
"no_methods must not leave an active source behind",
);
assert_rejected(
&client,
&server,
@@ -432,6 +442,42 @@ async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() {
assert!(!rendered.contains("digest"));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn cleanup_is_bounded_and_malformed_expired_jobs_do_not_block_later_rows() {
let registry = test_registry().await;
for index in 0..129 {
sqlx::query(
"insert into import_jobs (
id, workspace_id, kind, source_format, source_version, status,
preview_payload, created_operation_ids, error_text, created_at, expires_at, finished_at
) values (
$1, 'ws_default', 'openapi', 'openapi', null, 'pending',
'{\"source\": {\"source_id\": 1}}'::jsonb, '[]'::jsonb, null,
now() - interval '2 hours', now() - interval '1 hour', null
)",
)
.bind(format!("imp_cleanup_{index:03}"))
.execute(registry.pool())
.await
.unwrap();
}
let first = registry.cleanup_expired_import_jobs(128).await.unwrap();
assert_eq!(first.deleted_jobs, 128);
assert!(first.more_work);
let second = registry.cleanup_expired_import_jobs(128).await.unwrap();
assert_eq!(second.deleted_jobs, 1);
assert!(!second.more_work);
assert_eq!(
sqlx::query_scalar::<_, i64>("select count(*) from import_jobs")
.fetch_one(registry.pool())
.await
.unwrap(),
0,
);
}
async fn wait_for_source_lifecycle(
registry: &crank_registry::PostgresRegistry,
lifecycle: ArtifactSourceLifecycle,
@@ -746,11 +792,15 @@ async fn cancellation_detaches_and_restart_cleanup_recovers_a_dangling_source()
preview_task.abort();
let cancellation = preview_task.await.unwrap_err();
assert!(cancellation.is_cancelled());
wait_for_blocked_import_insert_to_stop(&observer).await;
// Dropping a sqlx query future does not synchronously cancel the backend
// statement. Release the artificial blocker first; the abandoned
// transaction can then finish and roll back before the detach fallback
// obtains another pooled connection.
sqlx::query("select pg_advisory_unlock(2147483001)")
.execute(&mut *lock_connection)
.await
.unwrap();
wait_for_blocked_import_insert_to_stop(&observer).await;
wait_for_specific_source_lifecycle(
&registry,
&cancelled_source_id,