fix(openapi): harden story 2.1 production lifecycle
This commit is contained in:
@@ -560,14 +560,15 @@ pub struct OpenApiImportCreateResponse {
|
||||
pub findings: Vec<crank_import::rest::ImportFinding>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct OpenApiImportCreatedOperation {
|
||||
pub operation_key: String,
|
||||
pub operation_id: String,
|
||||
pub name: String,
|
||||
pub version: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct OpenApiImportSkippedOperation {
|
||||
pub operation_key: String,
|
||||
pub name: String,
|
||||
|
||||
@@ -195,14 +195,6 @@ impl ApiError {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn source_unavailable() -> Self {
|
||||
Self::openapi_upload(OpenApiUploadLocale::En, "source_unavailable")
|
||||
}
|
||||
|
||||
pub(crate) fn source_integrity() -> Self {
|
||||
Self::openapi_upload(OpenApiUploadLocale::En, "source_integrity")
|
||||
}
|
||||
|
||||
pub(crate) fn rate_limited_with_context(message: impl Into<String>, context: Value) -> Self {
|
||||
Self::RateLimited {
|
||||
message: message.into(),
|
||||
|
||||
@@ -204,7 +204,6 @@ async fn run(
|
||||
verified_startup_secret_crypto(®istry, config.runtime.master_key.expose_secret())
|
||||
.await?;
|
||||
let artifact_store = open_reconciliation_store(config.storage_root.clone()).await?;
|
||||
registry.delete_expired_import_jobs().await?;
|
||||
let outbound_http_policy = crank_runtime::OutboundHttpPolicy::try_new_with_limits(
|
||||
config.runtime.outbound.allowed_hosts.clone(),
|
||||
config.runtime.outbound.denied_hosts.clone(),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
@@ -7,7 +8,7 @@ use async_trait::async_trait;
|
||||
use crank_artifacts::{
|
||||
ArtifactError, ArtifactStore, ReconciliationCursor, ReconciliationMutation,
|
||||
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
|
||||
ReconciliationScanStop,
|
||||
ReconciliationScanStop, TempScanCursor,
|
||||
};
|
||||
use crank_registry::{
|
||||
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
|
||||
@@ -25,6 +26,11 @@ pub const RECONCILIATION_LEASE: Duration = Duration::from_secs(5 * 60);
|
||||
pub const RECONCILIATION_TRAVERSAL_LIMIT: usize = 4_096;
|
||||
pub const RECONCILIATION_CANDIDATE_LIMIT: usize = 32;
|
||||
pub const RECONCILIATION_MUTATION_LIMIT: usize = 32;
|
||||
pub const RECONCILIATION_TICK_DEADLINE: Duration = Duration::from_secs(30);
|
||||
pub const TEMP_CLEANUP_GRACE: Duration = Duration::from_secs(60 * 60);
|
||||
pub const TEMP_CLEANUP_SCAN_LIMIT: usize = 512;
|
||||
pub const TEMP_CLEANUP_DELETE_LIMIT: usize = 16;
|
||||
pub const IMPORT_JOB_CLEANUP_LIMIT: u32 = 128;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum Phase {
|
||||
@@ -191,6 +197,8 @@ pub enum ReconciliationCoordinatorError {
|
||||
Backend,
|
||||
#[error("artifact reconciliation backend exceeded its bounded contract")]
|
||||
Bounds,
|
||||
#[error("artifact reconciliation backend exceeded its deadline")]
|
||||
Deadline,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -215,7 +223,7 @@ struct ReconciliationCoordinator<B: ReconciliationBackend> {
|
||||
#[derive(Clone)]
|
||||
struct PostgresReconciliationBackend {
|
||||
registry: PostgresRegistry,
|
||||
store: ArtifactStore,
|
||||
store: Arc<ArtifactStore>,
|
||||
}
|
||||
|
||||
enum PostgresReconciliationCursor {
|
||||
@@ -230,7 +238,7 @@ impl std::fmt::Debug for PostgresReconciliationCursor {
|
||||
}
|
||||
|
||||
impl PostgresReconciliationBackend {
|
||||
fn new(registry: PostgresRegistry, store: ArtifactStore) -> Self {
|
||||
fn new(registry: PostgresRegistry, store: Arc<ArtifactStore>) -> Self {
|
||||
Self { registry, store }
|
||||
}
|
||||
|
||||
@@ -245,7 +253,7 @@ impl PostgresReconciliationBackend {
|
||||
&self,
|
||||
artifact_ref: crank_artifacts::ArtifactRef,
|
||||
) -> Result<ReconciliationPresence, ReconciliationCoordinatorError> {
|
||||
let store = self.store.clone();
|
||||
let store = Arc::clone(&self.store);
|
||||
let result =
|
||||
tokio::task::spawn_blocking(move || store.reconciliation_presence(&artifact_ref))
|
||||
.await
|
||||
@@ -518,7 +526,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
|
||||
Phase::Recovery => ReconciliationNamespace::Quarantine,
|
||||
Phase::Sweep => ReconciliationNamespace::Final,
|
||||
};
|
||||
let store = self.store.clone();
|
||||
let store = Arc::clone(&self.store);
|
||||
let (mut scan, candidates) = tokio::task::spawn_blocking(move || {
|
||||
store.scan_reconciliation_namespace(
|
||||
namespace,
|
||||
@@ -560,7 +568,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
|
||||
Phase::Recovery => Duration::ZERO,
|
||||
Phase::Sweep => RECONCILIATION_GRACE,
|
||||
};
|
||||
let store = self.store.clone();
|
||||
let store = Arc::clone(&self.store);
|
||||
let registration = tokio::task::spawn_blocking(move || {
|
||||
let registration = store.register_reconciliation(
|
||||
&candidate,
|
||||
@@ -621,7 +629,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
|
||||
|
||||
step.mutation_attempts += 1;
|
||||
step.physical_mutation_possible = true;
|
||||
let store = self.store.clone();
|
||||
let store = Arc::clone(&self.store);
|
||||
let mutation = match tokio::task::spawn_blocking(move || match phase {
|
||||
Phase::Recovery => store.delete_quarantined_reconciliation(candidate),
|
||||
Phase::Sweep => store.quarantine_reconciliation(candidate),
|
||||
@@ -717,8 +725,8 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
|
||||
/// async executor.
|
||||
pub async fn open_reconciliation_store(
|
||||
root: PathBuf,
|
||||
) -> Result<ArtifactStore, ReconciliationCoordinatorError> {
|
||||
tokio::task::spawn_blocking(move || ArtifactStore::open(root))
|
||||
) -> Result<Arc<ArtifactStore>, ReconciliationCoordinatorError> {
|
||||
tokio::task::spawn_blocking(move || ArtifactStore::open(root).map(Arc::new))
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)
|
||||
@@ -726,10 +734,25 @@ pub async fn open_reconciliation_store(
|
||||
|
||||
/// Runs the immediate bounded startup cycle, then schedules 15-minute ticks.
|
||||
/// Recovery remains ahead of the final sweep even when it spans several ticks.
|
||||
pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: ArtifactStore) {
|
||||
let backend = PostgresReconciliationBackend::new(registry, store);
|
||||
pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: Arc<ArtifactStore>) {
|
||||
let cleanup_registry = registry.clone();
|
||||
let backend = PostgresReconciliationBackend::new(registry, Arc::clone(&store));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend);
|
||||
observe_tick(coordinator.tick().await);
|
||||
let mut temp_cursor = None;
|
||||
let startup_maintenance = async {
|
||||
observe_import_job_cleanup(&cleanup_registry).await;
|
||||
observe_tick(run_bounded_reconciliation_tick(&mut coordinator).await);
|
||||
temp_cursor = observe_stale_temp_cleanup(Arc::clone(&store), None).await;
|
||||
};
|
||||
if tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, startup_maintenance)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
warn!(
|
||||
name: "admin.artifact_maintenance.startup_deadline",
|
||||
"startup maintenance exceeded its shared deadline; remaining work will resume in background"
|
||||
);
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(RECONCILIATION_INTERVAL);
|
||||
@@ -738,11 +761,148 @@ pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: Ar
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
observe_tick(coordinator.tick().await);
|
||||
observe_import_job_cleanup(&cleanup_registry).await;
|
||||
observe_tick(run_bounded_reconciliation_tick(&mut coordinator).await);
|
||||
temp_cursor = observe_stale_temp_cleanup(Arc::clone(&store), temp_cursor).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn observe_import_job_cleanup(registry: &PostgresRegistry) {
|
||||
let cleanup = async {
|
||||
let mut deleted_jobs = 0_u64;
|
||||
let mut detached_sources = 0_u64;
|
||||
loop {
|
||||
let report = registry
|
||||
.cleanup_expired_import_jobs(IMPORT_JOB_CLEANUP_LIMIT)
|
||||
.await?;
|
||||
deleted_jobs = deleted_jobs.saturating_add(report.deleted_jobs);
|
||||
detached_sources = detached_sources.saturating_add(report.detached_sources);
|
||||
if !report.more_work {
|
||||
return Ok::<_, crank_registry::RegistryError>((deleted_jobs, detached_sources));
|
||||
}
|
||||
// Each pass commits independently. Yield before requesting the
|
||||
// next lock batch so normal import traffic can make progress.
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
};
|
||||
match tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, cleanup).await {
|
||||
Ok(Ok((deleted_jobs, detached_sources))) => info!(
|
||||
name: "admin.openapi_import_cleanup.tick",
|
||||
deleted_jobs,
|
||||
detached_sources,
|
||||
more_work = false,
|
||||
"OpenAPI import cleanup tick completed"
|
||||
),
|
||||
Ok(Err(_)) => warn!(
|
||||
name: "admin.openapi_import_cleanup.failed",
|
||||
error_category = "registry",
|
||||
"OpenAPI import cleanup tick will be retried"
|
||||
),
|
||||
Err(_) => warn!(
|
||||
name: "admin.openapi_import_cleanup.failed",
|
||||
error_category = "deadline",
|
||||
"OpenAPI import cleanup tick exceeded its deadline"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_bounded_reconciliation_tick<B: ReconciliationBackend>(
|
||||
coordinator: &mut ReconciliationCoordinator<B>,
|
||||
) -> Result<ReconciliationTickReport, ReconciliationCoordinatorError> {
|
||||
tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, coordinator.tick())
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Deadline)?
|
||||
}
|
||||
|
||||
async fn observe_stale_temp_cleanup(
|
||||
store: Arc<ArtifactStore>,
|
||||
cursor: Option<TempScanCursor>,
|
||||
) -> Option<TempScanCursor> {
|
||||
let progress = Arc::new(std::sync::Mutex::new(cursor));
|
||||
let cleanup_progress = Arc::clone(&progress);
|
||||
let cleanup = async move {
|
||||
let mut scanned = 0_usize;
|
||||
let mut omitted = 0_usize;
|
||||
let mut deleted = 0_usize;
|
||||
let mut retryable = 0_usize;
|
||||
loop {
|
||||
let scanner = Arc::clone(&store);
|
||||
let cursor = cleanup_progress
|
||||
.lock()
|
||||
.map_err(|_| ArtifactError::Storage)?
|
||||
.clone();
|
||||
let (scan, candidates, next_cursor) = tokio::task::spawn_blocking(move || {
|
||||
scanner.scan_stale_temps_after(
|
||||
TEMP_CLEANUP_GRACE,
|
||||
cursor,
|
||||
TEMP_CLEANUP_SCAN_LIMIT,
|
||||
TEMP_CLEANUP_DELETE_LIMIT,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ArtifactError::Storage)??;
|
||||
scanned = scanned.saturating_add(scan.scanned);
|
||||
omitted = omitted.saturating_add(scan.omitted);
|
||||
*cleanup_progress
|
||||
.lock()
|
||||
.map_err(|_| ArtifactError::Storage)? = Some(next_cursor);
|
||||
|
||||
for candidate in candidates {
|
||||
let store = Arc::clone(&store);
|
||||
match tokio::task::spawn_blocking(move || store.delete_stale_temp(candidate)).await
|
||||
{
|
||||
Ok(Ok(())) => deleted += 1,
|
||||
Ok(Err(_)) | Err(_) => retryable += 1,
|
||||
}
|
||||
}
|
||||
if scan.complete {
|
||||
return Ok::<_, ArtifactError>((
|
||||
scanned,
|
||||
omitted,
|
||||
deleted,
|
||||
retryable,
|
||||
cleanup_progress
|
||||
.lock()
|
||||
.map_err(|_| ArtifactError::Storage)?
|
||||
.clone(),
|
||||
));
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
};
|
||||
|
||||
match tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, cleanup).await {
|
||||
Ok(Ok((scanned, omitted, deleted, retryable, next_cursor))) => {
|
||||
info!(
|
||||
name: "admin.artifact_temp_cleanup.tick",
|
||||
scanned,
|
||||
omitted,
|
||||
deleted,
|
||||
retryable,
|
||||
"artifact temporary-file cleanup tick completed"
|
||||
);
|
||||
next_cursor
|
||||
}
|
||||
Ok(Err(_)) => {
|
||||
warn!(
|
||||
name: "admin.artifact_temp_cleanup.failed",
|
||||
error_category = "backend",
|
||||
"artifact temporary-file cleanup tick will be retried"
|
||||
);
|
||||
progress.lock().ok().and_then(|cursor| cursor.clone())
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
name: "admin.artifact_temp_cleanup.failed",
|
||||
error_category = "deadline",
|
||||
"artifact temporary-file cleanup tick exceeded its deadline"
|
||||
);
|
||||
progress.lock().ok().and_then(|cursor| cursor.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_tick(result: Result<ReconciliationTickReport, ReconciliationCoordinatorError>) {
|
||||
match result {
|
||||
Ok(report) => info!(
|
||||
@@ -772,6 +932,7 @@ fn observe_tick(result: Result<ReconciliationTickReport, ReconciliationCoordinat
|
||||
error_category = match error {
|
||||
ReconciliationCoordinatorError::Backend => "backend",
|
||||
ReconciliationCoordinatorError::Bounds => "bounds",
|
||||
ReconciliationCoordinatorError::Deadline => "deadline",
|
||||
},
|
||||
"artifact reconciliation tick will be retried"
|
||||
),
|
||||
|
||||
@@ -518,7 +518,7 @@ async fn production_backend_recovers_absence_before_sweep_and_restarts_after_mut
|
||||
assert_ne!(unsafe_name, later.artifact_ref().digest_hex());
|
||||
symlink("not-a-digest", shard.join(unsafe_name)).unwrap();
|
||||
|
||||
let backend = PostgresReconciliationBackend::new(registry.clone(), store.clone());
|
||||
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone()));
|
||||
let mut coordinator = ReconciliationCoordinator::with_tick_limits(
|
||||
backend,
|
||||
StepLimits {
|
||||
@@ -660,7 +660,7 @@ async fn expired_claim_cursor_prevents_unsafe_prefix_starvation_across_ticks() {
|
||||
.unwrap();
|
||||
remove_registered_artifact(&store, &absent);
|
||||
|
||||
let backend = PostgresReconciliationBackend::new(registry.clone(), store);
|
||||
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend);
|
||||
let first = coordinator.tick().await.unwrap();
|
||||
assert_eq!(first.candidates, RECONCILIATION_CANDIDATE_LIMIT);
|
||||
|
||||
@@ -24,23 +24,89 @@ pub async fn health() -> Json<serde_json::Value> {
|
||||
}
|
||||
|
||||
pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
match state.service.readiness().await {
|
||||
Ok(()) => (
|
||||
let checks = state.service.readiness().await;
|
||||
let postgres = if checks.postgres {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
let artifact_storage = if checks.artifact_storage {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
if checks.is_ready() {
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": "ready" }
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
})),
|
||||
),
|
||||
Err(error) => (
|
||||
)
|
||||
} else {
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": "not_ready" },
|
||||
"error": error.to_string()
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
})),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::service::ReadinessChecks;
|
||||
|
||||
#[test]
|
||||
fn readiness_checks_identify_a_postgres_failure() {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: false,
|
||||
artifact_storage: true,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
if checks.postgres {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
},
|
||||
"not_ready"
|
||||
);
|
||||
assert_eq!(
|
||||
if checks.artifact_storage {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
},
|
||||
"ready"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readiness_checks_identify_an_artifact_storage_failure() {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: true,
|
||||
artifact_storage: false,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
if checks.postgres {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
},
|
||||
"ready"
|
||||
);
|
||||
assert_eq!(
|
||||
if checks.artifact_storage {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
},
|
||||
"not_ready"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +70,16 @@ async fn parse_openapi_upload(
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "invalid_filename"))?;
|
||||
let mime_type = field
|
||||
.content_type()
|
||||
.map(ToString::to_string)
|
||||
// Axum exposes the raw header value here. Persist only the MIME
|
||||
// essence so parameters and case cannot make route and service
|
||||
// validation disagree.
|
||||
.map(|mime| {
|
||||
mime.split(';')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
})
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "invalid_media_type"))?;
|
||||
if !valid_upload_type(filename, &mime_type) {
|
||||
return Err(ApiError::openapi_upload(locale, "invalid_media_type"));
|
||||
@@ -123,35 +132,90 @@ fn valid_upload_type(filename: &str, mime_type: &str) -> bool {
|
||||
}
|
||||
|
||||
fn openapi_upload_locale(headers: &HeaderMap) -> OpenApiUploadLocale {
|
||||
let prefers_russian = headers
|
||||
let Some(value) = headers
|
||||
.get("accept-language")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| {
|
||||
value.split(',').any(|range| {
|
||||
let language = range.split(';').next().unwrap_or_default().trim();
|
||||
language.eq_ignore_ascii_case("ru")
|
||||
|| language.to_ascii_lowercase().starts_with("ru-")
|
||||
else {
|
||||
return OpenApiUploadLocale::En;
|
||||
};
|
||||
|
||||
// RFC 9110: highest q wins; ties preserve the header's order. Only the
|
||||
// locales served by this endpoint participate in negotiation.
|
||||
let mut preferred = (0_u16, usize::MAX, OpenApiUploadLocale::En);
|
||||
for (index, range) in value.split(',').enumerate() {
|
||||
let mut parts = range.split(';');
|
||||
let language = parts.next().unwrap_or_default().trim();
|
||||
let locale = if language.eq_ignore_ascii_case("ru")
|
||||
|| language.to_ascii_lowercase().starts_with("ru-")
|
||||
{
|
||||
Some(OpenApiUploadLocale::Ru)
|
||||
} else if language.eq_ignore_ascii_case("en")
|
||||
|| language.to_ascii_lowercase().starts_with("en-")
|
||||
|| language == "*"
|
||||
{
|
||||
Some(OpenApiUploadLocale::En)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let Some(locale) = locale else { continue };
|
||||
let quality = match parts
|
||||
.filter_map(|parameter| {
|
||||
let (name, value) = parameter.trim().split_once('=')?;
|
||||
name.eq_ignore_ascii_case("q").then_some(value.trim())
|
||||
})
|
||||
});
|
||||
if prefers_russian {
|
||||
OpenApiUploadLocale::Ru
|
||||
} else {
|
||||
OpenApiUploadLocale::En
|
||||
.next()
|
||||
{
|
||||
None => Some(1_000_u16),
|
||||
Some(value) => value
|
||||
.parse::<f32>()
|
||||
.ok()
|
||||
.filter(|quality| (0.0..=1.0).contains(quality) && *quality > 0.0)
|
||||
.map(|quality| (quality * 1_000.0).round() as u16),
|
||||
};
|
||||
let Some(quality) = quality else { continue };
|
||||
if quality > preferred.0 || (quality == preferred.0 && index < preferred.1) {
|
||||
preferred = (quality, index, locale);
|
||||
}
|
||||
}
|
||||
preferred.2
|
||||
}
|
||||
|
||||
pub async fn create_openapi_import(
|
||||
Path(path): Path<WorkspaceImportPath>,
|
||||
State(state): State<AppState>,
|
||||
headers: HeaderMap,
|
||||
Json(payload): Json<OpenApiImportCreatePayload>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let imported = state
|
||||
.service
|
||||
.create_openapi_import(
|
||||
.create_openapi_import_with_locale(
|
||||
&path.workspace_id.as_str().into(),
|
||||
&path.job_id.as_str().into(),
|
||||
payload,
|
||||
openapi_upload_locale(&headers),
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(json!(imported)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn locale(value: &str) -> OpenApiUploadLocale {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("accept-language", HeaderValue::from_str(value).unwrap());
|
||||
openapi_upload_locale(&headers)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accept_language_honours_quality_zero_and_header_order() {
|
||||
assert_eq!(locale("ru;q=0, en;q=0.5"), OpenApiUploadLocale::En);
|
||||
assert_eq!(locale("en;q=0.5, ru;q=0.5"), OpenApiUploadLocale::En);
|
||||
assert_eq!(locale("ru-RU;q=0.9, en;q=1"), OpenApiUploadLocale::En);
|
||||
assert_eq!(locale("ru, en;q=0.5"), OpenApiUploadLocale::Ru);
|
||||
assert_eq!(locale("ru;q=bad, en;q=0.5"), OpenApiUploadLocale::En);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ pub struct AdminService {
|
||||
pub struct AdminServiceBuilder {
|
||||
registry: PostgresRegistry,
|
||||
storage_root: PathBuf,
|
||||
artifact_store: Option<ArtifactStore>,
|
||||
artifact_store: Option<Arc<ArtifactStore>>,
|
||||
auth_settings: AuthSettings,
|
||||
secret_crypto: SecretCrypto,
|
||||
runtime: RuntimeExecutor,
|
||||
@@ -88,6 +88,18 @@ pub struct AdminServiceBuilder {
|
||||
|
||||
pub use crate::dto::*;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct ReadinessChecks {
|
||||
pub postgres: bool,
|
||||
pub artifact_storage: bool,
|
||||
}
|
||||
|
||||
impl ReadinessChecks {
|
||||
pub const fn is_ready(self) -> bool {
|
||||
self.postgres && self.artifact_storage
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AdminAuditContext {
|
||||
actor: AuditActor,
|
||||
@@ -124,9 +136,16 @@ pub(super) struct CredentialAuditRecord<'a> {
|
||||
}
|
||||
|
||||
impl AdminService {
|
||||
pub async fn readiness(&self) -> Result<(), ApiError> {
|
||||
self.registry.ping().await?;
|
||||
Ok(())
|
||||
pub async fn readiness(&self) -> ReadinessChecks {
|
||||
let postgres = self.registry.ping().await.is_ok();
|
||||
let store = Arc::clone(&self.artifact_store);
|
||||
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
|
||||
.await
|
||||
.is_ok_and(|result| result.is_ok());
|
||||
ReadinessChecks {
|
||||
postgres,
|
||||
artifact_storage,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -230,7 +249,7 @@ impl AdminServiceBuilder {
|
||||
/// Reuses the process-wide immutable artifact authority for OpenAPI
|
||||
/// ingress and reconciliation. Tests may omit this and use their private
|
||||
/// storage root instead.
|
||||
pub fn with_artifact_store(mut self, artifact_store: ArtifactStore) -> Self {
|
||||
pub fn with_artifact_store(mut self, artifact_store: Arc<ArtifactStore>) -> Self {
|
||||
self.artifact_store = Some(artifact_store);
|
||||
self
|
||||
}
|
||||
@@ -264,10 +283,9 @@ impl AdminServiceBuilder {
|
||||
pub fn build(self) -> AdminService {
|
||||
AdminService {
|
||||
registry: self.registry,
|
||||
artifact_store: Arc::new(
|
||||
self.artifact_store
|
||||
.unwrap_or_else(|| ArtifactStore::new(&self.storage_root)),
|
||||
),
|
||||
artifact_store: self
|
||||
.artifact_store
|
||||
.unwrap_or_else(|| Arc::new(ArtifactStore::new(&self.storage_root))),
|
||||
runtime: self.runtime,
|
||||
storage: LocalArtifactStorage::new(self.storage_root),
|
||||
auth_settings: self.auth_settings,
|
||||
|
||||
@@ -29,6 +29,10 @@ use crate::{
|
||||
};
|
||||
|
||||
const IMPORT_JOB_TTL_HOURS: i64 = 24;
|
||||
const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
const OPENAPI_PARSE_CONCURRENCY: usize = 4;
|
||||
static OPENAPI_PARSE_SLOTS: tokio::sync::Semaphore =
|
||||
tokio::sync::Semaphore::const_new(OPENAPI_PARSE_CONCURRENCY);
|
||||
|
||||
impl AdminService {
|
||||
#[instrument(skip(self, upload), fields(workspace_id = %workspace_id.as_str()))]
|
||||
@@ -38,7 +42,6 @@ impl AdminService {
|
||||
upload: OpenApiUpload,
|
||||
) -> Result<OpenApiImportPreviewResponse, ApiError> {
|
||||
self.ensure_workspace_exists(workspace_id).await?;
|
||||
self.registry.delete_expired_import_jobs().await?;
|
||||
|
||||
validate_openapi_upload(&upload)?;
|
||||
let OpenApiUpload {
|
||||
@@ -72,19 +75,38 @@ impl AdminService {
|
||||
source_id.clone(),
|
||||
source.updated_at,
|
||||
);
|
||||
let verified = self
|
||||
let verified = match self
|
||||
.registry
|
||||
.read_artifact_source(&self.artifact_store, workspace_id, &source_id)
|
||||
.await?;
|
||||
.read_artifact_source(
|
||||
std::sync::Arc::clone(&self.artifact_store),
|
||||
workspace_id,
|
||||
&source_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(verified) => verified,
|
||||
Err(error) => {
|
||||
detach_guard.detach_now().await;
|
||||
return Err(ApiError::from(error));
|
||||
}
|
||||
};
|
||||
if verified.source.blob.artifact_ref != *artifact.artifact_ref() {
|
||||
detach_guard.detach_now().await;
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let preview = parse_verified_preview(verified.bytes, locale).await?;
|
||||
let preview = match parse_verified_preview(verified.bytes, locale).await {
|
||||
Ok(preview) => preview,
|
||||
Err(error) => {
|
||||
detach_guard.detach_now().await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
if preview
|
||||
.groups
|
||||
.iter()
|
||||
.all(|group| group.operations.is_empty())
|
||||
{
|
||||
detach_guard.detach_now().await;
|
||||
return Err(ApiError::openapi_upload(locale, "no_methods"));
|
||||
}
|
||||
let source_envelope = ImportJobSourceEnvelope {
|
||||
@@ -93,15 +115,18 @@ impl AdminService {
|
||||
};
|
||||
let preview_value = serde_json::to_value(&preview)
|
||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
let preview_digest = preview_digest(&preview_value)?;
|
||||
let preview_payload = json!({
|
||||
"source": {
|
||||
"source_id": source_envelope.source_id.as_str(),
|
||||
"digest": source_envelope.digest.as_str(),
|
||||
},
|
||||
"preview": preview_value,
|
||||
"preview_digest": preview_digest,
|
||||
});
|
||||
|
||||
self.registry
|
||||
if let Err(error) = self
|
||||
.registry
|
||||
.create_import_job(CreateImportJobRequest {
|
||||
id: &job_id,
|
||||
workspace_id,
|
||||
@@ -114,7 +139,11 @@ impl AdminService {
|
||||
created_at: &now,
|
||||
expires_at: &expires_at,
|
||||
})
|
||||
.await?;
|
||||
.await
|
||||
{
|
||||
detach_guard.detach_now().await;
|
||||
return Err(ApiError::from(error));
|
||||
}
|
||||
detach_guard.disarm();
|
||||
|
||||
Ok(OpenApiImportPreviewResponse {
|
||||
@@ -132,9 +161,24 @@ impl AdminService {
|
||||
workspace_id: &WorkspaceId,
|
||||
job_id: &ImportJobId,
|
||||
payload: OpenApiImportCreatePayload,
|
||||
) -> Result<OpenApiImportCreateResponse, ApiError> {
|
||||
self.create_openapi_import_with_locale(
|
||||
workspace_id,
|
||||
job_id,
|
||||
payload,
|
||||
OpenApiUploadLocale::En,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn create_openapi_import_with_locale(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
job_id: &ImportJobId,
|
||||
payload: OpenApiImportCreatePayload,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<OpenApiImportCreateResponse, ApiError> {
|
||||
self.ensure_workspace_exists(workspace_id).await?;
|
||||
self.registry.delete_expired_import_jobs().await?;
|
||||
|
||||
if !matches!(payload.conflict_mode.as_str(), "skip" | "rename") {
|
||||
return Err(ApiError::validation(
|
||||
@@ -185,22 +229,56 @@ impl AdminService {
|
||||
application_key: &application_key,
|
||||
conflict_mode,
|
||||
operations: &[],
|
||||
pre_skipped: &[],
|
||||
finished_at: &finished_at,
|
||||
})
|
||||
.await?;
|
||||
return Ok(openapi_import_response(applied, Vec::new()));
|
||||
return Ok(openapi_import_response(applied));
|
||||
}
|
||||
|
||||
let source = import_job_source(&job.preview_payload)?;
|
||||
let verified = self
|
||||
let source = import_job_source(&job.preview_payload, locale)?;
|
||||
let verified = match self
|
||||
.registry
|
||||
.read_artifact_source(&self.artifact_store, workspace_id, &source.source_id)
|
||||
.read_artifact_source(
|
||||
std::sync::Arc::clone(&self.artifact_store),
|
||||
workspace_id,
|
||||
&source.source_id,
|
||||
)
|
||||
.await
|
||||
.map_err(openapi_source_error)?;
|
||||
{
|
||||
Ok(verified) => verified,
|
||||
Err(
|
||||
error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable),
|
||||
) => {
|
||||
// Another request may have finished and detached the source
|
||||
// after our unlocked Pending read. Re-read the job and ask
|
||||
// the registry for its canonical, locked replay instead of
|
||||
// exposing a false `source_unavailable` outcome.
|
||||
let latest = self.registry.get_import_job(workspace_id, job_id).await?;
|
||||
if latest.is_some_and(|latest| latest.status == ImportJobStatus::Completed) {
|
||||
let applied = self
|
||||
.registry
|
||||
.apply_import_job(ApplyImportJobRequest {
|
||||
id: job_id,
|
||||
workspace_id,
|
||||
application_key: &application_key,
|
||||
conflict_mode,
|
||||
operations: &[],
|
||||
pre_skipped: &[],
|
||||
finished_at: &finished_at,
|
||||
})
|
||||
.await?;
|
||||
return Ok(openapi_import_response(applied));
|
||||
}
|
||||
return Err(openapi_source_error(locale, error));
|
||||
}
|
||||
Err(error) => return Err(openapi_source_error(locale, error)),
|
||||
};
|
||||
if verified.source.blob.artifact_ref != source.digest {
|
||||
return Err(ApiError::source_integrity());
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let preview = parse_verified_preview(verified.bytes, OpenApiUploadLocale::En).await?;
|
||||
let preview = parse_verified_preview(verified.bytes, locale).await?;
|
||||
verify_preview_contract(&job.preview_payload, &preview, locale)?;
|
||||
let mut candidates = BTreeMap::new();
|
||||
for group in &preview.groups {
|
||||
for operation in &group.operations {
|
||||
@@ -213,7 +291,7 @@ impl AdminService {
|
||||
|
||||
for operation_key in selected {
|
||||
let Some(candidate) = candidates.get(&operation_key) else {
|
||||
skipped.push(OpenApiImportSkippedOperation {
|
||||
skipped.push(crank_registry::SkippedImportOperation {
|
||||
operation_key,
|
||||
name: String::new(),
|
||||
reason: "operation was not found in import preview".to_owned(),
|
||||
@@ -261,22 +339,21 @@ impl AdminService {
|
||||
application_key: &application_key,
|
||||
conflict_mode,
|
||||
operations: &operations,
|
||||
pre_skipped: &skipped,
|
||||
finished_at: &finished_at,
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(openapi_import_response(applied, skipped))
|
||||
Ok(openapi_import_response(applied))
|
||||
}
|
||||
}
|
||||
|
||||
fn openapi_import_response(
|
||||
applied: ImportJobApplyResult,
|
||||
mut skipped: Vec<OpenApiImportSkippedOperation>,
|
||||
) -> OpenApiImportCreateResponse {
|
||||
fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse {
|
||||
let created = applied
|
||||
.created
|
||||
.iter()
|
||||
.map(|operation| OpenApiImportCreatedOperation {
|
||||
operation_key: operation.operation_key.clone(),
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
name: operation.name.clone(),
|
||||
version: operation.version,
|
||||
@@ -300,22 +377,29 @@ fn openapi_import_response(
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for operation in applied.skipped {
|
||||
skipped.push(OpenApiImportSkippedOperation {
|
||||
operation_key: operation.operation_key.clone(),
|
||||
name: operation.name.clone(),
|
||||
reason: "operation with this name already exists".to_owned(),
|
||||
});
|
||||
findings.push(ImportFinding {
|
||||
code: operation.reason,
|
||||
severity: ImportFindingSeverity::Warning,
|
||||
message: format!(
|
||||
"Операция {} уже существует и была пропущена.",
|
||||
operation.name
|
||||
),
|
||||
operation_key: Some(operation.operation_key),
|
||||
});
|
||||
}
|
||||
let skipped = applied
|
||||
.skipped
|
||||
.into_iter()
|
||||
.map(|operation| {
|
||||
let reason = operation.reason.clone();
|
||||
let name = operation.name.clone();
|
||||
findings.push(ImportFinding {
|
||||
code: reason.clone(),
|
||||
severity: ImportFindingSeverity::Warning,
|
||||
message: if name.is_empty() {
|
||||
"Выбранная операция отсутствует в исходном preview и была пропущена.".to_owned()
|
||||
} else {
|
||||
format!("Операция {name} уже существует и была пропущена.")
|
||||
},
|
||||
operation_key: Some(operation.operation_key.clone()),
|
||||
});
|
||||
OpenApiImportSkippedOperation {
|
||||
operation_key: operation.operation_key.clone(),
|
||||
name: operation.name,
|
||||
reason,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
info!(
|
||||
name: "admin.openapi_import.completed",
|
||||
created = created.len(),
|
||||
@@ -361,14 +445,28 @@ async fn parse_verified_preview(
|
||||
bytes: Vec<u8>,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<crank_import::rest::ImportPreview, ApiError> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let started = tokio::time::Instant::now();
|
||||
let permit = tokio::time::timeout(OPENAPI_PARSE_DEADLINE, OPENAPI_PARSE_SLOTS.acquire())
|
||||
.await
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?;
|
||||
let parsing = tokio::task::spawn_blocking(move || {
|
||||
// Keep the permit inside the blocking task. Timing out the caller
|
||||
// cannot cancel CPU work already running, but abandoned parsers remain
|
||||
// globally bounded and release capacity when they actually finish.
|
||||
let _permit = permit;
|
||||
let document = std::str::from_utf8(&bytes)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "invalid_utf8"))?;
|
||||
crank_import::rest::preview_document(document)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
|
||||
});
|
||||
let remaining = OPENAPI_PARSE_DEADLINE
|
||||
.checked_sub(started.elapsed())
|
||||
.unwrap_or(std::time::Duration::ZERO);
|
||||
tokio::time::timeout(remaining, parsing)
|
||||
.await
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
|
||||
}
|
||||
|
||||
fn artifact_error(locale: OpenApiUploadLocale, error: ArtifactError) -> ApiError {
|
||||
@@ -383,30 +481,63 @@ fn artifact_error(locale: OpenApiUploadLocale, error: ArtifactError) -> ApiError
|
||||
}
|
||||
}
|
||||
|
||||
fn openapi_source_error(error: RegistryError) -> ApiError {
|
||||
fn openapi_source_error(locale: OpenApiUploadLocale, error: RegistryError) -> ApiError {
|
||||
match error {
|
||||
RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable => {
|
||||
ApiError::source_unavailable()
|
||||
ApiError::openapi_upload(locale, "source_unavailable")
|
||||
}
|
||||
RegistryError::SourceIntegrity => ApiError::source_integrity(),
|
||||
RegistryError::SourceIntegrity => ApiError::openapi_upload(locale, "source_integrity"),
|
||||
other => ApiError::from(other),
|
||||
}
|
||||
}
|
||||
|
||||
fn import_job_source(payload: &serde_json::Value) -> Result<ImportJobSourceEnvelope, ApiError> {
|
||||
fn preview_digest(preview: &serde_json::Value) -> Result<String, ApiError> {
|
||||
let canonical =
|
||||
serde_json::to_vec(preview).map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
Ok(format!("{:x}", Sha256::digest(canonical)))
|
||||
}
|
||||
|
||||
fn verify_preview_contract(
|
||||
payload: &serde_json::Value,
|
||||
preview: &crank_import::rest::ImportPreview,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<(), ApiError> {
|
||||
// Pre-fingerprint jobs are legacy rolling-upgrade records. They retain the
|
||||
// old reparse behavior; new jobs fail closed if parser output drifts or a
|
||||
// persisted preview has been changed.
|
||||
let Some(expected) = payload
|
||||
.get("preview_digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let actual = preview_digest(
|
||||
&serde_json::to_value(preview).map_err(|error| ApiError::internal(error.to_string()))?,
|
||||
)?;
|
||||
if actual == expected {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ApiError::openapi_upload(locale, "source_integrity"))
|
||||
}
|
||||
}
|
||||
|
||||
fn import_job_source(
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<ImportJobSourceEnvelope, ApiError> {
|
||||
let source = payload
|
||||
.get("source")
|
||||
.ok_or_else(ApiError::source_unavailable)?;
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let source_id = source
|
||||
.get("source_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or_else(ApiError::source_unavailable)?;
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let digest = source
|
||||
.get("digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| value.parse().ok())
|
||||
.ok_or_else(ApiError::source_integrity)?;
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
Ok(ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
@@ -440,6 +571,25 @@ impl SourceDetachGuard {
|
||||
fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
async fn detach_now(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
self.armed = false;
|
||||
// This is the normal error path: await the database transition so a
|
||||
// caller receives a completed cleanup before it can retry. `Drop`
|
||||
// remains only the cancellation/shutdown safety net.
|
||||
let _ = self
|
||||
.registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &self.workspace_id,
|
||||
source_id: &self.source_id,
|
||||
expected_updated_at: Some(self.expected_updated_at),
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SourceDetachGuard {
|
||||
|
||||
Reference in New Issue
Block a user