feat(openapi): complete upload preview and UI evidence
This commit is contained in:
@@ -4,6 +4,7 @@ use axum::{
|
||||
middleware,
|
||||
routing::{delete, get, post},
|
||||
};
|
||||
use crank_artifacts::MAX_ARTIFACT_BYTES;
|
||||
|
||||
use crate::{
|
||||
auth::{
|
||||
@@ -48,7 +49,11 @@ use crate::{
|
||||
pub fn build_app(state: AppState) -> Router {
|
||||
let workspace_router = Router::new()
|
||||
.route("/operations", get(list_operations).post(create_operation))
|
||||
.route("/imports/openapi/preview", post(preview_openapi_import))
|
||||
.route(
|
||||
"/imports/openapi/preview",
|
||||
post(preview_openapi_import)
|
||||
.layer(DefaultBodyLimit::max(MAX_ARTIFACT_BYTES + 32 * 1024)),
|
||||
)
|
||||
.route(
|
||||
"/imports/openapi/{job_id}/create",
|
||||
post(create_openapi_import),
|
||||
|
||||
@@ -524,9 +524,17 @@ pub struct LegacyYamlOperationDocument {
|
||||
pub operation: RegistryOperation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct OpenApiImportPreviewPayload {
|
||||
pub document: String,
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum OpenApiUploadLocale {
|
||||
En,
|
||||
Ru,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct OpenApiUpload {
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: String,
|
||||
pub locale: OpenApiUploadLocale,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
|
||||
+109
-4
@@ -11,6 +11,7 @@ use serde_json::{Value, json};
|
||||
use thiserror::Error;
|
||||
use tracing::{error, warn};
|
||||
|
||||
use crate::dto::OpenApiUploadLocale;
|
||||
use crate::storage::StorageError;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
@@ -96,6 +97,112 @@ impl ApiError {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn openapi_upload(locale: OpenApiUploadLocale, code: &'static str) -> Self {
|
||||
let russian = locale == OpenApiUploadLocale::Ru;
|
||||
let message = match (russian, code) {
|
||||
(_, "file_too_large") => {
|
||||
if russian {
|
||||
"файл OpenAPI превышает лимит 256 КиБ"
|
||||
} else {
|
||||
"OpenAPI file exceeds the 256 KiB limit"
|
||||
}
|
||||
}
|
||||
(_, "empty_file") => {
|
||||
if russian {
|
||||
"файл OpenAPI не должен быть пустым"
|
||||
} else {
|
||||
"OpenAPI file must not be empty"
|
||||
}
|
||||
}
|
||||
(_, "invalid_utf8") => {
|
||||
if russian {
|
||||
"файл OpenAPI должен быть в UTF-8"
|
||||
} else {
|
||||
"OpenAPI file must be UTF-8"
|
||||
}
|
||||
}
|
||||
(_, "invalid_media_type") => {
|
||||
if russian {
|
||||
"тип файла OpenAPI не поддерживается"
|
||||
} else {
|
||||
"OpenAPI file type is not supported"
|
||||
}
|
||||
}
|
||||
(_, "invalid_document") => {
|
||||
if russian {
|
||||
"некорректный или неподдерживаемый документ OpenAPI"
|
||||
} else {
|
||||
"OpenAPI document is invalid or unsupported"
|
||||
}
|
||||
}
|
||||
(_, "no_methods") => {
|
||||
if russian {
|
||||
"документ OpenAPI не содержит поддерживаемых методов"
|
||||
} else {
|
||||
"OpenAPI document contains no supported methods"
|
||||
}
|
||||
}
|
||||
(_, "source_integrity") => {
|
||||
if russian {
|
||||
"проверка целостности источника OpenAPI не пройдена"
|
||||
} else {
|
||||
"OpenAPI source integrity verification failed"
|
||||
}
|
||||
}
|
||||
(_, "source_unavailable") => {
|
||||
if russian {
|
||||
"источник OpenAPI недоступен"
|
||||
} else {
|
||||
"OpenAPI source is unavailable"
|
||||
}
|
||||
}
|
||||
(_, "parser_unavailable") | (_, "storage_unavailable") => {
|
||||
if russian {
|
||||
"обработка OpenAPI временно недоступна"
|
||||
} else {
|
||||
"OpenAPI processing is temporarily unavailable"
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if russian {
|
||||
"некорректная multipart-загрузка OpenAPI"
|
||||
} else {
|
||||
"invalid OpenAPI multipart upload"
|
||||
}
|
||||
}
|
||||
};
|
||||
let context = json!({ "error_code": format!("openapi_upload.{code}") });
|
||||
if code == "file_too_large" {
|
||||
Self::PayloadTooLarge {
|
||||
message: message.to_owned(),
|
||||
context: Some(context),
|
||||
}
|
||||
} else if matches!(code, "storage_unavailable" | "parser_unavailable") {
|
||||
Self::Internal {
|
||||
message: message.to_owned(),
|
||||
context: Some(context),
|
||||
}
|
||||
} else if matches!(code, "source_integrity" | "source_unavailable") {
|
||||
Self::Unprocessable {
|
||||
message: message.to_owned(),
|
||||
context: Some(context),
|
||||
}
|
||||
} else {
|
||||
Self::Validation {
|
||||
message: message.to_owned(),
|
||||
context: Some(context),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn source_unavailable() -> Self {
|
||||
Self::openapi_upload(OpenApiUploadLocale::En, "source_unavailable")
|
||||
}
|
||||
|
||||
pub(crate) fn source_integrity() -> Self {
|
||||
Self::openapi_upload(OpenApiUploadLocale::En, "source_integrity")
|
||||
}
|
||||
|
||||
pub(crate) fn rate_limited_with_context(message: impl Into<String>, context: Value) -> Self {
|
||||
Self::RateLimited {
|
||||
message: message.into(),
|
||||
@@ -588,17 +695,15 @@ impl From<RegistryError> for ApiError {
|
||||
format!("import job {job_id} was already applied with different parameters"),
|
||||
json!({ "job_id": job_id }),
|
||||
),
|
||||
RegistryError::SourceNotFound { source_id } => Self::not_found_with_context(
|
||||
RegistryError::SourceNotFound { .. } => Self::not_found_with_context(
|
||||
"artifact source was not found",
|
||||
json!({
|
||||
"source_id": source_id,
|
||||
"error_code": "artifact_source_not_found"
|
||||
}),
|
||||
),
|
||||
RegistryError::SourceConflict { source_id } => Self::conflict_with_context(
|
||||
RegistryError::SourceConflict { .. } => Self::conflict_with_context(
|
||||
"artifact source metadata or lifecycle conflicts with the request",
|
||||
json!({
|
||||
"source_id": source_id,
|
||||
"error_code": "artifact_source_conflict",
|
||||
"recovery": "reload"
|
||||
}),
|
||||
|
||||
@@ -204,6 +204,7 @@ async fn run(
|
||||
verified_startup_secret_crypto(®istry, config.runtime.master_key.expose_secret())
|
||||
.await?;
|
||||
let artifact_store = open_reconciliation_store(config.storage_root.clone()).await?;
|
||||
registry.delete_expired_import_jobs().await?;
|
||||
let outbound_http_policy = crank_runtime::OutboundHttpPolicy::try_new_with_limits(
|
||||
config.runtime.outbound.allowed_hosts.clone(),
|
||||
config.runtime.outbound.denied_hosts.clone(),
|
||||
@@ -224,6 +225,7 @@ async fn run(
|
||||
secret_crypto,
|
||||
runtime,
|
||||
)
|
||||
.with_artifact_store(artifact_store.clone())
|
||||
.with_public_base_url(base_url)
|
||||
.with_outbound_http_policy(outbound_http_policy)
|
||||
.with_identity_provider(std::sync::Arc::new(identity_provider))
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
use axum::{
|
||||
Json,
|
||||
extract::{Path, State},
|
||||
extract::{Multipart, Path, State, multipart::MultipartRejection},
|
||||
http::HeaderMap,
|
||||
response::IntoResponse,
|
||||
};
|
||||
use crank_artifacts::MAX_ARTIFACT_BYTES;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{
|
||||
error::ApiError,
|
||||
service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload},
|
||||
service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale},
|
||||
state::AppState,
|
||||
};
|
||||
|
||||
@@ -25,15 +28,118 @@ pub struct WorkspaceImportPath {
|
||||
pub async fn preview_openapi_import(
|
||||
Path(path): Path<WorkspacePath>,
|
||||
State(state): State<AppState>,
|
||||
Json(payload): Json<OpenApiImportPreviewPayload>,
|
||||
headers: HeaderMap,
|
||||
multipart: Result<Multipart, MultipartRejection>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let locale = openapi_upload_locale(&headers);
|
||||
let upload = parse_openapi_upload(
|
||||
multipart.map_err(|rejection| multipart_rejection(locale, rejection))?,
|
||||
locale,
|
||||
)
|
||||
.await?;
|
||||
let preview = state
|
||||
.service
|
||||
.preview_openapi_import(&path.workspace_id.as_str().into(), payload)
|
||||
.preview_openapi_import(&path.workspace_id.as_str().into(), upload)
|
||||
.await?;
|
||||
Ok(Json(json!(preview)))
|
||||
}
|
||||
|
||||
fn multipart_rejection(locale: OpenApiUploadLocale, rejection: MultipartRejection) -> ApiError {
|
||||
if rejection.into_response().status() == axum::http::StatusCode::PAYLOAD_TOO_LARGE {
|
||||
ApiError::openapi_upload(locale, "file_too_large")
|
||||
} else {
|
||||
ApiError::openapi_upload(locale, "malformed_multipart")
|
||||
}
|
||||
}
|
||||
|
||||
async fn parse_openapi_upload(
|
||||
mut multipart: Multipart,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<OpenApiUpload, ApiError> {
|
||||
let mut upload = None;
|
||||
while let Some(field) = multipart
|
||||
.next_field()
|
||||
.await
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "malformed_multipart"))?
|
||||
{
|
||||
if upload.is_some() || field.name() != Some("file") {
|
||||
return Err(ApiError::openapi_upload(locale, "malformed_multipart"));
|
||||
}
|
||||
let filename = field
|
||||
.file_name()
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "invalid_filename"))?;
|
||||
let mime_type = field
|
||||
.content_type()
|
||||
.map(ToString::to_string)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "invalid_media_type"))?;
|
||||
if !valid_upload_type(filename, &mime_type) {
|
||||
return Err(ApiError::openapi_upload(locale, "invalid_media_type"));
|
||||
}
|
||||
let mut bytes = Vec::with_capacity(8 * 1024);
|
||||
let mut field = field;
|
||||
while let Some(chunk) = field
|
||||
.chunk()
|
||||
.await
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "malformed_multipart"))?
|
||||
{
|
||||
if bytes.len().saturating_add(chunk.len()) > MAX_ARTIFACT_BYTES {
|
||||
return Err(ApiError::openapi_upload(locale, "file_too_large"));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
if bytes.is_empty() {
|
||||
return Err(ApiError::openapi_upload(locale, "empty_file"));
|
||||
}
|
||||
if std::str::from_utf8(&bytes).is_err() {
|
||||
return Err(ApiError::openapi_upload(locale, "invalid_utf8"));
|
||||
}
|
||||
upload = Some(OpenApiUpload {
|
||||
bytes,
|
||||
mime_type,
|
||||
locale,
|
||||
});
|
||||
}
|
||||
upload.ok_or_else(|| ApiError::openapi_upload(locale, "missing_file"))
|
||||
}
|
||||
|
||||
fn valid_upload_type(filename: &str, mime_type: &str) -> bool {
|
||||
let filename = filename.to_ascii_lowercase();
|
||||
let mime_type = mime_type.to_ascii_lowercase();
|
||||
match filename.rsplit_once('.') {
|
||||
Some((_, "yaml" | "yml")) => matches!(
|
||||
mime_type.as_str(),
|
||||
"application/yaml"
|
||||
| "application/x-yaml"
|
||||
| "text/yaml"
|
||||
| "text/x-yaml"
|
||||
| "application/octet-stream"
|
||||
),
|
||||
Some((_, "json")) => matches!(
|
||||
mime_type.as_str(),
|
||||
"application/json" | "application/openapi+json" | "application/octet-stream"
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn openapi_upload_locale(headers: &HeaderMap) -> OpenApiUploadLocale {
|
||||
let prefers_russian = headers
|
||||
.get("accept-language")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| {
|
||||
value.split(',').any(|range| {
|
||||
let language = range.split(';').next().unwrap_or_default().trim();
|
||||
language.eq_ignore_ascii_case("ru")
|
||||
|| language.to_ascii_lowercase().starts_with("ru-")
|
||||
})
|
||||
});
|
||||
if prefers_russian {
|
||||
OpenApiUploadLocale::Ru
|
||||
} else {
|
||||
OpenApiUploadLocale::En
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn create_openapi_import(
|
||||
Path(path): Path<WorkspaceImportPath>,
|
||||
State(state): State<AppState>,
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use crank_artifacts::ArtifactStore;
|
||||
use crank_core::{
|
||||
AuditActor, AuditEvent, AuditEventId, AuditSink, AuditTarget, AuditTargetKind, AuthProfile,
|
||||
CapabilityProfile, CommunityCapabilityProfile, CorrelationContext, EditionCapabilities,
|
||||
@@ -57,6 +58,7 @@ use operation_validation::{
|
||||
#[derive(Clone)]
|
||||
pub struct AdminService {
|
||||
registry: PostgresRegistry,
|
||||
artifact_store: Arc<ArtifactStore>,
|
||||
runtime: RuntimeExecutor,
|
||||
storage: LocalArtifactStorage,
|
||||
auth_settings: AuthSettings,
|
||||
@@ -72,6 +74,7 @@ pub struct AdminService {
|
||||
pub struct AdminServiceBuilder {
|
||||
registry: PostgresRegistry,
|
||||
storage_root: PathBuf,
|
||||
artifact_store: Option<ArtifactStore>,
|
||||
auth_settings: AuthSettings,
|
||||
secret_crypto: SecretCrypto,
|
||||
runtime: RuntimeExecutor,
|
||||
@@ -201,6 +204,7 @@ impl AdminServiceBuilder {
|
||||
Self {
|
||||
registry,
|
||||
storage_root,
|
||||
artifact_store: None,
|
||||
auth_settings,
|
||||
secret_crypto,
|
||||
runtime,
|
||||
@@ -223,6 +227,14 @@ impl AdminServiceBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Reuses the process-wide immutable artifact authority for OpenAPI
|
||||
/// ingress and reconciliation. Tests may omit this and use their private
|
||||
/// storage root instead.
|
||||
pub fn with_artifact_store(mut self, artifact_store: ArtifactStore) -> Self {
|
||||
self.artifact_store = Some(artifact_store);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_public_base_url(mut self, public_base_url: String) -> Self {
|
||||
self.public_base_url = public_base_url.trim_end_matches('/').to_owned();
|
||||
self
|
||||
@@ -252,6 +264,10 @@ impl AdminServiceBuilder {
|
||||
pub fn build(self) -> AdminService {
|
||||
AdminService {
|
||||
registry: self.registry,
|
||||
artifact_store: Arc::new(
|
||||
self.artifact_store
|
||||
.unwrap_or_else(|| ArtifactStore::new(&self.storage_root)),
|
||||
),
|
||||
runtime: self.runtime,
|
||||
storage: LocalArtifactStorage::new(self.storage_root),
|
||||
auth_settings: self.auth_settings,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crank_artifacts::{ArtifactError, MAX_ARTIFACT_BYTES};
|
||||
use crank_core::{
|
||||
ExecutionConfig, OperationSecurityLevel, Protocol, ToolQualityFinding, ToolQualitySeverity,
|
||||
WorkspaceId,
|
||||
@@ -8,8 +9,10 @@ use crank_import::rest::{
|
||||
ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
|
||||
};
|
||||
use crank_registry::{
|
||||
ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind,
|
||||
ImportJobStatus, ImportOperationDraft,
|
||||
ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity,
|
||||
CreateArtifactSourceRequest, CreateImportJobRequest, DetachArtifactSourceRequest,
|
||||
ImportConflictMode, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope,
|
||||
ImportJobStatus, ImportOperationDraft, RegistryError,
|
||||
};
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -20,30 +23,83 @@ use crate::{
|
||||
error::ApiError,
|
||||
service::{
|
||||
AdminService, OpenApiImportCreatePayload, OpenApiImportCreateResponse,
|
||||
OpenApiImportCreatedOperation, OpenApiImportPreviewPayload, OpenApiImportPreviewResponse,
|
||||
OpenApiImportSkippedOperation, OperationPayload, new_prefixed_id,
|
||||
OpenApiImportCreatedOperation, OpenApiImportPreviewResponse, OpenApiImportSkippedOperation,
|
||||
OpenApiUpload, OpenApiUploadLocale, OperationPayload, new_prefixed_id,
|
||||
},
|
||||
};
|
||||
|
||||
const IMPORT_JOB_TTL_HOURS: i64 = 24;
|
||||
|
||||
impl AdminService {
|
||||
#[instrument(skip(self, payload), fields(workspace_id = %workspace_id.as_str()))]
|
||||
#[instrument(skip(self, upload), fields(workspace_id = %workspace_id.as_str()))]
|
||||
pub async fn preview_openapi_import(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
payload: OpenApiImportPreviewPayload,
|
||||
upload: OpenApiUpload,
|
||||
) -> Result<OpenApiImportPreviewResponse, ApiError> {
|
||||
self.ensure_workspace_exists(workspace_id).await?;
|
||||
let _ = self.registry.delete_expired_import_jobs().await;
|
||||
self.registry.delete_expired_import_jobs().await?;
|
||||
|
||||
let preview = crank_import::rest::preview_document(&payload.document)
|
||||
.map_err(|error| ApiError::validation(error.to_string()))?;
|
||||
validate_openapi_upload(&upload)?;
|
||||
let OpenApiUpload {
|
||||
bytes,
|
||||
mime_type,
|
||||
locale,
|
||||
} = upload;
|
||||
let store = self.artifact_store.clone();
|
||||
let artifact = tokio::task::spawn_blocking(move || store.put_registered(&bytes))
|
||||
.await
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "storage_unavailable"))?
|
||||
.map_err(|error| artifact_error(locale, error))?;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let expires_at = now + Duration::hours(IMPORT_JOB_TTL_HOURS);
|
||||
let job_id = ImportJobId::new(new_prefixed_id("imp"));
|
||||
let preview_payload = serde_json::to_value(&preview)
|
||||
let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi"));
|
||||
let source = self
|
||||
.registry
|
||||
.create_artifact_source(CreateArtifactSourceRequest {
|
||||
workspace_id,
|
||||
source_id: &source_id,
|
||||
artifact: &artifact,
|
||||
mime_type: &mime_type,
|
||||
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||
created_at: now,
|
||||
})
|
||||
.await?;
|
||||
let mut detach_guard = SourceDetachGuard::new(
|
||||
self.registry.clone(),
|
||||
workspace_id.clone(),
|
||||
source_id.clone(),
|
||||
source.updated_at,
|
||||
);
|
||||
let verified = self
|
||||
.registry
|
||||
.read_artifact_source(&self.artifact_store, workspace_id, &source_id)
|
||||
.await?;
|
||||
if verified.source.blob.artifact_ref != *artifact.artifact_ref() {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let preview = parse_verified_preview(verified.bytes, locale).await?;
|
||||
if preview
|
||||
.groups
|
||||
.iter()
|
||||
.all(|group| group.operations.is_empty())
|
||||
{
|
||||
return Err(ApiError::openapi_upload(locale, "no_methods"));
|
||||
}
|
||||
let source_envelope = ImportJobSourceEnvelope {
|
||||
source_id,
|
||||
digest: artifact.artifact_ref().clone(),
|
||||
};
|
||||
let preview_value = serde_json::to_value(&preview)
|
||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
let preview_payload = json!({
|
||||
"source": {
|
||||
"source_id": source_envelope.source_id.as_str(),
|
||||
"digest": source_envelope.digest.as_str(),
|
||||
},
|
||||
"preview": preview_value,
|
||||
});
|
||||
|
||||
self.registry
|
||||
.create_import_job(CreateImportJobRequest {
|
||||
@@ -53,11 +109,13 @@ impl AdminService {
|
||||
source_format: &preview.source.format,
|
||||
source_version: preview.source.version.as_deref(),
|
||||
status: ImportJobStatus::Pending,
|
||||
source: &source_envelope,
|
||||
preview_payload: &preview_payload,
|
||||
created_at: &now,
|
||||
expires_at: &expires_at,
|
||||
})
|
||||
.await?;
|
||||
detach_guard.disarm();
|
||||
|
||||
Ok(OpenApiImportPreviewResponse {
|
||||
job_id: job_id.as_str().to_owned(),
|
||||
@@ -76,7 +134,7 @@ impl AdminService {
|
||||
payload: OpenApiImportCreatePayload,
|
||||
) -> Result<OpenApiImportCreateResponse, ApiError> {
|
||||
self.ensure_workspace_exists(workspace_id).await?;
|
||||
let _ = self.registry.delete_expired_import_jobs().await;
|
||||
self.registry.delete_expired_import_jobs().await?;
|
||||
|
||||
if !matches!(payload.conflict_mode.as_str(), "skip" | "rename") {
|
||||
return Err(ApiError::validation(
|
||||
@@ -101,13 +159,6 @@ impl AdminService {
|
||||
return Err(ApiError::validation("import job kind is not openapi"));
|
||||
}
|
||||
|
||||
let stored_preview = job
|
||||
.preview_payload
|
||||
.get("preview")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| job.preview_payload.clone());
|
||||
let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview)
|
||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
let selected = payload
|
||||
.selected_operation_keys
|
||||
.iter()
|
||||
@@ -118,7 +169,38 @@ impl AdminService {
|
||||
"selected_operation_keys must contain at least one operation",
|
||||
));
|
||||
}
|
||||
let finished_at = OffsetDateTime::now_utc();
|
||||
let application_key = openapi_application_key(&payload)?;
|
||||
let conflict_mode = if payload.conflict_mode == "skip" {
|
||||
ImportConflictMode::Skip
|
||||
} else {
|
||||
ImportConflictMode::Rename
|
||||
};
|
||||
if job.status == ImportJobStatus::Completed {
|
||||
let applied = self
|
||||
.registry
|
||||
.apply_import_job(ApplyImportJobRequest {
|
||||
id: job_id,
|
||||
workspace_id,
|
||||
application_key: &application_key,
|
||||
conflict_mode,
|
||||
operations: &[],
|
||||
finished_at: &finished_at,
|
||||
})
|
||||
.await?;
|
||||
return Ok(openapi_import_response(applied, Vec::new()));
|
||||
}
|
||||
|
||||
let source = import_job_source(&job.preview_payload)?;
|
||||
let verified = self
|
||||
.registry
|
||||
.read_artifact_source(&self.artifact_store, workspace_id, &source.source_id)
|
||||
.await
|
||||
.map_err(openapi_source_error)?;
|
||||
if verified.source.blob.artifact_ref != source.digest {
|
||||
return Err(ApiError::source_integrity());
|
||||
}
|
||||
let preview = parse_verified_preview(verified.bytes, OpenApiUploadLocale::En).await?;
|
||||
let mut candidates = BTreeMap::new();
|
||||
for group in &preview.groups {
|
||||
for operation in &group.operations {
|
||||
@@ -171,13 +253,6 @@ impl AdminService {
|
||||
});
|
||||
}
|
||||
|
||||
let finished_at = OffsetDateTime::now_utc();
|
||||
let application_key = openapi_application_key(&payload)?;
|
||||
let conflict_mode = if payload.conflict_mode == "skip" {
|
||||
ImportConflictMode::Skip
|
||||
} else {
|
||||
ImportConflictMode::Rename
|
||||
};
|
||||
let applied = self
|
||||
.registry
|
||||
.apply_import_job(ApplyImportJobRequest {
|
||||
@@ -190,20 +265,31 @@ impl AdminService {
|
||||
})
|
||||
.await?;
|
||||
|
||||
let created = applied
|
||||
.created
|
||||
.iter()
|
||||
.map(|operation| OpenApiImportCreatedOperation {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
name: operation.name.clone(),
|
||||
version: operation.version,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut findings = applied
|
||||
.created
|
||||
.iter()
|
||||
.filter_map(|operation| {
|
||||
operation.renamed_from.as_ref().map(|previous_name| ImportFinding {
|
||||
Ok(openapi_import_response(applied, skipped))
|
||||
}
|
||||
}
|
||||
|
||||
fn openapi_import_response(
|
||||
applied: ImportJobApplyResult,
|
||||
mut skipped: Vec<OpenApiImportSkippedOperation>,
|
||||
) -> OpenApiImportCreateResponse {
|
||||
let created = applied
|
||||
.created
|
||||
.iter()
|
||||
.map(|operation| OpenApiImportCreatedOperation {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
name: operation.name.clone(),
|
||||
version: operation.version,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut findings = applied
|
||||
.created
|
||||
.iter()
|
||||
.filter_map(|operation| {
|
||||
operation
|
||||
.renamed_from
|
||||
.as_ref()
|
||||
.map(|previous_name| ImportFinding {
|
||||
code: "operation_name_renamed".to_owned(),
|
||||
severity: ImportFindingSeverity::Info,
|
||||
message: format!(
|
||||
@@ -212,36 +298,171 @@ impl AdminService {
|
||||
),
|
||||
operation_key: Some(operation.operation_key.clone()),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for operation in applied.skipped {
|
||||
skipped.push(OpenApiImportSkippedOperation {
|
||||
operation_key: operation.operation_key.clone(),
|
||||
name: operation.name.clone(),
|
||||
reason: "operation with this name already exists".to_owned(),
|
||||
});
|
||||
findings.push(ImportFinding {
|
||||
code: operation.reason,
|
||||
severity: ImportFindingSeverity::Warning,
|
||||
message: format!(
|
||||
"Операция {} уже существует и была пропущена.",
|
||||
operation.name
|
||||
),
|
||||
operation_key: Some(operation.operation_key),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for operation in applied.skipped {
|
||||
skipped.push(OpenApiImportSkippedOperation {
|
||||
operation_key: operation.operation_key.clone(),
|
||||
name: operation.name.clone(),
|
||||
reason: "operation with this name already exists".to_owned(),
|
||||
});
|
||||
findings.push(ImportFinding {
|
||||
code: operation.reason,
|
||||
severity: ImportFindingSeverity::Warning,
|
||||
message: format!(
|
||||
"Операция {} уже существует и была пропущена.",
|
||||
operation.name
|
||||
),
|
||||
operation_key: Some(operation.operation_key),
|
||||
});
|
||||
}
|
||||
info!(
|
||||
name: "admin.openapi_import.completed",
|
||||
created = created.len(),
|
||||
skipped = skipped.len(),
|
||||
"openapi import created drafts"
|
||||
);
|
||||
OpenApiImportCreateResponse {
|
||||
created,
|
||||
skipped,
|
||||
findings,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_openapi_upload(upload: &OpenApiUpload) -> Result<(), ApiError> {
|
||||
if upload.bytes.is_empty() {
|
||||
return Err(ApiError::openapi_upload(upload.locale, "empty_file"));
|
||||
}
|
||||
if upload.bytes.len() > MAX_ARTIFACT_BYTES {
|
||||
return Err(ApiError::openapi_upload(upload.locale, "file_too_large"));
|
||||
}
|
||||
if !matches!(
|
||||
upload.mime_type.as_str(),
|
||||
"application/yaml"
|
||||
| "application/x-yaml"
|
||||
| "text/yaml"
|
||||
| "text/x-yaml"
|
||||
| "application/json"
|
||||
| "application/openapi+json"
|
||||
| "application/octet-stream"
|
||||
) {
|
||||
return Err(ApiError::openapi_upload(
|
||||
upload.locale,
|
||||
"invalid_media_type",
|
||||
));
|
||||
}
|
||||
if std::str::from_utf8(&upload.bytes).is_err() {
|
||||
return Err(ApiError::openapi_upload(upload.locale, "invalid_utf8"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn parse_verified_preview(
|
||||
bytes: Vec<u8>,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<crank_import::rest::ImportPreview, ApiError> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let document = std::str::from_utf8(&bytes)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "invalid_utf8"))?;
|
||||
crank_import::rest::preview_document(document)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?
|
||||
}
|
||||
|
||||
fn artifact_error(locale: OpenApiUploadLocale, error: ArtifactError) -> ApiError {
|
||||
match error {
|
||||
ArtifactError::EmptySource => ApiError::openapi_upload(locale, "empty_file"),
|
||||
ArtifactError::SourceTooLarge => ApiError::openapi_upload(locale, "file_too_large"),
|
||||
ArtifactError::Integrity => ApiError::openapi_upload(locale, "source_integrity"),
|
||||
ArtifactError::Storage
|
||||
| ArtifactError::NotFound
|
||||
| ArtifactError::InvalidReference
|
||||
| ArtifactError::UnsafeRoot => ApiError::openapi_upload(locale, "storage_unavailable"),
|
||||
}
|
||||
}
|
||||
|
||||
fn openapi_source_error(error: RegistryError) -> ApiError {
|
||||
match error {
|
||||
RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable => {
|
||||
ApiError::source_unavailable()
|
||||
}
|
||||
RegistryError::SourceIntegrity => ApiError::source_integrity(),
|
||||
other => ApiError::from(other),
|
||||
}
|
||||
}
|
||||
|
||||
fn import_job_source(payload: &serde_json::Value) -> Result<ImportJobSourceEnvelope, ApiError> {
|
||||
let source = payload
|
||||
.get("source")
|
||||
.ok_or_else(ApiError::source_unavailable)?;
|
||||
let source_id = source
|
||||
.get("source_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or_else(ApiError::source_unavailable)?;
|
||||
let digest = source
|
||||
.get("digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| value.parse().ok())
|
||||
.ok_or_else(ApiError::source_integrity)?;
|
||||
Ok(ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
})
|
||||
}
|
||||
|
||||
struct SourceDetachGuard {
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl SourceDetachGuard {
|
||||
fn new(
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
workspace_id,
|
||||
source_id,
|
||||
expected_updated_at,
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SourceDetachGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
let registry = self.registry.clone();
|
||||
let workspace_id = self.workspace_id.clone();
|
||||
let source_id = self.source_id.clone();
|
||||
let expected_updated_at = Some(self.expected_updated_at);
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
let _ = registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &workspace_id,
|
||||
source_id: &source_id,
|
||||
expected_updated_at,
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
});
|
||||
}
|
||||
info!(
|
||||
name: "admin.openapi_import.completed",
|
||||
created = created.len(),
|
||||
skipped = skipped.len(),
|
||||
"openapi import created drafts"
|
||||
);
|
||||
|
||||
Ok(OpenApiImportCreateResponse {
|
||||
created,
|
||||
skipped,
|
||||
findings,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ mod integration {
|
||||
mod logs_usage;
|
||||
mod onboarding;
|
||||
mod openapi_import;
|
||||
mod openapi_source;
|
||||
mod operation_lifecycle;
|
||||
mod operations_agents;
|
||||
mod request_context;
|
||||
|
||||
@@ -328,14 +328,18 @@ pub(super) async fn test_registry() -> PostgresRegistry {
|
||||
}
|
||||
|
||||
pub(super) fn test_storage_root(name: &str) -> std::path::PathBuf {
|
||||
env::temp_dir().join(format!(
|
||||
let root = env::temp_dir().join(format!(
|
||||
"crank_admin_api_{name}_{}_{}",
|
||||
std::process::id(),
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
))
|
||||
));
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
#[cfg(unix)]
|
||||
std::fs::set_permissions(&root, std::os::unix::fs::PermissionsExt::from_mode(0o700)).unwrap();
|
||||
root
|
||||
}
|
||||
|
||||
pub(super) fn test_auth_settings() -> AuthSettings {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
|
||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
||||
use crank_core::WorkspaceId;
|
||||
use crank_registry::ImportJobStatus;
|
||||
use serial_test::serial;
|
||||
@@ -51,12 +51,7 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiImportPreviewPayload {
|
||||
document: OPENAPI3.to_owned(),
|
||||
},
|
||||
)
|
||||
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -89,6 +84,24 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
assert_eq!(created.created[0].name, "latest_rates");
|
||||
assert!(created.skipped.is_empty());
|
||||
|
||||
let replayed = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replayed.created.len(), 1);
|
||||
assert_eq!(
|
||||
replayed.created[0].operation_id,
|
||||
created.created[0].operation_id
|
||||
);
|
||||
|
||||
let operations = service.list_operations(&workspace_id).await.unwrap();
|
||||
assert!(
|
||||
operations
|
||||
@@ -120,12 +133,7 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
);
|
||||
|
||||
let skip_preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiImportPreviewPayload {
|
||||
document: OPENAPI3.to_owned(),
|
||||
},
|
||||
)
|
||||
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||
.await
|
||||
.unwrap();
|
||||
let skipped = service
|
||||
@@ -147,12 +155,7 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
assert_eq!(skipped.findings[0].code, "operation_name_conflict");
|
||||
|
||||
let rename_preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiImportPreviewPayload {
|
||||
document: OPENAPI3.to_owned(),
|
||||
},
|
||||
)
|
||||
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||
.await
|
||||
.unwrap();
|
||||
let renamed = service
|
||||
@@ -185,12 +188,7 @@ async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiImportPreviewPayload {
|
||||
document: OPENAPI3.to_owned(),
|
||||
},
|
||||
)
|
||||
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||
.await
|
||||
.unwrap();
|
||||
let job_id = preview.job_id.as_str().into();
|
||||
@@ -232,3 +230,11 @@ async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
||||
.await;
|
||||
assert!(conflicting_replay.is_err());
|
||||
}
|
||||
|
||||
fn openapi_upload() -> OpenApiUpload {
|
||||
OpenApiUpload {
|
||||
bytes: OPENAPI3.as_bytes().to_vec(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,937 @@
|
||||
use std::{
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
||||
use crank_artifacts::MAX_ARTIFACT_BYTES;
|
||||
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
||||
use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
use opentelemetry::trace::TracerProvider as _;
|
||||
use opentelemetry_sdk::{
|
||||
error::OTelSdkResult,
|
||||
trace::{SdkTracerProvider, SpanData, SpanExporter},
|
||||
};
|
||||
use reqwest::multipart::{Form, Part};
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
use super::common::{
|
||||
authorized_client, build_test_app, spawn_admin_api, test_auth_settings, test_registry,
|
||||
test_secret_crypto, test_service, test_storage_root,
|
||||
};
|
||||
|
||||
mod apply_failures;
|
||||
|
||||
const OPENAPI: &str = r#"
|
||||
openapi: 3.0.3
|
||||
info: { title: Source authority }
|
||||
servers:
|
||||
- url: https://example.test
|
||||
paths:
|
||||
/health:
|
||||
get:
|
||||
operationId: sourceHealth
|
||||
responses:
|
||||
'200': { description: OK }
|
||||
"#;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn multipart_accepts_valid_yaml_and_json_through_the_outer_router() {
|
||||
let registry = test_registry().await;
|
||||
let app = build_test_app(registry, test_storage_root("openapi_valid_multipart"));
|
||||
let server = spawn_admin_api(app).await;
|
||||
let client = authorized_client(&server).await;
|
||||
|
||||
for (document, filename, mime_type) in [
|
||||
(OPENAPI.as_bytes(), "openapi.yaml", "application/yaml"),
|
||||
(
|
||||
br#"{"openapi":"3.0.3","info":{"title":"JSON"},"paths":{"/health":{"get":{"responses":{"200":{"description":"OK"}}}}}}"#
|
||||
.as_slice(),
|
||||
"openapi.json",
|
||||
"application/json",
|
||||
),
|
||||
] {
|
||||
let response = client
|
||||
.post(format!("{server}/imports/openapi/preview"))
|
||||
.multipart(Form::new().part(
|
||||
"file",
|
||||
file_part(document, filename, mime_type),
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = response.json::<Value>().await.unwrap();
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "{body}");
|
||||
assert!(body["job_id"].is_string());
|
||||
assert!(!body.to_string().contains("source_id"));
|
||||
assert!(!body.to_string().contains("digest"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn multipart_requires_an_owner_membership_before_reading_the_file() {
|
||||
let registry = test_registry().await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let foreign_workspace = WorkspaceId::new("ws_openapi_unauthorized");
|
||||
registry
|
||||
.create_workspace(CreateWorkspaceRequest {
|
||||
workspace: &Workspace {
|
||||
id: foreign_workspace.clone(),
|
||||
slug: "openapi-unauthorized".to_owned(),
|
||||
display_name: "OpenAPI Unauthorized".to_owned(),
|
||||
status: WorkspaceStatus::Active,
|
||||
settings: serde_json::json!({}),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let app = build_test_app(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_upload_authorization"),
|
||||
);
|
||||
let server = spawn_admin_api(app).await;
|
||||
let endpoint = format!("{server}/imports/openapi/preview");
|
||||
|
||||
let anonymous = reqwest::Client::new()
|
||||
.post(&endpoint)
|
||||
.header("content-type", "multipart/form-data; boundary=broken")
|
||||
.body("this body must not be parsed before authentication")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(anonymous.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
|
||||
let authorized = authorized_client(&server).await;
|
||||
let foreign_endpoint = endpoint.replace("ws_default", foreign_workspace.as_str());
|
||||
let forbidden = authorized
|
||||
.post(foreign_endpoint)
|
||||
.header("content-type", "multipart/form-data; boundary=broken")
|
||||
.body("this body must not be parsed before workspace authorization")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(forbidden.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from artifact_sources")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from import_jobs")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
|
||||
const CANARY: &str = "openapi-telemetry-secret-canary";
|
||||
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
recorder
|
||||
.install()
|
||||
.expect("isolated integration test metrics recorder");
|
||||
|
||||
let writer = SharedLogWriter::default();
|
||||
let exported = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = SdkTracerProvider::builder()
|
||||
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
|
||||
.build();
|
||||
let tracer = provider.tracer("admin-openapi-source-test");
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(tracing_subscriber::fmt::layer().with_writer(writer.clone()))
|
||||
.with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
tracing::dispatcher::set_global_default(dispatch)
|
||||
.expect("isolated integration test tracing subscriber");
|
||||
|
||||
let registry = test_registry().await;
|
||||
let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary"));
|
||||
let server = spawn_admin_api(app).await;
|
||||
let client = authorized_client(&server).await;
|
||||
let document = format!(
|
||||
"openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n /broken:\n get:\n description: {CANARY}\n responses: ["
|
||||
);
|
||||
let response = client
|
||||
.post(format!("{server}/imports/openapi/preview"))
|
||||
.multipart(Form::new().part(
|
||||
"file",
|
||||
file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
let trace_id = response
|
||||
.headers()
|
||||
.get("x-trace-id")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let body = response.text().await.unwrap();
|
||||
assert!(!body.contains(CANARY));
|
||||
let diagnostics: Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(diagnostics["error"]["trace_id"], trace_id);
|
||||
|
||||
provider.force_flush().unwrap();
|
||||
let logs = writer.output();
|
||||
assert!(!logs.contains(CANARY));
|
||||
assert!(logs.contains(&trace_id));
|
||||
let spans = exported.lock().unwrap();
|
||||
let rendered_spans = format!("{spans:?}");
|
||||
assert!(!rendered_spans.contains(CANARY));
|
||||
drop(spans);
|
||||
for (key, _, _, _) in snapshotter.snapshot().into_vec() {
|
||||
assert!(!key.key().name().contains(CANARY));
|
||||
assert!(
|
||||
!key.key()
|
||||
.labels()
|
||||
.any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) })
|
||||
);
|
||||
}
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() {
|
||||
let registry = test_registry().await;
|
||||
let app = build_test_app(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_multipart_boundary"),
|
||||
);
|
||||
let server = spawn_admin_api(app).await;
|
||||
let client = authorized_client(&server).await;
|
||||
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"file",
|
||||
file_part(b"sensitive-openapi-body", "openapi.txt", "text/plain"),
|
||||
),
|
||||
"ru-RU",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.invalid_media_type",
|
||||
Some("тип"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"file",
|
||||
file_part(OPENAPI.as_bytes(), "openapi.txt", "application/yaml"),
|
||||
),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.invalid_media_type",
|
||||
Some("not supported"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"file",
|
||||
file_part(OPENAPI.as_bytes(), "openapi.yaml", "application/json"),
|
||||
),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.invalid_media_type",
|
||||
Some("not supported"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new(),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.malformed_multipart",
|
||||
Some("multipart"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from import_jobs")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from operations")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"other",
|
||||
file_part(OPENAPI.as_bytes(), "openapi.yaml", "application/yaml"),
|
||||
),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.malformed_multipart",
|
||||
Some("multipart"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new()
|
||||
.part(
|
||||
"file",
|
||||
file_part(OPENAPI.as_bytes(), "one.yaml", "application/yaml"),
|
||||
)
|
||||
.part(
|
||||
"file",
|
||||
file_part(OPENAPI.as_bytes(), "two.yaml", "application/yaml"),
|
||||
),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.malformed_multipart",
|
||||
Some("multipart"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part("file", file_part(b"", "openapi.yaml", "application/yaml")),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.empty_file",
|
||||
Some("empty"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"file",
|
||||
file_part([0xff, 0xfe], "openapi.yaml", "application/yaml"),
|
||||
),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.invalid_utf8",
|
||||
Some("UTF-8"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"file",
|
||||
file_part(
|
||||
vec![b'x'; MAX_ARTIFACT_BYTES + 1],
|
||||
"openapi.yaml",
|
||||
"application/yaml",
|
||||
),
|
||||
),
|
||||
"en-US",
|
||||
reqwest::StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"openapi_upload.file_too_large",
|
||||
Some("256 KiB"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from import_jobs")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from artifact_sources")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"file",
|
||||
file_part(
|
||||
b"openapi: 3.0.3\ninfo: { title: Empty }\npaths: {}\n",
|
||||
"openapi.yaml",
|
||||
"application/yaml",
|
||||
),
|
||||
),
|
||||
"en-US",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.no_methods",
|
||||
Some("no supported methods"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part("file", file_part(b"", "openapi.yaml", "application/yaml")),
|
||||
"ru-RU",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.empty_file",
|
||||
Some("пуст"),
|
||||
)
|
||||
.await;
|
||||
assert_rejected(
|
||||
&client,
|
||||
&server,
|
||||
Form::new().part(
|
||||
"file",
|
||||
file_part(b"openapi: [", "openapi.yaml", "application/yaml"),
|
||||
),
|
||||
"ru-RU",
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"openapi_upload.invalid_document",
|
||||
Some("документ"),
|
||||
)
|
||||
.await;
|
||||
wait_for_source_lifecycle(®istry, ArtifactSourceLifecycle::Detached).await;
|
||||
|
||||
let mut exact_limit = OPENAPI.as_bytes().to_vec();
|
||||
let padding = MAX_ARTIFACT_BYTES
|
||||
.checked_sub(exact_limit.len())
|
||||
.expect("OpenAPI fixture must fit inside the exact-size boundary");
|
||||
exact_limit.extend(std::iter::repeat_n(b'#', padding));
|
||||
let response = client
|
||||
.post(format!("{server}/imports/openapi/preview"))
|
||||
.multipart(Form::new().part(
|
||||
"file",
|
||||
file_part(&exact_limit, "openapi.yaml", "application/yaml"),
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||
let body = response.json::<Value>().await.unwrap();
|
||||
let rendered = body.to_string();
|
||||
assert!(body["job_id"].is_string());
|
||||
assert!(!rendered.contains("source_id"));
|
||||
assert!(!rendered.contains("digest"));
|
||||
}
|
||||
|
||||
async fn wait_for_source_lifecycle(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
lifecycle: ArtifactSourceLifecycle,
|
||||
) {
|
||||
let expected = match lifecycle {
|
||||
ArtifactSourceLifecycle::Active => "active",
|
||||
ArtifactSourceLifecycle::Detached => "detached",
|
||||
};
|
||||
for _ in 0..100 {
|
||||
let found = sqlx::query_scalar::<_, bool>(
|
||||
"select exists(select 1 from artifact_sources where lifecycle = $1)",
|
||||
)
|
||||
.bind(expected)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
if found {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("artifact source did not reach {expected}");
|
||||
}
|
||||
|
||||
fn file_part(bytes: impl AsRef<[u8]>, filename: &str, mime_type: &str) -> Part {
|
||||
Part::bytes(bytes.as_ref().to_vec())
|
||||
.file_name(filename.to_owned())
|
||||
.mime_str(mime_type)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn assert_rejected(
|
||||
client: &reqwest::Client,
|
||||
server: &impl AsRef<str>,
|
||||
form: Form,
|
||||
language: &str,
|
||||
expected_status: reqwest::StatusCode,
|
||||
expected_code: &str,
|
||||
message_fragment: Option<&str>,
|
||||
) {
|
||||
let response = client
|
||||
.post(format!("{}/imports/openapi/preview", server.as_ref()))
|
||||
.header("accept-language", language)
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = response.json::<Value>().await.unwrap();
|
||||
assert_eq!(
|
||||
status, expected_status,
|
||||
"unexpected status while checking {expected_code}: {body}"
|
||||
);
|
||||
assert_eq!(body["error"]["code"], expected_code);
|
||||
assert!(body["error"]["request_id"].is_string());
|
||||
assert!(body["error"]["trace_id"].is_string());
|
||||
if let Some(fragment) = message_fragment {
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains(fragment)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn apply_rereads_the_scoped_verified_source_and_expiry_detaches_it() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_verified_source"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
);
|
||||
let service = service.clone();
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: OPENAPI.as_bytes().to_vec(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let job_id = preview.job_id.as_str().into();
|
||||
let job = registry
|
||||
.get_import_job(&workspace_id, &job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let source_id = job.preview_payload["source"]["source_id"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let source = registry
|
||||
.get_artifact_source(&workspace_id, &source_id.as_str().into())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(source.lifecycle, ArtifactSourceLifecycle::Active);
|
||||
|
||||
// The stored preview is presentation data only. A forged candidate cannot
|
||||
// create a Draft because apply reparses the source bytes.
|
||||
sqlx::query("update import_jobs set preview_payload = jsonb_set(preview_payload, '{preview}', '{\"groups\":[]}') where id = $1")
|
||||
.bind(job_id.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let imported = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&job_id,
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /health".to_owned()],
|
||||
server_url: None,
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(imported.created.len(), 1);
|
||||
let detached = registry
|
||||
.get_artifact_source(&workspace_id, &source_id.as_str().into())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(detached.lifecycle, ArtifactSourceLifecycle::Detached);
|
||||
|
||||
let second_preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: OPENAPI.as_bytes().to_vec(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let second_job_id = second_preview.job_id.as_str().into();
|
||||
let second_job = registry
|
||||
.get_import_job(&workspace_id, &second_job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let second_source_id = second_job.preview_payload["source"]["source_id"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
sqlx::query("update import_jobs set expires_at = $1 where id = $2")
|
||||
.bind(OffsetDateTime::now_utc() - Duration::minutes(1))
|
||||
.bind(second_job_id.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
registry.delete_expired_import_jobs().await.unwrap();
|
||||
let expired_job_source = registry
|
||||
.get_artifact_source(&workspace_id, &second_source_id.as_str().into())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
expired_job_source.lifecycle,
|
||||
ArtifactSourceLifecycle::Detached
|
||||
);
|
||||
|
||||
let legacy_preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: format!("{OPENAPI}\n# legacy-upgrade").into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let legacy_job_id: crank_registry::ImportJobId = legacy_preview.job_id.as_str().into();
|
||||
let legacy_job = registry
|
||||
.get_import_job(&workspace_id, &legacy_job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let legacy_source_id = legacy_job.preview_payload["source"]["source_id"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
sqlx::query(
|
||||
"update import_jobs
|
||||
set preview_payload = jsonb_set(
|
||||
preview_payload,
|
||||
'{source}',
|
||||
'{\"format\":\"openapi\",\"version\":\"3.0.3\"}'::jsonb
|
||||
),
|
||||
expires_at = now() - interval '1 minute'
|
||||
where id = $1",
|
||||
)
|
||||
.bind(legacy_job_id.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"update artifact_sources
|
||||
set created_at = now() - interval '10 minutes',
|
||||
updated_at = now() - interval '10 minutes'
|
||||
where workspace_id = $1 and source_id = $2",
|
||||
)
|
||||
.bind(workspace_id.as_str())
|
||||
.bind(&legacy_source_id)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
registry.delete_expired_import_jobs().await.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.get_import_job(&workspace_id, &legacy_job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
registry
|
||||
.get_artifact_source(&workspace_id, &legacy_source_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.lifecycle,
|
||||
ArtifactSourceLifecycle::Detached
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancellation_detaches_and_restart_cleanup_recovers_a_dangling_source() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_cancel_restart"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
);
|
||||
let schema = sqlx::query_scalar::<_, String>("select current_schema()")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let observer = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(crank_test_support::postgres_database_url().await)
|
||||
.await
|
||||
.unwrap();
|
||||
let lock_pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(crank_test_support::postgres_database_url().await)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("select set_config('search_path', $1, false)")
|
||||
.bind(&schema)
|
||||
.execute(&observer)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("select set_config('search_path', $1, false)")
|
||||
.bind(&schema)
|
||||
.execute(&lock_pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"create function block_openapi_import_insert() returns trigger language plpgsql as $$
|
||||
begin
|
||||
perform pg_advisory_xact_lock(2147483001);
|
||||
return new;
|
||||
end $$",
|
||||
)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"create trigger block_openapi_import_insert
|
||||
before insert on import_jobs
|
||||
for each row execute function block_openapi_import_insert()",
|
||||
)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut lock_connection = lock_pool.acquire().await.unwrap();
|
||||
sqlx::query("select pg_advisory_lock(2147483001)")
|
||||
.execute(&mut *lock_connection)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let preview_task = tokio::spawn({
|
||||
let service = service.clone();
|
||||
let workspace_id = workspace_id.clone();
|
||||
async move {
|
||||
service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: OPENAPI.as_bytes().to_vec(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
let cancelled_source_id =
|
||||
wait_for_source_lifecycle_in_pool(&observer, ArtifactSourceLifecycle::Active).await;
|
||||
assert!(!preview_task.is_finished());
|
||||
preview_task.abort();
|
||||
let cancellation = preview_task.await.unwrap_err();
|
||||
assert!(cancellation.is_cancelled());
|
||||
wait_for_blocked_import_insert_to_stop(&observer).await;
|
||||
sqlx::query("select pg_advisory_unlock(2147483001)")
|
||||
.execute(&mut *lock_connection)
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for_specific_source_lifecycle(
|
||||
®istry,
|
||||
&cancelled_source_id,
|
||||
ArtifactSourceLifecycle::Detached,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from import_jobs")
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
|
||||
sqlx::query("drop trigger block_openapi_import_insert on import_jobs")
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: format!("{OPENAPI}\n# restart-window").into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||||
let job = registry
|
||||
.get_import_job(&workspace_id, &job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let source_id = job.preview_payload["source"]["source_id"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
sqlx::query("delete from import_jobs where id = $1")
|
||||
.bind(job_id.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"update artifact_sources
|
||||
set created_at = now() - interval '10 minutes',
|
||||
updated_at = now() - interval '10 minutes'
|
||||
where workspace_id = $1 and source_id = $2",
|
||||
)
|
||||
.bind(workspace_id.as_str())
|
||||
.bind(&source_id)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
registry.delete_expired_import_jobs().await.unwrap();
|
||||
let recovered = registry
|
||||
.get_artifact_source(&workspace_id, &source_id.as_str().into())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(recovered.lifecycle, ArtifactSourceLifecycle::Detached);
|
||||
}
|
||||
|
||||
async fn wait_for_source_lifecycle_in_pool(
|
||||
pool: &sqlx::PgPool,
|
||||
lifecycle: ArtifactSourceLifecycle,
|
||||
) -> ArtifactSourceId {
|
||||
let expected = match lifecycle {
|
||||
ArtifactSourceLifecycle::Active => "active",
|
||||
ArtifactSourceLifecycle::Detached => "detached",
|
||||
};
|
||||
for _ in 0..100 {
|
||||
let found = sqlx::query_scalar::<_, Option<String>>(
|
||||
"select min(source_id) from artifact_sources where lifecycle = $1",
|
||||
)
|
||||
.bind(expected)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
if let Some(source_id) = found {
|
||||
return ArtifactSourceId::new(source_id);
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("artifact source did not reach {expected}");
|
||||
}
|
||||
|
||||
async fn wait_for_blocked_import_insert_to_stop(pool: &sqlx::PgPool) {
|
||||
for _ in 0..100 {
|
||||
let active = sqlx::query_scalar::<_, bool>(
|
||||
"select exists(
|
||||
select 1
|
||||
from pg_stat_activity
|
||||
where state = 'active'
|
||||
and query like 'insert into import_jobs%'
|
||||
)",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
if !active {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("cancelled import insert remained active in PostgreSQL");
|
||||
}
|
||||
|
||||
async fn wait_for_specific_source_lifecycle(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
source_id: &ArtifactSourceId,
|
||||
lifecycle: ArtifactSourceLifecycle,
|
||||
) {
|
||||
let expected = match lifecycle {
|
||||
ArtifactSourceLifecycle::Active => "active",
|
||||
ArtifactSourceLifecycle::Detached => "detached",
|
||||
};
|
||||
for _ in 0..100 {
|
||||
let found = sqlx::query_scalar::<_, bool>(
|
||||
"select exists(
|
||||
select 1
|
||||
from artifact_sources
|
||||
where source_id = $1 and lifecycle = $2
|
||||
)",
|
||||
)
|
||||
.bind(source_id.as_str())
|
||||
.bind(expected)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
if found {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("artifact source {source_id:?} did not reach {expected}");
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
||||
|
||||
impl SpanExporter for CapturingExporter {
|
||||
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
|
||||
self.0.lock().unwrap().extend(batch);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
use crank_artifacts::ArtifactRef;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn public_http_contract_is_safe_for_source_failures() {
|
||||
const SOURCE_CANARY: &str = "openapi-apply-source-secret-canary";
|
||||
|
||||
let registry = test_registry().await;
|
||||
let storage_root = test_storage_root("openapi_apply_source_errors");
|
||||
let app = build_test_app(registry.clone(), storage_root.clone());
|
||||
let server = spawn_admin_api(app).await;
|
||||
let client = authorized_client(&server).await;
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
|
||||
for (marker, expected_code, corrupt) in [
|
||||
("missing", "openapi_upload.source_unavailable", false),
|
||||
("corrupt", "openapi_upload.source_integrity", true),
|
||||
] {
|
||||
let document = format!("{OPENAPI}\n# {marker} {SOURCE_CANARY}");
|
||||
let preview_response = client
|
||||
.post(format!("{server}/imports/openapi/preview"))
|
||||
.multipart(Form::new().part(
|
||||
"file",
|
||||
file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(preview_response.status(), reqwest::StatusCode::OK);
|
||||
let preview = preview_response.json::<Value>().await.unwrap();
|
||||
let job_id: crank_registry::ImportJobId = preview["job_id"].as_str().unwrap().into();
|
||||
let job = registry
|
||||
.get_import_job(&workspace_id, &job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let artifact_ref: ArtifactRef = job.preview_payload["source"]["digest"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse()
|
||||
.unwrap();
|
||||
let path = artifact_path(&storage_root, &artifact_ref);
|
||||
if corrupt {
|
||||
std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o600))
|
||||
.unwrap();
|
||||
std::fs::write(&path, b"corrupt").unwrap();
|
||||
std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o400))
|
||||
.unwrap();
|
||||
} else {
|
||||
std::fs::remove_file(path).unwrap();
|
||||
}
|
||||
|
||||
let response = client
|
||||
.post(format!(
|
||||
"{server}/imports/openapi/{}/create",
|
||||
job_id.as_str()
|
||||
))
|
||||
.json(&serde_json::json!({
|
||||
"selected_operation_keys": ["GET /health"],
|
||||
"conflict_mode": "skip"
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = response.json::<Value>().await.unwrap();
|
||||
assert_eq!(status, reqwest::StatusCode::UNPROCESSABLE_ENTITY, "{body}");
|
||||
assert_eq!(body["error"]["code"], expected_code);
|
||||
assert!(body["error"]["request_id"].is_string());
|
||||
assert!(body["error"]["trace_id"].is_string());
|
||||
let rendered = body.to_string();
|
||||
assert!(!rendered.contains(SOURCE_CANARY));
|
||||
assert!(!rendered.contains("source_id"));
|
||||
assert!(!rendered.contains("digest"));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from operations where workspace_id = $1")
|
||||
.bind(workspace_id.as_str())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
0,
|
||||
"source failures must not create operations"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn missing_corrupt_changed_and_foreign_sources_create_no_drafts() {
|
||||
let registry = test_registry().await;
|
||||
let storage_root = test_storage_root("openapi_source_fail_closed");
|
||||
let service = test_service(
|
||||
registry.clone(),
|
||||
storage_root.clone(),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
|
||||
let (missing_job, _, missing_ref) =
|
||||
preview_job_source(®istry, &service, &workspace_id, "missing").await;
|
||||
std::fs::remove_file(artifact_path(&storage_root, &missing_ref)).unwrap();
|
||||
assert!(
|
||||
apply_health_operation(&service, &workspace_id, &missing_job)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let (corrupt_job, _, corrupt_ref) =
|
||||
preview_job_source(®istry, &service, &workspace_id, "corrupt").await;
|
||||
let corrupt_path = artifact_path(&storage_root, &corrupt_ref);
|
||||
std::fs::set_permissions(
|
||||
&corrupt_path,
|
||||
std::os::unix::fs::PermissionsExt::from_mode(0o600),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(&corrupt_path, b"corrupt-openapi-canary").unwrap();
|
||||
std::fs::set_permissions(
|
||||
&corrupt_path,
|
||||
std::os::unix::fs::PermissionsExt::from_mode(0o400),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
apply_health_operation(&service, &workspace_id, &corrupt_job)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let (changed_job, _, _) =
|
||||
preview_job_source(®istry, &service, &workspace_id, "changed").await;
|
||||
sqlx::query(
|
||||
"update import_jobs
|
||||
set preview_payload = jsonb_set(
|
||||
preview_payload,
|
||||
'{source,digest}',
|
||||
to_jsonb($1::text)
|
||||
)
|
||||
where id = $2",
|
||||
)
|
||||
.bind(format!("sha256:{}", "0".repeat(64)))
|
||||
.bind(changed_job.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
apply_health_operation(&service, &workspace_id, &changed_job)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let foreign_workspace = WorkspaceId::new("ws_openapi_foreign");
|
||||
registry
|
||||
.create_workspace(CreateWorkspaceRequest {
|
||||
workspace: &Workspace {
|
||||
id: foreign_workspace.clone(),
|
||||
slug: "openapi-foreign".to_owned(),
|
||||
display_name: "OpenAPI Foreign".to_owned(),
|
||||
status: WorkspaceStatus::Active,
|
||||
settings: serde_json::json!({}),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let (foreign_job, _, _) =
|
||||
preview_job_source(®istry, &service, &workspace_id, "foreign").await;
|
||||
sqlx::query("update import_jobs set workspace_id = $1 where id = $2")
|
||||
.bind(foreign_workspace.as_str())
|
||||
.bind(foreign_job.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
apply_health_operation(&service, &foreign_workspace, &foreign_job)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
assert!(
|
||||
service
|
||||
.list_operations(&workspace_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
service
|
||||
.list_operations(&foreign_workspace)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
async fn preview_job_source(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
service: &admin_api::service::AdminService,
|
||||
workspace_id: &WorkspaceId,
|
||||
marker: &str,
|
||||
) -> (crank_registry::ImportJobId, ArtifactSourceId, ArtifactRef) {
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: format!("{OPENAPI}\n# {marker}").into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||||
let job = registry
|
||||
.get_import_job(workspace_id, &job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let source = &job.preview_payload["source"];
|
||||
(
|
||||
job_id,
|
||||
ArtifactSourceId::new(source["source_id"].as_str().unwrap()),
|
||||
source["digest"].as_str().unwrap().parse().unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
fn artifact_path(root: &std::path::Path, artifact_ref: &ArtifactRef) -> std::path::PathBuf {
|
||||
root.join("sha256")
|
||||
.join(&artifact_ref.digest_hex()[..2])
|
||||
.join(artifact_ref.digest_hex())
|
||||
}
|
||||
|
||||
async fn apply_health_operation(
|
||||
service: &admin_api::service::AdminService,
|
||||
workspace_id: &WorkspaceId,
|
||||
job_id: &crank_registry::ImportJobId,
|
||||
) -> Result<admin_api::service::OpenApiImportCreateResponse, admin_api::error::ApiError> {
|
||||
service
|
||||
.create_openapi_import(
|
||||
workspace_id,
|
||||
job_id,
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /health".to_owned()],
|
||||
server_url: None,
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
Reference in New Issue
Block a user