use std::collections::{BTreeMap, BTreeSet}; use crank_artifacts::{ArtifactError, MAX_ARTIFACT_BYTES}; use crank_core::{ ExecutionConfig, OperationSecurityLevel, Protocol, ToolQualityFinding, ToolQualitySeverity, WorkspaceId, }; use crank_import::rest::{ ImportFinding, ImportFindingSeverity, ImportOperationCandidate, NORMALIZER_VERSION, PROJECTION_VERSION, operation_draft_from_candidate, }; use crank_registry::{ ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity, CreateArtifactSourceRequest, CreateImportJobRequest, DetachArtifactSourceRequest, ImportConflictMode, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope, ImportJobStatus, ImportOperationDraft, RegistryError, }; use serde_json::json; use sha2::{Digest, Sha256}; use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; use tracing::{info, instrument}; use crate::{ error::ApiError, service::{ AdminService, OpenApiImportCreatePayload, OpenApiImportCreateResponse, OpenApiImportCreatedOperation, OpenApiImportPreviewResponse, OpenApiImportSkippedOperation, OpenApiUpload, OpenApiUploadLocale, OperationPayload, new_prefixed_id, }, }; const IMPORT_JOB_TTL_HOURS: i64 = 24; const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); const OPENAPI_PARSE_CONCURRENCY: usize = 4; static OPENAPI_PARSE_SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(OPENAPI_PARSE_CONCURRENCY); impl AdminService { #[instrument(skip(self, upload), fields(workspace_id = %workspace_id.as_str()))] pub async fn preview_openapi_import( &self, workspace_id: &WorkspaceId, upload: OpenApiUpload, ) -> Result { self.ensure_workspace_exists(workspace_id).await?; validate_openapi_upload(&upload)?; let OpenApiUpload { bytes, mime_type, locale, } = upload; let store = self.artifact_store.clone(); let artifact = tokio::task::spawn_blocking(move || store.put_registered(&bytes)) .await .map_err(|_| ApiError::openapi_upload(locale, "storage_unavailable"))? .map_err(|error| artifact_error(locale, error))?; let now = OffsetDateTime::now_utc(); let expires_at = now + Duration::hours(IMPORT_JOB_TTL_HOURS); let job_id = ImportJobId::new(new_prefixed_id("imp")); let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi")); let source = self .registry .create_artifact_source(CreateArtifactSourceRequest { workspace_id, source_id: &source_id, artifact: &artifact, mime_type: &mime_type, sensitivity: ArtifactSourceSensitivity::Internal, created_at: now, }) .await?; let mut detach_guard = SourceDetachGuard::new( self.registry.clone(), workspace_id.clone(), source_id.clone(), source.updated_at, ); let verified = match self .registry .read_artifact_source( std::sync::Arc::clone(&self.artifact_store), workspace_id, &source_id, ) .await { Ok(verified) => verified, Err(error) => { detach_guard.detach_now().await; return Err(ApiError::from(error)); } }; if verified.source.blob.artifact_ref != *artifact.artifact_ref() { detach_guard.detach_now().await; return Err(ApiError::openapi_upload(locale, "source_integrity")); } let parsed = match parse_verified_preview( verified.bytes, artifact.artifact_ref().digest_hex().to_owned(), locale, false, ) .await { Ok(preview) => preview, Err(error) => { detach_guard.detach_now().await; return Err(error); } }; if parsed .preview .groups .iter() .all(|group| group.operations.is_empty()) { detach_guard.detach_now().await; return Err(ApiError::openapi_upload(locale, "no_methods")); } let source_envelope = ImportJobSourceEnvelope { source_id, digest: artifact.artifact_ref().clone(), }; let preview_value = serde_json::to_value(&parsed.preview) .map_err(|error| ApiError::internal(error.to_string()))?; let preview_digest = preview_digest(&preview_value)?; let preview_payload = json!({ "source": { "source_id": source_envelope.source_id.as_str(), "digest": source_envelope.digest.as_str(), }, "preview": preview_value, "preview_digest": preview_digest, "normalization": { "normalizer_version": NORMALIZER_VERSION, "projection_version": PROJECTION_VERSION, "ir_fingerprint": parsed.ir_fingerprint, }, }); if let Err(error) = self .registry .create_import_job(CreateImportJobRequest { id: &job_id, workspace_id, kind: ImportJobKind::OpenApi, source_format: &parsed.preview.source.format, source_version: parsed.preview.source.version.as_deref(), status: ImportJobStatus::Pending, source: &source_envelope, preview_payload: &preview_payload, created_at: &now, expires_at: &expires_at, }) .await { detach_guard.detach_now().await; return Err(ApiError::from(error)); } detach_guard.disarm(); Ok(OpenApiImportPreviewResponse { job_id: job_id.as_str().to_owned(), expires_at: expires_at .format(&Rfc3339) .map_err(|error| ApiError::internal(error.to_string()))?, preview: parsed.preview, }) } #[instrument(skip(self, payload), fields(workspace_id = %workspace_id.as_str(), job_id = %job_id.as_str()))] pub async fn create_openapi_import( &self, workspace_id: &WorkspaceId, job_id: &ImportJobId, payload: OpenApiImportCreatePayload, ) -> Result { self.create_openapi_import_with_locale( workspace_id, job_id, payload, OpenApiUploadLocale::En, ) .await } pub async fn create_openapi_import_with_locale( &self, workspace_id: &WorkspaceId, job_id: &ImportJobId, payload: OpenApiImportCreatePayload, locale: OpenApiUploadLocale, ) -> Result { self.ensure_workspace_exists(workspace_id).await?; if !matches!(payload.conflict_mode.as_str(), "skip" | "rename") { return Err(ApiError::validation( "unsupported conflict_mode; supported values are skip and rename", )); } let job = self .registry .get_import_job(workspace_id, job_id) .await? .ok_or_else(|| { ApiError::not_found_with_context( "import job was not found", json!({ "job_id": job_id.as_str() }), ) })?; if job.expires_at <= OffsetDateTime::now_utc() { return Err(ApiError::validation("import preview has expired")); } if job.kind != ImportJobKind::OpenApi { return Err(ApiError::validation("import job kind is not openapi")); } let selected = payload .selected_operation_keys .iter() .cloned() .collect::>(); if selected.is_empty() { return Err(ApiError::validation( "selected_operation_keys must contain at least one operation", )); } let finished_at = OffsetDateTime::now_utc(); let application_key = openapi_application_key(&payload)?; let conflict_mode = if payload.conflict_mode == "skip" { ImportConflictMode::Skip } else { ImportConflictMode::Rename }; if job.status == ImportJobStatus::Completed { let applied = self .registry .apply_import_job(ApplyImportJobRequest { id: job_id, workspace_id, application_key: &application_key, conflict_mode, operations: &[], pre_skipped: &[], finished_at: &finished_at, }) .await?; return Ok(openapi_import_response(applied)); } let source = import_job_source(&job.preview_payload, locale)?; let verified = match self .registry .read_artifact_source( std::sync::Arc::clone(&self.artifact_store), workspace_id, &source.source_id, ) .await { Ok(verified) => verified, Err( error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable), ) => { // Another request may have finished and detached the source // after our unlocked Pending read. Re-read the job and ask // the registry for its canonical, locked replay instead of // exposing a false `source_unavailable` outcome. let latest = self.registry.get_import_job(workspace_id, job_id).await?; if latest.is_some_and(|latest| latest.status == ImportJobStatus::Completed) { let applied = self .registry .apply_import_job(ApplyImportJobRequest { id: job_id, workspace_id, application_key: &application_key, conflict_mode, operations: &[], pre_skipped: &[], finished_at: &finished_at, }) .await?; return Ok(openapi_import_response(applied)); } return Err(openapi_source_error(locale, error)); } Err(error) => return Err(openapi_source_error(locale, error)), }; if verified.source.blob.artifact_ref != source.digest { return Err(ApiError::openapi_upload(locale, "source_integrity")); } let legacy_v1 = job.preview_payload.get("normalization").is_none(); let parsed = parse_verified_preview( verified.bytes, source.digest.digest_hex().to_owned(), locale, legacy_v1, ) .await?; verify_preview_contract(&job.preview_payload, &parsed, locale)?; if preview_has_blocker(&parsed.preview) { return Err(ApiError::openapi_upload(locale, "invalid_document")); } let mut candidates = BTreeMap::new(); for group in &parsed.preview.groups { for operation in &group.operations { candidates.insert(operation.key.clone(), operation); } } let mut skipped = Vec::new(); let mut operations = Vec::new(); for operation_key in selected { let Some(candidate) = candidates.get(&operation_key) else { skipped.push(crank_registry::SkippedImportOperation { operation_key, name: String::new(), reason: "operation was not found in import preview".to_owned(), }); continue; }; let mut draft = operation_draft_from_candidate(candidate, payload.server_url.as_deref()); attach_import_findings(&mut draft, candidate); let operation = self.new_operation_snapshot(OperationPayload { name: draft.name.clone(), display_name: draft.display_name.clone(), category: draft.category, protocol: Protocol::Rest, security_level: OperationSecurityLevel::Standard, target: crank_core::Target::Rest(draft.target), input_schema: draft.input_schema, output_schema: draft.output_schema, input_mapping: draft.input_mapping, output_mapping: draft.output_mapping, execution_config: ExecutionConfig { timeout_ms: 10_000, retry_policy: None, response_cache: None, idempotency: None, safety: None, approval_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, tool_description: draft.tool_description, wizard_state: draft.wizard_state, })?; operations.push(ImportOperationDraft { operation_key: candidate.key.clone(), operation, }); } let applied = self .registry .apply_import_job(ApplyImportJobRequest { id: job_id, workspace_id, application_key: &application_key, conflict_mode, operations: &operations, pre_skipped: &skipped, finished_at: &finished_at, }) .await?; Ok(openapi_import_response(applied)) } } fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse { let created = applied .created .iter() .map(|operation| OpenApiImportCreatedOperation { operation_key: operation.operation_key.clone(), operation_id: operation.operation_id.as_str().to_owned(), name: operation.name.clone(), version: operation.version, }) .collect::>(); let mut findings = applied .created .iter() .filter_map(|operation| { operation .renamed_from .as_ref() .map(|previous_name| ImportFinding { code: "operation_name_renamed".to_owned(), severity: ImportFindingSeverity::Info, message: format!( "Операция {previous_name} уже существует, новый черновик создан как {}.", operation.name ), operation_key: Some(operation.operation_key.clone()), }) }) .collect::>(); let skipped = applied .skipped .into_iter() .map(|operation| { let reason = operation.reason.clone(); let name = operation.name.clone(); findings.push(ImportFinding { code: reason.clone(), severity: ImportFindingSeverity::Warning, message: if name.is_empty() { "Выбранная операция отсутствует в исходном preview и была пропущена.".to_owned() } else { format!("Операция {name} уже существует и была пропущена.") }, operation_key: Some(operation.operation_key.clone()), }); OpenApiImportSkippedOperation { operation_key: operation.operation_key.clone(), name: operation.name, reason, } }) .collect::>(); info!( name: "admin.openapi_import.completed", created = created.len(), skipped = skipped.len(), "openapi import created drafts" ); OpenApiImportCreateResponse { created, skipped, findings, } } fn validate_openapi_upload(upload: &OpenApiUpload) -> Result<(), ApiError> { if upload.bytes.is_empty() { return Err(ApiError::openapi_upload(upload.locale, "empty_file")); } if upload.bytes.len() > MAX_ARTIFACT_BYTES { return Err(ApiError::openapi_upload(upload.locale, "file_too_large")); } if !matches!( upload.mime_type.as_str(), "application/yaml" | "application/x-yaml" | "text/yaml" | "text/x-yaml" | "application/json" | "application/openapi+json" | "application/octet-stream" ) { return Err(ApiError::openapi_upload( upload.locale, "invalid_media_type", )); } if std::str::from_utf8(&upload.bytes).is_err() { return Err(ApiError::openapi_upload(upload.locale, "invalid_utf8")); } Ok(()) } struct ParsedOpenApiPreview { preview: crank_import::rest::ImportPreview, ir_fingerprint: String, } async fn parse_verified_preview( bytes: Vec, digest: String, locale: OpenApiUploadLocale, legacy_v1: bool, ) -> Result { let started = tokio::time::Instant::now(); let permit = tokio::time::timeout(OPENAPI_PARSE_DEADLINE, OPENAPI_PARSE_SLOTS.acquire()) .await .map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))? .map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))?; let parsing = tokio::task::spawn_blocking(move || { // Keep the permit inside the blocking task. Timing out the caller // cannot cancel CPU work already running, but abandoned parsers remain // globally bounded and release capacity when they actually finish. let _permit = permit; let document = std::str::from_utf8(&bytes) .map_err(|_| ApiError::openapi_upload(locale, "invalid_utf8"))?; if legacy_v1 { return crank_import::rest::preview_document_legacy_v1(document) .map(|preview| ParsedOpenApiPreview { preview, ir_fingerprint: String::new(), }) .map_err(|_| ApiError::openapi_upload(locale, "invalid_document")); } let digest = crank_import::rest::SourceDigest::parse(digest) .map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?; let ir = crank_import::rest::normalize_verified_document(document, digest, &Default::default()) .map_err(|error| match error { crank_import::rest::ImportParseError::NoMethods => { ApiError::openapi_upload(locale, "no_methods") } _ => ApiError::openapi_upload(locale, "invalid_document"), })?; let ir_fingerprint = preview_digest( &serde_json::to_value(&ir) .map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))?, )?; Ok::<_, ApiError>(ParsedOpenApiPreview { preview: crank_import::rest::preview_from_ir(&ir), ir_fingerprint, }) }); let remaining = OPENAPI_PARSE_DEADLINE .checked_sub(started.elapsed()) .unwrap_or(std::time::Duration::ZERO); tokio::time::timeout(remaining, parsing) .await .map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))? .map_err(|_| ApiError::openapi_upload(locale, "parser_unavailable"))? } fn artifact_error(locale: OpenApiUploadLocale, error: ArtifactError) -> ApiError { match error { ArtifactError::EmptySource => ApiError::openapi_upload(locale, "empty_file"), ArtifactError::SourceTooLarge => ApiError::openapi_upload(locale, "file_too_large"), ArtifactError::Integrity => ApiError::openapi_upload(locale, "source_integrity"), ArtifactError::Storage | ArtifactError::NotFound | ArtifactError::InvalidReference | ArtifactError::UnsafeRoot => ApiError::openapi_upload(locale, "storage_unavailable"), } } fn openapi_source_error(locale: OpenApiUploadLocale, error: RegistryError) -> ApiError { match error { RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable => { ApiError::openapi_upload(locale, "source_unavailable") } RegistryError::SourceIntegrity => ApiError::openapi_upload(locale, "source_integrity"), other => ApiError::from(other), } } fn preview_digest(preview: &serde_json::Value) -> Result { let canonical = serde_json::to_vec(preview).map_err(|error| ApiError::internal(error.to_string()))?; Ok(format!("{:x}", Sha256::digest(canonical))) } fn verify_preview_contract( payload: &serde_json::Value, parsed: &ParsedOpenApiPreview, locale: OpenApiUploadLocale, ) -> Result<(), ApiError> { // Pre-fingerprint jobs are legacy rolling-upgrade records. They retain the // old reparse behavior; new jobs fail closed if parser output drifts or a // persisted preview has been changed. let expected = payload .get("preview_digest") .and_then(serde_json::Value::as_str); if payload.get("normalization").is_some() && expected.is_none() { return Err(ApiError::openapi_upload(locale, "source_integrity")); } let Some(expected) = expected else { return Ok(()); }; let actual = preview_digest( &serde_json::to_value(&parsed.preview) .map_err(|error| ApiError::internal(error.to_string()))?, )?; if actual == expected { if let Some(normalization) = payload.get("normalization") { let normalizer = normalization .get("normalizer_version") .and_then(serde_json::Value::as_str); let projection = normalization .get("projection_version") .and_then(serde_json::Value::as_str); let fingerprint = normalization .get("ir_fingerprint") .and_then(serde_json::Value::as_str); if normalizer != Some(NORMALIZER_VERSION) || projection != Some(PROJECTION_VERSION) || fingerprint != Some(parsed.ir_fingerprint.as_str()) { return Err(ApiError::openapi_upload(locale, "source_integrity")); } } Ok(()) } else { Err(ApiError::openapi_upload(locale, "source_integrity")) } } fn preview_has_blocker(preview: &crank_import::rest::ImportPreview) -> bool { preview .findings .iter() .chain( preview .groups .iter() .flat_map(|group| group.operations.iter()) .flat_map(|operation| operation.findings.iter()), ) .any(|finding| finding.severity == ImportFindingSeverity::Error) } fn import_job_source( payload: &serde_json::Value, locale: OpenApiUploadLocale, ) -> Result { let source = payload .get("source") .ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?; let source_id = source .get("source_id") .and_then(serde_json::Value::as_str) .filter(|value| value.len() <= 132) .ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?; let digest = source .get("digest") .and_then(serde_json::Value::as_str) .and_then(|value| value.parse().ok()) .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; Ok(ImportJobSourceEnvelope { source_id: ArtifactSourceId::new(source_id), digest, }) } struct SourceDetachGuard { registry: crank_registry::PostgresRegistry, workspace_id: WorkspaceId, source_id: ArtifactSourceId, expected_updated_at: OffsetDateTime, armed: bool, } impl SourceDetachGuard { fn new( registry: crank_registry::PostgresRegistry, workspace_id: WorkspaceId, source_id: ArtifactSourceId, expected_updated_at: OffsetDateTime, ) -> Self { Self { registry, workspace_id, source_id, expected_updated_at, armed: true, } } fn disarm(&mut self) { self.armed = false; } async fn detach_now(&mut self) { if !self.armed { return; } self.armed = false; // This is the normal error path: await the database transition so a // caller receives a completed cleanup before it can retry. `Drop` // remains only the cancellation/shutdown safety net. let _ = self .registry .detach_artifact_source(DetachArtifactSourceRequest { workspace_id: &self.workspace_id, source_id: &self.source_id, expected_updated_at: Some(self.expected_updated_at), detached_at: OffsetDateTime::now_utc(), }) .await; } } impl Drop for SourceDetachGuard { fn drop(&mut self) { if !self.armed { return; } let registry = self.registry.clone(); let workspace_id = self.workspace_id.clone(); let source_id = self.source_id.clone(); let expected_updated_at = Some(self.expected_updated_at); if let Ok(handle) = tokio::runtime::Handle::try_current() { handle.spawn(async move { let _ = registry .detach_artifact_source(DetachArtifactSourceRequest { workspace_id: &workspace_id, source_id: &source_id, expected_updated_at, detached_at: OffsetDateTime::now_utc(), }) .await; }); } } } fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result { let selected_operation_keys = payload .selected_operation_keys .iter() .cloned() .collect::>(); let canonical = serde_json::to_vec(&json!({ "selected_operation_keys": selected_operation_keys, "server_url": payload.server_url.as_deref(), "conflict_mode": payload.conflict_mode.as_str(), })) .map_err(|error| ApiError::internal(error.to_string()))?; Ok(format!("{:x}", Sha256::digest(canonical))) } fn attach_import_findings( draft: &mut crank_import::rest::RestImportCandidate, candidate: &ImportOperationCandidate, ) { let findings = candidate .findings .iter() .map(tool_quality_finding_from_import) .collect::>(); if findings.is_empty() { return; } let mut wizard_state = draft.wizard_state.take().unwrap_or_default(); wizard_state.import_findings = findings; draft.wizard_state = Some(wizard_state); } fn tool_quality_finding_from_import(finding: &ImportFinding) -> ToolQualityFinding { ToolQualityFinding { severity: match finding.severity { ImportFindingSeverity::Info => ToolQualitySeverity::Info, ImportFindingSeverity::Warning => ToolQualitySeverity::Warning, ImportFindingSeverity::Error => ToolQualitySeverity::Error, }, code: format!("openapi_import.{}", finding.code), message: finding.message.clone(), suggested_action: Some( "Откройте черновик в мастере и уточните описание, схемы или маппинг перед публикацией." .to_owned(), ), field_path: finding.operation_key.clone(), } }