use std::collections::{BTreeMap, BTreeSet}; use crank_artifacts::{ArtifactError, MAX_ARTIFACT_BYTES}; use crank_core::{ AuthProfileId, ExecutionConfig, OperationId, OperationSecurityLevel, OperationStatus, Protocol, SecretStatus, ToolQualityFinding, ToolQualitySeverity, WorkspaceId, }; use crank_import::rest::{ ImportFinding, ImportFindingSeverity, ImportNameConflict, ImportOperationCandidate, NORMALIZER_VERSION, PROJECTION_VERSION, canonical_external_document_uri, operation_draft_from_candidate, validate_document_source_safe, }; use crank_registry::{ ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity, CreateArtifactSourceRequest, CreateImportJobRequest, CreateImportPlanRequest, FinishImportJobRequest, ImportConflictMode, ImportJob, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope, ImportJobStatus, ImportOperationDraft, ImportPlanId, RegistryError, RegistryOperation, }; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; use tracing::{info, instrument}; use crate::{ error::ApiError, service::{ AdminService, OpenApiImportAuthProfileChoice, OpenApiImportCreatePayload, OpenApiImportCreateResponse, OpenApiImportCreatedOperation, OpenApiImportPlanApplyPayload, OpenApiImportPlanCandidateView, OpenApiImportPlanCounts, OpenApiImportPlanCreatePayload, OpenApiImportPlanCreateResponse, OpenApiImportPlanDecision, OpenApiImportPlanRevalidationResponse, OpenApiImportPreviewResponse, OpenApiImportSecurityProfileChoice, OpenApiImportSkippedOperation, OpenApiUpload, OpenApiUploadLocale, OperationPayload, new_prefixed_id, }, }; mod external_references; mod import_plan; mod job_contract; mod plan_apply; mod verified_preview; use external_references::ImportReplayContext; use import_plan::import_plan_repreview_error; use job_contract::{ SourceDetachGuard, import_job_dependencies, import_job_normalization_config, import_job_source, }; use verified_preview::{ ImportPreviewResolution, artifact_error, openapi_source_error, parse_verified_preview, preview_digest, preview_has_blocker, }; const IMPORT_JOB_TTL_HOURS: i64 = 24; const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); const OPENAPI_PARSE_CONCURRENCY: usize = 4; const MAX_IMPORT_PLAN_KEYS: usize = 2_000; const OPENAPI_SUPPORT_MATRIX_VERSION: &str = "1.0.4"; const MAX_IMPORT_PLAN_KEY_BYTES: usize = 4_096; const MAX_IMPORT_PLAN_SERVER_URL_BYTES: usize = 8_192; const MAX_IMPORT_PLAN_NAME_BYTES: usize = 256; const MAX_IMPORT_PLAN_ID_REF_BYTES: usize = 128; static OPENAPI_PARSE_SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(OPENAPI_PARSE_CONCURRENCY); fn preview_name_conflicts( preview: &crank_import::rest::ImportPreview, catalog_names: &[String], ) -> Vec { let operations = preview .groups .iter() .flat_map(|group| group.operations.iter()) .collect::>(); let mut operation_ids = BTreeMap::<&str, usize>::new(); for candidate in &operations { if let Some(id) = candidate.operation_id.as_deref() { *operation_ids.entry(id).or_default() += 1; } } let occupied = catalog_names .iter() .map(String::as_str) .collect::>(); let mut proposals = catalog_names.iter().cloned().collect::>(); proposals.extend( operations .iter() .map(|candidate| candidate.suggested_name.clone()), ); let mut conflicts = Vec::new(); for candidate in operations { let duplicate = candidate .operation_id .as_deref() .is_some_and(|id| operation_ids.get(id).is_some_and(|count| *count > 1)); let invalid = candidate.operation_id.as_deref().is_some_and(|id| { id.is_empty() || id.len() > MAX_IMPORT_PLAN_NAME_BYTES || !id.bytes().enumerate().all(|(index, ch)| { ch.is_ascii_alphabetic() || ch == b'_' || (index > 0 && ch.is_ascii_digit()) }) }); let taken = occupied.contains(candidate.suggested_name.as_str()); if !duplicate && !invalid && !taken { continue; } let base = candidate .suggested_name .chars() .take(240) .collect::(); let proposal = next_available_name(&base, &mut proposals); for conflict_type in [ duplicate.then_some("duplicate_name"), invalid.then_some("invalid_name"), taken.then_some("taken_name"), ] .into_iter() .flatten() { conflicts.push(ImportNameConflict { operation_key: candidate.key.clone(), conflict_type: conflict_type.to_owned(), proposed_name: proposal.clone(), source_location: candidate.source_location.as_ref().map(|location| { crank_import::rest::SourceLocation { pointer: if candidate.operation_id.is_some() { format!("{}/operationId", location.pointer) } else { location.pointer.clone() }, } }), }); } } conflicts } fn next_available_name(base: &str, occupied: &mut BTreeSet) -> String { // A catalog may contain more names than the preview candidate cap. With // occupied.len() + 1 suffixes, at least one candidate must be free. (2..=occupied.len() + 2) .map(|index| format!("{base}_{index}")) .find(|name| occupied.insert(name.clone())) .expect("finite occupied names always leave a free suffix") } fn attach_preview_readiness( preview: &mut crank_import::rest::ImportPreview, choices: &[OpenApiImportSecurityProfileChoice], ) { let document_blocked = preview.findings.iter().any(|finding| { finding.severity == ImportFindingSeverity::Error && finding.operation_key.is_none() }); for candidate in preview .groups .iter_mut() .flat_map(|group| group.operations.iter_mut()) { let blocked = document_blocked || candidate .findings .iter() .any(|finding| finding.severity == ImportFindingSeverity::Error); let alternatives = &candidate.security_requirements.alternatives; let public = alternatives.is_empty() || alternatives .iter() .any(|alternative| alternative.supported && alternative.schemes.is_empty()); candidate.status = if blocked { "blocked" } else if public { "actionable" } else if alternatives.iter().enumerate().any(|(index, alternative)| { alternative.supported && choices.iter().any(|choice| { choice.operation_key == candidate.key && choice.alternative_index == index && !choice.profiles.is_empty() }) }) { "unresolved" } else { "blocked" } .to_owned(); } } impl AdminService { #[instrument(skip(self, upload), fields(workspace_id = %workspace_id.as_str()))] pub async fn preview_openapi_import( &self, workspace_id: &WorkspaceId, upload: OpenApiUpload, ) -> Result { self.ensure_workspace_exists(workspace_id).await?; validate_openapi_upload(&upload)?; let OpenApiUpload { bytes: raw_bytes, mime_type, locale, } = upload; let raw_document = std::str::from_utf8(&raw_bytes) .map_err(|_| ApiError::openapi_upload(locale, "invalid_utf8"))?; validate_document_source_safe(raw_document, &self.external_reference_normalization) .map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))?; let now = OffsetDateTime::now_utc(); let mut dependencies = self .materialize_external_reference_snapshots(workspace_id, &raw_bytes, now) .await; if dependencies.unsafe_source { return Err(ApiError::openapi_upload(locale, "invalid_document")); } let source_digest = crank_import::rest::SourceDigest::parse(format!("{:x}", Sha256::digest(&raw_bytes))) .map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))?; if crank_import::rest::validate_resolved_bundle_source_safe( raw_document, source_digest, &dependencies.snapshots, &self.external_reference_normalization, ) .is_err() { dependencies.detach_all().await; return Err(ApiError::openapi_upload(locale, "invalid_document")); } let bytes = raw_bytes; let store = self.artifact_store.clone(); let artifact_result = tokio::task::spawn_blocking(move || store.put_registered(&bytes)) .await .map_err(|_| ApiError::openapi_upload(locale, "storage_unavailable")) .and_then(|result| result.map_err(|error| artifact_error(locale, error))); let artifact = match artifact_result { Ok(artifact) => artifact, Err(error) => { dependencies.detach_all().await; return Err(error); } }; let expires_at = now + Duration::hours(IMPORT_JOB_TTL_HOURS); let job_id = ImportJobId::new(new_prefixed_id("imp")); let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi")); let source_result = self .registry .create_artifact_source(CreateArtifactSourceRequest { workspace_id, source_id: &source_id, artifact: &artifact, mime_type: &mime_type, sensitivity: ArtifactSourceSensitivity::Internal, created_at: now, }) .await; let source = match source_result { Ok(source) => source, Err(error) => { dependencies.detach_all().await; return Err(ApiError::from(error)); } }; let mut detach_guard = SourceDetachGuard::new( self.registry.clone(), workspace_id.clone(), source_id.clone(), source.updated_at, ); let verified = match self .registry .read_artifact_source( std::sync::Arc::clone(&self.artifact_store), workspace_id, &source_id, ) .await { Ok(verified) => verified, Err(error) => { detach_guard.detach_now().await; return Err(ApiError::from(error)); } }; if verified.source.blob.artifact_ref != *artifact.artifact_ref() { detach_guard.detach_now().await; return Err(ApiError::openapi_upload(locale, "source_integrity")); } let parsed = match parse_verified_preview( verified.bytes, artifact.artifact_ref().digest_hex().to_owned(), dependencies.snapshots.clone(), self.external_reference_normalization.clone(), locale, false, ) .await { Ok(preview) => preview, Err(error) => { dependencies.detach_all().await; detach_guard.detach_now().await; return Err(error); } }; if parsed .preview .groups .iter() .all(|group| group.operations.is_empty()) { dependencies.detach_all().await; detach_guard.detach_now().await; return Err(ApiError::openapi_upload(locale, "no_methods")); } let candidate_count: usize = parsed .preview .groups .iter() .map(|group| group.operations.len()) .sum(); if candidate_count > MAX_IMPORT_PLAN_KEYS { dependencies.detach_all().await; detach_guard.detach_now().await; return Err(ApiError::validation_with_context( "OpenAPI preview has too many candidates", json!({ "error_code": "openapi_preview_too_many_candidates", "limit": MAX_IMPORT_PLAN_KEYS }), )); } let security_profile_choices = match self .security_profile_choices(workspace_id, &parsed.preview) .await { Ok(choices) => choices, Err(error) => { dependencies.detach_all().await; detach_guard.detach_now().await; return Err(error); } }; let catalog_names = match self.registry.list_import_catalog_names(workspace_id).await { Ok(names) => names, Err(error) => { dependencies.detach_all().await; detach_guard.detach_now().await; return Err(ApiError::from(error)); } }; let source_envelope = ImportJobSourceEnvelope { source_id, digest: artifact.artifact_ref().clone(), }; let preview_value = serde_json::to_value(&parsed.preview) .map_err(|error| ApiError::internal(error.to_string()))?; let preview_digest = preview_digest(&preview_value)?; let preview_payload = json!({ "source": { "source_id": source_envelope.source_id.as_str(), "digest": source_envelope.digest.as_str(), }, "dependencies": dependencies.payload(), "dependency_snapshots": dependencies.snapshot_payload(), "preview": preview_value, "preview_digest": preview_digest, "normalization": { "normalizer_version": NORMALIZER_VERSION, "projection_version": PROJECTION_VERSION, "ir_fingerprint": parsed.ir_fingerprint, "config": self.external_reference_normalization, }, }); if let Err(error) = self .registry .create_import_job(CreateImportJobRequest { id: &job_id, workspace_id, kind: ImportJobKind::OpenApi, source_format: &parsed.preview.source.format, source_version: parsed.preview.source.version.as_deref(), status: ImportJobStatus::Pending, source: &source_envelope, preview_payload: &preview_payload, created_at: &now, expires_at: &expires_at, }) .await { dependencies.detach_all().await; detach_guard.detach_now().await; return Err(ApiError::from(error)); } dependencies.disarm(); detach_guard.disarm(); let mut response_preview = parsed.preview; response_preview.conflicts = preview_name_conflicts(&response_preview, &catalog_names); attach_preview_readiness(&mut response_preview, &security_profile_choices); Ok(OpenApiImportPreviewResponse { job_id: job_id.as_str().to_owned(), expires_at: expires_at .format(&Rfc3339) .map_err(|error| ApiError::internal(error.to_string()))?, preview: response_preview, security_profile_choices, }) } async fn security_profile_choices( &self, workspace_id: &WorkspaceId, preview: &crank_import::rest::ImportPreview, ) -> Result, ApiError> { let profiles = self.registry.list_auth_profiles(workspace_id).await?; let mut available = Vec::new(); for profile in profiles { if profile.kind != profile.config.kind() { continue; } let mut usable = true; for secret_id in profile.config.secret_ids() { let Some(secret) = self.registry.get_secret(workspace_id, secret_id).await? else { usable = false; break; }; if secret.secret.status != SecretStatus::Active || self .registry .get_current_secret_version(workspace_id, secret_id) .await? .is_none() { usable = false; break; } } if usable { available.push(profile); } } let mut choices = Vec::new(); for operation in preview .groups .iter() .flat_map(|group| group.operations.iter()) { for (alternative_index, alternative) in operation .security_requirements .alternatives .iter() .enumerate() { if !alternative.supported || alternative.schemes.len() != 1 { continue; } let Some(scheme) = operation.security_schemes.get(&alternative.schemes[0]) else { continue; }; let profiles = available .iter() .filter(|profile| { import_plan::auth_config_matches_scheme(&profile.config, scheme) }) .map(|profile| OpenApiImportAuthProfileChoice { id: profile.id.as_str().to_owned(), name: profile.name.clone(), kind: profile.kind, }) .collect::>(); choices.push(OpenApiImportSecurityProfileChoice { operation_key: operation.key.clone(), alternative_index, schemes: alternative.schemes.clone(), profiles, }); } } Ok(choices) } #[instrument(skip(self, payload), fields(workspace_id = %workspace_id.as_str(), job_id = %job_id.as_str()))] pub async fn create_openapi_import( &self, workspace_id: &WorkspaceId, job_id: &ImportJobId, payload: OpenApiImportCreatePayload, ) -> Result { self.create_openapi_import_with_locale( workspace_id, job_id, payload, OpenApiUploadLocale::En, ) .await } pub async fn create_openapi_import_with_locale( &self, workspace_id: &WorkspaceId, job_id: &ImportJobId, payload: OpenApiImportCreatePayload, locale: OpenApiUploadLocale, ) -> Result { self.ensure_workspace_exists(workspace_id).await?; if !matches!(payload.conflict_mode.as_str(), "skip" | "rename") { return Err(ApiError::validation( "unsupported conflict_mode; supported values are skip and rename", )); } let job = self .registry .get_import_job(workspace_id, job_id) .await? .ok_or_else(|| { ApiError::not_found_with_context( "import job was not found", json!({ "job_id": job_id.as_str() }), ) })?; if job.expires_at <= OffsetDateTime::now_utc() { let message = match locale { OpenApiUploadLocale::Ru => "срок действия preview импорта истёк", OpenApiUploadLocale::En => "import preview has expired", }; return Err(ApiError::validation(message)); } if job.kind != ImportJobKind::OpenApi { return Err(ApiError::validation("import job kind is not openapi")); } let selected = payload .selected_operation_keys .iter() .cloned() .collect::>(); if selected.is_empty() { return Err(ApiError::validation( "selected_operation_keys must contain at least one operation", )); } let finished_at = OffsetDateTime::now_utc(); let application_key = openapi_application_key(&payload)?; let conflict_mode = if payload.conflict_mode == "skip" { ImportConflictMode::Skip } else { ImportConflictMode::Rename }; let replay_context = ImportReplayContext { workspace_id, job_id, application_key: &application_key, conflict_mode, finished_at: &finished_at, }; if job.status == ImportJobStatus::Completed { let applied = self .registry .apply_import_job(ApplyImportJobRequest { id: job_id, workspace_id, application_key: &application_key, conflict_mode, operations: &[], pre_skipped: &[], finished_at: &finished_at, }) .await?; return Ok(openapi_import_response(applied)); } let resolved_preview = match self .resolve_verified_openapi_preview(workspace_id, &job, locale) .await { Ok(resolved) => resolved, Err(ImportPreviewResolution::Fatal(error_code, error)) => { return self .fail_openapi_import_or_replay(&replay_context, error_code, error) .await; } Err(ImportPreviewResolution::NonFatal(error)) => return Err(error), }; let preview = resolved_preview.preview; if preview_has_blocker(&preview, &selected) { return self .fail_openapi_import_or_replay( &replay_context, "reference_resolution_blocked", ApiError::openapi_upload(locale, "invalid_document"), ) .await; } let mut candidates = BTreeMap::new(); for group in &preview.groups { for operation in &group.operations { candidates.insert(operation.key.clone(), operation); } } let mut skipped = Vec::new(); let mut operations = Vec::new(); for operation_key in selected { let Some(candidate) = candidates.get(&operation_key) else { skipped.push(crank_registry::SkippedImportOperation { operation_key, name: String::new(), reason: "operation was not found in import preview".to_owned(), }); continue; }; let mut draft = operation_draft_from_candidate(candidate, payload.server_url.as_deref()); attach_import_findings(&mut draft, candidate); let operation = self.new_operation_snapshot(OperationPayload { name: draft.name.clone(), display_name: draft.display_name.clone(), category: draft.category, protocol: Protocol::Rest, security_level: OperationSecurityLevel::Standard, target: crank_core::Target::Rest(draft.target), input_schema: draft.input_schema, output_schema: draft.output_schema, input_mapping: draft.input_mapping, output_mapping: draft.output_mapping, execution_config: ExecutionConfig { timeout_ms: 10_000, retry_policy: None, response_cache: None, idempotency: None, safety: None, approval_policy: None, execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, tool_description: draft.tool_description, wizard_state: draft.wizard_state, })?; operations.push(ImportOperationDraft { operation_key: candidate.key.clone(), operation, }); } let applied = self .registry .apply_import_job(ApplyImportJobRequest { id: job_id, workspace_id, application_key: &application_key, conflict_mode, operations: &operations, pre_skipped: &skipped, finished_at: &finished_at, }) .await?; Ok(openapi_import_response(applied)) } } fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse { let created = applied .created .iter() .map(|operation| OpenApiImportCreatedOperation { operation_key: operation.operation_key.clone(), operation_id: operation.operation_id.as_str().to_owned(), name: operation.name.clone(), version: operation.version, }) .collect::>(); let mut findings = applied .created .iter() .filter_map(|operation| { operation .renamed_from .as_ref() .map(|previous_name| ImportFinding { code: "operation_name_renamed".to_owned(), severity: ImportFindingSeverity::Info, message: format!( "Операция {previous_name} уже существует, новый черновик создан как {}.", operation.name ), operation_key: Some(operation.operation_key.clone()), source_location: None, }) }) .collect::>(); let skipped = applied .skipped .into_iter() .map(|operation| { let reason = operation.reason.clone(); let name = operation.name.clone(); findings.push(ImportFinding { code: reason.clone(), severity: ImportFindingSeverity::Warning, message: if name.is_empty() { "Выбранная операция отсутствует в исходном preview и была пропущена.".to_owned() } else { format!("Операция {name} уже существует и была пропущена.") }, operation_key: Some(operation.operation_key.clone()), source_location: None, }); OpenApiImportSkippedOperation { operation_key: operation.operation_key.clone(), name: operation.name, reason, } }) .collect::>(); info!( name: "admin.openapi_import.completed", created = created.len(), skipped = skipped.len(), "openapi import created drafts" ); OpenApiImportCreateResponse { created, skipped, findings, } } fn validate_openapi_upload(upload: &OpenApiUpload) -> Result<(), ApiError> { if upload.bytes.is_empty() { return Err(ApiError::openapi_upload(upload.locale, "empty_file")); } if upload.bytes.len() > MAX_ARTIFACT_BYTES { return Err(ApiError::openapi_upload(upload.locale, "file_too_large")); } if !matches!( upload.mime_type.as_str(), "application/yaml" | "application/x-yaml" | "text/yaml" | "text/x-yaml" | "application/json" | "application/openapi+json" | "application/octet-stream" ) { return Err(ApiError::openapi_upload( upload.locale, "invalid_media_type", )); } if std::str::from_utf8(&upload.bytes).is_err() { return Err(ApiError::openapi_upload(upload.locale, "invalid_utf8")); } Ok(()) } fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result { let selected_operation_keys = payload .selected_operation_keys .iter() .cloned() .collect::>(); let canonical = serde_json::to_vec(&json!({ "selected_operation_keys": selected_operation_keys, "server_url": payload.server_url.as_deref(), "conflict_mode": payload.conflict_mode.as_str(), })) .map_err(|error| ApiError::internal(error.to_string()))?; Ok(format!("{:x}", Sha256::digest(canonical))) } fn attach_import_findings( draft: &mut crank_import::rest::RestImportCandidate, candidate: &ImportOperationCandidate, ) { let findings = candidate .findings .iter() .map(tool_quality_finding_from_import) .collect::>(); if findings.is_empty() { return; } let mut wizard_state = draft.wizard_state.take().unwrap_or_default(); wizard_state.import_findings = findings; draft.wizard_state = Some(wizard_state); } fn tool_quality_finding_from_import(finding: &ImportFinding) -> ToolQualityFinding { ToolQualityFinding { severity: match finding.severity { ImportFindingSeverity::Info => ToolQualitySeverity::Info, ImportFindingSeverity::Warning => ToolQualitySeverity::Warning, ImportFindingSeverity::Error => ToolQualitySeverity::Error, }, code: format!("openapi_import.{}", finding.code), message: finding.message.clone(), suggested_action: Some( "Откройте черновик в мастере и уточните описание, схемы или маппинг перед публикацией." .to_owned(), ), field_path: finding.operation_key.clone(), } } #[cfg(test)] mod preview_name_tests { use super::next_available_name; use std::collections::BTreeSet; #[test] fn crowded_catalog_still_gets_a_free_deterministic_proposal() { let mut occupied = (2..=2_002) .map(|index| format!("shared_{index}")) .collect::>(); occupied.insert("shared".to_owned()); assert_eq!(next_available_name("shared", &mut occupied), "shared_2003"); assert_eq!(next_available_name("shared", &mut occupied), "shared_2004"); } }