diff --git a/Cargo.lock b/Cargo.lock index 41e51bc..ae4ded4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,6 +13,7 @@ dependencies = [ "base64", "crank-community-auth", "crank-core", + "crank-import", "crank-mapping", "crank-registry", "crank-runtime", @@ -596,6 +597,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "crank-import" +version = "0.3.1" +dependencies = [ + "crank-core", + "crank-mapping", + "crank-schema", + "serde", + "serde_json", + "serde_yaml", + "thiserror", +] + [[package]] name = "crank-mapping" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 1b39fc3..8b41aef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/crank-community-auth", "crates/crank-community-mcp", "crates/crank-core", + "crates/crank-import", "crates/crank-schema", "crates/crank-mapping", "crates/crank-registry", diff --git a/apps/admin-api/Cargo.toml b/apps/admin-api/Cargo.toml index 2544ec7..ed9d51f 100644 --- a/apps/admin-api/Cargo.toml +++ b/apps/admin-api/Cargo.toml @@ -16,6 +16,7 @@ axum-extra.workspace = true base64.workspace = true crank-community-auth = { path = "../../crates/crank-community-auth" } crank-core = { path = "../../crates/crank-core" } +crank-import = { path = "../../crates/crank-import" } crank-mapping = { path = "../../crates/crank-mapping" } crank-registry = { path = "../../crates/crank-registry" } crank-runtime = { path = "../../crates/crank-runtime" } diff --git a/apps/admin-api/src/app.rs b/apps/admin-api/src/app.rs index 26a64d2..34944a5 100644 --- a/apps/admin-api/src/app.rs +++ b/apps/admin-api/src/app.rs @@ -18,6 +18,7 @@ use crate::{ auth::{change_password, get_profile, get_session, login, logout, update_profile}, auth_profiles::{create_auth_profile, get_auth_profile, list_auth_profiles}, capabilities::get_capabilities, + imports::{create_openapi_import, preview_openapi_import}, observability::{get_agent_usage, get_log, get_operation_usage, get_usage, list_logs}, operations::{ analyze_operation_quality, archive_operation, create_operation, create_version, @@ -35,6 +36,11 @@ use crate::{ pub fn build_app(state: AppState) -> Router { let workspace_router = Router::new() .route("/operations", get(list_operations).post(create_operation)) + .route("/imports/openapi/preview", post(preview_openapi_import)) + .route( + "/imports/openapi/{job_id}/create", + post(create_openapi_import), + ) .route( "/operations/analyze-quality", post(analyze_operation_quality), diff --git a/apps/admin-api/src/dto.rs b/apps/admin-api/src/dto.rs index e2c7611..06de9e8 100644 --- a/apps/admin-api/src/dto.rs +++ b/apps/admin-api/src/dto.rs @@ -309,6 +309,48 @@ pub struct YamlOperationDocument { pub operation: RegistryOperation, } +#[derive(Clone, Debug, Deserialize)] +pub struct OpenApiImportPreviewPayload { + pub document: String, +} + +#[derive(Clone, Debug, Serialize)] +pub struct OpenApiImportPreviewResponse { + pub job_id: String, + pub expires_at: String, + pub preview: crank_import::rest::ImportPreview, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct OpenApiImportCreatePayload { + pub selected_operation_keys: Vec, + #[serde(default)] + pub server_url: Option, + #[serde(default = "default_openapi_conflict_mode")] + pub conflict_mode: String, +} + +#[derive(Clone, Debug, Serialize)] +pub struct OpenApiImportCreateResponse { + pub created: Vec, + pub skipped: Vec, + pub findings: Vec, +} + +#[derive(Clone, Debug, Serialize)] +pub struct OpenApiImportCreatedOperation { + pub operation_id: String, + pub name: String, + pub version: u32, +} + +#[derive(Clone, Debug, Serialize)] +pub struct OpenApiImportSkippedOperation { + pub operation_key: String, + pub name: String, + pub reason: String, +} + #[derive(Clone, Debug, Serialize)] pub struct CreatedOperationResponse { pub operation_id: String, @@ -433,6 +475,10 @@ fn default_operation_category() -> String { "general".to_owned() } +fn default_openapi_conflict_mode() -> String { + "rename".to_owned() +} + fn default_export_mode() -> ExportMode { ExportMode::Portable } diff --git a/apps/admin-api/src/error.rs b/apps/admin-api/src/error.rs index b94289f..fabf35f 100644 --- a/apps/admin-api/src/error.rs +++ b/apps/admin-api/src/error.rs @@ -347,6 +347,10 @@ impl From for ApiError { format!("yaml import job {job_id} was not found"), json!({ "job_id": job_id }), ), + RegistryError::ImportJobNotFound { job_id } => Self::not_found_with_context( + format!("import job {job_id} was not found"), + json!({ "job_id": job_id }), + ), RegistryError::Storage(_) | RegistryError::Serialization(_) => { Self::internal(value.to_string()) } diff --git a/apps/admin-api/src/routes.rs b/apps/admin-api/src/routes.rs index f07b9ee..25a1dee 100644 --- a/apps/admin-api/src/routes.rs +++ b/apps/admin-api/src/routes.rs @@ -3,6 +3,7 @@ pub mod agents; pub mod auth; pub mod auth_profiles; pub mod capabilities; +pub mod imports; pub mod observability; pub mod operations; pub mod secrets; diff --git a/apps/admin-api/src/routes/imports.rs b/apps/admin-api/src/routes/imports.rs new file mode 100644 index 0000000..8916635 --- /dev/null +++ b/apps/admin-api/src/routes/imports.rs @@ -0,0 +1,51 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::{ + error::ApiError, + service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload}, + state::AppState, +}; + +#[derive(Deserialize)] +pub struct WorkspacePath { + pub workspace_id: String, +} + +#[derive(Deserialize)] +pub struct WorkspaceImportPath { + pub workspace_id: String, + pub job_id: String, +} + +pub async fn preview_openapi_import( + Path(path): Path, + State(state): State, + Json(payload): Json, +) -> Result, ApiError> { + let preview = state + .service + .preview_openapi_import(&path.workspace_id.as_str().into(), payload) + .await?; + Ok(Json(json!(preview))) +} + +pub async fn create_openapi_import( + Path(path): Path, + State(state): State, + Json(payload): Json, +) -> Result, ApiError> { + let imported = state + .service + .create_openapi_import( + &path.workspace_id.as_str().into(), + &path.job_id.as_str().into(), + payload, + ) + .await?; + Ok(Json(json!(imported))) +} diff --git a/apps/admin-api/src/service.rs b/apps/admin-api/src/service.rs index ee3d6ee..ffbe704 100644 --- a/apps/admin-api/src/service.rs +++ b/apps/admin-api/src/service.rs @@ -27,6 +27,7 @@ mod api_keys; mod auth; mod demo; mod import_export; +mod imports; mod observability; mod operation_validation; mod operations; diff --git a/apps/admin-api/src/service/demo.rs b/apps/admin-api/src/service/demo.rs index 6042c88..ad91171 100644 --- a/apps/admin-api/src/service/demo.rs +++ b/apps/admin-api/src/service/demo.rs @@ -392,6 +392,7 @@ fn demo_rest_operation_payload() -> OperationPayload { input_sample: Some(input), output_sample: Some(output), test_input: Some(demo_rest_input_sample()), + import_findings: Vec::new(), }), } } diff --git a/apps/admin-api/src/service/imports.rs b/apps/admin-api/src/service/imports.rs new file mode 100644 index 0000000..f98f48a --- /dev/null +++ b/apps/admin-api/src/service/imports.rs @@ -0,0 +1,285 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crank_core::{ + ExecutionConfig, OperationSecurityLevel, Protocol, ToolQualityFinding, ToolQualitySeverity, + WorkspaceId, +}; +use crank_import::rest::{ + ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate, +}; +use crank_registry::{ + CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus, +}; +use serde_json::json; +use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; +use tracing::{info, instrument}; + +use crate::{ + error::ApiError, + service::{ + AdminService, OpenApiImportCreatePayload, OpenApiImportCreateResponse, + OpenApiImportCreatedOperation, OpenApiImportPreviewPayload, OpenApiImportPreviewResponse, + OpenApiImportSkippedOperation, OperationPayload, new_prefixed_id, + }, +}; + +const IMPORT_JOB_TTL_HOURS: i64 = 24; + +impl AdminService { + #[instrument(skip(self, payload), fields(workspace_id = %workspace_id.as_str()))] + pub async fn preview_openapi_import( + &self, + workspace_id: &WorkspaceId, + payload: OpenApiImportPreviewPayload, + ) -> Result { + self.ensure_workspace_exists(workspace_id).await?; + let _ = self.registry.delete_expired_import_jobs().await; + + let preview = crank_import::rest::preview_document(&payload.document) + .map_err(|error| ApiError::validation(error.to_string()))?; + let now = OffsetDateTime::now_utc(); + let expires_at = now + Duration::hours(IMPORT_JOB_TTL_HOURS); + let job_id = ImportJobId::new(new_prefixed_id("imp")); + let preview_payload = serde_json::to_value(&preview) + .map_err(|error| ApiError::internal(error.to_string()))?; + + self.registry + .create_import_job(CreateImportJobRequest { + id: &job_id, + workspace_id, + kind: ImportJobKind::OpenApi, + source_format: &preview.source.format, + source_version: preview.source.version.as_deref(), + status: ImportJobStatus::Completed, + preview_payload: &preview_payload, + created_at: &now, + expires_at: &expires_at, + }) + .await?; + + Ok(OpenApiImportPreviewResponse { + job_id: job_id.as_str().to_owned(), + expires_at: expires_at + .format(&Rfc3339) + .map_err(|error| ApiError::internal(error.to_string()))?, + preview, + }) + } + + #[instrument(skip(self, payload), fields(workspace_id = %workspace_id.as_str(), job_id = %job_id.as_str()))] + pub async fn create_openapi_import( + &self, + workspace_id: &WorkspaceId, + job_id: &ImportJobId, + payload: OpenApiImportCreatePayload, + ) -> Result { + self.ensure_workspace_exists(workspace_id).await?; + let _ = self.registry.delete_expired_import_jobs().await; + + if !matches!(payload.conflict_mode.as_str(), "skip" | "rename") { + return Err(ApiError::validation( + "unsupported conflict_mode; supported values are skip and rename", + )); + } + + let job = self + .registry + .get_import_job(workspace_id, job_id) + .await? + .ok_or_else(|| { + ApiError::not_found_with_context( + "import job was not found", + json!({ "job_id": job_id.as_str() }), + ) + })?; + if job.expires_at <= OffsetDateTime::now_utc() { + return Err(ApiError::validation("import preview has expired")); + } + if job.kind != ImportJobKind::OpenApi { + return Err(ApiError::validation("import job kind is not openapi")); + } + + let preview: crank_import::rest::ImportPreview = + serde_json::from_value(job.preview_payload.clone()) + .map_err(|error| ApiError::internal(error.to_string()))?; + let selected = payload + .selected_operation_keys + .iter() + .cloned() + .collect::>(); + if selected.is_empty() { + return Err(ApiError::validation( + "selected_operation_keys must contain at least one operation", + )); + } + + let mut candidates = BTreeMap::new(); + for group in &preview.groups { + for operation in &group.operations { + candidates.insert(operation.key.clone(), operation); + } + } + + let mut created = Vec::new(); + let mut skipped = Vec::new(); + let mut findings = Vec::new(); + let mut created_ids = Vec::new(); + + for operation_key in selected { + let Some(candidate) = candidates.get(&operation_key) else { + skipped.push(OpenApiImportSkippedOperation { + operation_key, + name: String::new(), + reason: "operation was not found in import preview".to_owned(), + }); + continue; + }; + let mut draft = + operation_draft_from_candidate(candidate, payload.server_url.as_deref()); + attach_import_findings(&mut draft, candidate); + if let Some(existing_name) = self + .find_operation_by_name(workspace_id, &draft.name) + .await? + .map(|operation| operation.name) + { + if payload.conflict_mode == "skip" { + skipped.push(OpenApiImportSkippedOperation { + operation_key: candidate.key.clone(), + name: draft.name.clone(), + reason: "operation with this name already exists".to_owned(), + }); + findings.push(ImportFinding { + code: "operation_name_conflict".to_owned(), + severity: ImportFindingSeverity::Warning, + message: format!( + "Операция {} уже существует и была пропущена.", + draft.name + ), + operation_key: Some(candidate.key.clone()), + }); + continue; + } + + let renamed = self + .next_available_operation_name(workspace_id, &draft.name) + .await?; + findings.push(ImportFinding { + code: "operation_name_renamed".to_owned(), + severity: ImportFindingSeverity::Info, + message: format!( + "Операция {existing_name} уже существует, новый черновик создан как {renamed}." + ), + operation_key: Some(candidate.key.clone()), + }); + draft.name = renamed; + } + + let payload = OperationPayload { + name: draft.name.clone(), + display_name: draft.display_name.clone(), + category: draft.category, + protocol: Protocol::Rest, + security_level: OperationSecurityLevel::Standard, + target: crank_core::Target::Rest(draft.target), + input_schema: draft.input_schema, + output_schema: draft.output_schema, + input_mapping: draft.input_mapping, + output_mapping: draft.output_mapping, + execution_config: ExecutionConfig { + timeout_ms: 10_000, + retry_policy: None, + response_cache: None, + idempotency: None, + safety: None, + auth_profile_ref: None, + headers: BTreeMap::new(), + }, + tool_description: draft.tool_description, + wizard_state: draft.wizard_state, + }; + let result = self.create_operation(workspace_id, payload).await?; + created_ids.push(result.operation_id.clone()); + created.push(OpenApiImportCreatedOperation { + operation_id: result.operation_id, + name: draft.name, + version: result.version, + }); + } + + let finished_at = OffsetDateTime::now_utc(); + self.registry + .finish_import_job(FinishImportJobRequest { + id: job_id, + status: ImportJobStatus::Completed, + created_operation_ids: &json!(created_ids), + error_text: None, + finished_at: &finished_at, + }) + .await?; + info!( + created = created.len(), + skipped = skipped.len(), + "openapi import created drafts" + ); + + Ok(OpenApiImportCreateResponse { + created, + skipped, + findings, + }) + } + + async fn next_available_operation_name( + &self, + workspace_id: &WorkspaceId, + base_name: &str, + ) -> Result { + for index in 2.. { + let candidate = format!("{base_name}_{index}"); + if self + .find_operation_by_name(workspace_id, &candidate) + .await? + .is_none() + { + return Ok(candidate); + } + } + + unreachable!() + } +} + +fn attach_import_findings( + draft: &mut crank_import::rest::RestImportCandidate, + candidate: &ImportOperationCandidate, +) { + let findings = candidate + .findings + .iter() + .map(tool_quality_finding_from_import) + .collect::>(); + if findings.is_empty() { + return; + } + + let mut wizard_state = draft.wizard_state.take().unwrap_or_default(); + wizard_state.import_findings = findings; + draft.wizard_state = Some(wizard_state); +} + +fn tool_quality_finding_from_import(finding: &ImportFinding) -> ToolQualityFinding { + ToolQualityFinding { + severity: match finding.severity { + ImportFindingSeverity::Info => ToolQualitySeverity::Info, + ImportFindingSeverity::Warning => ToolQualitySeverity::Warning, + ImportFindingSeverity::Error => ToolQualitySeverity::Error, + }, + code: format!("openapi_import.{}", finding.code), + message: finding.message.clone(), + suggested_action: Some( + "Откройте черновик в мастере и уточните описание, схемы или маппинг перед публикацией." + .to_owned(), + ), + field_path: finding.operation_key.clone(), + } +} diff --git a/apps/admin-api/tests/integration.rs b/apps/admin-api/tests/integration.rs index b65b800..f58bd88 100644 --- a/apps/admin-api/tests/integration.rs +++ b/apps/admin-api/tests/integration.rs @@ -2,6 +2,7 @@ mod integration { mod auth_rate_limit; mod common; mod community_access_usage; + mod openapi_import; mod operations_agents; mod secrets_import_auth; } diff --git a/apps/admin-api/tests/integration/openapi_import.rs b/apps/admin-api/tests/integration/openapi_import.rs new file mode 100644 index 0000000..55e146e --- /dev/null +++ b/apps/admin-api/tests/integration/openapi_import.rs @@ -0,0 +1,149 @@ +use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload}; +use crank_core::WorkspaceId; +use serial_test::serial; + +use super::common::{ + test_auth_settings, test_registry, test_secret_crypto, test_service, test_storage_root, +}; + +const OPENAPI3: &str = r#" +openapi: 3.0.3 +info: + title: Frankfurter API +servers: + - url: https://api.frankfurter.dev +paths: + /v2/latest: + get: + operationId: latestRates + summary: Получить последние курсы валют + description: Курсы. + tags: [currency] + parameters: + - name: base + in: query + required: true + schema: { type: string } + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + base: { type: string } + rates: + type: object +"#; + +#[tokio::test] +#[serial] +async fn previews_openapi_and_creates_draft_operations() { + let registry = test_registry().await; + let service = test_service( + registry, + test_storage_root("openapi_import"), + test_auth_settings(), + test_secret_crypto(), + ); + let workspace_id = WorkspaceId::new("ws_default"); + + let preview = service + .preview_openapi_import( + &workspace_id, + OpenApiImportPreviewPayload { + document: OPENAPI3.to_owned(), + }, + ) + .await + .unwrap(); + + assert_eq!(preview.preview.groups.len(), 1); + assert_eq!( + preview.preview.groups[0].operations[0].suggested_name, + "latest_rates" + ); + + let created = service + .create_openapi_import( + &workspace_id, + &preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /v2/latest".to_owned()], + server_url: Some("https://api.frankfurter.dev".to_owned()), + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + + assert_eq!(created.created.len(), 1); + assert_eq!(created.created[0].name, "latest_rates"); + assert!(created.skipped.is_empty()); + + let operations = service.list_operations(&workspace_id).await.unwrap(); + assert!( + operations + .iter() + .any(|operation| operation.name == "latest_rates") + ); + let imported = operations + .iter() + .find(|operation| operation.name == "latest_rates") + .unwrap(); + let version = service + .get_operation_version( + &workspace_id, + &imported.id.as_str().into(), + imported.current_draft_version, + ) + .await + .unwrap(); + let import_findings = &version + .snapshot + .wizard_state + .as_ref() + .unwrap() + .import_findings; + assert!( + import_findings + .iter() + .any(|finding| finding.code == "openapi_import.weak_tool_description") + ); + + let skipped = service + .create_openapi_import( + &workspace_id, + &preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /v2/latest".to_owned()], + server_url: Some("https://api.frankfurter.dev".to_owned()), + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + + assert!(skipped.created.is_empty()); + assert_eq!(skipped.skipped.len(), 1); + assert_eq!(skipped.skipped[0].name, "latest_rates"); + assert_eq!(skipped.findings[0].code, "operation_name_conflict"); + + let renamed = service + .create_openapi_import( + &workspace_id, + &preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /v2/latest".to_owned()], + server_url: Some("https://api.frankfurter.dev".to_owned()), + conflict_mode: "rename".to_owned(), + }, + ) + .await + .unwrap(); + + assert_eq!(renamed.created.len(), 1); + assert_eq!(renamed.created[0].name, "latest_rates_2"); + assert_eq!(renamed.findings[0].code, "operation_name_renamed"); +} diff --git a/crates/crank-core/src/operation.rs b/crates/crank-core/src/operation.rs index cc152f7..8af7f07 100644 --- a/crates/crank-core/src/operation.rs +++ b/crates/crank-core/src/operation.rs @@ -8,6 +8,7 @@ use crate::{ edition::OperationSecurityLevel, ids::{AuthProfileId, OperationId, SampleId}, protocol::{ExportMode, HttpMethod, Protocol}, + tool_quality::ToolQualityFinding, }; fn default_operation_category() -> String { @@ -180,6 +181,8 @@ pub struct WizardState { pub output_sample: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub test_input: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub import_findings: Vec, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/crates/crank-import/Cargo.toml b/crates/crank-import/Cargo.toml new file mode 100644 index 0000000..5b438ad --- /dev/null +++ b/crates/crank-import/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "crank-import" +edition.workspace = true +license.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +crank-core = { path = "../crank-core" } +crank-mapping = { path = "../crank-mapping" } +crank-schema = { path = "../crank-schema" } +serde.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true +thiserror.workspace = true + diff --git a/crates/crank-import/src/lib.rs b/crates/crank-import/src/lib.rs new file mode 100644 index 0000000..2a7079e --- /dev/null +++ b/crates/crank-import/src/lib.rs @@ -0,0 +1 @@ +pub mod rest; diff --git a/crates/crank-import/src/rest/mapping.rs b/crates/crank-import/src/rest/mapping.rs new file mode 100644 index 0000000..12d4b38 --- /dev/null +++ b/crates/crank-import/src/rest/mapping.rs @@ -0,0 +1,79 @@ +use crank_mapping::{MappingRule, MappingSet}; +use crank_schema::{Schema, SchemaKind}; + +use crate::rest::model::{RestImportParameter, RestParameterLocation}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BodyFieldMapping { + pub input_name: String, + pub body_path: String, + pub required: bool, +} + +pub fn input_mapping( + parameters: &[RestImportParameter], + body_fields: &[BodyFieldMapping], +) -> MappingSet { + let mut rules = Vec::new(); + for parameter in parameters { + let target_root = match parameter.location { + RestParameterLocation::Path => "$.request.path", + RestParameterLocation::Query => "$.request.query", + RestParameterLocation::Header => "$.request.headers", + }; + rules.push(MappingRule { + source: format!("$.mcp.{}", parameter.name), + target: format!("{target_root}.{}", parameter.name), + required: parameter.required, + default_value: None, + transform: None, + condition: None, + notes: None, + }); + } + for field in body_fields { + let target = if field.body_path.is_empty() { + "$.request.body".to_owned() + } else { + format!("$.request.body.{}", field.body_path) + }; + rules.push(MappingRule { + source: format!("$.mcp.{}", field.input_name), + target, + required: field.required, + default_value: None, + transform: None, + condition: None, + notes: None, + }); + } + MappingSet { rules } +} + +pub fn output_mapping(output_schema: &Schema) -> MappingSet { + let mut rules = Vec::new(); + if output_schema.kind == SchemaKind::Object && !output_schema.fields.is_empty() { + for (name, field) in &output_schema.fields { + rules.push(MappingRule { + source: format!("$.response.body.{name}"), + target: format!("$.output.{name}"), + required: field.required, + default_value: None, + transform: None, + condition: None, + notes: None, + }); + } + } else { + rules.push(MappingRule { + source: "$.response.body".to_owned(), + target: "$.output".to_owned(), + required: true, + default_value: None, + transform: None, + condition: None, + notes: None, + }); + } + MappingSet { rules } +} diff --git a/crates/crank-import/src/rest/mod.rs b/crates/crank-import/src/rest/mod.rs new file mode 100644 index 0000000..04a510d --- /dev/null +++ b/crates/crank-import/src/rest/mod.rs @@ -0,0 +1,17 @@ +mod mapping; +pub mod model; +mod naming; +mod normalize; +mod openapi3; +mod payload; +mod recommendations; +mod schema; +mod swagger2; + +pub use model::{ + ImportFinding, ImportFindingSeverity, ImportGroupPreview, ImportOperationCandidate, + ImportPreview, ImportSourcePreview, RestImportCandidate, RestImportDocument, + RestImportOperation, RestImportParameter, RestParameterLocation, +}; +pub use normalize::preview_document; +pub use payload::operation_draft_from_candidate; diff --git a/crates/crank-import/src/rest/model.rs b/crates/crank-import/src/rest/model.rs new file mode 100644 index 0000000..a46a6e7 --- /dev/null +++ b/crates/crank-import/src/rest/model.rs @@ -0,0 +1,123 @@ +use crank_core::{HttpMethod, RestTarget, ToolDescription, WizardState}; +use crank_mapping::MappingSet; +use crank_schema::Schema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportFindingSeverity { + Info, + Warning, + Error, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportFinding { + pub code: String, + pub severity: ImportFindingSeverity, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operation_key: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImportSourcePreview { + pub format: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + pub title: String, + #[serde(default)] + pub servers: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImportPreview { + pub source: ImportSourcePreview, + pub groups: Vec, + #[serde(default)] + pub findings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImportGroupPreview { + pub key: String, + pub title: String, + pub operations: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImportOperationCandidate { + pub key: String, + pub method: HttpMethod, + pub path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub operation_id: Option, + pub suggested_name: String, + pub suggested_display_name: String, + pub description: String, + pub category: String, + pub input_fields: usize, + pub output_fields: usize, + #[serde(default)] + pub server_urls: Vec, + #[serde(default)] + pub findings: Vec, + pub draft: RestImportCandidate, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RestImportCandidate { + pub name: String, + pub display_name: String, + pub category: String, + pub target: RestTarget, + pub input_schema: Schema, + pub output_schema: Schema, + pub input_mapping: MappingSet, + pub output_mapping: MappingSet, + pub tool_description: ToolDescription, + pub wizard_state: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RestImportDocument { + pub format: String, + pub version: Option, + pub title: String, + pub servers: Vec, + pub operations: Vec, + pub findings: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RestImportOperation { + pub key: String, + pub method: HttpMethod, + pub path: String, + pub operation_id: Option, + pub summary: Option, + pub description: Option, + pub tags: Vec, + pub parameters: Vec, + pub request_body_schema: Option, + pub response_schema: Option, + pub servers: Vec, + pub findings: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RestImportParameter { + pub name: String, + pub location: RestParameterLocation, + pub required: bool, + pub description: Option, + pub schema: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RestParameterLocation { + Path, + Query, + Header, +} diff --git a/crates/crank-import/src/rest/naming.rs b/crates/crank-import/src/rest/naming.rs new file mode 100644 index 0000000..77cec98 --- /dev/null +++ b/crates/crank-import/src/rest/naming.rs @@ -0,0 +1,62 @@ +use std::collections::BTreeSet; + +pub fn snake_name(seed: &str) -> String { + let mut out = String::new(); + let mut prev_underscore = false; + + for ch in seed.chars() { + if ch.is_ascii_alphanumeric() { + if ch.is_ascii_uppercase() && !out.is_empty() && !prev_underscore { + out.push('_'); + } + out.push(ch.to_ascii_lowercase()); + prev_underscore = false; + } else if !out.is_empty() && !prev_underscore { + out.push('_'); + prev_underscore = true; + } + } + + let trimmed = out.trim_matches('_').to_owned(); + if trimmed.is_empty() { + "imported_operation".to_owned() + } else if trimmed.chars().next().is_some_and(|ch| ch.is_ascii_digit()) { + format!("op_{trimmed}") + } else { + trimmed + } +} + +pub fn unique_name(base: &str, used: &mut BTreeSet) -> String { + let clean = snake_name(base); + if used.insert(clean.clone()) { + return clean; + } + + for index in 2.. { + let candidate = format!("{clean}_{index}"); + if used.insert(candidate.clone()) { + return candidate; + } + } + + unreachable!() +} + +pub fn title_from_name(name: &str) -> String { + name.split('_') + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } + }) + .collect::>() + .join(" ") +} + +pub fn fallback_operation_seed(method: &str, path: &str) -> String { + format!("{method}_{path}") +} diff --git a/crates/crank-import/src/rest/normalize.rs b/crates/crank-import/src/rest/normalize.rs new file mode 100644 index 0000000..45ca924 --- /dev/null +++ b/crates/crank-import/src/rest/normalize.rs @@ -0,0 +1,111 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; +use thiserror::Error; + +use crate::rest::{ + model::{ImportGroupPreview, ImportPreview, RestImportDocument}, + openapi3, + payload::candidate_from_operation, + swagger2, +}; + +#[derive(Debug, Error)] +pub enum ImportParseError { + #[error("document is not valid YAML or JSON: {0}")] + InvalidDocument(String), + #[error("unsupported OpenAPI document")] + UnsupportedDocument, +} + +pub fn preview_document(document: &str) -> Result { + let yaml: serde_yaml::Value = serde_yaml::from_str(document) + .map_err(|error| ImportParseError::InvalidDocument(error.to_string()))?; + let root = serde_json::to_value(yaml) + .map_err(|error| ImportParseError::InvalidDocument(error.to_string()))?; + + let normalized = if root.get("openapi").is_some() { + openapi3::parse_document(&root)? + } else if root.get("swagger").and_then(Value::as_str) == Some("2.0") { + swagger2::parse_document(&root)? + } else { + return Err(ImportParseError::UnsupportedDocument); + }; + + Ok(preview_from_document(normalized)) +} + +fn preview_from_document(document: RestImportDocument) -> ImportPreview { + let mut groups: BTreeMap = BTreeMap::new(); + let mut used_names = BTreeSet::new(); + + for operation in &document.operations { + let candidate = candidate_from_operation(operation, &document.servers, &mut used_names); + let group_title = operation + .tags + .first() + .cloned() + .unwrap_or_else(|| "Без группы".to_owned()); + let group_key = crate::rest::naming::snake_name(&group_title); + groups + .entry(group_key.clone()) + .or_insert_with(|| ImportGroupPreview { + key: group_key, + title: group_title, + operations: Vec::new(), + }) + .operations + .push(candidate); + } + + ImportPreview { + source: crate::rest::model::ImportSourcePreview { + format: document.format, + version: document.version, + title: document.title, + servers: document.servers, + }, + groups: groups.into_values().collect(), + findings: document.findings, + } +} + +pub(crate) fn resolve_local_ref(root: &Value, value: &Value, depth: usize) -> Value { + if depth > 12 { + return value.clone(); + } + if let Some(reference) = value.get("$ref").and_then(Value::as_str) { + if let Some(resolved) = pointer(root, reference) { + return resolve_local_ref(root, resolved, depth + 1); + } + return value.clone(); + } + match value { + Value::Object(map) => { + let mut out = serde_json::Map::new(); + for (key, item) in map { + out.insert(key.clone(), resolve_local_ref(root, item, depth + 1)); + } + Value::Object(out) + } + Value::Array(items) => Value::Array( + items + .iter() + .map(|item| resolve_local_ref(root, item, depth + 1)) + .collect(), + ), + _ => value.clone(), + } +} + +fn pointer<'a>(root: &'a Value, reference: &str) -> Option<&'a Value> { + if !reference.starts_with("#/") { + return None; + } + let mut current = root; + for part in reference.trim_start_matches("#/").split('/') { + let part = part.replace("~1", "/").replace("~0", "~"); + current = current.get(&part)?; + } + Some(current) +} diff --git a/crates/crank-import/src/rest/openapi3.rs b/crates/crank-import/src/rest/openapi3.rs new file mode 100644 index 0000000..de3395a --- /dev/null +++ b/crates/crank-import/src/rest/openapi3.rs @@ -0,0 +1,228 @@ +use crank_core::HttpMethod; +use serde_json::Value; + +use crate::rest::{ + model::{ + ImportFinding, RestImportDocument, RestImportOperation, RestImportParameter, + RestParameterLocation, + }, + normalize::{ImportParseError, resolve_local_ref}, + recommendations::document_finding, +}; + +pub fn parse_document(root: &Value) -> Result { + let version = root + .get("openapi") + .and_then(Value::as_str) + .map(ToOwned::to_owned); + let title = root + .pointer("/info/title") + .and_then(Value::as_str) + .unwrap_or("Imported API") + .to_owned(); + let servers = root + .get("servers") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(|item| item.get("url").and_then(Value::as_str)) + .map(|url| url.trim_end_matches('/').to_owned()) + .collect::>() + }) + .unwrap_or_default(); + let mut findings = Vec::new(); + if servers.is_empty() { + findings.push(document_finding( + "missing_servers", + "В документе не указаны servers, base URL нужно будет выбрать вручную.", + )); + } else if servers.len() > 1 { + findings.push(document_finding( + "multiple_servers", + "В документе несколько servers, при импорте нужно выбрать нужный base URL.", + )); + } + + let mut operations = Vec::new(); + let paths = root + .get("paths") + .and_then(Value::as_object) + .ok_or(ImportParseError::UnsupportedDocument)?; + + for (path, path_item) in paths { + let path_parameters = parameters(root, path_item.get("parameters")); + for method_name in ["get", "post", "put", "patch", "delete"] { + let Some(operation_value) = path_item.get(method_name) else { + continue; + }; + let Some(method) = method_from_lower(method_name) else { + continue; + }; + + let mut operation_parameters = path_parameters.clone(); + operation_parameters.extend(parameters(root, operation_value.get("parameters"))); + let operation_servers = operation_value + .get("servers") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(|item| item.get("url").and_then(Value::as_str)) + .map(|url| url.trim_end_matches('/').to_owned()) + .collect::>() + }) + .unwrap_or_default(); + + operations.push(RestImportOperation { + key: format!("{} {}", method_name.to_uppercase(), path), + method, + path: path.clone(), + operation_id: operation_value + .get("operationId") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + summary: operation_value + .get("summary") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + description: operation_value + .get("description") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + tags: operation_value + .get("tags") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(Value::as_str) + .map(ToOwned::to_owned) + .collect() + }) + .unwrap_or_default(), + parameters: operation_parameters, + request_body_schema: request_body_schema(root, operation_value), + response_schema: response_schema(root, operation_value), + servers: operation_servers, + findings: operation_findings(operation_value), + }); + } + } + + Ok(RestImportDocument { + format: "openapi".to_owned(), + version, + title, + servers, + operations, + findings, + }) +} + +fn parameters(root: &Value, value: Option<&Value>) -> Vec { + value + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(|item| { + let item = resolve_local_ref(root, item, 0); + let location = match item.get("in").and_then(Value::as_str)? { + "path" => RestParameterLocation::Path, + "query" => RestParameterLocation::Query, + "header" => RestParameterLocation::Header, + _ => return None, + }; + Some(RestImportParameter { + name: item.get("name").and_then(Value::as_str)?.to_owned(), + location, + required: item + .get("required") + .and_then(Value::as_bool) + .unwrap_or(false) + || location == RestParameterLocation::Path, + description: item + .get("description") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + schema: item + .get("schema") + .map(|schema| resolve_local_ref(root, schema, 0)), + }) + }) + .collect() + }) + .unwrap_or_default() +} + +fn request_body_schema(root: &Value, operation: &Value) -> Option { + let body = resolve_local_ref(root, operation.get("requestBody")?, 0); + let content = body.get("content")?.as_object()?; + for content_type in ["application/json", "application/*+json"] { + if let Some(schema) = content + .get(content_type) + .and_then(|media| media.get("schema")) + { + return Some(resolve_local_ref(root, schema, 0)); + } + } + content + .iter() + .find(|(content_type, _)| content_type.contains("json")) + .and_then(|(_, media)| media.get("schema")) + .map(|schema| resolve_local_ref(root, schema, 0)) +} + +fn response_schema(root: &Value, operation: &Value) -> Option { + let responses = operation.get("responses")?.as_object()?; + for code in ["200", "201", "202", "default"] { + let Some(response) = responses.get(code) else { + continue; + }; + let response = resolve_local_ref(root, response, 0); + let Some(content) = response.get("content").and_then(Value::as_object) else { + continue; + }; + for content_type in ["application/json", "application/*+json"] { + if let Some(schema) = content + .get(content_type) + .and_then(|media| media.get("schema")) + { + return Some(resolve_local_ref(root, schema, 0)); + } + } + if let Some(schema) = content + .iter() + .find(|(content_type, _)| content_type.contains("json")) + .and_then(|(_, media)| media.get("schema")) + { + return Some(resolve_local_ref(root, schema, 0)); + } + } + None +} + +fn operation_findings(operation: &Value) -> Vec { + let mut findings = Vec::new(); + if operation.get("requestBody").is_some() + && request_body_schema(&Value::Null, operation).is_none() + { + findings.push(document_finding( + "unsupported_request_body", + "У метода есть requestBody, но JSON schema не найдена.", + )); + } + findings +} + +fn method_from_lower(value: &str) -> Option { + match value { + "get" => Some(HttpMethod::Get), + "post" => Some(HttpMethod::Post), + "put" => Some(HttpMethod::Put), + "patch" => Some(HttpMethod::Patch), + "delete" => Some(HttpMethod::Delete), + _ => None, + } +} diff --git a/crates/crank-import/src/rest/payload.rs b/crates/crank-import/src/rest/payload.rs new file mode 100644 index 0000000..1bdca5e --- /dev/null +++ b/crates/crank-import/src/rest/payload.rs @@ -0,0 +1,242 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crank_core::{ + ExecutionConfig, HttpMethod, OperationSafetyClass, OperationSafetyPolicy, Protocol, RestTarget, + ToolDescription, ToolExample, WizardState, +}; +use serde_json::json; + +use crate::rest::{ + mapping::{BodyFieldMapping, input_mapping, output_mapping}, + model::{ + ImportOperationCandidate, RestImportCandidate, RestImportOperation, RestParameterLocation, + }, + naming::{fallback_operation_seed, title_from_name, unique_name}, + recommendations::{candidate_recommendations, operation_recommendations}, + schema::{any_object, field_count, object_with_fields, schema_from_openapi}, +}; + +pub fn candidate_from_operation( + operation: &RestImportOperation, + document_servers: &[String], + used_names: &mut BTreeSet, +) -> ImportOperationCandidate { + let method_name = method_as_str(operation.method); + let seed = operation + .operation_id + .as_deref() + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| fallback_operation_seed(method_name, &operation.path)); + let name = unique_name(&seed, used_names); + let display_name = operation + .summary + .as_deref() + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| title_from_name(&name)); + let description = operation + .description + .as_deref() + .or(operation.summary.as_deref()) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| format!("Выполняет {method_name} {}", operation.path)); + let category = operation + .tags + .first() + .cloned() + .unwrap_or_else(|| "imported".to_owned()); + let server_urls = if operation.servers.is_empty() { + document_servers.to_vec() + } else { + operation.servers.clone() + }; + let base_url = server_urls.first().cloned().unwrap_or_default(); + let mut input_fields = BTreeMap::new(); + let mut used_input_field_names = BTreeSet::new(); + for parameter in &operation.parameters { + used_input_field_names.insert(parameter.name.clone()); + input_fields.insert( + parameter.name.clone(), + schema_from_openapi( + parameter.schema.as_ref(), + parameter.required || parameter.location == RestParameterLocation::Path, + parameter.description.clone(), + ), + ); + } + let body_fields = body_input_fields( + operation.request_body_schema.as_ref(), + &mut used_input_field_names, + &mut input_fields, + ); + let input_schema = object_with_fields( + Some("Входные параметры MCP-инструмента".to_owned()), + input_fields, + ); + let output_schema = operation + .response_schema + .as_ref() + .map(|schema| schema_from_openapi(Some(schema), true, Some("Ответ API".to_owned()))) + .unwrap_or_else(|| any_object(Some("Ответ API".to_owned()))); + let input_mapping = input_mapping(&operation.parameters, &body_fields); + let output_mapping = output_mapping(&output_schema); + let mut findings = operation_recommendations(operation); + findings.extend(candidate_recommendations( + operation, + &name, + &description, + &input_schema, + &output_schema, + )); + + let draft = RestImportCandidate { + name: name.clone(), + display_name: display_name.clone(), + category: category.clone(), + target: RestTarget { + base_url, + method: operation.method, + path_template: operation.path.clone(), + static_headers: BTreeMap::new(), + }, + input_schema: input_schema.clone(), + output_schema: output_schema.clone(), + input_mapping, + output_mapping, + tool_description: ToolDescription { + title: display_name.clone(), + description: description.clone(), + tags: operation.tags.clone(), + examples: vec![ToolExample { input: json!({}) }], + }, + wizard_state: Some(WizardState { + input_sample: None, + output_sample: None, + test_input: None, + import_findings: Vec::new(), + }), + }; + + ImportOperationCandidate { + key: operation.key.clone(), + method: operation.method, + path: operation.path.clone(), + operation_id: operation.operation_id.clone(), + suggested_name: name, + suggested_display_name: display_name, + description, + category, + input_fields: field_count(&input_schema), + output_fields: field_count(&output_schema), + server_urls, + findings, + draft, + } +} + +fn body_input_fields( + request_body_schema: Option<&serde_json::Value>, + used_names: &mut BTreeSet, + input_fields: &mut BTreeMap, +) -> Vec { + let Some(request_body_schema) = request_body_schema else { + return Vec::new(); + }; + let body_schema = schema_from_openapi( + Some(request_body_schema), + true, + Some("Тело запроса".to_owned()), + ); + if body_schema.kind != crank_schema::SchemaKind::Object || body_schema.fields.is_empty() { + let input_name = unique_body_input_name("body", used_names); + input_fields.insert(input_name.clone(), body_schema); + return vec![BodyFieldMapping { + input_name, + body_path: String::new(), + required: true, + }]; + } + + let mut mappings = Vec::new(); + for (field_name, field_schema) in body_schema.fields { + let input_name = unique_body_input_name(&field_name, used_names); + mappings.push(BodyFieldMapping { + input_name: input_name.clone(), + body_path: field_name, + required: field_schema.required, + }); + input_fields.insert(input_name, field_schema); + } + mappings +} + +fn unique_body_input_name(name: &str, used_names: &mut BTreeSet) -> String { + if used_names.insert(name.to_owned()) { + return name.to_owned(); + } + let prefixed = format!("body_{name}"); + if used_names.insert(prefixed.clone()) { + return prefixed; + } + for index in 2.. { + let candidate = format!("body_{name}_{index}"); + if used_names.insert(candidate.clone()) { + return candidate; + } + } + unreachable!() +} + +pub fn operation_draft_from_candidate( + candidate: &ImportOperationCandidate, + server_url: Option<&str>, +) -> RestImportCandidate { + let mut draft = candidate.draft.clone(); + if let Some(server_url) = server_url.filter(|value| !value.trim().is_empty()) { + draft.target.base_url = server_url.trim().trim_end_matches('/').to_owned(); + } + draft +} + +fn method_as_str(method: HttpMethod) -> &'static str { + match method { + HttpMethod::Get => "GET", + HttpMethod::Post => "POST", + HttpMethod::Put => "PUT", + HttpMethod::Patch => "PATCH", + HttpMethod::Delete => "DELETE", + } +} + +#[allow(dead_code)] +fn safety_for_method(method: HttpMethod) -> OperationSafetyPolicy { + let class = match method { + HttpMethod::Get => OperationSafetyClass::Read, + HttpMethod::Post | HttpMethod::Put | HttpMethod::Patch => OperationSafetyClass::Write, + HttpMethod::Delete => OperationSafetyClass::Destructive, + }; + OperationSafetyPolicy { + class, + confirmation: None, + } +} + +#[allow(dead_code)] +fn default_execution_config() -> ExecutionConfig { + ExecutionConfig { + timeout_ms: 10_000, + retry_policy: None, + response_cache: None, + idempotency: None, + safety: None, + auth_profile_ref: None, + headers: BTreeMap::new(), + } +} + +#[allow(dead_code)] +fn _protocol() -> Protocol { + Protocol::Rest +} diff --git a/crates/crank-import/src/rest/recommendations.rs b/crates/crank-import/src/rest/recommendations.rs new file mode 100644 index 0000000..6460319 --- /dev/null +++ b/crates/crank-import/src/rest/recommendations.rs @@ -0,0 +1,213 @@ +use crank_schema::{Schema, SchemaKind}; + +use crate::rest::model::{ImportFinding, ImportFindingSeverity, RestImportOperation}; + +pub fn document_finding(code: &str, message: impl Into) -> ImportFinding { + ImportFinding { + code: code.to_owned(), + severity: ImportFindingSeverity::Warning, + message: message.into(), + operation_key: None, + } +} + +pub fn operation_finding( + operation_key: &str, + code: &str, + message: impl Into, +) -> ImportFinding { + ImportFinding { + code: code.to_owned(), + severity: ImportFindingSeverity::Warning, + message: message.into(), + operation_key: Some(operation_key.to_owned()), + } +} + +pub fn operation_info( + operation_key: &str, + code: &str, + message: impl Into, +) -> ImportFinding { + ImportFinding { + code: code.to_owned(), + severity: ImportFindingSeverity::Info, + message: message.into(), + operation_key: Some(operation_key.to_owned()), + } +} + +pub fn operation_recommendations(operation: &RestImportOperation) -> Vec { + let mut findings = operation.findings.clone(); + if operation + .operation_id + .as_deref() + .unwrap_or_default() + .trim() + .is_empty() + { + findings.push(operation_finding( + &operation.key, + "missing_operation_id", + "У метода нет operationId, имя инструмента будет сгенерировано из метода и пути.", + )); + } + if operation + .summary + .as_deref() + .unwrap_or_default() + .trim() + .is_empty() + { + findings.push(operation_finding( + &operation.key, + "missing_summary", + "У метода нет summary, отображаемое имя будет сгенерировано автоматически.", + )); + } + if operation + .description + .as_deref() + .unwrap_or_default() + .trim() + .is_empty() + { + findings.push(operation_finding( + &operation.key, + "missing_description", + "У метода нет description. Перед публикацией лучше описать, когда агенту стоит вызывать этот инструмент.", + )); + } + if operation.response_schema.is_none() { + findings.push(operation_finding( + &operation.key, + "missing_response_schema", + "У метода не найдена схема успешного ответа, результат будет описан как общий объект.", + )); + } + if operation.parameters.len() > 12 { + findings.push(operation_finding( + &operation.key, + "too_many_inputs", + "У метода много входных параметров. Проверьте, не стоит ли разделить инструмент на более узкие сценарии.", + )); + } + let undocumented_parameters = operation + .parameters + .iter() + .filter(|parameter| { + parameter + .description + .as_deref() + .unwrap_or_default() + .trim() + .is_empty() + }) + .count(); + if undocumented_parameters > 0 { + findings.push(operation_finding( + &operation.key, + "parameter_descriptions_missing", + format!( + "У {undocumented_parameters} входных параметров нет описания. Модели будет сложнее понять, какие значения туда передавать." + ), + )); + } + findings +} + +pub fn candidate_recommendations( + operation: &RestImportOperation, + suggested_name: &str, + description: &str, + input_schema: &Schema, + output_schema: &Schema, +) -> Vec { + let mut findings = Vec::new(); + let input_fields = field_count(input_schema); + let output_fields = field_count(output_schema); + + if input_fields == 0 { + findings.push(operation_info( + &operation.key, + "no_input_fields", + "У инструмента нет входных параметров. Это нормально для справочных методов, но проверьте, что агенту не нужно передавать фильтры.", + )); + } + if output_fields == 0 { + findings.push(operation_finding( + &operation.key, + "empty_output_schema", + "В ответе не найдено отдельных полей. Перед публикацией проверьте схему ответа и маппинг результата.", + )); + } + if output_fields > 12 { + findings.push(operation_finding( + &operation.key, + "too_many_output_fields", + format!( + "В ответ инструмента попадает много полей: {output_fields}. Лучше вернуть только данные, которые нужны агенту для ответа пользователю." + ), + )); + } + if is_weak_description(operation, description) { + findings.push(operation_finding( + &operation.key, + "weak_tool_description", + "Описание инструмента слишком короткое или техническое. Перед публикацией добавьте, когда агент должен вызывать инструмент и что будет в успешном ответе.", + )); + } + if weak_name(suggested_name) { + findings.push(operation_finding( + &operation.key, + "weak_tool_name", + format!( + "Имя инструмента `{suggested_name}` выглядит слишком общим. Лучше использовать имя с конкретным действием и объектом." + ), + )); + } + + findings +} + +fn is_weak_description(operation: &RestImportOperation, description: &str) -> bool { + let normalized = description.trim(); + if normalized.chars().count() < 40 { + return true; + } + if normalized.starts_with("Выполняет ") { + return true; + } + operation + .summary + .as_deref() + .map(|summary| summary.trim() == normalized) + .unwrap_or(false) +} + +fn weak_name(name: &str) -> bool { + let parts = name + .split('_') + .filter(|part| !part.is_empty()) + .collect::>(); + if parts.len() < 2 { + return true; + } + let weak_terms = [ + "api", + "data", + "item", + "items", + "object", + "operation", + "request", + ]; + parts.iter().any(|part| weak_terms.contains(part)) +} + +fn field_count(schema: &Schema) -> usize { + match schema.kind { + SchemaKind::Object => schema.fields.len(), + _ => 0, + } +} diff --git a/crates/crank-import/src/rest/schema.rs b/crates/crank-import/src/rest/schema.rs new file mode 100644 index 0000000..42df4bb --- /dev/null +++ b/crates/crank-import/src/rest/schema.rs @@ -0,0 +1,203 @@ +use std::collections::BTreeMap; + +use crank_schema::{Schema, SchemaKind}; +use serde_json::Value; + +pub fn any_object(description: Option) -> Schema { + Schema { + kind: SchemaKind::Object, + description, + required: true, + nullable: false, + default_value: None, + fields: BTreeMap::new(), + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} + +pub fn field_count(schema: &Schema) -> usize { + match schema.kind { + SchemaKind::Object => schema.fields.len(), + _ => 0, + } +} + +pub fn schema_from_openapi( + value: Option<&Value>, + required: bool, + description: Option, +) -> Schema { + let Some(value) = value else { + return primitive(SchemaKind::String, required, description); + }; + let resolved = collapse_composition(value); + + if let Some(values) = resolved.get("enum").and_then(Value::as_array) { + let enum_values = values + .iter() + .filter_map(Value::as_str) + .map(ToOwned::to_owned) + .collect::>(); + if !enum_values.is_empty() { + return Schema { + kind: SchemaKind::Enum, + description: description.or_else(|| text(&resolved, "description")), + required, + nullable: nullable(&resolved), + default_value: resolved.get("default").cloned(), + fields: BTreeMap::new(), + items: None, + enum_values, + variants: Vec::new(), + }; + } + } + + match type_name(&resolved).unwrap_or("object") { + "object" => object_schema(&resolved, required, description), + "array" => Schema { + kind: SchemaKind::Array, + description: description.or_else(|| text(&resolved, "description")), + required, + nullable: nullable(&resolved), + default_value: resolved.get("default").cloned(), + fields: BTreeMap::new(), + items: Some(Box::new(schema_from_openapi( + resolved.get("items"), + true, + None, + ))), + enum_values: Vec::new(), + variants: Vec::new(), + }, + "integer" => primitive( + SchemaKind::Integer, + required, + description.or_else(|| text(&resolved, "description")), + ), + "number" => primitive( + SchemaKind::Number, + required, + description.or_else(|| text(&resolved, "description")), + ), + "boolean" => primitive( + SchemaKind::Boolean, + required, + description.or_else(|| text(&resolved, "description")), + ), + "null" => primitive( + SchemaKind::Null, + required, + description.or_else(|| text(&resolved, "description")), + ), + _ => primitive( + SchemaKind::String, + required, + description.or_else(|| text(&resolved, "description")), + ), + } +} + +pub fn object_with_fields(description: Option, fields: BTreeMap) -> Schema { + Schema { + kind: SchemaKind::Object, + description, + required: true, + nullable: false, + default_value: None, + fields, + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} + +fn object_schema(value: &Value, required: bool, description: Option) -> Schema { + let required_fields = value + .get("required") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(Value::as_str) + .collect::>() + }) + .unwrap_or_default(); + let mut fields = BTreeMap::new(); + + if let Some(properties) = value.get("properties").and_then(Value::as_object) { + for (name, schema) in properties { + fields.insert( + name.clone(), + schema_from_openapi( + Some(schema), + required_fields.contains(name.as_str()), + text(schema, "description"), + ), + ); + } + } + + Schema { + kind: SchemaKind::Object, + description: description.or_else(|| text(value, "description")), + required, + nullable: nullable(value), + default_value: value.get("default").cloned(), + fields, + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} + +fn primitive(kind: SchemaKind, required: bool, description: Option) -> Schema { + Schema { + kind, + description, + required, + nullable: false, + default_value: None, + fields: BTreeMap::new(), + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} + +fn type_name(value: &Value) -> Option<&str> { + match value.get("type") { + Some(Value::String(value)) => Some(value.as_str()), + Some(Value::Array(values)) => values.iter().find_map(Value::as_str), + _ if value.get("properties").is_some() => Some("object"), + _ if value.get("items").is_some() => Some("array"), + _ => None, + } +} + +fn nullable(value: &Value) -> bool { + value + .get("nullable") + .and_then(Value::as_bool) + .unwrap_or(false) +} + +fn text(value: &Value, key: &str) -> Option { + value + .get(key) + .and_then(Value::as_str) + .map(ToOwned::to_owned) +} + +fn collapse_composition(value: &Value) -> Value { + for key in ["allOf", "oneOf", "anyOf"] { + if let Some(items) = value.get(key).and_then(Value::as_array) { + if let Some(first) = items.first() { + return first.clone(); + } + } + } + value.clone() +} diff --git a/crates/crank-import/src/rest/swagger2.rs b/crates/crank-import/src/rest/swagger2.rs new file mode 100644 index 0000000..61bb401 --- /dev/null +++ b/crates/crank-import/src/rest/swagger2.rs @@ -0,0 +1,234 @@ +use crank_core::HttpMethod; +use serde_json::Value; + +use crate::rest::{ + model::{RestImportDocument, RestImportOperation, RestImportParameter, RestParameterLocation}, + normalize::{ImportParseError, resolve_local_ref}, + recommendations::{document_finding, operation_finding}, +}; + +pub fn parse_document(root: &Value) -> Result { + let title = root + .pointer("/info/title") + .and_then(Value::as_str) + .unwrap_or("Imported API") + .to_owned(); + let servers = swagger_servers(root); + let mut findings = Vec::new(); + if servers.is_empty() { + findings.push(document_finding( + "missing_servers", + "В Swagger 2.0 документе не указаны host/schemes, base URL нужно будет выбрать вручную.", + )); + } + + let mut operations = Vec::new(); + let paths = root + .get("paths") + .and_then(Value::as_object) + .ok_or(ImportParseError::UnsupportedDocument)?; + + for (path, path_item) in paths { + let path_parameters = parameters(root, path_item.get("parameters")); + for method_name in ["get", "post", "put", "patch", "delete"] { + let Some(operation_value) = path_item.get(method_name) else { + continue; + }; + let Some(method) = method_from_lower(method_name) else { + continue; + }; + let mut operation_parameters = path_parameters.clone(); + operation_parameters.extend(parameters(root, operation_value.get("parameters"))); + let request_body_schema = operation_parameters + .iter() + .find(|parameter| parameter.name == "body") + .and_then(|parameter| parameter.schema.clone()); + let operation_parameters = operation_parameters + .into_iter() + .filter(|parameter| parameter.name != "body") + .collect::>(); + + operations.push(RestImportOperation { + key: format!("{} {}", method_name.to_uppercase(), path), + method, + path: path.clone(), + operation_id: operation_value + .get("operationId") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + summary: operation_value + .get("summary") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + description: operation_value + .get("description") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + tags: operation_value + .get("tags") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(Value::as_str) + .map(ToOwned::to_owned) + .collect() + }) + .unwrap_or_default(), + parameters: operation_parameters, + request_body_schema, + response_schema: response_schema(root, operation_value), + servers: Vec::new(), + findings: swagger_operation_findings(path, operation_value), + }); + } + } + + Ok(RestImportDocument { + format: "swagger".to_owned(), + version: Some("2.0".to_owned()), + title, + servers, + operations, + findings, + }) +} + +fn swagger_servers(root: &Value) -> Vec { + let Some(host) = root.get("host").and_then(Value::as_str) else { + return Vec::new(); + }; + let base_path = root.get("basePath").and_then(Value::as_str).unwrap_or(""); + let schemes = root + .get("schemes") + .and_then(Value::as_array) + .map(|items| items.iter().filter_map(Value::as_str).collect::>()) + .filter(|items| !items.is_empty()) + .unwrap_or_else(|| vec!["https"]); + + schemes + .into_iter() + .map(|scheme| { + format!("{scheme}://{host}{base_path}") + .trim_end_matches('/') + .to_owned() + }) + .collect() +} + +fn parameters(root: &Value, value: Option<&Value>) -> Vec { + value + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(|item| { + let item = resolve_local_ref(root, item, 0); + let raw_location = item.get("in").and_then(Value::as_str)?; + if raw_location == "body" { + return Some(RestImportParameter { + name: "body".to_owned(), + location: RestParameterLocation::Query, + required: item + .get("required") + .and_then(Value::as_bool) + .unwrap_or(false), + description: item + .get("description") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + schema: item + .get("schema") + .map(|schema| resolve_local_ref(root, schema, 0)), + }); + } + let location = match raw_location { + "path" => RestParameterLocation::Path, + "query" => RestParameterLocation::Query, + "header" => RestParameterLocation::Header, + _ => return None, + }; + Some(RestImportParameter { + name: item.get("name").and_then(Value::as_str)?.to_owned(), + location, + required: item + .get("required") + .and_then(Value::as_bool) + .unwrap_or(false) + || location == RestParameterLocation::Path, + description: item + .get("description") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + schema: swagger_parameter_schema(root, &item), + }) + }) + .collect() + }) + .unwrap_or_default() +} + +fn swagger_parameter_schema(root: &Value, parameter: &Value) -> Option { + if let Some(schema) = parameter.get("schema") { + return Some(resolve_local_ref(root, schema, 0)); + } + let mut schema = serde_json::Map::new(); + for key in ["type", "format", "items", "enum", "default", "description"] { + if let Some(value) = parameter.get(key) { + schema.insert(key.to_owned(), resolve_local_ref(root, value, 0)); + } + } + if schema.is_empty() { + None + } else { + Some(Value::Object(schema)) + } +} + +fn response_schema(root: &Value, operation: &Value) -> Option { + let responses = operation.get("responses")?.as_object()?; + for code in ["200", "201", "202", "default"] { + let Some(response) = responses.get(code) else { + continue; + }; + let response = resolve_local_ref(root, response, 0); + if let Some(schema) = response.get("schema") { + return Some(resolve_local_ref(root, schema, 0)); + } + } + None +} + +fn swagger_operation_findings( + path: &str, + operation: &Value, +) -> Vec { + let mut findings = Vec::new(); + if operation + .get("consumes") + .and_then(Value::as_array) + .is_some_and(|items| { + !items + .iter() + .any(|item| item.as_str().is_some_and(|value| value.contains("json"))) + }) + { + findings.push(operation_finding( + path, + "unsupported_consumes", + "Метод использует content type без JSON. Проверьте настройки тела запроса после импорта.", + )); + } + findings +} + +fn method_from_lower(value: &str) -> Option { + match value { + "get" => Some(HttpMethod::Get), + "post" => Some(HttpMethod::Post), + "put" => Some(HttpMethod::Put), + "patch" => Some(HttpMethod::Patch), + "delete" => Some(HttpMethod::Delete), + _ => None, + } +} diff --git a/crates/crank-import/tests/unit.rs b/crates/crank-import/tests/unit.rs new file mode 100644 index 0000000..456654a --- /dev/null +++ b/crates/crank-import/tests/unit.rs @@ -0,0 +1,243 @@ +mod unit { + use crank_core::HttpMethod; + use crank_import::rest::preview_document; + + const OPENAPI3: &str = r#" +openapi: 3.0.3 +info: + title: Frankfurter API +servers: + - url: https://api.frankfurter.dev +paths: + /v2/latest: + get: + operationId: getLatestRates + summary: Получить последние курсы + description: Возвращает последние курсы валют для базовой валюты. + tags: [currency] + parameters: + - name: base + in: query + required: true + schema: + type: string + - name: symbols + in: query + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + required: [amount, base] + properties: + amount: + type: number + base: + type: string +"#; + + const SWAGGER2: &str = r#" +swagger: "2.0" +info: + title: Pet API +host: petstore.example.com +basePath: /api +schemes: [https] +paths: + /pets/{id}: + get: + operationId: getPet + summary: Получить питомца + tags: [pets] + parameters: + - name: id + in: path + required: true + type: string + responses: + '200': + description: OK + schema: + $ref: '#/definitions/Pet' +definitions: + Pet: + type: object + required: [id, name] + properties: + id: + type: string + name: + type: string +"#; + + #[test] + fn previews_openapi3_rest_operations_grouped_by_tag() { + let preview = preview_document(OPENAPI3).unwrap(); + + assert_eq!(preview.source.format, "openapi"); + assert_eq!(preview.source.servers, vec!["https://api.frankfurter.dev"]); + assert_eq!(preview.groups.len(), 1); + assert_eq!(preview.groups[0].key, "currency"); + let operation = &preview.groups[0].operations[0]; + assert_eq!(operation.method, HttpMethod::Get); + assert_eq!(operation.suggested_name, "get_latest_rates"); + assert_eq!(operation.input_fields, 2); + assert_eq!(operation.output_fields, 2); + assert_eq!(operation.draft.target.path_template, "/v2/latest"); + assert_eq!(operation.draft.input_mapping.rules.len(), 2); + } + + #[test] + fn previews_swagger2_and_resolves_definitions() { + let preview = preview_document(SWAGGER2).unwrap(); + + assert_eq!(preview.source.format, "swagger"); + assert_eq!( + preview.source.servers, + vec!["https://petstore.example.com/api"] + ); + let operation = &preview.groups[0].operations[0]; + assert_eq!(operation.suggested_name, "get_pet"); + assert_eq!(operation.input_fields, 1); + assert_eq!(operation.output_fields, 2); + assert_eq!(operation.draft.target.path_template, "/pets/{id}"); + assert_eq!( + operation.draft.input_mapping.rules[0].target, + "$.request.path.id" + ); + } + + #[test] + fn reports_missing_descriptions_as_recommendations() { + let document = r#" +openapi: 3.0.3 +info: { title: Minimal API } +paths: + /items: + get: + responses: + '204': { description: Empty } +"#; + + let preview = preview_document(document).unwrap(); + let operation = &preview.groups[0].operations[0]; + let codes = operation + .findings + .iter() + .map(|finding| finding.code.as_str()) + .collect::>(); + + assert!(codes.contains(&"missing_operation_id")); + assert!(codes.contains(&"missing_summary")); + assert!(codes.contains(&"missing_description")); + assert!(codes.contains(&"missing_response_schema")); + } + + #[test] + fn expands_json_request_body_object_into_tool_inputs() { + let document = r#" +openapi: 3.0.3 +info: { title: CRM API } +servers: + - url: https://crm.example.test +paths: + /leads: + post: + operationId: createLead + summary: Создать лид + description: Создает лид в CRM. + tags: [crm] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [email] + properties: + email: { type: string } + name: { type: string } + responses: + '201': + description: Created + content: + application/json: + schema: + type: object + properties: + id: { type: string } +"#; + + let preview = preview_document(document).unwrap(); + let operation = &preview.groups[0].operations[0]; + let input_fields = &operation.draft.input_schema.fields; + let targets = operation + .draft + .input_mapping + .rules + .iter() + .map(|rule| rule.target.as_str()) + .collect::>(); + + assert!(input_fields.contains_key("email")); + assert!(input_fields.contains_key("name")); + assert!(targets.contains(&"$.request.body.email")); + assert!(targets.contains(&"$.request.body.name")); + } + + #[test] + fn reports_tool_quality_recommendations_for_imported_operations() { + let document = r#" +openapi: 3.0.3 +info: { title: Wide API } +paths: + /items: + get: + operationId: getItems + summary: Get items + description: Get items. + parameters: + - name: page + in: query + schema: { type: integer } + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + field01: { type: string } + field02: { type: string } + field03: { type: string } + field04: { type: string } + field05: { type: string } + field06: { type: string } + field07: { type: string } + field08: { type: string } + field09: { type: string } + field10: { type: string } + field11: { type: string } + field12: { type: string } + field13: { type: string } +"#; + + let preview = preview_document(document).unwrap(); + let operation = &preview.groups[0].operations[0]; + let codes = operation + .findings + .iter() + .map(|finding| finding.code.as_str()) + .collect::>(); + + assert!(codes.contains(&"parameter_descriptions_missing")); + assert!(codes.contains(&"weak_tool_description")); + assert!(codes.contains(&"weak_tool_name")); + assert!(codes.contains(&"too_many_output_fields")); + } +} diff --git a/crates/crank-registry/src/error.rs b/crates/crank-registry/src/error.rs index 29d14d6..49b3449 100644 --- a/crates/crank-registry/src/error.rs +++ b/crates/crank-registry/src/error.rs @@ -72,6 +72,8 @@ pub enum RegistryError { AuthProfileNotFound { auth_profile_id: String }, #[error("yaml import job {job_id} was not found")] YamlImportJobNotFound { job_id: String }, + #[error("import job {job_id} was not found")] + ImportJobNotFound { job_id: String }, #[error("unsupported enum representation for field {field}")] InvalidEnumRepresentation { field: &'static str }, #[error("invalid numeric value for field {field}: {value}")] diff --git a/crates/crank-registry/src/lib.rs b/crates/crank-registry/src/lib.rs index e19f452..9be46e2 100644 --- a/crates/crank-registry/src/lib.rs +++ b/crates/crank-registry/src/lib.rs @@ -10,10 +10,11 @@ pub use ext::{ExtensionMigration, RegistryExtension, apply_extension_migrations} pub mod records { pub use crate::model::{ AgentSummary, AgentVersionRecord, AuthUserRecord, DescriptorKind, DescriptorMetadata, - InvitationRecord, InvocationLogRecord, MembershipRecord, OperationAgentRef, - OperationSampleMetadata, OperationSummary, OperationUsageSummary, OperationVersionRecord, - Page, PlatformApiKeyRecord, PublishedAgentTool, RegistryOperation, SampleKind, - SecretRecord, SecretVersionRecord, SessionRecord, UsageAgentBreakdown, UsageBucket, + ImportJob, ImportJobId, ImportJobKind, ImportJobStatus, InvitationRecord, + InvocationLogRecord, MembershipRecord, OperationAgentRef, OperationSampleMetadata, + OperationSummary, OperationUsageSummary, OperationVersionRecord, Page, + PlatformApiKeyRecord, PublishedAgentTool, RegistryOperation, SampleKind, SecretRecord, + SecretVersionRecord, SessionRecord, UsageAgentBreakdown, UsageBucket, UsageOperationBreakdown, UsageRollupRecord, UsageSummary, UsageTimelinePoint, WorkspaceMembershipRecord, WorkspaceRecord, WorkspaceUpstream, WorkspaceUpstreamId, YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus, @@ -22,13 +23,13 @@ pub mod records { pub mod requests { pub use crate::model::{ - CreateAgentDraftVersionRequest, CreateAgentRequest, CreateInvitationRequest, - CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest, - CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, - ListInvocationLogsQuery, PublishAgentRequest, PublishRequest, RotateSecretRequest, - SaveAgentBindingsRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest, - SaveSampleMetadataRequest, SaveWorkspaceUpstreamRequest, UpdateWorkspaceRequest, - UsageQuery, + CreateAgentDraftVersionRequest, CreateAgentRequest, CreateImportJobRequest, + CreateInvitationRequest, CreateInvocationLogRequest, CreatePlatformApiKeyRequest, + CreateSecretRequest, CreateVersionRequest, CreateWorkspaceRequest, + CreateYamlImportJobRequest, FinishImportJobRequest, ListInvocationLogsQuery, + PublishAgentRequest, PublishRequest, RotateSecretRequest, SaveAgentBindingsRequest, + SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest, + SaveWorkspaceUpstreamRequest, UpdateWorkspaceRequest, UsageQuery, }; } @@ -39,18 +40,19 @@ pub mod infrastructure { pub use model::{ AgentSummary, AgentVersionRecord, AuthUserRecord, CreateAgentDraftVersionRequest, - CreateAgentRequest, CreateInvitationRequest, CreateInvocationLogRequest, - CreatePlatformApiKeyRequest, CreateSecretRequest, CreateVersionRequest, CreateWorkspaceRequest, - CreateYamlImportJobRequest, DescriptorKind, DescriptorMetadata, InvitationRecord, - InvocationLogRecord, ListInvocationLogsQuery, MembershipRecord, OperationAgentRef, - OperationSampleMetadata, OperationSummary, OperationUsageSummary, OperationVersionRecord, Page, - PlatformApiKeyRecord, PublishAgentRequest, PublishRequest, PublishedAgentTool, - RegistryOperation, RotateSecretRequest, SampleKind, SaveAgentBindingsRequest, - SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest, - SaveWorkspaceUpstreamRequest, SecretRecord, SecretVersionRecord, SessionRecord, - UpdateWorkspaceRequest, UsageAgentBreakdown, UsageBucket, UsageOperationBreakdown, UsageQuery, - UsageRollupRecord, UsageSummary, UsageTimelinePoint, WorkspaceMembershipRecord, - WorkspaceRecord, WorkspaceUpstream, WorkspaceUpstreamId, YamlImportJob, - YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus, + CreateAgentRequest, CreateImportJobRequest, CreateInvitationRequest, + CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest, + CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, DescriptorKind, + DescriptorMetadata, FinishImportJobRequest, ImportJob, ImportJobId, ImportJobKind, + ImportJobStatus, InvitationRecord, InvocationLogRecord, ListInvocationLogsQuery, + MembershipRecord, OperationAgentRef, OperationSampleMetadata, OperationSummary, + OperationUsageSummary, OperationVersionRecord, Page, PlatformApiKeyRecord, PublishAgentRequest, + PublishRequest, PublishedAgentTool, RegistryOperation, RotateSecretRequest, SampleKind, + SaveAgentBindingsRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest, + SaveSampleMetadataRequest, SaveWorkspaceUpstreamRequest, SecretRecord, SecretVersionRecord, + SessionRecord, UpdateWorkspaceRequest, UsageAgentBreakdown, UsageBucket, + UsageOperationBreakdown, UsageQuery, UsageRollupRecord, UsageSummary, UsageTimelinePoint, + WorkspaceMembershipRecord, WorkspaceRecord, WorkspaceUpstream, WorkspaceUpstreamId, + YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus, }; pub use postgres::{PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry}; diff --git a/crates/crank-registry/src/migrations.rs b/crates/crank-registry/src/migrations.rs index b7acea6..e6d28ac 100644 --- a/crates/crank-registry/src/migrations.rs +++ b/crates/crank-registry/src/migrations.rs @@ -451,6 +451,25 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { .execute(pool) .await?; + query( + "create table if not exists import_jobs ( + id text primary key, + workspace_id text not null references workspaces(id) on delete cascade, + kind text not null, + source_format text not null, + source_version text null, + status text not null, + preview_payload jsonb not null, + created_operation_ids jsonb not null default '[]'::jsonb, + error_text text null, + created_at timestamptz not null, + expires_at timestamptz not null, + finished_at timestamptz null + )", + ) + .execute(pool) + .await?; + query( "create table if not exists agents ( id text primary key, diff --git a/crates/crank-registry/src/model.rs b/crates/crank-registry/src/model.rs index 7309219..d53454b 100644 --- a/crates/crank-registry/src/model.rs +++ b/crates/crank-registry/src/model.rs @@ -41,6 +41,7 @@ macro_rules! define_registry_id { } define_registry_id!(YamlImportJobId); +define_registry_id!(ImportJobId); define_registry_id!(WorkspaceUpstreamId); pub type RegistryOperation = Operation; @@ -326,6 +327,61 @@ pub struct YamlImportJobCompletion { pub finished_at: OffsetDateTime, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportJobStatus { + Pending, + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportJobKind { + OpenApi, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImportJob { + pub id: ImportJobId, + pub workspace_id: WorkspaceId, + pub kind: ImportJobKind, + pub source_format: String, + pub source_version: Option, + pub status: ImportJobStatus, + pub preview_payload: Value, + pub created_operation_ids: Value, + pub error_text: Option, + #[serde(with = "time::serde::rfc3339")] + pub created_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339")] + pub expires_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339::option")] + pub finished_at: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct CreateImportJobRequest<'a> { + pub id: &'a ImportJobId, + pub workspace_id: &'a WorkspaceId, + pub kind: ImportJobKind, + pub source_format: &'a str, + pub source_version: Option<&'a str>, + pub status: ImportJobStatus, + pub preview_payload: &'a Value, + pub created_at: &'a OffsetDateTime, + pub expires_at: &'a OffsetDateTime, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct FinishImportJobRequest<'a> { + pub id: &'a ImportJobId, + pub status: ImportJobStatus, + pub created_operation_ids: &'a Value, + pub error_text: Option<&'a str>, + pub finished_at: &'a OffsetDateTime, +} + #[derive(Clone, Debug, PartialEq)] pub struct CreateInvocationLogRequest<'a> { pub log: &'a InvocationLog, diff --git a/crates/crank-registry/src/postgres/import_job.rs b/crates/crank-registry/src/postgres/import_job.rs new file mode 100644 index 0000000..f81d11e --- /dev/null +++ b/crates/crank-registry/src/postgres/import_job.rs @@ -0,0 +1,107 @@ +use super::*; + +impl PostgresRegistry { + pub async fn create_import_job( + &self, + request: CreateImportJobRequest<'_>, + ) -> Result<(), RegistryError> { + sqlx::query( + "insert into import_jobs ( + id, + workspace_id, + kind, + source_format, + source_version, + status, + preview_payload, + created_operation_ids, + error_text, + created_at, + expires_at, + finished_at + ) values ( + $1, $2, $3, $4, $5, $6, $7, '[]'::jsonb, null, $8::timestamptz, $9::timestamptz, null + )", + ) + .bind(request.id.as_str()) + .bind(request.workspace_id.as_str()) + .bind(serialize_enum_text(&request.kind, "kind")?) + .bind(request.source_format) + .bind(request.source_version) + .bind(serialize_enum_text(&request.status, "status")?) + .bind(request.preview_payload) + .bind(request.created_at) + .bind(request.expires_at) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn get_import_job( + &self, + workspace_id: &WorkspaceId, + job_id: &ImportJobId, + ) -> Result, RegistryError> { + let row = sqlx::query( + "select + id, + workspace_id, + kind, + source_format, + source_version, + status, + preview_payload, + created_operation_ids, + error_text, + created_at, + expires_at, + finished_at + from import_jobs + where id = $1 and workspace_id = $2", + ) + .bind(job_id.as_str()) + .bind(workspace_id.as_str()) + .fetch_optional(&self.pool) + .await?; + + row.as_ref().map(map_import_job).transpose() + } + + pub async fn finish_import_job( + &self, + request: FinishImportJobRequest<'_>, + ) -> Result<(), RegistryError> { + let result = sqlx::query( + "update import_jobs + set status = $2, + created_operation_ids = $3, + error_text = $4, + finished_at = $5::timestamptz + where id = $1", + ) + .bind(request.id.as_str()) + .bind(serialize_enum_text(&request.status, "status")?) + .bind(request.created_operation_ids) + .bind(request.error_text) + .bind(request.finished_at) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Err(RegistryError::ImportJobNotFound { + job_id: request.id.as_str().to_owned(), + }); + } + + Ok(()) + } + + pub async fn delete_expired_import_jobs(&self) -> Result { + let result = sqlx::query("delete from import_jobs where expires_at < now()") + .execute(&self.pool) + .await?; + + Ok(result.rows_affected()) + } +} diff --git a/crates/crank-registry/src/postgres/mod.rs b/crates/crank-registry/src/postgres/mod.rs index 567ef6e..e377fa6 100644 --- a/crates/crank-registry/src/postgres/mod.rs +++ b/crates/crank-registry/src/postgres/mod.rs @@ -2,6 +2,7 @@ mod agent; mod api_key; mod auth; mod connection; +mod import_job; mod observability; mod operation; mod operation_artifact; @@ -28,9 +29,10 @@ use crate::{ error::RegistryError, model::{ AgentSummary, AgentVersionRecord, AuthUserRecord, CreateAgentDraftVersionRequest, - CreateAgentRequest, CreateInvitationRequest, CreateInvocationLogRequest, - CreatePlatformApiKeyRequest, CreateSecretRequest, CreateVersionRequest, - CreateWorkspaceRequest, CreateYamlImportJobRequest, DescriptorMetadata, InvitationRecord, + CreateAgentRequest, CreateImportJobRequest, CreateInvitationRequest, + CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest, + CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, + DescriptorMetadata, FinishImportJobRequest, ImportJob, ImportJobId, InvitationRecord, InvocationLogRecord, ListInvocationLogsQuery, MembershipRecord, OperationAgentRef, OperationSampleMetadata, OperationSummary, OperationUsageSummary, OperationVersionRecord, PlatformApiKeyRecord, PublishAgentRequest, PublishRequest, PublishedAgentTool, @@ -661,6 +663,23 @@ fn map_yaml_import_job(row: &PgRow) -> Result { }) } +fn map_import_job(row: &PgRow) -> Result { + Ok(ImportJob { + id: ImportJobId::new(row.try_get::("id")?), + workspace_id: WorkspaceId::new(row.try_get::("workspace_id")?), + kind: deserialize_enum_text(&row.try_get::("kind")?, "kind")?, + source_format: row.try_get("source_format")?, + source_version: row.try_get("source_version")?, + status: deserialize_enum_text(&row.try_get::("status")?, "status")?, + preview_payload: row.try_get("preview_payload")?, + created_operation_ids: row.try_get("created_operation_ids")?, + error_text: row.try_get("error_text")?, + created_at: row.try_get("created_at")?, + expires_at: row.try_get("expires_at")?, + finished_at: row.try_get("finished_at")?, + }) +} + fn map_usage_operation_breakdown(row: &PgRow) -> Result { Ok(UsageOperationBreakdown { operation_id: OperationId::new(row.try_get::("operation_id")?), diff --git a/crates/crank-registry/tests/integration/common.rs b/crates/crank-registry/tests/integration/common.rs index 980d416..7a8af8a 100644 --- a/crates/crank-registry/tests/integration/common.rs +++ b/crates/crank-registry/tests/integration/common.rs @@ -156,6 +156,7 @@ pub(super) fn test_operation(id: &str, version: u32, status: OperationStatus) -> input_sample: Some(json!({ "email": format!("lead-{version}@example.com") })), output_sample: Some(json!({ "id": format!("lead_{version}") })), test_input: Some(json!({ "email": format!("test-v{version}@example.com") })), + import_findings: Vec::new(), }), created_at: timestamp("2026-03-25T11:58:00Z"), updated_at: timestamp(&format!("2026-03-25T12:{version:02}:00Z")),