From 2719c5f7bf9618b75ace83faab89af13843c8e0a Mon Sep 17 00:00:00 2001 From: github-ops Date: Sat, 20 Jun 2026 19:53:42 +0000 Subject: [PATCH] Add tool identity quality checks --- apps/admin-api/src/service.rs | 5 +- crates/crank-core/src/lib.rs | 4 +- crates/crank-core/src/tool_quality.rs | 103 +++++++++++++++++++ crates/crank-core/tests/unit/tool_quality.rs | 92 ++++++++++++++++- docs/admin-api.md | 17 +++ 5 files changed, 218 insertions(+), 3 deletions(-) diff --git a/apps/admin-api/src/service.rs b/apps/admin-api/src/service.rs index 1309465..8774f82 100644 --- a/apps/admin-api/src/service.rs +++ b/apps/admin-api/src/service.rs @@ -1452,7 +1452,10 @@ impl AdminService { self.validate_operation_payload(&payload)?; self.ensure_workspace_exists(workspace_id).await?; - Ok(crank_core::ToolQualityReport::empty()) + Ok(crank_core::analyze_tool_identity_quality( + &payload.name, + &payload.tool_description, + )) } #[instrument(skip(self, payload), fields(operation_id = %operation_id.as_str(), protocol = ?payload.operation.protocol))] diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index f84675c..777cde0 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -62,5 +62,7 @@ pub use operation::{ }; pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion}; -pub use tool_quality::{ToolQualityFinding, ToolQualityReport, ToolQualitySeverity}; +pub use tool_quality::{ + ToolQualityFinding, ToolQualityReport, ToolQualitySeverity, analyze_tool_identity_quality, +}; pub use workspace::{Workspace, WorkspaceStatus}; diff --git a/crates/crank-core/src/tool_quality.rs b/crates/crank-core/src/tool_quality.rs index 07ef5cd..cd739ec 100644 --- a/crates/crank-core/src/tool_quality.rs +++ b/crates/crank-core/src/tool_quality.rs @@ -1,5 +1,11 @@ use serde::{Deserialize, Serialize}; +use crate::ToolDescription; + +const MAX_TOOL_NAME_LEN: usize = 64; +const MIN_DESCRIPTION_CHARS: usize = 40; +const GENERIC_TOOL_NAMES: &[&str] = &["call_api", "manage", "execute", "request"]; + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ToolQualitySeverity { @@ -38,3 +44,100 @@ impl ToolQualityReport { Self::new(Vec::new()) } } + +pub fn analyze_tool_identity_quality( + tool_name: &str, + tool_description: &ToolDescription, +) -> ToolQualityReport { + let mut findings = Vec::new(); + let normalized_name = tool_name.trim(); + + if normalized_name.is_empty() { + findings.push(finding( + ToolQualitySeverity::Error, + "tool_name_missing", + "Имя инструмента обязательно.", + Some("Укажите короткое техническое имя в snake_case, например get_customer_profile."), + Some("name"), + )); + } else { + if normalized_name.chars().count() > MAX_TOOL_NAME_LEN { + findings.push(finding( + ToolQualitySeverity::Warning, + "tool_name_too_long", + "Имя инструмента слишком длинное.", + Some("Сократите имя до понятного действия и объекта."), + Some("name"), + )); + } + + if !is_safe_tool_name(normalized_name) { + findings.push(finding( + ToolQualitySeverity::Error, + "tool_name_invalid_format", + "Имя инструмента должно быть в безопасном snake_case формате.", + Some("Используйте только строчные английские буквы, цифры и подчеркивания."), + Some("name"), + )); + } + + if GENERIC_TOOL_NAMES.contains(&normalized_name) { + findings.push(finding( + ToolQualitySeverity::Warning, + "tool_name_too_generic", + "Имя инструмента слишком общее.", + Some("Назовите действие и объект явно, например create_crm_lead или get_ticket_status."), + Some("name"), + )); + } + } + + let description = tool_description.description.trim(); + if description.is_empty() { + findings.push(finding( + ToolQualitySeverity::Error, + "tool_description_missing", + "Описание инструмента обязательно.", + Some("Опишите, когда агенту нужно использовать этот инструмент и какой результат считается успешным."), + Some("tool_description.description"), + )); + } else if description.chars().count() < MIN_DESCRIPTION_CHARS { + findings.push(finding( + ToolQualitySeverity::Warning, + "tool_description_too_short", + "Описание инструмента слишком короткое.", + Some("Добавьте назначение, условия применения и ожидаемый успешный результат."), + Some("tool_description.description"), + )); + } + + ToolQualityReport::new(findings) +} + +fn is_safe_tool_name(value: &str) -> bool { + let mut chars = value.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !first.is_ascii_lowercase() { + return false; + } + + chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') +} + +fn finding( + severity: ToolQualitySeverity, + code: &str, + message: &str, + suggested_action: Option<&str>, + field_path: Option<&str>, +) -> ToolQualityFinding { + ToolQualityFinding { + severity, + code: code.to_owned(), + message: message.to_owned(), + suggested_action: suggested_action.map(ToOwned::to_owned), + field_path: field_path.map(ToOwned::to_owned), + } +} diff --git a/crates/crank-core/tests/unit/tool_quality.rs b/crates/crank-core/tests/unit/tool_quality.rs index 29d3a63..4252cdf 100644 --- a/crates/crank-core/tests/unit/tool_quality.rs +++ b/crates/crank-core/tests/unit/tool_quality.rs @@ -1,4 +1,7 @@ -use crank_core::{ToolQualityFinding, ToolQualityReport, ToolQualitySeverity}; +use crank_core::{ + ToolDescription, ToolQualityFinding, ToolQualityReport, ToolQualitySeverity, + analyze_tool_identity_quality, +}; #[test] fn serializes_tool_quality_finding_contract() { @@ -30,3 +33,90 @@ fn report_is_blocking_when_error_finding_exists() { assert!(report.blocking); assert_eq!(report.findings.len(), 1); } + +#[test] +fn reports_missing_tool_name_as_blocking_error() { + let report = analyze_tool_identity_quality( + " ", + &ToolDescription { + title: "Create lead".to_owned(), + description: "Создает новый лид в CRM, когда пользователь просит добавить потенциального клиента.".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + ); + + assert!(report.blocking); + assert!(has_finding( + &report, + "tool_name_missing", + ToolQualitySeverity::Error + )); +} + +#[test] +fn warns_about_generic_tool_name() { + let report = analyze_tool_identity_quality( + "call_api", + &ToolDescription { + title: "Call API".to_owned(), + description: "Получает данные клиента по идентификатору, когда пользователю нужна карточка клиента.".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + ); + + assert!(!report.blocking); + assert!(has_finding( + &report, + "tool_name_too_generic", + ToolQualitySeverity::Warning + )); +} + +#[test] +fn reports_unsafe_tool_name_format_as_blocking_error() { + let report = analyze_tool_identity_quality( + "Create Lead!", + &ToolDescription { + title: "Create lead".to_owned(), + description: "Создает новый лид в CRM, когда пользователь просит добавить потенциального клиента.".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + ); + + assert!(report.blocking); + assert!(has_finding( + &report, + "tool_name_invalid_format", + ToolQualitySeverity::Error + )); +} + +#[test] +fn warns_about_short_description() { + let report = analyze_tool_identity_quality( + "create_lead", + &ToolDescription { + title: "Create lead".to_owned(), + description: "Создает лид.".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + ); + + assert!(!report.blocking); + assert!(has_finding( + &report, + "tool_description_too_short", + ToolQualitySeverity::Warning + )); +} + +fn has_finding(report: &ToolQualityReport, code: &str, severity: ToolQualitySeverity) -> bool { + report + .findings + .iter() + .any(|finding| finding.code == code && finding.severity == severity) +} diff --git a/docs/admin-api.md b/docs/admin-api.md index 5551824..853c75e 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -53,6 +53,23 @@ Community работает с одним bootstrap workspace. API не подд - `POST /api/admin/workspaces/{workspace_id}/operations/{operation_id}/archive` - `POST /api/admin/workspaces/{workspace_id}/operations/{operation_id}/test-runs` - `GET /api/admin/workspaces/{workspace_id}/operations/{operation_id}/export` + +`analyze-quality` принимает такой же draft payload, как создание операции, и возвращает отчет: + +```json +{ + "blocking": false, + "findings": [ + { + "severity": "warning", + "code": "tool_description_too_short", + "message": "Описание инструмента слишком короткое.", + "suggested_action": "Добавьте назначение, условия применения и ожидаемый успешный результат.", + "field_path": "tool_description.description" + } + ] +} +``` - `POST /api/admin/workspaces/{workspace_id}/operations/import` Community принимает только `protocol = rest`.