github-ops
384 lines
14 KiB
Rust
384 lines
14 KiB
Rust
use serde::{Deserialize, Serialize};
|
||
|
||
use crate::ToolDescription;
|
||
|
||
const MAX_TOOL_NAME_LEN: usize = 64;
|
||
const MIN_DESCRIPTION_CHARS: usize = 40;
|
||
const GENERIC_TOOL_NAMES: &[&str] = &["call_api", "manage", "execute", "request"];
|
||
const MULTI_ACTION_FIELD_NAMES: &[&str] = &["action", "mode", "operation", "type"];
|
||
const ENUM_LIKE_FIELD_NAMES: &[&str] = &["mode", "type", "status", "category"];
|
||
const MANY_PROJECTED_FIELDS_THRESHOLD: usize = 12;
|
||
const LARGE_AGENT_TOOL_CATALOG_THRESHOLD: usize = 8;
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum ToolQualitySeverity {
|
||
Info,
|
||
Warning,
|
||
Error,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct ToolQualityFinding {
|
||
pub severity: ToolQualitySeverity,
|
||
pub code: String,
|
||
pub message: String,
|
||
#[serde(skip_serializing_if = "Option::is_none")]
|
||
pub suggested_action: Option<String>,
|
||
#[serde(skip_serializing_if = "Option::is_none")]
|
||
pub field_path: Option<String>,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct ToolQualityReport {
|
||
pub blocking: bool,
|
||
pub findings: Vec<ToolQualityFinding>,
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
#[serde(rename_all = "snake_case")]
|
||
pub enum ToolQualitySchemaKind {
|
||
Object,
|
||
Array,
|
||
String,
|
||
Integer,
|
||
Number,
|
||
Boolean,
|
||
Enum,
|
||
Null,
|
||
Oneof,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct ToolQualitySchemaNode {
|
||
pub kind: ToolQualitySchemaKind,
|
||
#[serde(skip_serializing_if = "Option::is_none")]
|
||
pub description: Option<String>,
|
||
pub fields: Vec<(String, ToolQualitySchemaNode)>,
|
||
#[serde(skip_serializing_if = "Option::is_none")]
|
||
pub items: Option<Box<ToolQualitySchemaNode>>,
|
||
pub enum_values: Vec<String>,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct ToolQualityMappingRule {
|
||
pub source: String,
|
||
pub target: String,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||
pub struct ToolQualityMappingSet {
|
||
pub rules: Vec<ToolQualityMappingRule>,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct ToolQualityCatalogTool {
|
||
pub name: String,
|
||
pub display_name: String,
|
||
pub description: String,
|
||
}
|
||
|
||
impl ToolQualityReport {
|
||
pub fn new(findings: Vec<ToolQualityFinding>) -> Self {
|
||
let blocking = findings
|
||
.iter()
|
||
.any(|finding| finding.severity == ToolQualitySeverity::Error);
|
||
|
||
Self { blocking, findings }
|
||
}
|
||
|
||
pub fn empty() -> Self {
|
||
Self::new(Vec::new())
|
||
}
|
||
}
|
||
|
||
impl ToolQualitySchemaNode {
|
||
pub fn object(description: Option<&str>, fields: Vec<(&str, ToolQualitySchemaNode)>) -> Self {
|
||
Self {
|
||
kind: ToolQualitySchemaKind::Object,
|
||
description: description.map(ToOwned::to_owned),
|
||
fields: fields
|
||
.into_iter()
|
||
.map(|(name, schema)| (name.to_owned(), schema))
|
||
.collect(),
|
||
items: None,
|
||
enum_values: Vec::new(),
|
||
}
|
||
}
|
||
|
||
pub fn string(description: Option<&str>) -> Self {
|
||
Self {
|
||
kind: ToolQualitySchemaKind::String,
|
||
description: description.map(ToOwned::to_owned),
|
||
fields: Vec::new(),
|
||
items: None,
|
||
enum_values: Vec::new(),
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn analyze_tool_identity_quality(
|
||
tool_name: &str,
|
||
tool_description: &ToolDescription,
|
||
) -> ToolQualityReport {
|
||
let mut findings = Vec::new();
|
||
let normalized_name = tool_name.trim();
|
||
|
||
if normalized_name.is_empty() {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Error,
|
||
"tool_name_missing",
|
||
"Имя инструмента обязательно.",
|
||
Some("Укажите короткое техническое имя в snake_case, например get_customer_profile."),
|
||
Some("name"),
|
||
));
|
||
} else {
|
||
if normalized_name.chars().count() > MAX_TOOL_NAME_LEN {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"tool_name_too_long",
|
||
"Имя инструмента слишком длинное.",
|
||
Some("Сократите имя до понятного действия и объекта."),
|
||
Some("name"),
|
||
));
|
||
}
|
||
|
||
if !is_safe_tool_name(normalized_name) {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Error,
|
||
"tool_name_invalid_format",
|
||
"Имя инструмента должно быть в безопасном snake_case формате.",
|
||
Some("Используйте только строчные английские буквы, цифры и подчеркивания."),
|
||
Some("name"),
|
||
));
|
||
}
|
||
|
||
if GENERIC_TOOL_NAMES.contains(&normalized_name) {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"tool_name_too_generic",
|
||
"Имя инструмента слишком общее.",
|
||
Some("Назовите действие и объект явно, например create_crm_lead или get_ticket_status."),
|
||
Some("name"),
|
||
));
|
||
}
|
||
}
|
||
|
||
let description = tool_description.description.trim();
|
||
if description.is_empty() {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Error,
|
||
"tool_description_missing",
|
||
"Описание инструмента обязательно.",
|
||
Some("Опишите, когда агенту нужно использовать этот инструмент и какой результат считается успешным."),
|
||
Some("tool_description.description"),
|
||
));
|
||
} else if description.chars().count() < MIN_DESCRIPTION_CHARS {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"tool_description_too_short",
|
||
"Описание инструмента слишком короткое.",
|
||
Some("Добавьте назначение, условия применения и ожидаемый успешный результат."),
|
||
Some("tool_description.description"),
|
||
));
|
||
}
|
||
|
||
ToolQualityReport::new(findings)
|
||
}
|
||
|
||
pub fn analyze_tool_schema_quality(
|
||
root_path: &str,
|
||
schema: &ToolQualitySchemaNode,
|
||
) -> ToolQualityReport {
|
||
let mut findings = Vec::new();
|
||
analyze_schema_node(root_path, schema, &mut findings);
|
||
ToolQualityReport::new(findings)
|
||
}
|
||
|
||
pub fn analyze_tool_response_projection_quality(
|
||
output_mapping: &ToolQualityMappingSet,
|
||
) -> ToolQualityReport {
|
||
let mut findings = Vec::new();
|
||
|
||
for (index, rule) in output_mapping.rules.iter().enumerate() {
|
||
let source = rule.source.trim();
|
||
if source == "$.response.body" || source == "$.response.data" {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"response_projection_full_body",
|
||
"Результат инструмента отдает агенту весь ответ API.",
|
||
Some("Выберите только поля, которые нужны агенту для ответа пользователю или следующего действия."),
|
||
Some(&format!("output_mapping.rules.{index}.source")),
|
||
));
|
||
}
|
||
}
|
||
|
||
if output_mapping.rules.len() >= MANY_PROJECTED_FIELDS_THRESHOLD {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Info,
|
||
"response_projection_many_fields",
|
||
"В результат инструмента попадает много полей.",
|
||
Some("Проверьте, что каждое поле действительно нужно агенту. Лишние данные увеличивают риск ошибок и галлюцинаций."),
|
||
Some("output_mapping.rules"),
|
||
));
|
||
}
|
||
|
||
ToolQualityReport::new(findings)
|
||
}
|
||
|
||
pub fn analyze_agent_tool_catalog_quality(tools: &[ToolQualityCatalogTool]) -> ToolQualityReport {
|
||
let mut findings = Vec::new();
|
||
|
||
if tools.len() > LARGE_AGENT_TOOL_CATALOG_THRESHOLD {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"agent_catalog_too_large",
|
||
"В каталоге агента много инструментов.",
|
||
Some("Оставьте только инструменты, которые нужны этому агенту для конкретной задачи."),
|
||
Some("agent.operations"),
|
||
));
|
||
}
|
||
|
||
for (left_index, left) in tools.iter().enumerate() {
|
||
for (right_offset, right) in tools[left_index + 1..].iter().enumerate() {
|
||
let right_index = left_index + right_offset + 1;
|
||
if tools_are_similar(left, right) {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"agent_catalog_similar_tools",
|
||
"У агента есть похожие инструменты.",
|
||
Some("Переименуйте инструменты точнее или оставьте один вариант, чтобы модель не выбирала наугад."),
|
||
Some(&format!("agent.operations.{left_index},{right_index}")),
|
||
));
|
||
}
|
||
}
|
||
}
|
||
|
||
ToolQualityReport::new(findings)
|
||
}
|
||
|
||
fn is_safe_tool_name(value: &str) -> bool {
|
||
let mut chars = value.chars();
|
||
let Some(first) = chars.next() else {
|
||
return false;
|
||
};
|
||
if !first.is_ascii_lowercase() {
|
||
return false;
|
||
}
|
||
|
||
chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_')
|
||
}
|
||
|
||
fn tools_are_similar(left: &ToolQualityCatalogTool, right: &ToolQualityCatalogTool) -> bool {
|
||
let left_tokens = catalog_tool_tokens(left);
|
||
let right_tokens = catalog_tool_tokens(right);
|
||
if left_tokens.is_empty() || right_tokens.is_empty() {
|
||
return false;
|
||
}
|
||
|
||
let shared = left_tokens
|
||
.iter()
|
||
.filter(|token| right_tokens.contains(token))
|
||
.count();
|
||
let smaller = left_tokens.len().min(right_tokens.len());
|
||
|
||
shared >= 2 && shared * 100 / smaller >= 60
|
||
}
|
||
|
||
fn catalog_tool_tokens(tool: &ToolQualityCatalogTool) -> Vec<String> {
|
||
let text = format!("{} {} {}", tool.name, tool.display_name, tool.description);
|
||
let mut tokens = Vec::new();
|
||
for token in text
|
||
.to_lowercase()
|
||
.split(|ch: char| !(ch.is_ascii_alphanumeric() || ('а'..='я').contains(&ch) || ch == 'ё'))
|
||
{
|
||
if token.len() < 3 || tokens.iter().any(|existing| existing == token) {
|
||
continue;
|
||
}
|
||
tokens.push(token.to_owned());
|
||
}
|
||
tokens
|
||
}
|
||
|
||
fn analyze_schema_node(
|
||
path: &str,
|
||
schema: &ToolQualitySchemaNode,
|
||
findings: &mut Vec<ToolQualityFinding>,
|
||
) {
|
||
if schema.kind == ToolQualitySchemaKind::Object && schema.fields.is_empty() {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"schema_object_without_fields",
|
||
"Схема объекта не содержит параметров.",
|
||
Some("Добавьте только те параметры, которые действительно нужны агенту для вызова инструмента."),
|
||
Some(path),
|
||
));
|
||
}
|
||
|
||
for (field_name, field_schema) in &schema.fields {
|
||
let field_path = format!("{path}.fields.{field_name}");
|
||
|
||
if is_missing_description(field_schema) {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"schema_field_missing_description",
|
||
"У параметра нет описания.",
|
||
Some("Опишите назначение параметра понятным языком, чтобы модель не угадывала его смысл."),
|
||
Some(&format!("{field_path}.description")),
|
||
));
|
||
}
|
||
|
||
if MULTI_ACTION_FIELD_NAMES.contains(&field_name.as_str()) {
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Warning,
|
||
"schema_multi_action_parameter",
|
||
"Параметр выглядит как переключатель нескольких действий.",
|
||
Some("Лучше разделить разные действия на отдельные MCP инструменты с узкими именами и схемами."),
|
||
Some(&field_path),
|
||
));
|
||
}
|
||
|
||
if ENUM_LIKE_FIELD_NAMES.contains(&field_name.as_str())
|
||
&& field_schema.kind == ToolQualitySchemaKind::String
|
||
&& field_schema.enum_values.is_empty()
|
||
{
|
||
findings.push(finding(
|
||
ToolQualitySeverity::Info,
|
||
"schema_enum_recommended",
|
||
"Для параметра с ограниченным набором значений лучше указать enum.",
|
||
Some("Задайте допустимые значения явно, чтобы агент не придумывал варианты."),
|
||
Some(&field_path),
|
||
));
|
||
}
|
||
|
||
analyze_schema_node(&field_path, field_schema, findings);
|
||
}
|
||
|
||
if let Some(items) = &schema.items {
|
||
analyze_schema_node(&format!("{path}.items"), items, findings);
|
||
}
|
||
}
|
||
|
||
fn is_missing_description(schema: &ToolQualitySchemaNode) -> bool {
|
||
schema
|
||
.description
|
||
.as_deref()
|
||
.is_none_or(|description| description.trim().is_empty())
|
||
}
|
||
|
||
fn finding(
|
||
severity: ToolQualitySeverity,
|
||
code: &str,
|
||
message: &str,
|
||
suggested_action: Option<&str>,
|
||
field_path: Option<&str>,
|
||
) -> ToolQualityFinding {
|
||
ToolQualityFinding {
|
||
severity,
|
||
code: code.to_owned(),
|
||
message: message.to_owned(),
|
||
suggested_action: suggested_action.map(ToOwned::to_owned),
|
||
field_path: field_path.map(ToOwned::to_owned),
|
||
}
|
||
}
|