From 7099faad2b7881aeccfa6604de43d29c086c5b28 Mon Sep 17 00:00:00 2001 From: github-ops Date: Sat, 20 Jun 2026 21:09:45 +0000 Subject: [PATCH] Warn about confusing agent tool catalogs --- apps/ui/html/agents.html | 11 +++ apps/ui/js/agents.js | 51 +++++++++++++ apps/ui/js/i18n.js | 6 ++ crates/crank-core/src/lib.rs | 8 +- crates/crank-core/src/tool_quality.rs | 70 ++++++++++++++++++ crates/crank-core/tests/unit/tool_quality.rs | 78 +++++++++++++++++++- 6 files changed, 218 insertions(+), 6 deletions(-) diff --git a/apps/ui/html/agents.html b/apps/ui/html/agents.html index fe89031..a8a30f4 100644 --- a/apps/ui/html/agents.html +++ b/apps/ui/html/agents.html @@ -316,6 +316,17 @@
Select the MCP tools available to this agent.
+
diff --git a/apps/ui/js/agents.js b/apps/ui/js/agents.js index b57ac55..646095f 100644 --- a/apps/ui/js/agents.js +++ b/apps/ui/js/agents.js @@ -195,6 +195,35 @@ document.addEventListener('alpine:init', function() { return this.tKey(this.isCommunityBuild ? 'agents.drawer.operations_sub_community' : 'agents.drawer.operations_sub'); }, + get selectedOperations() { + var selected = this.form.selectedOps; + return this.operations.filter(function(operation) { + return selected.includes(operation.id); + }); + }, + + get agentToolFindings() { + var findings = []; + var selected = this.selectedOperations; + if (selected.length > 8) { + findings.push(this.tKey('agents.drawer.finding.too_many_tools')); + } + + for (var leftIndex = 0; leftIndex < selected.length; leftIndex += 1) { + for (var rightIndex = leftIndex + 1; rightIndex < selected.length; rightIndex += 1) { + if (this.operationsLookSimilar(selected[leftIndex], selected[rightIndex])) { + findings.push(this.tfKey('agents.drawer.finding.similar_tools', { + left: selected[leftIndex].display_name || selected[leftIndex].name, + right: selected[rightIndex].display_name || selected[rightIndex].name, + })); + return findings; + } + } + } + + return findings; + }, + get filteredOps() { var query = this.opSearch.toLowerCase().trim(); if (!query) return this.operations; @@ -267,6 +296,28 @@ document.addEventListener('alpine:init', function() { return this.form.selectedOps.includes(operationId); }, + operationsLookSimilar(left, right) { + var leftTokens = this.operationTokens(left); + var rightTokens = this.operationTokens(right); + if (!leftTokens.length || !rightTokens.length) return false; + var shared = leftTokens.filter(function(token) { return rightTokens.includes(token); }).length; + var smaller = Math.min(leftTokens.length, rightTokens.length); + return shared >= 2 && (shared / smaller) >= 0.6; + }, + + operationTokens(operation) { + var text = [ + operation.name || '', + operation.display_name || '', + ].join(' ').toLowerCase(); + var tokens = []; + text.split(/[^a-z0-9а-яё]+/i).forEach(function(token) { + if (token.length < 3 || tokens.includes(token)) return; + tokens.push(token); + }); + return tokens; + }, + async saveAgent() { var self = this; if ( diff --git a/apps/ui/js/i18n.js b/apps/ui/js/i18n.js index 983f9d4..964c3ff 100644 --- a/apps/ui/js/i18n.js +++ b/apps/ui/js/i18n.js @@ -765,6 +765,9 @@ var TRANSLATIONS = { 'agents.drawer.selected': '{count} selected', 'agents.drawer.operations_sub': 'Select the MCP tools available to this agent.', 'agents.drawer.operations_sub_community': 'Select the MCP tools available to this agent.', + 'agents.drawer.finding.title': 'Recommendation', + 'agents.drawer.finding.too_many_tools': 'This agent has many tools. Keep only the tools needed for one concrete task.', + 'agents.drawer.finding.similar_tools': 'Tools “{left}” and “{right}” look similar. Rename them more precisely or keep one of them.', 'agents.drawer.filter_ops': 'Filter operations…', 'agents.drawer.ops_no_match': 'No operations match "{query}"', 'agents.drawer.ops_selected': '{count} operations selected', @@ -1577,6 +1580,9 @@ var TRANSLATIONS = { 'agents.drawer.selected': 'Выбрано: {count}', 'agents.drawer.operations_sub': 'Выберите MCP инструменты, которые будут доступны для этого агента.', 'agents.drawer.operations_sub_community': 'Выберите MCP инструменты, которые будут доступны для этого агента.', + 'agents.drawer.finding.title': 'Рекомендация', + 'agents.drawer.finding.too_many_tools': 'У агента выбрано много инструментов. Оставьте только те, которые нужны для одной конкретной задачи.', + 'agents.drawer.finding.similar_tools': 'Инструменты «{left}» и «{right}» похожи. Переименуйте их точнее или оставьте один вариант.', 'agents.drawer.filter_ops': 'Фильтр операций…', 'agents.drawer.ops_no_match': 'Нет операций по запросу "{query}"', 'agents.drawer.ops_selected': 'Выбрано операций: {count}', diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index dfaccdd..46a169b 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -63,9 +63,9 @@ pub use operation::{ pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion}; pub use tool_quality::{ - ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualityReport, - ToolQualitySchemaKind, ToolQualitySchemaNode, ToolQualitySeverity, - analyze_tool_identity_quality, analyze_tool_response_projection_quality, - analyze_tool_schema_quality, + ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet, + ToolQualityReport, ToolQualitySchemaKind, ToolQualitySchemaNode, ToolQualitySeverity, + analyze_agent_tool_catalog_quality, analyze_tool_identity_quality, + analyze_tool_response_projection_quality, analyze_tool_schema_quality, }; pub use workspace::{Workspace, WorkspaceStatus}; diff --git a/crates/crank-core/src/tool_quality.rs b/crates/crank-core/src/tool_quality.rs index 89722be..669eec1 100644 --- a/crates/crank-core/src/tool_quality.rs +++ b/crates/crank-core/src/tool_quality.rs @@ -8,6 +8,7 @@ const GENERIC_TOOL_NAMES: &[&str] = &["call_api", "manage", "execute", "request" const MULTI_ACTION_FIELD_NAMES: &[&str] = &["action", "mode", "operation", "type"]; const ENUM_LIKE_FIELD_NAMES: &[&str] = &["mode", "type", "status", "category"]; const MANY_PROJECTED_FIELDS_THRESHOLD: usize = 12; +const LARGE_AGENT_TOOL_CATALOG_THRESHOLD: usize = 8; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -70,6 +71,13 @@ pub struct ToolQualityMappingSet { pub rules: Vec, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolQualityCatalogTool { + pub name: String, + pub display_name: String, + pub description: String, +} + impl ToolQualityReport { pub fn new(findings: Vec) -> Self { let blocking = findings @@ -218,6 +226,37 @@ pub fn analyze_tool_response_projection_quality( ToolQualityReport::new(findings) } +pub fn analyze_agent_tool_catalog_quality(tools: &[ToolQualityCatalogTool]) -> ToolQualityReport { + let mut findings = Vec::new(); + + if tools.len() > LARGE_AGENT_TOOL_CATALOG_THRESHOLD { + findings.push(finding( + ToolQualitySeverity::Warning, + "agent_catalog_too_large", + "В каталоге агента много инструментов.", + Some("Оставьте только инструменты, которые нужны этому агенту для конкретной задачи."), + Some("agent.operations"), + )); + } + + for (left_index, left) in tools.iter().enumerate() { + for (right_offset, right) in tools[left_index + 1..].iter().enumerate() { + let right_index = left_index + right_offset + 1; + if tools_are_similar(left, right) { + findings.push(finding( + ToolQualitySeverity::Warning, + "agent_catalog_similar_tools", + "У агента есть похожие инструменты.", + Some("Переименуйте инструменты точнее или оставьте один вариант, чтобы модель не выбирала наугад."), + Some(&format!("agent.operations.{left_index},{right_index}")), + )); + } + } + } + + ToolQualityReport::new(findings) +} + fn is_safe_tool_name(value: &str) -> bool { let mut chars = value.chars(); let Some(first) = chars.next() else { @@ -230,6 +269,37 @@ fn is_safe_tool_name(value: &str) -> bool { chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_') } +fn tools_are_similar(left: &ToolQualityCatalogTool, right: &ToolQualityCatalogTool) -> bool { + let left_tokens = catalog_tool_tokens(left); + let right_tokens = catalog_tool_tokens(right); + if left_tokens.is_empty() || right_tokens.is_empty() { + return false; + } + + let shared = left_tokens + .iter() + .filter(|token| right_tokens.contains(token)) + .count(); + let smaller = left_tokens.len().min(right_tokens.len()); + + shared >= 2 && shared * 100 / smaller >= 60 +} + +fn catalog_tool_tokens(tool: &ToolQualityCatalogTool) -> Vec { + let text = format!("{} {} {}", tool.name, tool.display_name, tool.description); + let mut tokens = Vec::new(); + for token in text + .to_lowercase() + .split(|ch: char| !(ch.is_ascii_alphanumeric() || ('а'..='я').contains(&ch) || ch == 'ё')) + { + if token.len() < 3 || tokens.iter().any(|existing| existing == token) { + continue; + } + tokens.push(token.to_owned()); + } + tokens +} + fn analyze_schema_node( path: &str, schema: &ToolQualitySchemaNode, diff --git a/crates/crank-core/tests/unit/tool_quality.rs b/crates/crank-core/tests/unit/tool_quality.rs index 62d9a3b..1a1f7f3 100644 --- a/crates/crank-core/tests/unit/tool_quality.rs +++ b/crates/crank-core/tests/unit/tool_quality.rs @@ -1,6 +1,7 @@ use crank_core::{ - ToolDescription, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet, - ToolQualityReport, ToolQualitySchemaNode, ToolQualitySeverity, analyze_tool_identity_quality, + ToolDescription, ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, + ToolQualityMappingSet, ToolQualityReport, ToolQualitySchemaNode, ToolQualitySeverity, + analyze_agent_tool_catalog_quality, analyze_tool_identity_quality, analyze_tool_response_projection_quality, analyze_tool_schema_quality, }; @@ -235,6 +236,79 @@ fn marks_large_output_projection_as_info() { )); } +#[test] +fn warns_about_large_agent_tool_catalog() { + let tools: Vec = (0..9) + .map(|index| { + catalog_tool( + &format!("tool_{index}"), + &format!("Tool {index}"), + "Does one task", + ) + }) + .collect(); + + let report = analyze_agent_tool_catalog_quality(&tools); + + assert!(has_finding( + &report, + "agent_catalog_too_large", + ToolQualitySeverity::Warning + )); +} + +#[test] +fn warns_about_similar_agent_tools() { + let tools = vec![ + catalog_tool( + "get_exchange_rate", + "Получить курс валюты", + "Получает последний курс валюты", + ), + catalog_tool( + "fetch_exchange_rate", + "Запросить курс валюты", + "Получает текущий курс валюты", + ), + ]; + + let report = analyze_agent_tool_catalog_quality(&tools); + + assert!(has_finding( + &report, + "agent_catalog_similar_tools", + ToolQualitySeverity::Warning + )); +} + +#[test] +fn accepts_small_catalog_with_distinct_tools() { + let tools = vec![ + catalog_tool( + "get_exchange_rate", + "Получить курс валюты", + "Получает курс валюты", + ), + catalog_tool( + "get_weather_forecast", + "Получить прогноз погоды", + "Получает прогноз погоды", + ), + ]; + + let report = analyze_agent_tool_catalog_quality(&tools); + + assert!(report.findings.is_empty()); +} + +fn catalog_tool(name: &str, display_name: &str, description: &str) -> ToolQualityCatalogTool { + ToolQualityCatalogTool { + name: name.to_owned(), + display_name: display_name.to_owned(), + description: description.to_owned(), + } +} + fn has_finding(report: &ToolQualityReport, code: &str, severity: ToolQualitySeverity) -> bool { report .findings