Warn about confusing agent tool catalogs

crates/crank-core/src/lib.rs

@@ -63,9 +63,9 @@ pub use operation::{
 pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
 pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
 pub use tool_quality::{
-    ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualityReport,
-    ToolQualitySchemaKind, ToolQualitySchemaNode, ToolQualitySeverity,
-    analyze_tool_identity_quality, analyze_tool_response_projection_quality,
-    analyze_tool_schema_quality,
+    ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
+    ToolQualityReport, ToolQualitySchemaKind, ToolQualitySchemaNode, ToolQualitySeverity,
+    analyze_agent_tool_catalog_quality, analyze_tool_identity_quality,
+    analyze_tool_response_projection_quality, analyze_tool_schema_quality,
 };
 pub use workspace::{Workspace, WorkspaceStatus};

crates/crank-core/src/tool_quality.rs

@@ -8,6 +8,7 @@ const GENERIC_TOOL_NAMES: &[&str] = &["call_api", "manage", "execute", "request"
 const MULTI_ACTION_FIELD_NAMES: &[&str] = &["action", "mode", "operation", "type"];
 const ENUM_LIKE_FIELD_NAMES: &[&str] = &["mode", "type", "status", "category"];
 const MANY_PROJECTED_FIELDS_THRESHOLD: usize = 12;
+const LARGE_AGENT_TOOL_CATALOG_THRESHOLD: usize = 8;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
@@ -70,6 +71,13 @@ pub struct ToolQualityMappingSet {
     pub rules: Vec<ToolQualityMappingRule>,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ToolQualityCatalogTool {
+    pub name: String,
+    pub display_name: String,
+    pub description: String,
+}
+
 impl ToolQualityReport {
     pub fn new(findings: Vec<ToolQualityFinding>) -> Self {
         let blocking = findings
@@ -218,6 +226,37 @@ pub fn analyze_tool_response_projection_quality(
     ToolQualityReport::new(findings)
 }
 
+pub fn analyze_agent_tool_catalog_quality(tools: &[ToolQualityCatalogTool]) -> ToolQualityReport {
+    let mut findings = Vec::new();
+
+    if tools.len() > LARGE_AGENT_TOOL_CATALOG_THRESHOLD {
+        findings.push(finding(
+            ToolQualitySeverity::Warning,
+            "agent_catalog_too_large",
+            "В каталоге агента много инструментов.",
+            Some("Оставьте только инструменты, которые нужны этому агенту для конкретной задачи."),
+            Some("agent.operations"),
+        ));
+    }
+
+    for (left_index, left) in tools.iter().enumerate() {
+        for (right_offset, right) in tools[left_index + 1..].iter().enumerate() {
+            let right_index = left_index + right_offset + 1;
+            if tools_are_similar(left, right) {
+                findings.push(finding(
+                    ToolQualitySeverity::Warning,
+                    "agent_catalog_similar_tools",
+                    "У агента есть похожие инструменты.",
+                    Some("Переименуйте инструменты точнее или оставьте один вариант, чтобы модель не выбирала наугад."),
+                    Some(&format!("agent.operations.{left_index},{right_index}")),
+                ));
+            }
+        }
+    }
+
+    ToolQualityReport::new(findings)
+}
+
 fn is_safe_tool_name(value: &str) -> bool {
     let mut chars = value.chars();
     let Some(first) = chars.next() else {
@@ -230,6 +269,37 @@ fn is_safe_tool_name(value: &str) -> bool {
     chars.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_')
 }
 
+fn tools_are_similar(left: &ToolQualityCatalogTool, right: &ToolQualityCatalogTool) -> bool {
+    let left_tokens = catalog_tool_tokens(left);
+    let right_tokens = catalog_tool_tokens(right);
+    if left_tokens.is_empty() || right_tokens.is_empty() {
+        return false;
+    }
+
+    let shared = left_tokens
+        .iter()
+        .filter(|token| right_tokens.contains(token))
+        .count();
+    let smaller = left_tokens.len().min(right_tokens.len());
+
+    shared >= 2 && shared * 100 / smaller >= 60
+}
+
+fn catalog_tool_tokens(tool: &ToolQualityCatalogTool) -> Vec<String> {
+    let text = format!("{} {} {}", tool.name, tool.display_name, tool.description);
+    let mut tokens = Vec::new();
+    for token in text
+        .to_lowercase()
+        .split(|ch: char| !(ch.is_ascii_alphanumeric() || ('а'..='я').contains(&ch) || ch == 'ё'))
+    {
+        if token.len() < 3 || tokens.iter().any(|existing| existing == token) {
+            continue;
+        }
+        tokens.push(token.to_owned());
+    }
+    tokens
+}
+
 fn analyze_schema_node(
     path: &str,
     schema: &ToolQualitySchemaNode,

crates/crank-core/tests/unit/tool_quality.rs

@@ -1,6 +1,7 @@
 use crank_core::{
-    ToolDescription, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
-    ToolQualityReport, ToolQualitySchemaNode, ToolQualitySeverity, analyze_tool_identity_quality,
+    ToolDescription, ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule,
+    ToolQualityMappingSet, ToolQualityReport, ToolQualitySchemaNode, ToolQualitySeverity,
+    analyze_agent_tool_catalog_quality, analyze_tool_identity_quality,
     analyze_tool_response_projection_quality, analyze_tool_schema_quality,
 };
 
@@ -235,6 +236,79 @@ fn marks_large_output_projection_as_info() {
     ));
 }
 
+#[test]
+fn warns_about_large_agent_tool_catalog() {
+    let tools: Vec<ToolQualityCatalogTool> = (0..9)
+        .map(|index| {
+            catalog_tool(
+                &format!("tool_{index}"),
+                &format!("Tool {index}"),
+                "Does one task",
+            )
+        })
+        .collect();
+
+    let report = analyze_agent_tool_catalog_quality(&tools);
+
+    assert!(has_finding(
+        &report,
+        "agent_catalog_too_large",
+        ToolQualitySeverity::Warning
+    ));
+}
+
+#[test]
+fn warns_about_similar_agent_tools() {
+    let tools = vec![
+        catalog_tool(
+            "get_exchange_rate",
+            "Получить курс валюты",
+            "Получает последний курс валюты",
+        ),
+        catalog_tool(
+            "fetch_exchange_rate",
+            "Запросить курс валюты",
+            "Получает текущий курс валюты",
+        ),
+    ];
+
+    let report = analyze_agent_tool_catalog_quality(&tools);
+
+    assert!(has_finding(
+        &report,
+        "agent_catalog_similar_tools",
+        ToolQualitySeverity::Warning
+    ));
+}
+
+#[test]
+fn accepts_small_catalog_with_distinct_tools() {
+    let tools = vec![
+        catalog_tool(
+            "get_exchange_rate",
+            "Получить курс валюты",
+            "Получает курс валюты",
+        ),
+        catalog_tool(
+            "get_weather_forecast",
+            "Получить прогноз погоды",
+            "Получает прогноз погоды",
+        ),
+    ];
+
+    let report = analyze_agent_tool_catalog_quality(&tools);
+
+    assert!(report.findings.is_empty());
+}
+
+fn catalog_tool(name: &str, display_name: &str, description: &str) -> ToolQualityCatalogTool {
+    ToolQualityCatalogTool {
+        name: name.to_owned(),
+        display_name: display_name.to_owned(),
+        description: description.to_owned(),
+    }
+}
+
 fn has_finding(report: &ToolQualityReport, code: &str, severity: ToolQualitySeverity) -> bool {
     report
         .findings