diff --git a/apps/admin-api/src/service.rs b/apps/admin-api/src/service.rs index daa8fc9..cd4310e 100644 --- a/apps/admin-api/src/service.rs +++ b/apps/admin-api/src/service.rs @@ -12,10 +12,11 @@ use crank_core::{ NoopAuditSink, OperationId, OperationSecurityLevel, OperationStatus, OwnerOnlyPolicyEngine, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyScope, PlatformApiKeyStatus, PolicyEngine, ProductEdition, Protocol, ResponseCachePolicy, SampleId, Samples, Secret, SecretId, SecretKind, - SecretStatus, Target, ToolQualitySchemaKind, ToolQualitySchemaNode, UsagePeriod, UserId, - UserSessionId, WizardState, Workspace, WorkspaceId, WorkspaceStatus, + SecretStatus, Target, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind, + ToolQualitySchemaNode, UsagePeriod, UserId, UserSessionId, WizardState, Workspace, WorkspaceId, + WorkspaceStatus, }; -use crank_mapping::{JsonPathRoot, MappingSet, infer_mapping_from_samples}; +use crank_mapping::{JsonPathRoot, MappingRule, MappingSet, infer_mapping_from_samples}; use crank_registry::{ AgentSummary, AgentVersionRecord, CreateAgentDraftVersionRequest, CreateAgentRequest, CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest, @@ -1459,6 +1460,9 @@ impl AdminService { findings.extend( crank_core::analyze_tool_schema_quality("input_schema", &input_schema).findings, ); + let output_mapping = tool_quality_mapping_set(&payload.output_mapping); + findings + .extend(crank_core::analyze_tool_response_projection_quality(&output_mapping).findings); Ok(crank_core::ToolQualityReport::new(findings)) } @@ -3832,6 +3836,23 @@ fn tool_quality_schema_kind(kind: &SchemaKind) -> ToolQualitySchemaKind { } } +fn tool_quality_mapping_set(mapping: &MappingSet) -> ToolQualityMappingSet { + ToolQualityMappingSet { + rules: mapping + .rules + .iter() + .map(tool_quality_mapping_rule) + .collect(), + } +} + +fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule { + ToolQualityMappingRule { + source: rule.source.clone(), + target: rule.target.clone(), + } +} + #[cfg(test)] #[allow(clippy::items_after_test_module)] mod tests { diff --git a/apps/ui/css/wizard.css b/apps/ui/css/wizard.css index 7a4d284..8f3048c 100644 --- a/apps/ui/css/wizard.css +++ b/apps/ui/css/wizard.css @@ -1478,6 +1478,53 @@ min-height: 54px; } +.quality-findings { + display: grid; + gap: 10px; +} + +.quality-finding { + background: var(--surface-subtle); + border: 1px solid var(--border); + border-radius: 10px; + padding: 12px 14px; +} + +.quality-finding.warning { + background: rgba(210, 153, 31, 0.08); + border-color: rgba(210, 153, 31, 0.45); +} + +.quality-finding.error { + background: rgba(219, 83, 74, 0.08); + border-color: rgba(219, 83, 74, 0.45); +} + +.quality-finding.info { + background: rgba(57, 128, 247, 0.07); + border-color: rgba(57, 128, 247, 0.35); +} + +.quality-finding-title { + color: var(--text-primary); + font-size: 13px; + font-weight: 700; +} + +.quality-finding-body, +.quality-finding-action { + color: var(--text-secondary); + font-size: 12px; + line-height: 1.55; + margin-top: 4px; +} + +.quality-finding-path { + color: var(--text-muted); + font-size: 11px; + margin-top: 6px; +} + @media (max-width: 760px) { .agent-preview-summary { grid-template-columns: 1fr; diff --git a/apps/ui/html/wizard/step5.html b/apps/ui/html/wizard/step5.html index 73fa2f9..5166c93 100644 --- a/apps/ui/html/wizard/step5.html +++ b/apps/ui/html/wizard/step5.html @@ -154,6 +154,26 @@ tls: +
+
+
+ + + + +
+
+
Проверка качества инструмента
+
Проверьте имя, описание, схемы и результат, который увидит агент.
+
+ +
+
+
Проверка еще не запускалась.
+ +
+
+
diff --git a/apps/ui/js/api.js b/apps/ui/js/api.js index e95cdc3..6332141 100644 --- a/apps/ui/js/api.js +++ b/apps/ui/js/api.js @@ -194,6 +194,9 @@ createOperation: function(workspaceId, payload) { return post('/workspaces/' + encodeURIComponent(workspaceId) + '/operations', payload); }, + analyzeOperationQuality: function(workspaceId, payload) { + return post('/workspaces/' + encodeURIComponent(workspaceId) + '/operations/analyze-quality', payload); + }, updateOperation: function(workspaceId, operationId, payload) { return patch('/workspaces/' + encodeURIComponent(workspaceId) + '/operations/' + encodeURIComponent(operationId), payload); }, diff --git a/apps/ui/js/i18n.js b/apps/ui/js/i18n.js index 3712b04..983f9d4 100644 --- a/apps/ui/js/i18n.js +++ b/apps/ui/js/i18n.js @@ -638,6 +638,7 @@ var TRANSLATIONS = { 'wizard.busy.output_sample': 'Saving output example…', 'wizard.busy.generate': 'Rebuilding settings…', 'wizard.busy.test': 'Running test…', + 'wizard.busy.quality': 'Checking quality…', 'wizard.busy.export_yaml': 'Exporting YAML…', 'wizard.busy.import_yaml': 'Importing YAML…', 'wizard.busy.publish': 'Publishing operation…', @@ -664,6 +665,19 @@ var TRANSLATIONS = { 'wizard.test.response_copied_body': 'The latest test response was copied into the output example.', 'wizard.agent_preview.copied': 'Schema copied', 'wizard.agent_preview.copied_body': 'The MCP tool schema was copied to the clipboard.', + 'wizard.quality.title': 'Tool quality check', + 'wizard.quality.subtitle': 'Check the name, description, schemas and the result visible to the agent.', + 'wizard.quality.action': 'Check quality', + 'wizard.quality.empty': 'Quality check has not run yet.', + 'wizard.quality.no_findings': 'No issues found.', + 'wizard.quality.checked_title': 'Quality check completed', + 'wizard.quality.checked_body': 'Review the recommendations before publishing.', + 'wizard.quality.blocked_title': 'Quality check found blocking issues', + 'wizard.quality.blocked_body': 'Fix error-level findings before publishing.', + 'wizard.quality.blocking_error': 'Fix blocking quality findings before publishing.', + 'wizard.quality.severity_error': 'Error', + 'wizard.quality.severity_warning': 'Warning', + 'wizard.quality.severity_info': 'Info', 'wizard.boolean.yes': 'yes', 'wizard.boolean.no': 'no', 'wizard.yaml.exported': 'YAML exported', @@ -1436,11 +1450,12 @@ var TRANSLATIONS = { 'wizard.busy.output_sample': 'Сохранение выходного примера…', 'wizard.busy.generate': 'Пересборка настроек…', 'wizard.busy.test': 'Запуск теста…', + 'wizard.busy.quality': 'Проверка качества…', 'wizard.busy.export_yaml': 'Экспорт YAML…', 'wizard.busy.import_yaml': 'Импорт YAML…', 'wizard.busy.publish': 'Публикация операции…', 'wizard.busy.descriptor': 'Загрузка descriptor set…', - 'wizard.busy.wsdl': 'Inspection WSDL…', + 'wizard.busy.wsdl': 'Проверка WSDL…', 'wizard.sample.input_saved': 'Входной пример сохранен', 'wizard.sample.input_saved_body': 'Входной пример сохранен для этой операции.', 'wizard.sample.output_saved': 'Выходной пример сохранен', @@ -1462,6 +1477,19 @@ var TRANSLATIONS = { 'wizard.test.response_copied_body': 'Последний тестовый ответ скопирован в выходной пример.', 'wizard.agent_preview.copied': 'Схема скопирована', 'wizard.agent_preview.copied_body': 'Схема MCP инструмента скопирована в буфер обмена.', + 'wizard.quality.title': 'Проверка качества инструмента', + 'wizard.quality.subtitle': 'Проверьте имя, описание, схемы и результат, который увидит агент.', + 'wizard.quality.action': 'Проверить качество', + 'wizard.quality.empty': 'Проверка еще не запускалась.', + 'wizard.quality.no_findings': 'Замечаний нет.', + 'wizard.quality.checked_title': 'Проверка качества завершена', + 'wizard.quality.checked_body': 'Посмотрите рекомендации перед публикацией.', + 'wizard.quality.blocked_title': 'Проверка нашла блокирующие ошибки', + 'wizard.quality.blocked_body': 'Исправьте замечания уровня «Ошибка» перед публикацией.', + 'wizard.quality.blocking_error': 'Исправьте блокирующие замечания качества перед публикацией.', + 'wizard.quality.severity_error': 'Ошибка', + 'wizard.quality.severity_warning': 'Предупреждение', + 'wizard.quality.severity_info': 'Информация', 'wizard.boolean.yes': 'да', 'wizard.boolean.no': 'нет', 'wizard.yaml.exported': 'YAML экспортирован', diff --git a/apps/ui/js/wizard-live.js b/apps/ui/js/wizard-live.js index 717507c..f768b37 100644 --- a/apps/ui/js/wizard-live.js +++ b/apps/ui/js/wizard-live.js @@ -98,6 +98,7 @@ function bindWizardLiveActions() { bindLiveAction('wizard-upload-output-sample', tKey('wizard.busy.output_sample'), uploadOutputSampleFromWizard); bindLiveAction('wizard-generate-draft', tKey('wizard.busy.generate'), generateDraftFromWizard); bindLiveAction('wizard-run-test', tKey('wizard.busy.test'), runWizardTest); + bindLiveAction('wizard-run-quality', tKey('wizard.busy.quality'), analyzeWizardQuality); bindClick('wizard-copy-test-response', copyTestResponseToOutputSample); bindClick('wizard-copy-agent-preview', copyAgentFacingPreview); bindLiveAction('wizard-export-yaml', tKey('wizard.busy.export_yaml'), exportWizardYaml); @@ -398,6 +399,74 @@ function copyAgentFacingPreview() { } } +function severityLabel(severity) { + if (severity === 'error') return tKey('wizard.quality.severity_error'); + if (severity === 'warning') return tKey('wizard.quality.severity_warning'); + return tKey('wizard.quality.severity_info'); +} + +function renderQualityFindings(report) { + var empty = document.getElementById('wizard-quality-empty'); + var list = document.getElementById('wizard-quality-findings'); + if (!empty || !list) return; + + list.replaceChildren(); + var findings = report && Array.isArray(report.findings) ? report.findings : []; + if (!findings.length) { + empty.textContent = tKey('wizard.quality.no_findings'); + empty.hidden = false; + list.hidden = true; + return; + } + + empty.hidden = true; + list.hidden = false; + findings.forEach(function(finding) { + var item = document.createElement('div'); + item.className = 'quality-finding ' + (finding.severity || 'info'); + item.dataset.code = finding.code || ''; + + var title = document.createElement('div'); + title.className = 'quality-finding-title'; + title.textContent = severityLabel(finding.severity) + ': ' + (finding.message || finding.code || ''); + item.appendChild(title); + + if (finding.suggested_action) { + var action = document.createElement('div'); + action.className = 'quality-finding-action'; + action.textContent = finding.suggested_action; + item.appendChild(action); + } + + if (finding.field_path) { + var path = document.createElement('div'); + path.className = 'quality-finding-path'; + path.textContent = finding.field_path; + item.appendChild(path); + } + + list.appendChild(item); + }); +} + +async function analyzeWizardQuality() { + if (!wizardWorkspaceId) { + throw new Error(tKey('wizard.error.no_workspace')); + } + + var report = await window.CrankApi.analyzeOperationQuality( + wizardWorkspaceId, + collectWizardPayload() + ); + renderQualityFindings(report); + showWizardLiveStatus( + report.blocking ? tKey('wizard.quality.blocked_title') : tKey('wizard.quality.checked_title'), + report.blocking ? tKey('wizard.quality.blocked_body') : tKey('wizard.quality.checked_body'), + !!report.blocking + ); + return report; +} + async function exportWizardYaml() { await persistCurrentDraft(true); var yaml = await window.CrankApi.exportOperation(wizardWorkspaceId, wizardEditId, { @@ -423,6 +492,10 @@ async function importWizardYaml() { } async function publishWizardOperation() { + var report = await analyzeWizardQuality(); + if (report.blocking) { + throw new Error(tKey('wizard.quality.blocking_error')); + } await persistCurrentDraft(true); var published = await window.CrankApi.publishOperation( wizardWorkspaceId, diff --git a/apps/ui/tests/e2e/wizard.spec.js b/apps/ui/tests/e2e/wizard.spec.js index ba39fe6..1cc05b8 100644 --- a/apps/ui/tests/e2e/wizard.spec.js +++ b/apps/ui/tests/e2e/wizard.spec.js @@ -490,6 +490,29 @@ test('wizard shows agent-facing MCP preview from current draft fields', async ({ }, ); + await page.route( + `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/analyze-quality`, + async (route) => { + const payload = route.request().postDataJSON(); + expect(payload.output_mapping.rules.length).toBeGreaterThan(0); + await route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + blocking: false, + findings: [ + { + severity: 'warning', + code: 'response_projection_full_body', + message: 'Результат инструмента отдает агенту весь ответ API.', + suggested_action: 'Выберите только поля, которые нужны агенту.', + field_path: 'output_mapping.rules.0.source', + }, + ], + }), + }); + }, + ); + await page.goto(`/wizard/?mode=edit&operationId=${operationId}`); await page.locator('#btn-continue').click(); await page.locator('#btn-continue').click(); @@ -502,6 +525,12 @@ test('wizard shows agent-facing MCP preview from current draft fields', async ({ await expect(page.locator('#agent-preview-tool-call')).toHaveValue(/"name": "frankfurter_monthly_rates"/); await expect(page.locator('#agent-preview-tool-call')).toHaveValue(/"from": "USD"/); await expect(page.locator('#agent-preview-success-response')).toHaveValue(/0\.91/); + await page.evaluate(() => window.CrankWizardShell.doGoToStep(5)); + await expect(page.locator('#step-panel-5')).toBeVisible(); + await page.locator('#wizard-run-quality').scrollIntoViewIfNeeded(); + await page.locator('#wizard-run-quality').click(); + await expect(page.locator('#wizard-quality-findings')).toContainText(/весь ответ API|full response/i); + await expect(page.locator('#wizard-quality-findings')).toContainText('output_mapping.rules.0.source'); }); test('wizard edit mode preserves explicit request mapping targets on save', async ({ page }) => { diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index f0652ee..dfaccdd 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -63,7 +63,9 @@ pub use operation::{ pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol}; pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion}; pub use tool_quality::{ - ToolQualityFinding, ToolQualityReport, ToolQualitySchemaKind, ToolQualitySchemaNode, - ToolQualitySeverity, analyze_tool_identity_quality, analyze_tool_schema_quality, + ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualityReport, + ToolQualitySchemaKind, ToolQualitySchemaNode, ToolQualitySeverity, + analyze_tool_identity_quality, analyze_tool_response_projection_quality, + analyze_tool_schema_quality, }; pub use workspace::{Workspace, WorkspaceStatus}; diff --git a/crates/crank-core/src/tool_quality.rs b/crates/crank-core/src/tool_quality.rs index c05d3a4..89722be 100644 --- a/crates/crank-core/src/tool_quality.rs +++ b/crates/crank-core/src/tool_quality.rs @@ -7,6 +7,7 @@ const MIN_DESCRIPTION_CHARS: usize = 40; const GENERIC_TOOL_NAMES: &[&str] = &["call_api", "manage", "execute", "request"]; const MULTI_ACTION_FIELD_NAMES: &[&str] = &["action", "mode", "operation", "type"]; const ENUM_LIKE_FIELD_NAMES: &[&str] = &["mode", "type", "status", "category"]; +const MANY_PROJECTED_FIELDS_THRESHOLD: usize = 12; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -58,6 +59,17 @@ pub struct ToolQualitySchemaNode { pub enum_values: Vec, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ToolQualityMappingRule { + pub source: String, + pub target: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct ToolQualityMappingSet { + pub rules: Vec, +} + impl ToolQualityReport { pub fn new(findings: Vec) -> Self { let blocking = findings @@ -175,6 +187,37 @@ pub fn analyze_tool_schema_quality( ToolQualityReport::new(findings) } +pub fn analyze_tool_response_projection_quality( + output_mapping: &ToolQualityMappingSet, +) -> ToolQualityReport { + let mut findings = Vec::new(); + + for (index, rule) in output_mapping.rules.iter().enumerate() { + let source = rule.source.trim(); + if source == "$.response.body" || source == "$.response.data" { + findings.push(finding( + ToolQualitySeverity::Warning, + "response_projection_full_body", + "Результат инструмента отдает агенту весь ответ API.", + Some("Выберите только поля, которые нужны агенту для ответа пользователю или следующего действия."), + Some(&format!("output_mapping.rules.{index}.source")), + )); + } + } + + if output_mapping.rules.len() >= MANY_PROJECTED_FIELDS_THRESHOLD { + findings.push(finding( + ToolQualitySeverity::Info, + "response_projection_many_fields", + "В результат инструмента попадает много полей.", + Some("Проверьте, что каждое поле действительно нужно агенту. Лишние данные увеличивают риск ошибок и галлюцинаций."), + Some("output_mapping.rules"), + )); + } + + ToolQualityReport::new(findings) +} + fn is_safe_tool_name(value: &str) -> bool { let mut chars = value.chars(); let Some(first) = chars.next() else { diff --git a/crates/crank-core/tests/unit/tool_quality.rs b/crates/crank-core/tests/unit/tool_quality.rs index 6ea01dc..62d9a3b 100644 --- a/crates/crank-core/tests/unit/tool_quality.rs +++ b/crates/crank-core/tests/unit/tool_quality.rs @@ -1,6 +1,7 @@ use crank_core::{ - ToolDescription, ToolQualityFinding, ToolQualityReport, ToolQualitySchemaNode, - ToolQualitySeverity, analyze_tool_identity_quality, analyze_tool_schema_quality, + ToolDescription, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet, + ToolQualityReport, ToolQualitySchemaNode, ToolQualitySeverity, analyze_tool_identity_quality, + analyze_tool_response_projection_quality, analyze_tool_schema_quality, }; #[test] @@ -181,6 +182,59 @@ fn recommends_enum_for_mode_like_string_parameter() { )); } +#[test] +fn warns_when_output_mapping_returns_full_response_body() { + let mapping = ToolQualityMappingSet { + rules: vec![ToolQualityMappingRule { + source: "$.response.body".to_owned(), + target: "$.output".to_owned(), + }], + }; + + let report = analyze_tool_response_projection_quality(&mapping); + + assert!(has_finding( + &report, + "response_projection_full_body", + ToolQualitySeverity::Warning + )); +} + +#[test] +fn accepts_output_mapping_with_selected_fields() { + let mapping = ToolQualityMappingSet { + rules: vec![ToolQualityMappingRule { + source: "$.response.body.rates.EUR".to_owned(), + target: "$.output.rate".to_owned(), + }], + }; + + let report = analyze_tool_response_projection_quality(&mapping); + + assert!(!report.blocking); + assert!(report.findings.is_empty()); +} + +#[test] +fn marks_large_output_projection_as_info() { + let mapping = ToolQualityMappingSet { + rules: (0..12) + .map(|index| ToolQualityMappingRule { + source: format!("$.response.body.field_{index}"), + target: format!("$.output.field_{index}"), + }) + .collect(), + }; + + let report = analyze_tool_response_projection_quality(&mapping); + + assert!(has_finding( + &report, + "response_projection_many_fields", + ToolQualitySeverity::Info + )); +} + fn has_finding(report: &ToolQualityReport, code: &str, severity: ToolQualitySeverity) -> bool { report .findings