diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df7464f..7f169f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,11 @@ npx playwright test [`docs/capability-baseline/manifest.json`](./docs/capability-baseline/manifest.json), sanitized results и SHA-256. Нельзя вручную объявлять pass для failed, flaky, skipped, not-run или manual-only evidence. +- Для записи результатов этапа и разделения исторического capability baseline + и текущего completion evidence следуйте + [правилам evidence разработки](./docs/development-evidence.md). Перед этапом + фиксируйте полный clean Git baseline; проверенную реализацию указывайте + отдельной ревизией. - Не добавляйте секреты, токены, приватные адреса и локальные настройки. - Не коммитьте `AGENTS.md`, `TASKS.md`, `.env` и временные файлы. - Для изменений SQL-запросов обновляйте `.sqlx`, если это требуется SQLx. diff --git a/apps/admin-api/src/service/agents.rs b/apps/admin-api/src/service/agents.rs index f9daa01..7359f2b 100644 --- a/apps/admin-api/src/service/agents.rs +++ b/apps/admin-api/src/service/agents.rs @@ -5,9 +5,8 @@ use crank_core::{ ToolAccessMode, ToolSelectionPolicy, UsagePeriod, WorkspaceId, search_tool_catalog, }; use crank_registry::{ - AgentStateExpectation, AgentVersionRecord, CreateAgentDraftVersionRequest, CreateAgentRequest, - PublishAgentRequest, SaveAgentCatalogConfigRequest, UpdateAgentSummaryRequest, UsageBucket, - UsageQuery, + AgentStateExpectation, AgentVersionRecord, CreateAgentRequest, PublishAgentRequest, + SaveAgentCatalogConfigRequest, UpdateAgentSummaryRequest, UsageBucket, UsageQuery, }; use serde_json::json; use sha2::{Digest, Sha256}; @@ -471,8 +470,9 @@ impl AdminService { payload: AgentCatalogPayload, expected_state: Option<&AgentStateExpectation>, ) -> Result { + let agent = self.get_agent(workspace_id, agent_id).await?; let current_version = self - .ensure_editable_agent_version(workspace_id, agent_id) + .get_agent_version(workspace_id, agent_id, agent.current_draft_version) .await?; let (payload, requested_policy) = payload.into_parts(); let bindings = payload @@ -492,8 +492,9 @@ impl AdminService { .unwrap_or_else(|| current_version.snapshot.tool_selection_policy.clone()); validate_tool_selection_policy(&tool_selection_policy, &bindings)?; - self.registry - .save_agent_catalog_config(SaveAgentCatalogConfigRequest { + let saved_version = self + .registry + .save_agent_catalog_config_editable(SaveAgentCatalogConfigRequest { workspace_id, agent_id, agent_version: current_version.version, @@ -505,47 +506,12 @@ impl AdminService { info!( name: "admin.agent.bindings_saved", agent_id = %agent_id.as_str(), - version = current_version.version, + version = saved_version, binding_count = bindings.len(), "agent bindings saved" ); - self.get_agent_version(workspace_id, agent_id, current_version.version) - .await - } - - async fn ensure_editable_agent_version( - &self, - workspace_id: &WorkspaceId, - agent_id: &AgentId, - ) -> Result { - let agent = self.get_agent(workspace_id, agent_id).await?; - let current = self - .get_agent_version(workspace_id, agent_id, agent.current_draft_version) - .await?; - if agent.latest_published_version != Some(agent.current_draft_version) { - return Ok(current); - } - - let now = OffsetDateTime::now_utc(); - let draft = AgentVersion { - agent_id: agent_id.clone(), - version: current.version + 1, - status: AgentStatus::Draft, - instructions: current.snapshot.instructions.clone(), - tool_selection_policy: current.snapshot.tool_selection_policy.clone(), - created_at: now, - }; - self.registry - .create_agent_draft_version(CreateAgentDraftVersionRequest { - workspace_id, - agent_id, - version: &draft, - bindings: ¤t.bindings, - updated_at: &now, - }) - .await?; - self.get_agent_version(workspace_id, agent_id, draft.version) + self.get_agent_version(workspace_id, agent_id, saved_version) .await } diff --git a/apps/admin-api/tests/integration.rs b/apps/admin-api/tests/integration.rs index eb3215b..a6ff2f7 100644 --- a/apps/admin-api/tests/integration.rs +++ b/apps/admin-api/tests/integration.rs @@ -1,5 +1,6 @@ mod integration { mod agent_catalog; + mod agent_catalog_edit; mod auth_rate_limit; mod common; mod community_access_usage; diff --git a/apps/admin-api/tests/integration/agent_catalog_edit.rs b/apps/admin-api/tests/integration/agent_catalog_edit.rs new file mode 100644 index 0000000..1986044 --- /dev/null +++ b/apps/admin-api/tests/integration/agent_catalog_edit.rs @@ -0,0 +1,889 @@ +use super::common::*; + +use crank_core::{Workspace, WorkspaceId, WorkspaceStatus}; +use crank_registry::{CreateWorkspaceRequest, PostgresRegistry}; +use serde_json::{Value, json}; +use serial_test::serial; +use time::OffsetDateTime; + +const DEFAULT_WORKSPACE_ID: &str = "ws_default"; + +fn binding(operation_id: &str, tool_name: &str, tool_title: &str) -> Value { + json!({ + "operation_id": operation_id, + "operation_version": 1, + "tool_name": tool_name, + "tool_title": tool_title, + "tool_description_override": null, + "enabled": true + }) +} + +async fn create_draft_operation( + client: &reqwest::Client, + base_url: &TestServer, + upstream: &str, + name: &str, +) -> String { + let operation = assert_success_json( + client + .post(format!("{base_url}/operations")) + .json(&test_operation_payload(upstream, name)) + .send() + .await + .unwrap(), + ) + .await; + operation["operation_id"].as_str().unwrap().to_owned() +} + +async fn create_published_operation( + client: &reqwest::Client, + base_url: &TestServer, + upstream: &str, + name: &str, +) -> String { + let operation_id = create_draft_operation(client, base_url, upstream, name).await; + let etag = operation_etag(client, base_url, &operation_id).await; + assert_success_json( + client + .post(format!("{base_url}/operations/{operation_id}/publish")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!({ "version": 1 })) + .send() + .await + .unwrap(), + ) + .await; + operation_id +} + +async fn create_published_agent( + client: &reqwest::Client, + base_url: &TestServer, + slug: &str, + operation_id: &str, + tool_name: &str, +) -> (String, Value) { + let agent = assert_success_json( + client + .post(format!("{base_url}/agents")) + .json(&json!({ + "slug": slug, + "display_name": slug, + "description": "agent catalog edit fixture", + "instructions": { "system_prompt": "Use only published bindings" }, + "tool_selection_policy": {} + })) + .send() + .await + .unwrap(), + ) + .await; + let agent_id = agent["agent_id"].as_str().unwrap().to_owned(); + + let etag = agent_etag(client, base_url, &agent_id).await; + assert_success_json( + client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!([binding(operation_id, tool_name, "Base Tool")])) + .send() + .await + .unwrap(), + ) + .await; + + let etag = agent_etag(client, base_url, &agent_id).await; + assert_success_json( + client + .post(format!("{base_url}/agents/{agent_id}/publish")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!({ "version": 1 })) + .send() + .await + .unwrap(), + ) + .await; + + let published_v1 = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}/versions/1")) + .send() + .await + .unwrap(), + ) + .await; + + (agent_id, published_v1) +} + +async fn agent_version_count(db: &PostgresRegistry, agent_id: &str) -> i64 { + sqlx::query_scalar("select count(*) from agent_versions where agent_id = $1") + .bind(agent_id) + .fetch_one(db.pool()) + .await + .unwrap() +} + +async fn assert_no_durable_draft( + client: &reqwest::Client, + db: &PostgresRegistry, + base_url: &TestServer, + agent_id: &str, + published_v1: &Value, +) { + let summary = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!( + summary["current_draft_version"], 1, + "a rejected edit must not fork a Draft Version: {summary}" + ); + assert_eq!(summary["latest_published_version"], 1, "{summary}"); + assert_eq!(summary["status"], "published", "{summary}"); + assert_eq!( + agent_version_count(db, agent_id).await, + 1, + "a rejected edit must not persist a new Agent Version row" + ); + + let published_after = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}/versions/1")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!( + &published_after, published_v1, + "the Published snapshot and its bindings must stay unchanged after a rejected edit" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn published_agent_first_edit_forks_one_draft_and_keeps_published_snapshot() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_first"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let operation_id = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_first_tool").await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-first", + &operation_id, + "edit_first_tool", + ) + .await; + + let summary_before = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!(summary_before["current_draft_version"], 1); + assert_eq!(summary_before["latest_published_version"], 1); + assert_eq!(summary_before["status"], "published"); + + let desired_policy = json!({ + "mode": "search", + "groups": [{ + "id": "crm-group", + "name": "CRM", + "description": "CRM tools", + "tool_names": ["edit_first_tool"] + }], + "search": { "max_results": 5 } + }); + + let edit_etag = agent_etag(&client, &base_url, &agent_id).await; + let response = client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, edit_etag) + .json(&json!({ + "bindings": [binding(&operation_id, "edit_first_tool", "Edited Tool")], + "tool_selection_policy": desired_policy + })) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.json::().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::OK, + "the first edit of a Published Agent must fork and save under the original If-Match: {body}" + ); + assert_eq!(body["version"], 2, "{body}"); + + let summary_after = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!(summary_after["current_draft_version"], 2, "{summary_after}"); + assert_eq!( + summary_after["latest_published_version"], 1, + "{summary_after}" + ); + assert_eq!(summary_after["status"], "published", "{summary_after}"); + + assert_eq!( + agent_version_count(&db, &agent_id).await, + 2, + "the first edit must create exactly one additional Draft Version" + ); + + let draft = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}/versions/2")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!(draft["status"], "draft", "{draft}"); + assert_eq!( + draft["snapshot"]["instructions"], + published_v1["snapshot"]["instructions"] + ); + assert_eq!(draft["bindings"].as_array().unwrap().len(), 1, "{draft}"); + assert_eq!(draft["bindings"][0]["tool_title"], "Edited Tool", "{draft}"); + assert_eq!( + draft["snapshot"]["tool_selection_policy"]["mode"], "search", + "{draft}" + ); + assert_eq!( + draft["snapshot"]["tool_selection_policy"]["search"]["max_results"], 5, + "{draft}" + ); + + let published_after = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}/versions/1")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!( + published_after, published_v1, + "the Published v1 snapshot must not change when the Draft is forked" + ); + + let next_etag = agent_etag(&client, &base_url, &agent_id).await; + let second_response = client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, next_etag) + .json(&json!({ + "bindings": [binding(&operation_id, "edit_first_tool", "Edited Again")], + "tool_selection_policy": desired_policy + })) + .send() + .await + .unwrap(); + let second_status = second_response.status(); + let second_body = second_response.text().await.unwrap(); + assert_eq!( + second_status, + reqwest::StatusCode::OK, + "a subsequent valid edit on the Draft must succeed: {second_body}" + ); + assert_eq!( + agent_version_count(&db, &agent_id).await, + 2, + "a subsequent valid edit must stay on the existing Draft v2" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn rejected_invalid_policy_leaves_no_durable_draft() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_invalid_policy"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let operation_id = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_policy_tool") + .await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-policy", + &operation_id, + "edit_policy_tool", + ) + .await; + + let etag = agent_etag(&client, &base_url, &agent_id).await; + let response = client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!({ + "bindings": [], + "tool_selection_policy": { + "mode": "direct", + "groups": [{ + "id": "group-1", + "name": "Group", + "description": "groups require search mode", + "tool_names": [] + }] + } + })) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.text().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::BAD_REQUEST, + "a grouping policy in direct mode must be rejected: {body}" + ); + + assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn rejected_unpublished_operation_leaves_no_durable_draft() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_draft_op"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let live_operation = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_live_tool").await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-draft-op", + &live_operation, + "edit_live_tool", + ) + .await; + + let draft_operation = + create_draft_operation(&client, &base_url, &upstream_base_url, "edit_draft_tool").await; + + let etag = agent_etag(&client, &base_url, &agent_id).await; + let response = client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!([binding( + &draft_operation, + "edit_draft_tool", + "Draft Tool" + )])) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.text().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::UNPROCESSABLE_ENTITY, + "binding an unpublished Operation must be rejected: {body}" + ); + let body: Value = serde_json::from_str(&body).unwrap(); + assert_eq!( + body["error"]["code"], "agent_binding_not_published", + "{body}" + ); + + assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn rejected_archived_operation_leaves_no_durable_draft() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_archived_op"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let live_operation = create_published_operation( + &client, + &base_url, + &upstream_base_url, + "edit_live_tool_arch", + ) + .await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-archived-op", + &live_operation, + "edit_live_tool_arch", + ) + .await; + + let archived_operation = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_archived_tool") + .await; + let operation_etag = operation_etag(&client, &base_url, &archived_operation).await; + let archived = client + .post(format!( + "{base_url}/operations/{archived_operation}/archive" + )) + .header(reqwest::header::IF_MATCH, operation_etag) + .send() + .await + .unwrap(); + let archived_status = archived.status(); + let archived_body = archived.text().await.unwrap(); + assert!(archived_status.is_success(), "{archived_body}"); + + let etag = agent_etag(&client, &base_url, &agent_id).await; + let response = client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!([binding( + &archived_operation, + "edit_archived_tool", + "Archived Tool" + )])) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.text().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::UNPROCESSABLE_ENTITY, + "binding an archived Operation must be rejected: {body}" + ); + let body: Value = serde_json::from_str(&body).unwrap(); + assert_eq!( + body["error"]["code"], "agent_binding_archived_operation", + "{body}" + ); + + assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn rejected_foreign_operation_leaves_no_durable_draft() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_foreign_op"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await; + let client = authorized_client(&base_url).await; + + let live_operation = create_published_operation( + &client, + &base_url, + &upstream_base_url, + "edit_live_tool_foreign", + ) + .await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-foreign-op", + &live_operation, + "edit_live_tool_foreign", + ) + .await; + + let foreign_workspace = WorkspaceId::new("ws_agent_edit_foreign"); + let now = OffsetDateTime::now_utc(); + db.create_workspace(CreateWorkspaceRequest { + workspace: &Workspace { + id: foreign_workspace.clone(), + slug: "agent-edit-foreign".to_owned(), + display_name: "Agent Edit Foreign".to_owned(), + status: WorkspaceStatus::Active, + settings: json!({}), + created_at: now, + updated_at: now, + }, + }) + .await + .unwrap(); + + let foreign_service = test_service( + db.clone(), + test_storage_root("agent_catalog_edit_foreign_service"), + test_auth_settings(), + test_secret_crypto(), + ); + let foreign_operation = foreign_service + .create_operation( + &foreign_workspace, + test_operation_payload(&upstream_base_url, "edit_foreign_tool"), + ) + .await + .unwrap(); + let foreign_operation_id = foreign_operation.operation_id; + + let etag = agent_etag(&client, &base_url, &agent_id).await; + let response = client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!([binding( + &foreign_operation_id, + "edit_foreign_tool", + "Foreign Tool" + )])) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.text().await.unwrap(); + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "binding an Operation from another workspace must be rejected: {body}" + ); + let body: Value = serde_json::from_str(&body).unwrap(); + assert_eq!( + body["error"]["code"], "agent_binding_scope_denied", + "{body}" + ); + + assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn concurrent_first_edits_with_same_etag_have_one_winner() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_concurrent"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let operation_id = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_tool").await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-race", + &operation_id, + "edit_race_tool", + ) + .await; + + // Single original precondition taken once from the Published state. + let original_etag = agent_etag(&client, &base_url, &agent_id).await; + let endpoint = format!("{base_url}/agents/{agent_id}/bindings"); + let payload_a = json!([binding(&operation_id, "edit_race_tool", "Race Winner A")]); + let payload_b = json!([binding(&operation_id, "edit_race_tool", "Race Winner B")]); + + let spawn_edit = |payload: Value, etag: String, endpoint: String| { + let client = client.clone(); + tokio::spawn(async move { + let response = client + .post(endpoint) + .header(reqwest::header::IF_MATCH, etag) + .json(&payload) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.json::().await.unwrap(); + (status, body) + }) + }; + + let first = spawn_edit(payload_a, original_etag.clone(), endpoint.clone()); + let second = spawn_edit(payload_b, original_etag, endpoint); + let (first, second) = tokio::join!(first, second); + let results = vec![first.unwrap(), second.unwrap()]; + + let successes = results + .iter() + .filter(|(status, _)| status.is_success()) + .count(); + let conflicts = results + .iter() + .filter(|(status, _)| *status == reqwest::StatusCode::CONFLICT) + .count(); + assert_eq!( + successes, 1, + "exactly one concurrent first edit must win: {results:?}" + ); + assert_eq!( + conflicts, 1, + "the losing concurrent first edit must report a stable conflict: {results:?}" + ); + + let conflict_body = results + .iter() + .find(|(status, _)| *status == reqwest::StatusCode::CONFLICT) + .map(|(_, body)| body) + .unwrap(); + assert_eq!( + conflict_body["error"]["code"], "agent_stale_revision", + "{conflict_body}" + ); + + let summary = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!(summary["current_draft_version"], 2, "{summary}"); + assert_eq!(summary["latest_published_version"], 1, "{summary}"); + assert_eq!( + agent_version_count(&db, &agent_id).await, + 2, + "the losing concurrent edit must not fork a second Draft" + ); + + let published_after = assert_success_json( + client + .get(format!("{base_url}/agents/{agent_id}/versions/1")) + .send() + .await + .unwrap(), + ) + .await; + assert_eq!(published_after, published_v1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn first_edit_after_operation_archive_leaves_no_durable_draft() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_archive_ordering"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let live_operation = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_order_live").await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-order", + &live_operation, + "edit_order_live", + ) + .await; + + let archived_operation = create_published_operation( + &client, + &base_url, + &upstream_base_url, + "edit_order_archived", + ) + .await; + let operation_etag = operation_etag(&client, &base_url, &archived_operation).await; + let archived = client + .post(format!( + "{base_url}/operations/{archived_operation}/archive" + )) + .header(reqwest::header::IF_MATCH, operation_etag) + .send() + .await + .unwrap(); + let archived_status = archived.status(); + let archived_body = archived.text().await.unwrap(); + assert!(archived_status.is_success(), "{archived_body}"); + + let etag = agent_etag(&client, &base_url, &agent_id).await; + let response = client + .post(format!("{base_url}/agents/{agent_id}/bindings")) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!([binding( + &archived_operation, + "edit_order_archived", + "Archived Order Tool" + )])) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.text().await.unwrap(); + assert!( + status.is_client_error(), + "a first edit that references an archived Operation must be rejected: {status}: {body}" + ); + + assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn first_edit_rejects_when_operation_archives_during_save() { + let registry = test_registry().await; + let db = registry.clone(); + let storage_root = test_storage_root("agent_catalog_edit_archive_race"); + let upstream_base_url = spawn_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let live_operation = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_live").await; + let (agent_id, published_v1) = create_published_agent( + &client, + &base_url, + "edit-archive-race", + &live_operation, + "edit_race_live", + ) + .await; + + let target_operation = + create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_target") + .await; + + // Read the edit precondition before taking the Operation lock; Agent detail + // may itself inspect bound Operations. + let etag_response = client + .get(format!("{base_url}/agents/{agent_id}")) + .send() + .await + .unwrap(); + let etag_status = etag_response.status(); + let etag = etag_response + .headers() + .get(reqwest::header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let etag_body = etag_response.text().await.unwrap(); + assert_eq!( + etag_status, + reqwest::StatusCode::OK, + "reading the Agent ETag before the archive race must succeed: {etag_body}" + ); + let etag = etag.expect("Agent detail must expose an ETag"); + + // Use an independent pool for the holder and pg_stat_activity monitor. The + // fixture registry has one connection, which the request needs to reach the + // Operation lock wait. + let lock_registry = + PostgresRegistry::connect_with_options((*db.pool().connect_options()).clone()) + .await + .unwrap(); + + // Hold the target Operation row lock from a separate connection so the edit + // transaction cannot complete its binding write until the archive wins. + let mut holder = lock_registry.pool().begin().await.unwrap(); + sqlx::query("select id from operations where workspace_id = $1 and id = $2 for update") + .bind(DEFAULT_WORKSPACE_ID) + .bind(&target_operation) + .fetch_one(&mut *holder) + .await + .unwrap(); + let holder_pid: i32 = sqlx::query_scalar("select pg_backend_pid()") + .fetch_one(&mut *holder) + .await + .unwrap(); + let edit_client = client.clone(); + let endpoint = format!("{base_url}/agents/{agent_id}/bindings"); + let edit_target = target_operation.clone(); + let edit = tokio::spawn(async move { + let response = edit_client + .post(endpoint) + .header(reqwest::header::IF_MATCH, etag) + .json(&json!([binding( + &edit_target, + "edit_race_target", + "Archive Race Tool" + )])) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.text().await.unwrap(); + (status, body) + }); + + // Wait for PostgreSQL to report the request blocked by this exact holder. + // If the request completes first (the baseline CAS path), keep going so the + // durable-draft assertion below reports that regression directly. + let lock_wait = async { + loop { + if edit.is_finished() { + return false; + } + let blocked: bool = sqlx::query_scalar( + "select exists ( + select 1 + from pg_stat_activity + where wait_event_type = 'Lock' + and $1 = any(pg_blocking_pids(pid)) + and query ilike '%select status from operations%' + )", + ) + .bind(holder_pid) + .fetch_one(lock_registry.pool()) + .await + .unwrap(); + if blocked { + return true; + } + tokio::task::yield_now().await; + } + }; + let saw_operation_lock_wait = + match tokio::time::timeout(std::time::Duration::from_secs(10), lock_wait).await { + Ok(saw_wait) => saw_wait, + Err(_) => { + edit.abort(); + holder.rollback().await.unwrap(); + let _ = edit.await; + panic!("timed out waiting for the edit to block on the held Operation row lock"); + } + }; + + sqlx::query( + "update operations + set status = 'archived', updated_at = now() + where workspace_id = $1 and id = $2", + ) + .bind(DEFAULT_WORKSPACE_ID) + .bind(&target_operation) + .execute(&mut *holder) + .await + .unwrap(); + holder.commit().await.unwrap(); + + let (status, body) = edit.await.unwrap(); + assert!( + status.is_client_error(), + "an edit racing an Operation archive must be rejected: {status}: {body}" + ); + + assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; + assert!( + saw_operation_lock_wait, + "a corrected save path must wait on the holder's Operation row lock before archive commits" + ); +} diff --git a/apps/admin-api/tests/unit/execution_parity.rs b/apps/admin-api/tests/unit/execution_parity.rs index 25c4656..3e03c7e 100644 --- a/apps/admin-api/tests/unit/execution_parity.rs +++ b/apps/admin-api/tests/unit/execution_parity.rs @@ -48,3 +48,28 @@ fn admin_and_mcp_preserve_dispatch_uncertainty_without_changing_the_code() { assert!(!mcp.recoverable); } } + +#[test] +fn admin_and_mcp_project_new_uncertain_upstream_codes_consistently() { + for locale in [ExecutionLocale::En, ExecutionLocale::Ru] { + for code in [ + ExecutionErrorCode::UpstreamRateLimited, + ExecutionErrorCode::UpstreamServerError, + ExecutionErrorCode::UpstreamResponseTooLarge, + ExecutionErrorCode::UpstreamStatusError, + ] { + let failure = ExecutionFailure::new(code, CorrelationContext::generate()) + .with_dispatch_uncertainty(); + let admin = execution_test_failure_localized(&failure, locale); + let mcp = tool_error_contract_from_failure(&failure, locale); + + assert_eq!(admin["code"], code.as_str()); + assert_eq!(admin["retryability"], "manual_reconcile"); + assert_eq!(admin["outcome_certainty"], "outcome_unknown"); + assert_eq!(admin["retryability"], mcp.retryability); + assert_eq!(admin["outcome_certainty"], mcp.outcome_certainty); + assert_eq!(mcp.error_code, code.as_str()); + assert!(!mcp.recoverable); + } + } +} diff --git a/apps/mcp-server/tests/integration.rs b/apps/mcp-server/tests/integration.rs index b8ca776..10c6668 100644 --- a/apps/mcp-server/tests/integration.rs +++ b/apps/mcp-server/tests/integration.rs @@ -1,6 +1,7 @@ mod integration { mod catalog_access; mod common; + mod execution_certainty; mod execution_stages; mod jsonrpc_correlation; mod request_context; diff --git a/apps/mcp-server/tests/integration/execution_certainty.rs b/apps/mcp-server/tests/integration/execution_certainty.rs new file mode 100644 index 0000000..74e4d02 --- /dev/null +++ b/apps/mcp-server/tests/integration/execution_certainty.rs @@ -0,0 +1,145 @@ +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use std::time::Duration; + +use axum::{Router, http::StatusCode, routing::post}; +use crank_core::{ + ExecutionErrorCode, InvocationSource, OutcomeCertainty, PlatformApiKeyScope, Retryability, + WorkspaceId, +}; +use crank_registry::{ListInvocationLogsQuery, PublishRequest}; +use serde_json::{Value, json}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tokio::net::TcpListener; + +use super::common::{ + agent_mcp_url, build_test_app, create_platform_api_key, initialize_session, + post_jsonrpc_response, publish_agent_for_operation, spawn_mcp_server, test_operation, + test_registry, test_workspace_id, +}; + +const REQUEST_ID: &str = "req_mcp_execution_certainty"; + +#[tokio::test] +async fn ambiguous_post_failure_is_reported_and_persisted_as_unknown() { + let registry = test_registry().await; + let upstream_calls = Arc::new(AtomicUsize::new(0)); + let upstream_base_url = spawn_503_upstream(Arc::clone(&upstream_calls)).await; + let operation = test_operation(&upstream_base_url, "mcp_execution_certainty"); + + registry + .create_operation(&test_workspace_id(), &operation, Some("test")) + .await + .unwrap(); + registry + .publish_operation(PublishRequest { + workspace_id: &test_workspace_id(), + operation_id: &operation.id, + version: 1, + published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(), + published_by: Some("test"), + }) + .await + .unwrap(); + publish_agent_for_operation(®istry, &operation, "mcp-certainty-agent").await; + let api_key = create_platform_api_key( + ®istry, + "mcp-certainty-agent", + "mcp-certainty-key", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let app = build_test_app(registry.clone(), Duration::ZERO, None); + let base_url = spawn_mcp_server(app).await; + let client = reqwest::Client::new(); + let mcp_url = agent_mcp_url(&base_url, "mcp-certainty-agent"); + let session_id = initialize_session(&client, &mcp_url, &api_key).await; + + let response = post_jsonrpc_response( + &client, + &mcp_url, + &api_key, + Some(&session_id), + Some(REQUEST_ID), + json!({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": operation.name, + "arguments": { "email": "user@example.com" } + } + }), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let response: Value = response.json().await.unwrap(); + let tool_result = &response["result"]; + let error = &tool_result["structuredContent"]["error"]; + assert_eq!(tool_result["isError"], true); + assert_eq!(error["code"], "upstream_server_error"); + assert_eq!(error["upstream_status"], 503); + assert_eq!(error["retryability"], "manual_reconcile"); + assert_eq!(error["outcome_certainty"], "outcome_unknown"); + assert_eq!(error["recoverable"], false); + assert_eq!(upstream_calls.load(Ordering::SeqCst), 1); + + let logs = registry + .list_invocation_logs(ListInvocationLogsQuery { + workspace_id: &WorkspaceId::new("ws_default"), + level: None, + status: None, + outcome_group: None, + search_text: None, + source: Some(InvocationSource::AgentToolCall), + operation_id: Some(&operation.id), + agent_id: None, + created_after: None, + created_before: None, + cursor_created_at: None, + cursor_id: None, + limit: 10, + }) + .await + .unwrap(); + let invocation = logs + .iter() + .find(|record| record.log.request_id.as_deref() == Some(REQUEST_ID)) + .expect("Invocation History entry for this MCP request"); + assert_eq!(invocation.log.tool_name, operation.name); + assert_eq!(invocation.log.operation_version, Some(1)); + assert_eq!(invocation.log.status_code, Some(503)); + assert_eq!( + invocation.log.execution_error_code, + Some(ExecutionErrorCode::UpstreamServerError) + ); + assert_eq!( + invocation.log.retryability, + Some(Retryability::ManualReconcile) + ); + assert_eq!( + invocation.log.outcome_certainty, + Some(OutcomeCertainty::OutcomeUnknown) + ); +} + +async fn spawn_503_upstream(calls: Arc) -> String { + let app = Router::new().route( + "/crm/leads", + post(move || { + let calls = Arc::clone(&calls); + async move { + calls.fetch_add(1, Ordering::SeqCst); + (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{address}") +} diff --git a/crates/crank-core/src/execution.rs b/crates/crank-core/src/execution.rs index d6cf5b1..7befc99 100644 --- a/crates/crank-core/src/execution.rs +++ b/crates/crank-core/src/execution.rs @@ -500,11 +500,16 @@ impl ExecutionFailure { self.code, ExecutionErrorCode::UpstreamTimeout | ExecutionErrorCode::UpstreamTransportError + | ExecutionErrorCode::UpstreamRateLimited + | ExecutionErrorCode::UpstreamServerError + | ExecutionErrorCode::UpstreamResponseTooLarge + | ExecutionErrorCode::UpstreamStatusError | ExecutionErrorCode::IdempotencyOutcomeUnknown | ExecutionErrorCode::PersistenceUnavailable ) { self.retryability = Retryability::ManualReconcile; self.outcome_certainty = OutcomeCertainty::OutcomeUnknown; + self.retry_after_ms = None; } self } diff --git a/crates/crank-registry/src/postgres/agent.rs b/crates/crank-registry/src/postgres/agent.rs index 9e2f2c6..cb6d2a3 100644 --- a/crates/crank-registry/src/postgres/agent.rs +++ b/crates/crank-registry/src/postgres/agent.rs @@ -265,16 +265,111 @@ impl PostgresRegistry { &self, request: SaveAgentCatalogConfigRequest<'_>, ) -> Result<(), RegistryError> { + self.save_agent_catalog_config_inner(request, false) + .await + .map(|_| ()) + } + + pub async fn save_agent_catalog_config_editable( + &self, + request: SaveAgentCatalogConfigRequest<'_>, + ) -> Result { + self.save_agent_catalog_config_inner(request, true).await + } + + async fn save_agent_catalog_config_inner( + &self, + request: SaveAgentCatalogConfigRequest<'_>, + allow_fork: bool, + ) -> Result { let mut tx = self.pool.begin().await?; - Self::lock_agent_and_validate_expected_state( + let agent_row = Self::lock_agent_and_validate_expected_state( &mut tx, request.workspace_id, request.agent_id, request.expected_state, ) .await?; - Self::reject_published_agent_version(&mut tx, request.agent_id, request.agent_version) - .await?; + + if allow_fork { + let status = + deserialize_enum_text::(agent_row.try_get("status")?, "status")?; + if status == AgentStatus::Archived { + return Err(RegistryError::InvalidAgentTransition { + agent_id: request.agent_id.as_str().to_owned(), + from: "archived".to_owned(), + action: "edit", + }); + } + let current_draft_version = from_db_version( + agent_row.try_get("current_draft_version")?, + "current_draft_version", + )?; + if request.agent_version != current_draft_version { + return Err(RegistryError::InvalidAgentVersionSequence { + agent_id: request.agent_id.as_str().to_owned(), + expected: current_draft_version, + actual: request.agent_version, + }); + } + let latest_published_version = agent_row + .try_get::, _>("latest_published_version")? + .map(|value| from_db_version(value, "latest_published_version")) + .transpose()?; + + if latest_published_version == Some(current_draft_version) { + let instructions = sqlx::query( + "select instructions_json + from agent_versions + where agent_id = $1 and version = $2", + ) + .bind(request.agent_id.as_str()) + .bind(to_db_version(current_draft_version)) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| RegistryError::AgentNotFound { + agent_id: request.agent_id.as_str().to_owned(), + })? + .try_get::("instructions_json")?; + + let now = time::OffsetDateTime::now_utc(); + let draft = AgentVersion { + agent_id: request.agent_id.clone(), + version: current_draft_version + 1, + status: AgentStatus::Draft, + instructions, + tool_selection_policy: request.tool_selection_policy.clone(), + created_at: now, + }; + insert_agent_version_row(&mut tx, &draft).await?; + replace_agent_bindings_rows( + &mut tx, + request.workspace_id, + request.agent_id, + draft.version, + request.bindings, + ) + .await?; + sqlx::query( + "update agents + set current_draft_version = $3, + updated_at = $4::timestamptz + where workspace_id = $1 and id = $2", + ) + .bind(request.workspace_id.as_str()) + .bind(request.agent_id.as_str()) + .bind(to_db_version(draft.version)) + .bind(now) + .execute(&mut *tx) + .await?; + tx.commit().await?; + return Ok(draft.version); + } + } else { + Self::reject_published_agent_version(&mut tx, request.agent_id, request.agent_version) + .await?; + } + let updated = sqlx::query( "update agent_versions set tool_selection_policy_json = $3 @@ -309,7 +404,7 @@ impl PostgresRegistry { .execute(&mut *tx) .await?; tx.commit().await?; - Ok(()) + Ok(request.agent_version) } async fn lock_agent_and_validate_expected_state( @@ -317,7 +412,7 @@ impl PostgresRegistry { workspace_id: &WorkspaceId, agent_id: &AgentId, expected_state: Option<&AgentStateExpectation>, - ) -> Result<(), RegistryError> { + ) -> Result { let row = sqlx::query( "select status, @@ -364,7 +459,7 @@ impl PostgresRegistry { } } - Ok(()) + Ok(row) } async fn reject_published_agent_version( diff --git a/crates/crank-runtime/src/execution_failure.rs b/crates/crank-runtime/src/execution_failure.rs index 0af69bd..e747726 100644 --- a/crates/crank-runtime/src/execution_failure.rs +++ b/crates/crank-runtime/src/execution_failure.rs @@ -38,10 +38,13 @@ fn normalize_runtime_error_with_effect( ) -> ExecutionFailure { let mut failure = ExecutionFailure::new(error_code(error), correlation.clone()); match error { - RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus { - status, .. - }) => { + RuntimeError::ProtocolAdapter( + adapter @ ProtocolAdapterError::UnexpectedStatus { status, .. }, + ) => { failure = failure.with_upstream_status(*status); + if may_mutate && ambiguous_after_dispatch(adapter) { + failure = failure.with_dispatch_uncertainty(); + } } RuntimeError::ProtocolAdapter(adapter) if may_mutate && ambiguous_after_dispatch(adapter) => diff --git a/crates/crank-runtime/tests/execution_certainty.rs b/crates/crank-runtime/tests/execution_certainty.rs new file mode 100644 index 0000000..9c8a4a6 --- /dev/null +++ b/crates/crank-runtime/tests/execution_certainty.rs @@ -0,0 +1,244 @@ +use std::collections::BTreeMap; + +use crank_core::{ + CorrelationContext, DispatchEvidence, ExecutionErrorCode, ExecutionFailure, HttpMethod, + OutcomeCertainty, Protocol, ProtocolAdapterError, RestTarget, Retryability, Target, + ToolDescription, +}; +use crank_mapping::MappingSet; +use crank_runtime::{RuntimeError, RuntimeOperation, normalize_runtime_error_for_operation}; +use crank_schema::{Schema, SchemaKind}; + +const MUTATING_METHODS: [HttpMethod; 4] = [ + HttpMethod::Post, + HttpMethod::Put, + HttpMethod::Patch, + HttpMethod::Delete, +]; +const READ_METHODS: [HttpMethod; 2] = [HttpMethod::Get, HttpMethod::Head]; +const UNCERTAIN_STATUSES: [u16; 4] = [408, 429, 500, 503]; + +#[test] +fn ambiguous_upstream_failures_are_unknown_only_for_dispatched_mutations() { + let mut failures = Vec::new(); + for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) { + for dispatch in [ + DispatchEvidence::MayHaveDispatched, + DispatchEvidence::NotDispatched, + ] { + for status in UNCERTAIN_STATUSES { + let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus { + status, + dispatch, + }); + if let Err(failure) = assert_normalization( + method, + &error, + is_ambiguous_mutation(method, dispatch), + Some(status), + format!("{method:?} status {status} {dispatch:?}"), + ) { + failures.push(failure); + } + } + + for adapter in [ + ProtocolAdapterError::ResponseTooLarge { dispatch }, + ProtocolAdapterError::InvalidResponse { dispatch }, + ] { + let error = RuntimeError::ProtocolAdapter(adapter); + if let Err(failure) = assert_normalization( + method, + &error, + is_ambiguous_mutation(method, dispatch), + None, + format!("{method:?} {error:?}"), + ) { + failures.push(failure); + } + } + } + } + assert!( + failures.is_empty(), + "unexpected normalization rows:\n{}", + failures.join("\n") + ); +} + +#[test] +fn nonambiguous_statuses_keep_their_existing_descriptor() { + let mut failures = Vec::new(); + for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) { + for status in [401, 404] { + let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus { + status, + dispatch: DispatchEvidence::MayHaveDispatched, + }); + if let Err(failure) = assert_normalization( + method, + &error, + false, + Some(status), + format!("{method:?} nonambiguous status {status}"), + ) { + failures.push(failure); + } + } + } + assert!( + failures.is_empty(), + "unexpected normalization rows:\n{}", + failures.join("\n") + ); +} + +#[test] +fn dispatch_uncertainty_allowlist_covers_uncertain_upstream_codes_and_drops_retry_delay() { + let uncertain_codes = [ + ExecutionErrorCode::UpstreamRateLimited, + ExecutionErrorCode::UpstreamServerError, + ExecutionErrorCode::UpstreamResponseTooLarge, + ExecutionErrorCode::UpstreamStatusError, + ]; + let mut failures = Vec::new(); + for code in uncertain_codes { + let failure = ExecutionFailure::new(code, CorrelationContext::generate()); + let failure = if matches!(code.retryability(), Retryability::AfterDelay) { + failure + .try_with_retry_after_ms(1_000) + .expect("retry delay is valid for the original descriptor") + } else { + failure + } + .with_dispatch_uncertainty(); + + if failure.error_code() != code + || failure.retryability() != Retryability::ManualReconcile + || failure.outcome_certainty() != OutcomeCertainty::OutcomeUnknown + || failure.retry_after_ms().is_some() + { + failures.push(format!( + "{code:?}: actual code={:?}, retryability={:?}, certainty={:?}, retry_after_ms={:?}", + failure.error_code(), + failure.retryability(), + failure.outcome_certainty(), + failure.retry_after_ms(), + )); + } + } + assert!( + failures.is_empty(), + "uncertain code allowlist failures:\n{}", + failures.join("\n") + ); +} + +fn assert_normalization( + method: HttpMethod, + error: &RuntimeError, + unknown: bool, + status: Option, + context: String, +) -> Result<(), String> { + let correlation = CorrelationContext::generate(); + let failure = normalize_runtime_error_for_operation(error, &correlation, &operation(method)); + let code = expected_code(error); + let expected = ExecutionFailure::new(code, correlation.clone()); + let expected_retryability = if unknown { + Retryability::ManualReconcile + } else { + expected.retryability() + }; + let expected_certainty = if unknown { + OutcomeCertainty::OutcomeUnknown + } else { + expected.outcome_certainty() + }; + + if failure.error_code() != code + || failure.retryability() != expected_retryability + || failure.outcome_certainty() != expected_certainty + || failure.upstream_status() != status + || failure.retry_after_ms().is_some() + { + return Err(format!( + "{context}: code={:?} (expected {code:?}), retryability={:?} (expected {expected_retryability:?}), certainty={:?} (expected {expected_certainty:?}), upstream_status={:?} (expected {status:?}), retry_after_ms={:?}", + failure.error_code(), + failure.retryability(), + failure.outcome_certainty(), + failure.upstream_status(), + failure.retry_after_ms(), + )); + } + Ok(()) +} + +fn expected_code(error: &RuntimeError) -> ExecutionErrorCode { + match error { + RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus { + status, .. + }) => match status { + 401 | 403 => ExecutionErrorCode::UpstreamAuthError, + 404 => ExecutionErrorCode::UpstreamNotFound, + 408 | 429 => ExecutionErrorCode::UpstreamRateLimited, + 500..=599 => ExecutionErrorCode::UpstreamServerError, + _ => ExecutionErrorCode::UpstreamStatusError, + }, + RuntimeError::ProtocolAdapter(ProtocolAdapterError::ResponseTooLarge { .. }) => { + ExecutionErrorCode::UpstreamResponseTooLarge + } + RuntimeError::ProtocolAdapter(ProtocolAdapterError::InvalidResponse { .. }) => { + ExecutionErrorCode::UpstreamStatusError + } + other => panic!("unexpected test fixture: {other:?}"), + } +} + +fn is_ambiguous_mutation(method: HttpMethod, dispatch: DispatchEvidence) -> bool { + MUTATING_METHODS.contains(&method) && dispatch == DispatchEvidence::MayHaveDispatched +} + +fn operation(method: HttpMethod) -> RuntimeOperation { + RuntimeOperation { + operation_id: crank_core::OperationId::new("certainty_test"), + operation_version: 1, + tool_name: "certainty_test".to_owned(), + protocol: Protocol::Rest, + target: Target::Rest(RestTarget { + base_url: "https://example.invalid".to_owned(), + method, + path_template: "/items".to_owned(), + static_headers: BTreeMap::new(), + request_contract: None, + response_contract: Vec::new(), + multipart_field_mime_types: BTreeMap::new(), + }), + input_schema: empty_schema(), + output_schema: empty_schema(), + input_mapping: MappingSet::default(), + output_mapping: MappingSet::default(), + execution_config: crank_core::ExecutionConfig::default(), + tool_description: ToolDescription { + title: "Certainty test".to_owned(), + description: "Execution certainty fixture".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + } +} + +fn empty_schema() -> Schema { + Schema { + kind: SchemaKind::Object, + format: None, + description: None, + required: true, + nullable: false, + default_value: None, + fields: BTreeMap::new(), + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} diff --git a/docs/README.md b/docs/README.md index 2334df2..bf0bcb0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,8 @@ Crank превращает REST API endpoint-ы в MCP-инструменты, - [Capability Baseline](./capability-baseline/manifest.json) — versioned SHA-256 snapshot фактических UI, Admin API и MCP flows, taxonomy, checklist и sanitized evidence results. +- [Evidence разработки](./development-evidence.md) — clean baseline, проверенная + revision реализации и отдельная запись текущего completion evidence. - [Schema Capability Baseline](./schemas/capability-baseline.schema.json) — fail-closed manifest contract. - [Настройки окружения](./runtime-config.md) diff --git a/docs/development-evidence.md b/docs/development-evidence.md new file mode 100644 index 0000000..2653ad9 --- /dev/null +++ b/docs/development-evidence.md @@ -0,0 +1,75 @@ +# Evidence разработки + +Этот порядок помогает восстановить состояние кода и результат каждого +проверяемого этапа. Он не заменяет [Capability Baseline](./capability-baseline/manifest.json): +baseline хранит версионированный снимок capability Inventory и исторические +результаты, а completion evidence указывает, что именно проверено для текущего +этапа или завершения. + +## Ревизии этапа + +До начала этапа запишите полный Git commit `baseline_commit` и убедитесь, что +`git status --short` пуст. Это проверка tracked, staged и обычных untracked +изменений; игнорируемые локальные файлы не входят в Git baseline. Не записывайте +`baseline + uncommitted worktree` как revision. + +После изменений сохраните проверенный implementation revision отдельным полным +commit hash. Проверки, которые подтверждают завершение этапа, должны относиться +к этому commit при чистом worktree. Укажите использованные конфигурацию и +класс окружения: одинаковый commit сам по себе не описывает их. + +Если нужен RED до коммита, зафиксируйте его как промежуточное наблюдение: укажите +`baseline_commit`, SHA-256 сохранённого diff/artifact, включающего изменения +(в том числе untracked test files), команду и её фактический результат. Не +представляйте baseline вместе с незакоммиченным diff как implementation revision. +После RED повторите нужные проверки на отдельном чистом implementation revision. + +## Запись completion evidence + +Для текущего завершения используйте отдельный versioned index и sanitized +evidence-файлы с устойчивыми repository-relative ссылками. Запись должна позволять +повторно определить проверенное состояние и результат, не сохраняя секреты, +локальные абсолютные пути или произвольные payloads. Для каждой проверки укажите: + +- идентификатор этапа/запуска и полный `baseline_commit`; +- полный `implementation_revision` проверенного commit; +- конфигурацию/toolchain и класс окружения; +- точную команду, exit code и фактические counts `passed`, `failed`, `skipped` + (или эквивалентный summary для команды без тестового счётчика); +- repository-relative путь к sanitised log/report и его SHA-256. + +Если результаты не относятся к одной и той же ревизии, конфигурации или +окружению, записывайте их раздельно. Не выводите PASS из exit code без проверки +счётчиков и пропусков; сохраняйте fail, flaky, skipped, not-run и manual-only +как наблюдались. Новое completion evidence не переписывает historical baseline +или его manual failures. + +Изменение canonical Capability Inventory status/evidence по-прежнему требует +обновить versioned `docs/capability-baseline/manifest.json`, sanitized +`docs/capability-baseline/results.json`, `baseline_version` и SHA-256 согласно +его schema и проверкам. Отдельный completion index не ослабляет и не обходит +этот gate. + +Компактный шаблон записи: + +```yaml +id: +baseline_commit: +implementation_revision: +environment: + class: + config: +command: +exit_code: +counts: + passed: + failed: + skipped: +evidence: + path: + sha256: +``` + +Для промежуточного RED вместо `implementation_revision` укажите +`diff_sha256` и ссылку на сохранённый diff/artifact; не выдавайте его за +проверенный implementation commit. diff --git a/docs/epic-1-completion/evidence/agent-probe.cjs b/docs/epic-1-completion/evidence/agent-probe.cjs new file mode 100644 index 0000000..bb71e91 --- /dev/null +++ b/docs/epic-1-completion/evidence/agent-probe.cjs @@ -0,0 +1,40 @@ +const assert = require('node:assert/strict'); +const { chromium } = require(process.env.CRANK_PROBE_PLAYWRIGHT || require('node:path').resolve(__dirname, '../../../apps/ui/node_modules/@playwright/test')); +(async () => { + const browser = await chromium.launch(); + try { + const page = await browser.newPage(); + await page.goto('http://127.0.0.1:3300/login'); + await page.locator('#email').fill('owner@crank.local'); + await page.locator('#password').fill('change-me-admin-password'); + await page.locator('.btn-signin').click(); + await page.waitForURL('http://127.0.0.1:3300/'); + const result = await page.evaluate(async () => { + const api = window.CrankApi; + const session = await api.getSession(); + const csrf = await api.refreshSessionCsrf(); + session.csrf_token = csrf.csrf_token; window.CrankAuth.replaceSession(session); + const ws = session.current_workspace_id || session.memberships[0].workspace.id; + const agents = await api.listAgents(ws); + const list = Array.isArray(agents) ? agents : (agents.agents || agents.items); + const source = list.find(a => a.latest_published_version != null); + if (!source) throw new Error('No published fixture Agent'); + const sid = source.agent_id || source.id; + const version = await fetch('/api/admin/workspaces/'+ws+'/agents/'+sid+'/versions/'+source.latest_published_version).then(r=>r.json()); + const bindings = version.bindings.map(b => ({operation_id:b.operation_id,operation_version:b.operation_version,tool_name:b.tool_name,tool_title:b.tool_title,tool_description_override:b.tool_description_override,enabled:b.enabled})); + const created = await api.createAgent(ws,{slug:'retro-probe-'+Date.now(),display_name:'Retro probe',description:'Synthetic regression probe',instructions:{},tool_selection_policy:{}}); + const id = created.agent_id || created.id; + await api.saveAgentBindings(ws,id,bindings); + await api.publishAgent(ws,id,{version:1}); + const before=await api.getAgent(ws,id); + const publishedBefore=await fetch('/api/admin/workspaces/'+ws+'/agents/'+id+'/versions/1').then(r=>r.json()); + let errorCode=null; let saved=null; + try { saved=await api.saveAgentBindings(ws,id,bindings); } catch(e) { errorCode=e.errorCode || e.code || e.message; } + const after=await api.getAgent(ws,id); + const publishedAfter=await fetch('/api/admin/workspaces/'+ws+'/agents/'+id+'/versions/1').then(r=>r.json()); + return {before_draft:before.current_draft_version,after_draft:after.current_draft_version,saved_version:saved && saved.version,published_unchanged:JSON.stringify(publishedBefore)===JSON.stringify(publishedAfter),error:errorCode}; + }); + console.log(JSON.stringify(result)); + assert.equal(result.error,null); assert.equal(result.before_draft,1); assert.equal(result.after_draft,2); assert.equal(result.saved_version,2); assert.equal(result.published_unchanged,true); + } finally { await browser.close(); } +})().catch(e=>{console.error(e.message);process.exitCode=1;}); diff --git a/scripts/check-rust-code-health.sh b/scripts/check-rust-code-health.sh index febf37e..8d95799 100755 --- a/scripts/check-rust-code-health.sh +++ b/scripts/check-rust-code-health.sh @@ -19,6 +19,12 @@ declare -A FILE_LIMITS=( ) status=0 +source_roots=() +for source_dir in apps crates; do + if [[ -d "$ROOT_DIR/$source_dir" ]]; then + source_roots+=("$ROOT_DIR/$source_dir") + fi +done echo "Rust code health: checking file size limits" @@ -32,10 +38,11 @@ while IFS= read -r file; do status=1 fi done < <( - find "$ROOT_DIR" \ - -path "$ROOT_DIR/target" -prune -o \ - -path "$ROOT_DIR/apps/ui/node_modules" -prune -o \ - -name '*.rs' -print | sort + for source_root in "${source_roots[@]}"; do + find "$source_root" \ + -type d \( -name target -o -name node_modules \) -prune -o \ + -type f -name '*.rs' -print + done | sort ) echo "Rust code health: checking large inline test modules" diff --git a/tests/unit/test_check_rust_code_health.py b/tests/unit/test_check_rust_code_health.py index 660d147..5256d19 100644 --- a/tests/unit/test_check_rust_code_health.py +++ b/tests/unit/test_check_rust_code_health.py @@ -41,7 +41,7 @@ class RustCodeHealthCheckTests(unittest.TestCase): self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("checking file size limits", result.stdout) - def test_fails_for_new_large_rust_file(self) -> None: + def test_fails_for_new_untracked_large_rust_file_in_source_scope(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.prepare_root(root) @@ -55,6 +55,46 @@ class RustCodeHealthCheckTests(unittest.TestCase): self.assertNotEqual(result.returncode, 0) self.assertIn("apps/demo/src/large.rs has 21 lines", result.stderr) + def test_ignores_large_rust_in_nested_bmad_worktree(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.prepare_root(root) + worktree = ( + root + / ".bmad-loop" + / "runs" + / "epic-1" + / "worktrees" + / "fixture" + ) + worktree.mkdir(parents=True) + (worktree / "large.rs").write_text( + "\n".join("// line" for _ in range(21)) + "\n", + encoding="utf-8", + ) + + result = self.run_checker(root) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_ignores_rust_under_nested_target_and_node_modules(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.prepare_root(root) + generated_dirs = ( + root / "apps" / "demo" / "target" / "generated", + root / "apps" / "demo" / "src" / "node_modules" / "fixture", + root / "crates" / "demo" / "node_modules" / "fixture", + ) + large_source = "\n".join("// generated" for _ in range(21)) + "\n" + for directory in generated_dirs: + directory.mkdir(parents=True) + (directory / "large.rs").write_text(large_source, encoding="utf-8") + + result = self.run_checker(root) + + self.assertEqual(result.returncode, 0, result.stderr) + def test_fails_for_large_inline_test_module(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp)