use super::common::*; use crank_core::{Workspace, WorkspaceId, WorkspaceStatus}; use crank_registry::{CreateWorkspaceRequest, PostgresRegistry}; use serde_json::{Value, json}; use serial_test::serial; use time::OffsetDateTime; const DEFAULT_WORKSPACE_ID: &str = "ws_default"; fn binding(operation_id: &str, tool_name: &str, tool_title: &str) -> Value { json!({ "operation_id": operation_id, "operation_version": 1, "tool_name": tool_name, "tool_title": tool_title, "tool_description_override": null, "enabled": true }) } async fn create_draft_operation( client: &reqwest::Client, base_url: &TestServer, upstream: &str, name: &str, ) -> String { let operation = assert_success_json( client .post(format!("{base_url}/operations")) .json(&test_operation_payload(upstream, name)) .send() .await .unwrap(), ) .await; operation["operation_id"].as_str().unwrap().to_owned() } async fn create_published_operation( client: &reqwest::Client, base_url: &TestServer, upstream: &str, name: &str, ) -> String { let operation_id = create_draft_operation(client, base_url, upstream, name).await; let etag = operation_etag(client, base_url, &operation_id).await; assert_success_json( client .post(format!("{base_url}/operations/{operation_id}/publish")) .header(reqwest::header::IF_MATCH, etag) .json(&json!({ "version": 1 })) .send() .await .unwrap(), ) .await; operation_id } async fn create_published_agent( client: &reqwest::Client, base_url: &TestServer, slug: &str, operation_id: &str, tool_name: &str, ) -> (String, Value) { let agent = assert_success_json( client .post(format!("{base_url}/agents")) .json(&json!({ "slug": slug, "display_name": slug, "description": "agent catalog edit fixture", "instructions": { "system_prompt": "Use only published bindings" }, "tool_selection_policy": {} })) .send() .await .unwrap(), ) .await; let agent_id = agent["agent_id"].as_str().unwrap().to_owned(); let etag = agent_etag(client, base_url, &agent_id).await; assert_success_json( client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, etag) .json(&json!([binding(operation_id, tool_name, "Base Tool")])) .send() .await .unwrap(), ) .await; let etag = agent_etag(client, base_url, &agent_id).await; assert_success_json( client .post(format!("{base_url}/agents/{agent_id}/publish")) .header(reqwest::header::IF_MATCH, etag) .json(&json!({ "version": 1 })) .send() .await .unwrap(), ) .await; let published_v1 = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}/versions/1")) .send() .await .unwrap(), ) .await; (agent_id, published_v1) } async fn agent_version_count(db: &PostgresRegistry, agent_id: &str) -> i64 { sqlx::query_scalar("select count(*) from agent_versions where agent_id = $1") .bind(agent_id) .fetch_one(db.pool()) .await .unwrap() } async fn assert_no_durable_draft( client: &reqwest::Client, db: &PostgresRegistry, base_url: &TestServer, agent_id: &str, published_v1: &Value, ) { let summary = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}")) .send() .await .unwrap(), ) .await; assert_eq!( summary["current_draft_version"], 1, "a rejected edit must not fork a Draft Version: {summary}" ); assert_eq!(summary["latest_published_version"], 1, "{summary}"); assert_eq!(summary["status"], "published", "{summary}"); assert_eq!( agent_version_count(db, agent_id).await, 1, "a rejected edit must not persist a new Agent Version row" ); let published_after = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}/versions/1")) .send() .await .unwrap(), ) .await; assert_eq!( &published_after, published_v1, "the Published snapshot and its bindings must stay unchanged after a rejected edit" ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn published_agent_first_edit_forks_one_draft_and_keeps_published_snapshot() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_first"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let client = authorized_client(&base_url).await; let operation_id = create_published_operation(&client, &base_url, &upstream_base_url, "edit_first_tool").await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-first", &operation_id, "edit_first_tool", ) .await; let summary_before = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}")) .send() .await .unwrap(), ) .await; assert_eq!(summary_before["current_draft_version"], 1); assert_eq!(summary_before["latest_published_version"], 1); assert_eq!(summary_before["status"], "published"); let desired_policy = json!({ "mode": "search", "groups": [{ "id": "crm-group", "name": "CRM", "description": "CRM tools", "tool_names": ["edit_first_tool"] }], "search": { "max_results": 5 } }); let edit_etag = agent_etag(&client, &base_url, &agent_id).await; let response = client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, edit_etag) .json(&json!({ "bindings": [binding(&operation_id, "edit_first_tool", "Edited Tool")], "tool_selection_policy": desired_policy })) .send() .await .unwrap(); let status = response.status(); let body = response.json::().await.unwrap(); assert_eq!( status, reqwest::StatusCode::OK, "the first edit of a Published Agent must fork and save under the original If-Match: {body}" ); assert_eq!(body["version"], 2, "{body}"); let summary_after = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}")) .send() .await .unwrap(), ) .await; assert_eq!(summary_after["current_draft_version"], 2, "{summary_after}"); assert_eq!( summary_after["latest_published_version"], 1, "{summary_after}" ); assert_eq!(summary_after["status"], "published", "{summary_after}"); assert_eq!( agent_version_count(&db, &agent_id).await, 2, "the first edit must create exactly one additional Draft Version" ); let draft = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}/versions/2")) .send() .await .unwrap(), ) .await; assert_eq!(draft["status"], "draft", "{draft}"); assert_eq!( draft["snapshot"]["instructions"], published_v1["snapshot"]["instructions"] ); assert_eq!(draft["bindings"].as_array().unwrap().len(), 1, "{draft}"); assert_eq!(draft["bindings"][0]["tool_title"], "Edited Tool", "{draft}"); assert_eq!( draft["snapshot"]["tool_selection_policy"]["mode"], "search", "{draft}" ); assert_eq!( draft["snapshot"]["tool_selection_policy"]["search"]["max_results"], 5, "{draft}" ); let published_after = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}/versions/1")) .send() .await .unwrap(), ) .await; assert_eq!( published_after, published_v1, "the Published v1 snapshot must not change when the Draft is forked" ); let next_etag = agent_etag(&client, &base_url, &agent_id).await; let second_response = client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, next_etag) .json(&json!({ "bindings": [binding(&operation_id, "edit_first_tool", "Edited Again")], "tool_selection_policy": desired_policy })) .send() .await .unwrap(); let second_status = second_response.status(); let second_body = second_response.text().await.unwrap(); assert_eq!( second_status, reqwest::StatusCode::OK, "a subsequent valid edit on the Draft must succeed: {second_body}" ); assert_eq!( agent_version_count(&db, &agent_id).await, 2, "a subsequent valid edit must stay on the existing Draft v2" ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn rejected_invalid_policy_leaves_no_durable_draft() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_invalid_policy"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let client = authorized_client(&base_url).await; let operation_id = create_published_operation(&client, &base_url, &upstream_base_url, "edit_policy_tool") .await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-policy", &operation_id, "edit_policy_tool", ) .await; let etag = agent_etag(&client, &base_url, &agent_id).await; let response = client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, etag) .json(&json!({ "bindings": [], "tool_selection_policy": { "mode": "direct", "groups": [{ "id": "group-1", "name": "Group", "description": "groups require search mode", "tool_names": [] }] } })) .send() .await .unwrap(); let status = response.status(); let body = response.text().await.unwrap(); assert_eq!( status, reqwest::StatusCode::BAD_REQUEST, "a grouping policy in direct mode must be rejected: {body}" ); assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn rejected_unpublished_operation_leaves_no_durable_draft() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_draft_op"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let client = authorized_client(&base_url).await; let live_operation = create_published_operation(&client, &base_url, &upstream_base_url, "edit_live_tool").await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-draft-op", &live_operation, "edit_live_tool", ) .await; let draft_operation = create_draft_operation(&client, &base_url, &upstream_base_url, "edit_draft_tool").await; let etag = agent_etag(&client, &base_url, &agent_id).await; let response = client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, etag) .json(&json!([binding( &draft_operation, "edit_draft_tool", "Draft Tool" )])) .send() .await .unwrap(); let status = response.status(); let body = response.text().await.unwrap(); assert_eq!( status, reqwest::StatusCode::UNPROCESSABLE_ENTITY, "binding an unpublished Operation must be rejected: {body}" ); let body: Value = serde_json::from_str(&body).unwrap(); assert_eq!( body["error"]["code"], "agent_binding_not_published", "{body}" ); assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn rejected_archived_operation_leaves_no_durable_draft() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_archived_op"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let client = authorized_client(&base_url).await; let live_operation = create_published_operation( &client, &base_url, &upstream_base_url, "edit_live_tool_arch", ) .await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-archived-op", &live_operation, "edit_live_tool_arch", ) .await; let archived_operation = create_published_operation(&client, &base_url, &upstream_base_url, "edit_archived_tool") .await; let operation_etag = operation_etag(&client, &base_url, &archived_operation).await; let archived = client .post(format!( "{base_url}/operations/{archived_operation}/archive" )) .header(reqwest::header::IF_MATCH, operation_etag) .send() .await .unwrap(); let archived_status = archived.status(); let archived_body = archived.text().await.unwrap(); assert!(archived_status.is_success(), "{archived_body}"); let etag = agent_etag(&client, &base_url, &agent_id).await; let response = client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, etag) .json(&json!([binding( &archived_operation, "edit_archived_tool", "Archived Tool" )])) .send() .await .unwrap(); let status = response.status(); let body = response.text().await.unwrap(); assert_eq!( status, reqwest::StatusCode::UNPROCESSABLE_ENTITY, "binding an archived Operation must be rejected: {body}" ); let body: Value = serde_json::from_str(&body).unwrap(); assert_eq!( body["error"]["code"], "agent_binding_archived_operation", "{body}" ); assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn rejected_foreign_operation_leaves_no_durable_draft() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_foreign_op"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await; let client = authorized_client(&base_url).await; let live_operation = create_published_operation( &client, &base_url, &upstream_base_url, "edit_live_tool_foreign", ) .await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-foreign-op", &live_operation, "edit_live_tool_foreign", ) .await; let foreign_workspace = WorkspaceId::new("ws_agent_edit_foreign"); let now = OffsetDateTime::now_utc(); db.create_workspace(CreateWorkspaceRequest { workspace: &Workspace { id: foreign_workspace.clone(), slug: "agent-edit-foreign".to_owned(), display_name: "Agent Edit Foreign".to_owned(), status: WorkspaceStatus::Active, settings: json!({}), created_at: now, updated_at: now, }, }) .await .unwrap(); let foreign_service = test_service( db.clone(), test_storage_root("agent_catalog_edit_foreign_service"), test_auth_settings(), test_secret_crypto(), ); let foreign_operation = foreign_service .create_operation( &foreign_workspace, test_operation_payload(&upstream_base_url, "edit_foreign_tool"), ) .await .unwrap(); let foreign_operation_id = foreign_operation.operation_id; let etag = agent_etag(&client, &base_url, &agent_id).await; let response = client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, etag) .json(&json!([binding( &foreign_operation_id, "edit_foreign_tool", "Foreign Tool" )])) .send() .await .unwrap(); let status = response.status(); let body = response.text().await.unwrap(); assert_eq!( status, reqwest::StatusCode::NOT_FOUND, "binding an Operation from another workspace must be rejected: {body}" ); let body: Value = serde_json::from_str(&body).unwrap(); assert_eq!( body["error"]["code"], "agent_binding_scope_denied", "{body}" ); assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn concurrent_first_edits_with_same_etag_have_one_winner() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_concurrent"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let client = authorized_client(&base_url).await; let operation_id = create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_tool").await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-race", &operation_id, "edit_race_tool", ) .await; // Single original precondition taken once from the Published state. let original_etag = agent_etag(&client, &base_url, &agent_id).await; let endpoint = format!("{base_url}/agents/{agent_id}/bindings"); let payload_a = json!([binding(&operation_id, "edit_race_tool", "Race Winner A")]); let payload_b = json!([binding(&operation_id, "edit_race_tool", "Race Winner B")]); let spawn_edit = |payload: Value, etag: String, endpoint: String| { let client = client.clone(); tokio::spawn(async move { let response = client .post(endpoint) .header(reqwest::header::IF_MATCH, etag) .json(&payload) .send() .await .unwrap(); let status = response.status(); let body = response.json::().await.unwrap(); (status, body) }) }; let first = spawn_edit(payload_a, original_etag.clone(), endpoint.clone()); let second = spawn_edit(payload_b, original_etag, endpoint); let (first, second) = tokio::join!(first, second); let results = vec![first.unwrap(), second.unwrap()]; let successes = results .iter() .filter(|(status, _)| status.is_success()) .count(); let conflicts = results .iter() .filter(|(status, _)| *status == reqwest::StatusCode::CONFLICT) .count(); assert_eq!( successes, 1, "exactly one concurrent first edit must win: {results:?}" ); assert_eq!( conflicts, 1, "the losing concurrent first edit must report a stable conflict: {results:?}" ); let conflict_body = results .iter() .find(|(status, _)| *status == reqwest::StatusCode::CONFLICT) .map(|(_, body)| body) .unwrap(); assert_eq!( conflict_body["error"]["code"], "agent_stale_revision", "{conflict_body}" ); let summary = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}")) .send() .await .unwrap(), ) .await; assert_eq!(summary["current_draft_version"], 2, "{summary}"); assert_eq!(summary["latest_published_version"], 1, "{summary}"); assert_eq!( agent_version_count(&db, &agent_id).await, 2, "the losing concurrent edit must not fork a second Draft" ); let published_after = assert_success_json( client .get(format!("{base_url}/agents/{agent_id}/versions/1")) .send() .await .unwrap(), ) .await; assert_eq!(published_after, published_v1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn first_edit_after_operation_archive_leaves_no_durable_draft() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_archive_ordering"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let client = authorized_client(&base_url).await; let live_operation = create_published_operation(&client, &base_url, &upstream_base_url, "edit_order_live").await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-order", &live_operation, "edit_order_live", ) .await; let archived_operation = create_published_operation( &client, &base_url, &upstream_base_url, "edit_order_archived", ) .await; let operation_etag = operation_etag(&client, &base_url, &archived_operation).await; let archived = client .post(format!( "{base_url}/operations/{archived_operation}/archive" )) .header(reqwest::header::IF_MATCH, operation_etag) .send() .await .unwrap(); let archived_status = archived.status(); let archived_body = archived.text().await.unwrap(); assert!(archived_status.is_success(), "{archived_body}"); let etag = agent_etag(&client, &base_url, &agent_id).await; let response = client .post(format!("{base_url}/agents/{agent_id}/bindings")) .header(reqwest::header::IF_MATCH, etag) .json(&json!([binding( &archived_operation, "edit_order_archived", "Archived Order Tool" )])) .send() .await .unwrap(); let status = response.status(); let body = response.text().await.unwrap(); assert!( status.is_client_error(), "a first edit that references an archived Operation must be rejected: {status}: {body}" ); assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn first_edit_rejects_when_operation_archives_during_save() { let registry = test_registry().await; let db = registry.clone(); let storage_root = test_storage_root("agent_catalog_edit_archive_race"); let upstream_base_url = spawn_upstream_server().await; let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; let client = authorized_client(&base_url).await; let live_operation = create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_live").await; let (agent_id, published_v1) = create_published_agent( &client, &base_url, "edit-archive-race", &live_operation, "edit_race_live", ) .await; let target_operation = create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_target") .await; // Read the edit precondition before taking the Operation lock; Agent detail // may itself inspect bound Operations. let etag_response = client .get(format!("{base_url}/agents/{agent_id}")) .send() .await .unwrap(); let etag_status = etag_response.status(); let etag = etag_response .headers() .get(reqwest::header::ETAG) .and_then(|value| value.to_str().ok()) .map(str::to_owned); let etag_body = etag_response.text().await.unwrap(); assert_eq!( etag_status, reqwest::StatusCode::OK, "reading the Agent ETag before the archive race must succeed: {etag_body}" ); let etag = etag.expect("Agent detail must expose an ETag"); // Use an independent pool for the holder and pg_stat_activity monitor. The // fixture registry has one connection, which the request needs to reach the // Operation lock wait. let lock_registry = PostgresRegistry::connect_with_options((*db.pool().connect_options()).clone()) .await .unwrap(); // Hold the target Operation row lock from a separate connection so the edit // transaction cannot complete its binding write until the archive wins. let mut holder = lock_registry.pool().begin().await.unwrap(); sqlx::query("select id from operations where workspace_id = $1 and id = $2 for update") .bind(DEFAULT_WORKSPACE_ID) .bind(&target_operation) .fetch_one(&mut *holder) .await .unwrap(); let holder_pid: i32 = sqlx::query_scalar("select pg_backend_pid()") .fetch_one(&mut *holder) .await .unwrap(); let edit_client = client.clone(); let endpoint = format!("{base_url}/agents/{agent_id}/bindings"); let edit_target = target_operation.clone(); let edit = tokio::spawn(async move { let response = edit_client .post(endpoint) .header(reqwest::header::IF_MATCH, etag) .json(&json!([binding( &edit_target, "edit_race_target", "Archive Race Tool" )])) .send() .await .unwrap(); let status = response.status(); let body = response.text().await.unwrap(); (status, body) }); // Wait for PostgreSQL to report the request blocked by this exact holder. // If the request completes first (the baseline CAS path), keep going so the // durable-draft assertion below reports that regression directly. let lock_wait = async { loop { if edit.is_finished() { return false; } let blocked: bool = sqlx::query_scalar( "select exists ( select 1 from pg_stat_activity where wait_event_type = 'Lock' and $1 = any(pg_blocking_pids(pid)) and query ilike '%select status from operations%' )", ) .bind(holder_pid) .fetch_one(lock_registry.pool()) .await .unwrap(); if blocked { return true; } tokio::task::yield_now().await; } }; let saw_operation_lock_wait = match tokio::time::timeout(std::time::Duration::from_secs(10), lock_wait).await { Ok(saw_wait) => saw_wait, Err(_) => { edit.abort(); holder.rollback().await.unwrap(); let _ = edit.await; panic!("timed out waiting for the edit to block on the held Operation row lock"); } }; sqlx::query( "update operations set status = 'archived', updated_at = now() where workspace_id = $1 and id = $2", ) .bind(DEFAULT_WORKSPACE_ID) .bind(&target_operation) .execute(&mut *holder) .await .unwrap(); holder.commit().await.unwrap(); let (status, body) = edit.await.unwrap(); assert!( status.is_client_error(), "an edit racing an Operation archive must be rejected: {status}: {body}" ); assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await; assert!( saw_operation_lock_wait, "a corrected save path must wait on the holder's Operation row lock before archive commits" ); }