Files

890 lines
29 KiB
Rust

use super::common::*;
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
use crank_registry::{CreateWorkspaceRequest, PostgresRegistry};
use serde_json::{Value, json};
use serial_test::serial;
use time::OffsetDateTime;
const DEFAULT_WORKSPACE_ID: &str = "ws_default";
fn binding(operation_id: &str, tool_name: &str, tool_title: &str) -> Value {
json!({
"operation_id": operation_id,
"operation_version": 1,
"tool_name": tool_name,
"tool_title": tool_title,
"tool_description_override": null,
"enabled": true
})
}
async fn create_draft_operation(
client: &reqwest::Client,
base_url: &TestServer,
upstream: &str,
name: &str,
) -> String {
let operation = assert_success_json(
client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(upstream, name))
.send()
.await
.unwrap(),
)
.await;
operation["operation_id"].as_str().unwrap().to_owned()
}
async fn create_published_operation(
client: &reqwest::Client,
base_url: &TestServer,
upstream: &str,
name: &str,
) -> String {
let operation_id = create_draft_operation(client, base_url, upstream, name).await;
let etag = operation_etag(client, base_url, &operation_id).await;
assert_success_json(
client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap(),
)
.await;
operation_id
}
async fn create_published_agent(
client: &reqwest::Client,
base_url: &TestServer,
slug: &str,
operation_id: &str,
tool_name: &str,
) -> (String, Value) {
let agent = assert_success_json(
client
.post(format!("{base_url}/agents"))
.json(&json!({
"slug": slug,
"display_name": slug,
"description": "agent catalog edit fixture",
"instructions": { "system_prompt": "Use only published bindings" },
"tool_selection_policy": {}
}))
.send()
.await
.unwrap(),
)
.await;
let agent_id = agent["agent_id"].as_str().unwrap().to_owned();
let etag = agent_etag(client, base_url, &agent_id).await;
assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!([binding(operation_id, tool_name, "Base Tool")]))
.send()
.await
.unwrap(),
)
.await;
let etag = agent_etag(client, base_url, &agent_id).await;
assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap(),
)
.await;
let published_v1 = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
.send()
.await
.unwrap(),
)
.await;
(agent_id, published_v1)
}
async fn agent_version_count(db: &PostgresRegistry, agent_id: &str) -> i64 {
sqlx::query_scalar("select count(*) from agent_versions where agent_id = $1")
.bind(agent_id)
.fetch_one(db.pool())
.await
.unwrap()
}
async fn assert_no_durable_draft(
client: &reqwest::Client,
db: &PostgresRegistry,
base_url: &TestServer,
agent_id: &str,
published_v1: &Value,
) {
let summary = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
summary["current_draft_version"], 1,
"a rejected edit must not fork a Draft Version: {summary}"
);
assert_eq!(summary["latest_published_version"], 1, "{summary}");
assert_eq!(summary["status"], "published", "{summary}");
assert_eq!(
agent_version_count(db, agent_id).await,
1,
"a rejected edit must not persist a new Agent Version row"
);
let published_after = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
&published_after, published_v1,
"the Published snapshot and its bindings must stay unchanged after a rejected edit"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn published_agent_first_edit_forks_one_draft_and_keeps_published_snapshot() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_first");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let operation_id =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_first_tool").await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-first",
&operation_id,
"edit_first_tool",
)
.await;
let summary_before = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(summary_before["current_draft_version"], 1);
assert_eq!(summary_before["latest_published_version"], 1);
assert_eq!(summary_before["status"], "published");
let desired_policy = json!({
"mode": "search",
"groups": [{
"id": "crm-group",
"name": "CRM",
"description": "CRM tools",
"tool_names": ["edit_first_tool"]
}],
"search": { "max_results": 5 }
});
let edit_etag = agent_etag(&client, &base_url, &agent_id).await;
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, edit_etag)
.json(&json!({
"bindings": [binding(&operation_id, "edit_first_tool", "Edited Tool")],
"tool_selection_policy": desired_policy
}))
.send()
.await
.unwrap();
let status = response.status();
let body = response.json::<Value>().await.unwrap();
assert_eq!(
status,
reqwest::StatusCode::OK,
"the first edit of a Published Agent must fork and save under the original If-Match: {body}"
);
assert_eq!(body["version"], 2, "{body}");
let summary_after = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(summary_after["current_draft_version"], 2, "{summary_after}");
assert_eq!(
summary_after["latest_published_version"], 1,
"{summary_after}"
);
assert_eq!(summary_after["status"], "published", "{summary_after}");
assert_eq!(
agent_version_count(&db, &agent_id).await,
2,
"the first edit must create exactly one additional Draft Version"
);
let draft = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}/versions/2"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(draft["status"], "draft", "{draft}");
assert_eq!(
draft["snapshot"]["instructions"],
published_v1["snapshot"]["instructions"]
);
assert_eq!(draft["bindings"].as_array().unwrap().len(), 1, "{draft}");
assert_eq!(draft["bindings"][0]["tool_title"], "Edited Tool", "{draft}");
assert_eq!(
draft["snapshot"]["tool_selection_policy"]["mode"], "search",
"{draft}"
);
assert_eq!(
draft["snapshot"]["tool_selection_policy"]["search"]["max_results"], 5,
"{draft}"
);
let published_after = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
published_after, published_v1,
"the Published v1 snapshot must not change when the Draft is forked"
);
let next_etag = agent_etag(&client, &base_url, &agent_id).await;
let second_response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, next_etag)
.json(&json!({
"bindings": [binding(&operation_id, "edit_first_tool", "Edited Again")],
"tool_selection_policy": desired_policy
}))
.send()
.await
.unwrap();
let second_status = second_response.status();
let second_body = second_response.text().await.unwrap();
assert_eq!(
second_status,
reqwest::StatusCode::OK,
"a subsequent valid edit on the Draft must succeed: {second_body}"
);
assert_eq!(
agent_version_count(&db, &agent_id).await,
2,
"a subsequent valid edit must stay on the existing Draft v2"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn rejected_invalid_policy_leaves_no_durable_draft() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_invalid_policy");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let operation_id =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_policy_tool")
.await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-policy",
&operation_id,
"edit_policy_tool",
)
.await;
let etag = agent_etag(&client, &base_url, &agent_id).await;
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!({
"bindings": [],
"tool_selection_policy": {
"mode": "direct",
"groups": [{
"id": "group-1",
"name": "Group",
"description": "groups require search mode",
"tool_names": []
}]
}
}))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status,
reqwest::StatusCode::BAD_REQUEST,
"a grouping policy in direct mode must be rejected: {body}"
);
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn rejected_unpublished_operation_leaves_no_durable_draft() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_draft_op");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let live_operation =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_live_tool").await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-draft-op",
&live_operation,
"edit_live_tool",
)
.await;
let draft_operation =
create_draft_operation(&client, &base_url, &upstream_base_url, "edit_draft_tool").await;
let etag = agent_etag(&client, &base_url, &agent_id).await;
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!([binding(
&draft_operation,
"edit_draft_tool",
"Draft Tool"
)]))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status,
reqwest::StatusCode::UNPROCESSABLE_ENTITY,
"binding an unpublished Operation must be rejected: {body}"
);
let body: Value = serde_json::from_str(&body).unwrap();
assert_eq!(
body["error"]["code"], "agent_binding_not_published",
"{body}"
);
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn rejected_archived_operation_leaves_no_durable_draft() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_archived_op");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let live_operation = create_published_operation(
&client,
&base_url,
&upstream_base_url,
"edit_live_tool_arch",
)
.await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-archived-op",
&live_operation,
"edit_live_tool_arch",
)
.await;
let archived_operation =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_archived_tool")
.await;
let operation_etag = operation_etag(&client, &base_url, &archived_operation).await;
let archived = client
.post(format!(
"{base_url}/operations/{archived_operation}/archive"
))
.header(reqwest::header::IF_MATCH, operation_etag)
.send()
.await
.unwrap();
let archived_status = archived.status();
let archived_body = archived.text().await.unwrap();
assert!(archived_status.is_success(), "{archived_body}");
let etag = agent_etag(&client, &base_url, &agent_id).await;
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!([binding(
&archived_operation,
"edit_archived_tool",
"Archived Tool"
)]))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status,
reqwest::StatusCode::UNPROCESSABLE_ENTITY,
"binding an archived Operation must be rejected: {body}"
);
let body: Value = serde_json::from_str(&body).unwrap();
assert_eq!(
body["error"]["code"], "agent_binding_archived_operation",
"{body}"
);
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn rejected_foreign_operation_leaves_no_durable_draft() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_foreign_op");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await;
let client = authorized_client(&base_url).await;
let live_operation = create_published_operation(
&client,
&base_url,
&upstream_base_url,
"edit_live_tool_foreign",
)
.await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-foreign-op",
&live_operation,
"edit_live_tool_foreign",
)
.await;
let foreign_workspace = WorkspaceId::new("ws_agent_edit_foreign");
let now = OffsetDateTime::now_utc();
db.create_workspace(CreateWorkspaceRequest {
workspace: &Workspace {
id: foreign_workspace.clone(),
slug: "agent-edit-foreign".to_owned(),
display_name: "Agent Edit Foreign".to_owned(),
status: WorkspaceStatus::Active,
settings: json!({}),
created_at: now,
updated_at: now,
},
})
.await
.unwrap();
let foreign_service = test_service(
db.clone(),
test_storage_root("agent_catalog_edit_foreign_service"),
test_auth_settings(),
test_secret_crypto(),
);
let foreign_operation = foreign_service
.create_operation(
&foreign_workspace,
test_operation_payload(&upstream_base_url, "edit_foreign_tool"),
)
.await
.unwrap();
let foreign_operation_id = foreign_operation.operation_id;
let etag = agent_etag(&client, &base_url, &agent_id).await;
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!([binding(
&foreign_operation_id,
"edit_foreign_tool",
"Foreign Tool"
)]))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"binding an Operation from another workspace must be rejected: {body}"
);
let body: Value = serde_json::from_str(&body).unwrap();
assert_eq!(
body["error"]["code"], "agent_binding_scope_denied",
"{body}"
);
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn concurrent_first_edits_with_same_etag_have_one_winner() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_concurrent");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let operation_id =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_tool").await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-race",
&operation_id,
"edit_race_tool",
)
.await;
// Single original precondition taken once from the Published state.
let original_etag = agent_etag(&client, &base_url, &agent_id).await;
let endpoint = format!("{base_url}/agents/{agent_id}/bindings");
let payload_a = json!([binding(&operation_id, "edit_race_tool", "Race Winner A")]);
let payload_b = json!([binding(&operation_id, "edit_race_tool", "Race Winner B")]);
let spawn_edit = |payload: Value, etag: String, endpoint: String| {
let client = client.clone();
tokio::spawn(async move {
let response = client
.post(endpoint)
.header(reqwest::header::IF_MATCH, etag)
.json(&payload)
.send()
.await
.unwrap();
let status = response.status();
let body = response.json::<Value>().await.unwrap();
(status, body)
})
};
let first = spawn_edit(payload_a, original_etag.clone(), endpoint.clone());
let second = spawn_edit(payload_b, original_etag, endpoint);
let (first, second) = tokio::join!(first, second);
let results = vec![first.unwrap(), second.unwrap()];
let successes = results
.iter()
.filter(|(status, _)| status.is_success())
.count();
let conflicts = results
.iter()
.filter(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
.count();
assert_eq!(
successes, 1,
"exactly one concurrent first edit must win: {results:?}"
);
assert_eq!(
conflicts, 1,
"the losing concurrent first edit must report a stable conflict: {results:?}"
);
let conflict_body = results
.iter()
.find(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
.map(|(_, body)| body)
.unwrap();
assert_eq!(
conflict_body["error"]["code"], "agent_stale_revision",
"{conflict_body}"
);
let summary = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(summary["current_draft_version"], 2, "{summary}");
assert_eq!(summary["latest_published_version"], 1, "{summary}");
assert_eq!(
agent_version_count(&db, &agent_id).await,
2,
"the losing concurrent edit must not fork a second Draft"
);
let published_after = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(published_after, published_v1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn first_edit_after_operation_archive_leaves_no_durable_draft() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_archive_ordering");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let live_operation =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_order_live").await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-order",
&live_operation,
"edit_order_live",
)
.await;
let archived_operation = create_published_operation(
&client,
&base_url,
&upstream_base_url,
"edit_order_archived",
)
.await;
let operation_etag = operation_etag(&client, &base_url, &archived_operation).await;
let archived = client
.post(format!(
"{base_url}/operations/{archived_operation}/archive"
))
.header(reqwest::header::IF_MATCH, operation_etag)
.send()
.await
.unwrap();
let archived_status = archived.status();
let archived_body = archived.text().await.unwrap();
assert!(archived_status.is_success(), "{archived_body}");
let etag = agent_etag(&client, &base_url, &agent_id).await;
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(reqwest::header::IF_MATCH, etag)
.json(&json!([binding(
&archived_operation,
"edit_order_archived",
"Archived Order Tool"
)]))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert!(
status.is_client_error(),
"a first edit that references an archived Operation must be rejected: {status}: {body}"
);
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn first_edit_rejects_when_operation_archives_during_save() {
let registry = test_registry().await;
let db = registry.clone();
let storage_root = test_storage_root("agent_catalog_edit_archive_race");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let live_operation =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_live").await;
let (agent_id, published_v1) = create_published_agent(
&client,
&base_url,
"edit-archive-race",
&live_operation,
"edit_race_live",
)
.await;
let target_operation =
create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_target")
.await;
// Read the edit precondition before taking the Operation lock; Agent detail
// may itself inspect bound Operations.
let etag_response = client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap();
let etag_status = etag_response.status();
let etag = etag_response
.headers()
.get(reqwest::header::ETAG)
.and_then(|value| value.to_str().ok())
.map(str::to_owned);
let etag_body = etag_response.text().await.unwrap();
assert_eq!(
etag_status,
reqwest::StatusCode::OK,
"reading the Agent ETag before the archive race must succeed: {etag_body}"
);
let etag = etag.expect("Agent detail must expose an ETag");
// Use an independent pool for the holder and pg_stat_activity monitor. The
// fixture registry has one connection, which the request needs to reach the
// Operation lock wait.
let lock_registry =
PostgresRegistry::connect_with_options((*db.pool().connect_options()).clone())
.await
.unwrap();
// Hold the target Operation row lock from a separate connection so the edit
// transaction cannot complete its binding write until the archive wins.
let mut holder = lock_registry.pool().begin().await.unwrap();
sqlx::query("select id from operations where workspace_id = $1 and id = $2 for update")
.bind(DEFAULT_WORKSPACE_ID)
.bind(&target_operation)
.fetch_one(&mut *holder)
.await
.unwrap();
let holder_pid: i32 = sqlx::query_scalar("select pg_backend_pid()")
.fetch_one(&mut *holder)
.await
.unwrap();
let edit_client = client.clone();
let endpoint = format!("{base_url}/agents/{agent_id}/bindings");
let edit_target = target_operation.clone();
let edit = tokio::spawn(async move {
let response = edit_client
.post(endpoint)
.header(reqwest::header::IF_MATCH, etag)
.json(&json!([binding(
&edit_target,
"edit_race_target",
"Archive Race Tool"
)]))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
(status, body)
});
// Wait for PostgreSQL to report the request blocked by this exact holder.
// If the request completes first (the baseline CAS path), keep going so the
// durable-draft assertion below reports that regression directly.
let lock_wait = async {
loop {
if edit.is_finished() {
return false;
}
let blocked: bool = sqlx::query_scalar(
"select exists (
select 1
from pg_stat_activity
where wait_event_type = 'Lock'
and $1 = any(pg_blocking_pids(pid))
and query ilike '%select status from operations%'
)",
)
.bind(holder_pid)
.fetch_one(lock_registry.pool())
.await
.unwrap();
if blocked {
return true;
}
tokio::task::yield_now().await;
}
};
let saw_operation_lock_wait =
match tokio::time::timeout(std::time::Duration::from_secs(10), lock_wait).await {
Ok(saw_wait) => saw_wait,
Err(_) => {
edit.abort();
holder.rollback().await.unwrap();
let _ = edit.await;
panic!("timed out waiting for the edit to block on the held Operation row lock");
}
};
sqlx::query(
"update operations
set status = 'archived', updated_at = now()
where workspace_id = $1 and id = $2",
)
.bind(DEFAULT_WORKSPACE_ID)
.bind(&target_operation)
.execute(&mut *holder)
.await
.unwrap();
holder.commit().await.unwrap();
let (status, body) = edit.await.unwrap();
assert!(
status.is_client_error(),
"an edit racing an Operation archive must be rejected: {status}: {body}"
);
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
assert!(
saw_operation_lock_wait,
"a corrected save path must wait on the holder's Operation row lock before archive commits"
);
}