fix(epic-1): repair atomic agent edits and dispatch certainty
This commit is contained in:
1 parent
82b215dd74
commit
7aa854de26
16 files changed
+1600
-57
No files matched your search
@@ -38,6 +38,11 @@ npx playwright test
|
|||||||
[`docs/capability-baseline/manifest.json`](./docs/capability-baseline/manifest.json),
|
[`docs/capability-baseline/manifest.json`](./docs/capability-baseline/manifest.json),
|
||||||
sanitized results и SHA-256. Нельзя вручную объявлять pass для failed, flaky,
|
sanitized results и SHA-256. Нельзя вручную объявлять pass для failed, flaky,
|
||||||
skipped, not-run или manual-only evidence.
|
skipped, not-run или manual-only evidence.
|
||||||
|
- Для записи результатов этапа и разделения исторического capability baseline
|
||||||
|
и текущего completion evidence следуйте
|
||||||
|
[правилам evidence разработки](./docs/development-evidence.md). Перед этапом
|
||||||
|
фиксируйте полный clean Git baseline; проверенную реализацию указывайте
|
||||||
|
отдельной ревизией.
|
||||||
- Не добавляйте секреты, токены, приватные адреса и локальные настройки.
|
- Не добавляйте секреты, токены, приватные адреса и локальные настройки.
|
||||||
- Не коммитьте `AGENTS.md`, `TASKS.md`, `.env` и временные файлы.
|
- Не коммитьте `AGENTS.md`, `TASKS.md`, `.env` и временные файлы.
|
||||||
- Для изменений SQL-запросов обновляйте `.sqlx`, если это требуется SQLx.
|
- Для изменений SQL-запросов обновляйте `.sqlx`, если это требуется SQLx.
|
||||||
|
|||||||
@@ -5,9 +5,8 @@ use crank_core::{
|
|||||||
ToolAccessMode, ToolSelectionPolicy, UsagePeriod, WorkspaceId, search_tool_catalog,
|
ToolAccessMode, ToolSelectionPolicy, UsagePeriod, WorkspaceId, search_tool_catalog,
|
||||||
};
|
};
|
||||||
use crank_registry::{
|
use crank_registry::{
|
||||||
AgentStateExpectation, AgentVersionRecord, CreateAgentDraftVersionRequest, CreateAgentRequest,
|
AgentStateExpectation, AgentVersionRecord, CreateAgentRequest, PublishAgentRequest,
|
||||||
PublishAgentRequest, SaveAgentCatalogConfigRequest, UpdateAgentSummaryRequest, UsageBucket,
|
SaveAgentCatalogConfigRequest, UpdateAgentSummaryRequest, UsageBucket, UsageQuery,
|
||||||
UsageQuery,
|
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
@@ -471,8 +470,9 @@ impl AdminService {
|
|||||||
payload: AgentCatalogPayload,
|
payload: AgentCatalogPayload,
|
||||||
expected_state: Option<&AgentStateExpectation>,
|
expected_state: Option<&AgentStateExpectation>,
|
||||||
) -> Result<AgentVersionRecord, ApiError> {
|
) -> Result<AgentVersionRecord, ApiError> {
|
||||||
|
let agent = self.get_agent(workspace_id, agent_id).await?;
|
||||||
let current_version = self
|
let current_version = self
|
||||||
.ensure_editable_agent_version(workspace_id, agent_id)
|
.get_agent_version(workspace_id, agent_id, agent.current_draft_version)
|
||||||
.await?;
|
.await?;
|
||||||
let (payload, requested_policy) = payload.into_parts();
|
let (payload, requested_policy) = payload.into_parts();
|
||||||
let bindings = payload
|
let bindings = payload
|
||||||
@@ -492,8 +492,9 @@ impl AdminService {
|
|||||||
.unwrap_or_else(|| current_version.snapshot.tool_selection_policy.clone());
|
.unwrap_or_else(|| current_version.snapshot.tool_selection_policy.clone());
|
||||||
validate_tool_selection_policy(&tool_selection_policy, &bindings)?;
|
validate_tool_selection_policy(&tool_selection_policy, &bindings)?;
|
||||||
|
|
||||||
self.registry
|
let saved_version = self
|
||||||
.save_agent_catalog_config(SaveAgentCatalogConfigRequest {
|
.registry
|
||||||
|
.save_agent_catalog_config_editable(SaveAgentCatalogConfigRequest {
|
||||||
workspace_id,
|
workspace_id,
|
||||||
agent_id,
|
agent_id,
|
||||||
agent_version: current_version.version,
|
agent_version: current_version.version,
|
||||||
@@ -505,47 +506,12 @@ impl AdminService {
|
|||||||
info!(
|
info!(
|
||||||
name: "admin.agent.bindings_saved",
|
name: "admin.agent.bindings_saved",
|
||||||
agent_id = %agent_id.as_str(),
|
agent_id = %agent_id.as_str(),
|
||||||
version = current_version.version,
|
version = saved_version,
|
||||||
binding_count = bindings.len(),
|
binding_count = bindings.len(),
|
||||||
"agent bindings saved"
|
"agent bindings saved"
|
||||||
);
|
);
|
||||||
|
|
||||||
self.get_agent_version(workspace_id, agent_id, current_version.version)
|
self.get_agent_version(workspace_id, agent_id, saved_version)
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn ensure_editable_agent_version(
|
|
||||||
&self,
|
|
||||||
workspace_id: &WorkspaceId,
|
|
||||||
agent_id: &AgentId,
|
|
||||||
) -> Result<AgentVersionRecord, ApiError> {
|
|
||||||
let agent = self.get_agent(workspace_id, agent_id).await?;
|
|
||||||
let current = self
|
|
||||||
.get_agent_version(workspace_id, agent_id, agent.current_draft_version)
|
|
||||||
.await?;
|
|
||||||
if agent.latest_published_version != Some(agent.current_draft_version) {
|
|
||||||
return Ok(current);
|
|
||||||
}
|
|
||||||
|
|
||||||
let now = OffsetDateTime::now_utc();
|
|
||||||
let draft = AgentVersion {
|
|
||||||
agent_id: agent_id.clone(),
|
|
||||||
version: current.version + 1,
|
|
||||||
status: AgentStatus::Draft,
|
|
||||||
instructions: current.snapshot.instructions.clone(),
|
|
||||||
tool_selection_policy: current.snapshot.tool_selection_policy.clone(),
|
|
||||||
created_at: now,
|
|
||||||
};
|
|
||||||
self.registry
|
|
||||||
.create_agent_draft_version(CreateAgentDraftVersionRequest {
|
|
||||||
workspace_id,
|
|
||||||
agent_id,
|
|
||||||
version: &draft,
|
|
||||||
bindings: ¤t.bindings,
|
|
||||||
updated_at: &now,
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
self.get_agent_version(workspace_id, agent_id, draft.version)
|
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
mod integration {
|
mod integration {
|
||||||
mod agent_catalog;
|
mod agent_catalog;
|
||||||
|
mod agent_catalog_edit;
|
||||||
mod auth_rate_limit;
|
mod auth_rate_limit;
|
||||||
mod common;
|
mod common;
|
||||||
mod community_access_usage;
|
mod community_access_usage;
|
||||||
|
|||||||
@@ -0,0 +1,889 @@
|
|||||||
|
use super::common::*;
|
||||||
|
|
||||||
|
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
||||||
|
use crank_registry::{CreateWorkspaceRequest, PostgresRegistry};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use serial_test::serial;
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
|
const DEFAULT_WORKSPACE_ID: &str = "ws_default";
|
||||||
|
|
||||||
|
fn binding(operation_id: &str, tool_name: &str, tool_title: &str) -> Value {
|
||||||
|
json!({
|
||||||
|
"operation_id": operation_id,
|
||||||
|
"operation_version": 1,
|
||||||
|
"tool_name": tool_name,
|
||||||
|
"tool_title": tool_title,
|
||||||
|
"tool_description_override": null,
|
||||||
|
"enabled": true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_draft_operation(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
base_url: &TestServer,
|
||||||
|
upstream: &str,
|
||||||
|
name: &str,
|
||||||
|
) -> String {
|
||||||
|
let operation = assert_success_json(
|
||||||
|
client
|
||||||
|
.post(format!("{base_url}/operations"))
|
||||||
|
.json(&test_operation_payload(upstream, name))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
operation["operation_id"].as_str().unwrap().to_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_published_operation(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
base_url: &TestServer,
|
||||||
|
upstream: &str,
|
||||||
|
name: &str,
|
||||||
|
) -> String {
|
||||||
|
let operation_id = create_draft_operation(client, base_url, upstream, name).await;
|
||||||
|
let etag = operation_etag(client, base_url, &operation_id).await;
|
||||||
|
assert_success_json(
|
||||||
|
client
|
||||||
|
.post(format!("{base_url}/operations/{operation_id}/publish"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!({ "version": 1 }))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
operation_id
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_published_agent(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
base_url: &TestServer,
|
||||||
|
slug: &str,
|
||||||
|
operation_id: &str,
|
||||||
|
tool_name: &str,
|
||||||
|
) -> (String, Value) {
|
||||||
|
let agent = assert_success_json(
|
||||||
|
client
|
||||||
|
.post(format!("{base_url}/agents"))
|
||||||
|
.json(&json!({
|
||||||
|
"slug": slug,
|
||||||
|
"display_name": slug,
|
||||||
|
"description": "agent catalog edit fixture",
|
||||||
|
"instructions": { "system_prompt": "Use only published bindings" },
|
||||||
|
"tool_selection_policy": {}
|
||||||
|
}))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let agent_id = agent["agent_id"].as_str().unwrap().to_owned();
|
||||||
|
|
||||||
|
let etag = agent_etag(client, base_url, &agent_id).await;
|
||||||
|
assert_success_json(
|
||||||
|
client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!([binding(operation_id, tool_name, "Base Tool")]))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let etag = agent_etag(client, base_url, &agent_id).await;
|
||||||
|
assert_success_json(
|
||||||
|
client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/publish"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!({ "version": 1 }))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let published_v1 = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
(agent_id, published_v1)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn agent_version_count(db: &PostgresRegistry, agent_id: &str) -> i64 {
|
||||||
|
sqlx::query_scalar("select count(*) from agent_versions where agent_id = $1")
|
||||||
|
.bind(agent_id)
|
||||||
|
.fetch_one(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_no_durable_draft(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
db: &PostgresRegistry,
|
||||||
|
base_url: &TestServer,
|
||||||
|
agent_id: &str,
|
||||||
|
published_v1: &Value,
|
||||||
|
) {
|
||||||
|
let summary = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
summary["current_draft_version"], 1,
|
||||||
|
"a rejected edit must not fork a Draft Version: {summary}"
|
||||||
|
);
|
||||||
|
assert_eq!(summary["latest_published_version"], 1, "{summary}");
|
||||||
|
assert_eq!(summary["status"], "published", "{summary}");
|
||||||
|
assert_eq!(
|
||||||
|
agent_version_count(db, agent_id).await,
|
||||||
|
1,
|
||||||
|
"a rejected edit must not persist a new Agent Version row"
|
||||||
|
);
|
||||||
|
|
||||||
|
let published_after = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
&published_after, published_v1,
|
||||||
|
"the Published snapshot and its bindings must stay unchanged after a rejected edit"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn published_agent_first_edit_forks_one_draft_and_keeps_published_snapshot() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_first");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let operation_id =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_first_tool").await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-first",
|
||||||
|
&operation_id,
|
||||||
|
"edit_first_tool",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let summary_before = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(summary_before["current_draft_version"], 1);
|
||||||
|
assert_eq!(summary_before["latest_published_version"], 1);
|
||||||
|
assert_eq!(summary_before["status"], "published");
|
||||||
|
|
||||||
|
let desired_policy = json!({
|
||||||
|
"mode": "search",
|
||||||
|
"groups": [{
|
||||||
|
"id": "crm-group",
|
||||||
|
"name": "CRM",
|
||||||
|
"description": "CRM tools",
|
||||||
|
"tool_names": ["edit_first_tool"]
|
||||||
|
}],
|
||||||
|
"search": { "max_results": 5 }
|
||||||
|
});
|
||||||
|
|
||||||
|
let edit_etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let response = client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, edit_etag)
|
||||||
|
.json(&json!({
|
||||||
|
"bindings": [binding(&operation_id, "edit_first_tool", "Edited Tool")],
|
||||||
|
"tool_selection_policy": desired_policy
|
||||||
|
}))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.json::<Value>().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
reqwest::StatusCode::OK,
|
||||||
|
"the first edit of a Published Agent must fork and save under the original If-Match: {body}"
|
||||||
|
);
|
||||||
|
assert_eq!(body["version"], 2, "{body}");
|
||||||
|
|
||||||
|
let summary_after = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(summary_after["current_draft_version"], 2, "{summary_after}");
|
||||||
|
assert_eq!(
|
||||||
|
summary_after["latest_published_version"], 1,
|
||||||
|
"{summary_after}"
|
||||||
|
);
|
||||||
|
assert_eq!(summary_after["status"], "published", "{summary_after}");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
agent_version_count(&db, &agent_id).await,
|
||||||
|
2,
|
||||||
|
"the first edit must create exactly one additional Draft Version"
|
||||||
|
);
|
||||||
|
|
||||||
|
let draft = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}/versions/2"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(draft["status"], "draft", "{draft}");
|
||||||
|
assert_eq!(
|
||||||
|
draft["snapshot"]["instructions"],
|
||||||
|
published_v1["snapshot"]["instructions"]
|
||||||
|
);
|
||||||
|
assert_eq!(draft["bindings"].as_array().unwrap().len(), 1, "{draft}");
|
||||||
|
assert_eq!(draft["bindings"][0]["tool_title"], "Edited Tool", "{draft}");
|
||||||
|
assert_eq!(
|
||||||
|
draft["snapshot"]["tool_selection_policy"]["mode"], "search",
|
||||||
|
"{draft}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
draft["snapshot"]["tool_selection_policy"]["search"]["max_results"], 5,
|
||||||
|
"{draft}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let published_after = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
published_after, published_v1,
|
||||||
|
"the Published v1 snapshot must not change when the Draft is forked"
|
||||||
|
);
|
||||||
|
|
||||||
|
let next_etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let second_response = client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, next_etag)
|
||||||
|
.json(&json!({
|
||||||
|
"bindings": [binding(&operation_id, "edit_first_tool", "Edited Again")],
|
||||||
|
"tool_selection_policy": desired_policy
|
||||||
|
}))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let second_status = second_response.status();
|
||||||
|
let second_body = second_response.text().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
second_status,
|
||||||
|
reqwest::StatusCode::OK,
|
||||||
|
"a subsequent valid edit on the Draft must succeed: {second_body}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
agent_version_count(&db, &agent_id).await,
|
||||||
|
2,
|
||||||
|
"a subsequent valid edit must stay on the existing Draft v2"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn rejected_invalid_policy_leaves_no_durable_draft() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_invalid_policy");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let operation_id =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_policy_tool")
|
||||||
|
.await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-policy",
|
||||||
|
&operation_id,
|
||||||
|
"edit_policy_tool",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let response = client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!({
|
||||||
|
"bindings": [],
|
||||||
|
"tool_selection_policy": {
|
||||||
|
"mode": "direct",
|
||||||
|
"groups": [{
|
||||||
|
"id": "group-1",
|
||||||
|
"name": "Group",
|
||||||
|
"description": "groups require search mode",
|
||||||
|
"tool_names": []
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
reqwest::StatusCode::BAD_REQUEST,
|
||||||
|
"a grouping policy in direct mode must be rejected: {body}"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn rejected_unpublished_operation_leaves_no_durable_draft() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_draft_op");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let live_operation =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_live_tool").await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-draft-op",
|
||||||
|
&live_operation,
|
||||||
|
"edit_live_tool",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let draft_operation =
|
||||||
|
create_draft_operation(&client, &base_url, &upstream_base_url, "edit_draft_tool").await;
|
||||||
|
|
||||||
|
let etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let response = client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!([binding(
|
||||||
|
&draft_operation,
|
||||||
|
"edit_draft_tool",
|
||||||
|
"Draft Tool"
|
||||||
|
)]))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
reqwest::StatusCode::UNPROCESSABLE_ENTITY,
|
||||||
|
"binding an unpublished Operation must be rejected: {body}"
|
||||||
|
);
|
||||||
|
let body: Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
body["error"]["code"], "agent_binding_not_published",
|
||||||
|
"{body}"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn rejected_archived_operation_leaves_no_durable_draft() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_archived_op");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let live_operation = create_published_operation(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
&upstream_base_url,
|
||||||
|
"edit_live_tool_arch",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-archived-op",
|
||||||
|
&live_operation,
|
||||||
|
"edit_live_tool_arch",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let archived_operation =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_archived_tool")
|
||||||
|
.await;
|
||||||
|
let operation_etag = operation_etag(&client, &base_url, &archived_operation).await;
|
||||||
|
let archived = client
|
||||||
|
.post(format!(
|
||||||
|
"{base_url}/operations/{archived_operation}/archive"
|
||||||
|
))
|
||||||
|
.header(reqwest::header::IF_MATCH, operation_etag)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let archived_status = archived.status();
|
||||||
|
let archived_body = archived.text().await.unwrap();
|
||||||
|
assert!(archived_status.is_success(), "{archived_body}");
|
||||||
|
|
||||||
|
let etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let response = client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!([binding(
|
||||||
|
&archived_operation,
|
||||||
|
"edit_archived_tool",
|
||||||
|
"Archived Tool"
|
||||||
|
)]))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
reqwest::StatusCode::UNPROCESSABLE_ENTITY,
|
||||||
|
"binding an archived Operation must be rejected: {body}"
|
||||||
|
);
|
||||||
|
let body: Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
body["error"]["code"], "agent_binding_archived_operation",
|
||||||
|
"{body}"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn rejected_foreign_operation_leaves_no_durable_draft() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_foreign_op");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let live_operation = create_published_operation(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
&upstream_base_url,
|
||||||
|
"edit_live_tool_foreign",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-foreign-op",
|
||||||
|
&live_operation,
|
||||||
|
"edit_live_tool_foreign",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let foreign_workspace = WorkspaceId::new("ws_agent_edit_foreign");
|
||||||
|
let now = OffsetDateTime::now_utc();
|
||||||
|
db.create_workspace(CreateWorkspaceRequest {
|
||||||
|
workspace: &Workspace {
|
||||||
|
id: foreign_workspace.clone(),
|
||||||
|
slug: "agent-edit-foreign".to_owned(),
|
||||||
|
display_name: "Agent Edit Foreign".to_owned(),
|
||||||
|
status: WorkspaceStatus::Active,
|
||||||
|
settings: json!({}),
|
||||||
|
created_at: now,
|
||||||
|
updated_at: now,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let foreign_service = test_service(
|
||||||
|
db.clone(),
|
||||||
|
test_storage_root("agent_catalog_edit_foreign_service"),
|
||||||
|
test_auth_settings(),
|
||||||
|
test_secret_crypto(),
|
||||||
|
);
|
||||||
|
let foreign_operation = foreign_service
|
||||||
|
.create_operation(
|
||||||
|
&foreign_workspace,
|
||||||
|
test_operation_payload(&upstream_base_url, "edit_foreign_tool"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let foreign_operation_id = foreign_operation.operation_id;
|
||||||
|
|
||||||
|
let etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let response = client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!([binding(
|
||||||
|
&foreign_operation_id,
|
||||||
|
"edit_foreign_tool",
|
||||||
|
"Foreign Tool"
|
||||||
|
)]))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
reqwest::StatusCode::NOT_FOUND,
|
||||||
|
"binding an Operation from another workspace must be rejected: {body}"
|
||||||
|
);
|
||||||
|
let body: Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
body["error"]["code"], "agent_binding_scope_denied",
|
||||||
|
"{body}"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn concurrent_first_edits_with_same_etag_have_one_winner() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_concurrent");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let operation_id =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_tool").await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-race",
|
||||||
|
&operation_id,
|
||||||
|
"edit_race_tool",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// Single original precondition taken once from the Published state.
|
||||||
|
let original_etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let endpoint = format!("{base_url}/agents/{agent_id}/bindings");
|
||||||
|
let payload_a = json!([binding(&operation_id, "edit_race_tool", "Race Winner A")]);
|
||||||
|
let payload_b = json!([binding(&operation_id, "edit_race_tool", "Race Winner B")]);
|
||||||
|
|
||||||
|
let spawn_edit = |payload: Value, etag: String, endpoint: String| {
|
||||||
|
let client = client.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let response = client
|
||||||
|
.post(endpoint)
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&payload)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.json::<Value>().await.unwrap();
|
||||||
|
(status, body)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
|
||||||
|
let first = spawn_edit(payload_a, original_etag.clone(), endpoint.clone());
|
||||||
|
let second = spawn_edit(payload_b, original_etag, endpoint);
|
||||||
|
let (first, second) = tokio::join!(first, second);
|
||||||
|
let results = vec![first.unwrap(), second.unwrap()];
|
||||||
|
|
||||||
|
let successes = results
|
||||||
|
.iter()
|
||||||
|
.filter(|(status, _)| status.is_success())
|
||||||
|
.count();
|
||||||
|
let conflicts = results
|
||||||
|
.iter()
|
||||||
|
.filter(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
|
||||||
|
.count();
|
||||||
|
assert_eq!(
|
||||||
|
successes, 1,
|
||||||
|
"exactly one concurrent first edit must win: {results:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
conflicts, 1,
|
||||||
|
"the losing concurrent first edit must report a stable conflict: {results:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let conflict_body = results
|
||||||
|
.iter()
|
||||||
|
.find(|(status, _)| *status == reqwest::StatusCode::CONFLICT)
|
||||||
|
.map(|(_, body)| body)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
conflict_body["error"]["code"], "agent_stale_revision",
|
||||||
|
"{conflict_body}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let summary = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(summary["current_draft_version"], 2, "{summary}");
|
||||||
|
assert_eq!(summary["latest_published_version"], 1, "{summary}");
|
||||||
|
assert_eq!(
|
||||||
|
agent_version_count(&db, &agent_id).await,
|
||||||
|
2,
|
||||||
|
"the losing concurrent edit must not fork a second Draft"
|
||||||
|
);
|
||||||
|
|
||||||
|
let published_after = assert_success_json(
|
||||||
|
client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}/versions/1"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(published_after, published_v1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn first_edit_after_operation_archive_leaves_no_durable_draft() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_archive_ordering");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let live_operation =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_order_live").await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-order",
|
||||||
|
&live_operation,
|
||||||
|
"edit_order_live",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let archived_operation = create_published_operation(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
&upstream_base_url,
|
||||||
|
"edit_order_archived",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let operation_etag = operation_etag(&client, &base_url, &archived_operation).await;
|
||||||
|
let archived = client
|
||||||
|
.post(format!(
|
||||||
|
"{base_url}/operations/{archived_operation}/archive"
|
||||||
|
))
|
||||||
|
.header(reqwest::header::IF_MATCH, operation_etag)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let archived_status = archived.status();
|
||||||
|
let archived_body = archived.text().await.unwrap();
|
||||||
|
assert!(archived_status.is_success(), "{archived_body}");
|
||||||
|
|
||||||
|
let etag = agent_etag(&client, &base_url, &agent_id).await;
|
||||||
|
let response = client
|
||||||
|
.post(format!("{base_url}/agents/{agent_id}/bindings"))
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!([binding(
|
||||||
|
&archived_operation,
|
||||||
|
"edit_order_archived",
|
||||||
|
"Archived Order Tool"
|
||||||
|
)]))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
assert!(
|
||||||
|
status.is_client_error(),
|
||||||
|
"a first edit that references an archived Operation must be rejected: {status}: {body}"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn first_edit_rejects_when_operation_archives_during_save() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let db = registry.clone();
|
||||||
|
let storage_root = test_storage_root("agent_catalog_edit_archive_race");
|
||||||
|
let upstream_base_url = spawn_upstream_server().await;
|
||||||
|
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||||
|
let client = authorized_client(&base_url).await;
|
||||||
|
|
||||||
|
let live_operation =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_live").await;
|
||||||
|
let (agent_id, published_v1) = create_published_agent(
|
||||||
|
&client,
|
||||||
|
&base_url,
|
||||||
|
"edit-archive-race",
|
||||||
|
&live_operation,
|
||||||
|
"edit_race_live",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let target_operation =
|
||||||
|
create_published_operation(&client, &base_url, &upstream_base_url, "edit_race_target")
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// Read the edit precondition before taking the Operation lock; Agent detail
|
||||||
|
// may itself inspect bound Operations.
|
||||||
|
let etag_response = client
|
||||||
|
.get(format!("{base_url}/agents/{agent_id}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let etag_status = etag_response.status();
|
||||||
|
let etag = etag_response
|
||||||
|
.headers()
|
||||||
|
.get(reqwest::header::ETAG)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.map(str::to_owned);
|
||||||
|
let etag_body = etag_response.text().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
etag_status,
|
||||||
|
reqwest::StatusCode::OK,
|
||||||
|
"reading the Agent ETag before the archive race must succeed: {etag_body}"
|
||||||
|
);
|
||||||
|
let etag = etag.expect("Agent detail must expose an ETag");
|
||||||
|
|
||||||
|
// Use an independent pool for the holder and pg_stat_activity monitor. The
|
||||||
|
// fixture registry has one connection, which the request needs to reach the
|
||||||
|
// Operation lock wait.
|
||||||
|
let lock_registry =
|
||||||
|
PostgresRegistry::connect_with_options((*db.pool().connect_options()).clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Hold the target Operation row lock from a separate connection so the edit
|
||||||
|
// transaction cannot complete its binding write until the archive wins.
|
||||||
|
let mut holder = lock_registry.pool().begin().await.unwrap();
|
||||||
|
sqlx::query("select id from operations where workspace_id = $1 and id = $2 for update")
|
||||||
|
.bind(DEFAULT_WORKSPACE_ID)
|
||||||
|
.bind(&target_operation)
|
||||||
|
.fetch_one(&mut *holder)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let holder_pid: i32 = sqlx::query_scalar("select pg_backend_pid()")
|
||||||
|
.fetch_one(&mut *holder)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let edit_client = client.clone();
|
||||||
|
let endpoint = format!("{base_url}/agents/{agent_id}/bindings");
|
||||||
|
let edit_target = target_operation.clone();
|
||||||
|
let edit = tokio::spawn(async move {
|
||||||
|
let response = edit_client
|
||||||
|
.post(endpoint)
|
||||||
|
.header(reqwest::header::IF_MATCH, etag)
|
||||||
|
.json(&json!([binding(
|
||||||
|
&edit_target,
|
||||||
|
"edit_race_target",
|
||||||
|
"Archive Race Tool"
|
||||||
|
)]))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = response.status();
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
(status, body)
|
||||||
|
});
|
||||||
|
|
||||||
|
// Wait for PostgreSQL to report the request blocked by this exact holder.
|
||||||
|
// If the request completes first (the baseline CAS path), keep going so the
|
||||||
|
// durable-draft assertion below reports that regression directly.
|
||||||
|
let lock_wait = async {
|
||||||
|
loop {
|
||||||
|
if edit.is_finished() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let blocked: bool = sqlx::query_scalar(
|
||||||
|
"select exists (
|
||||||
|
select 1
|
||||||
|
from pg_stat_activity
|
||||||
|
where wait_event_type = 'Lock'
|
||||||
|
and $1 = any(pg_blocking_pids(pid))
|
||||||
|
and query ilike '%select status from operations%'
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.bind(holder_pid)
|
||||||
|
.fetch_one(lock_registry.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
if blocked {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let saw_operation_lock_wait =
|
||||||
|
match tokio::time::timeout(std::time::Duration::from_secs(10), lock_wait).await {
|
||||||
|
Ok(saw_wait) => saw_wait,
|
||||||
|
Err(_) => {
|
||||||
|
edit.abort();
|
||||||
|
holder.rollback().await.unwrap();
|
||||||
|
let _ = edit.await;
|
||||||
|
panic!("timed out waiting for the edit to block on the held Operation row lock");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
|
"update operations
|
||||||
|
set status = 'archived', updated_at = now()
|
||||||
|
where workspace_id = $1 and id = $2",
|
||||||
|
)
|
||||||
|
.bind(DEFAULT_WORKSPACE_ID)
|
||||||
|
.bind(&target_operation)
|
||||||
|
.execute(&mut *holder)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
holder.commit().await.unwrap();
|
||||||
|
|
||||||
|
let (status, body) = edit.await.unwrap();
|
||||||
|
assert!(
|
||||||
|
status.is_client_error(),
|
||||||
|
"an edit racing an Operation archive must be rejected: {status}: {body}"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_no_durable_draft(&client, &db, &base_url, &agent_id, &published_v1).await;
|
||||||
|
assert!(
|
||||||
|
saw_operation_lock_wait,
|
||||||
|
"a corrected save path must wait on the holder's Operation row lock before archive commits"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -48,3 +48,28 @@ fn admin_and_mcp_preserve_dispatch_uncertainty_without_changing_the_code() {
|
|||||||
assert!(!mcp.recoverable);
|
assert!(!mcp.recoverable);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admin_and_mcp_project_new_uncertain_upstream_codes_consistently() {
|
||||||
|
for locale in [ExecutionLocale::En, ExecutionLocale::Ru] {
|
||||||
|
for code in [
|
||||||
|
ExecutionErrorCode::UpstreamRateLimited,
|
||||||
|
ExecutionErrorCode::UpstreamServerError,
|
||||||
|
ExecutionErrorCode::UpstreamResponseTooLarge,
|
||||||
|
ExecutionErrorCode::UpstreamStatusError,
|
||||||
|
] {
|
||||||
|
let failure = ExecutionFailure::new(code, CorrelationContext::generate())
|
||||||
|
.with_dispatch_uncertainty();
|
||||||
|
let admin = execution_test_failure_localized(&failure, locale);
|
||||||
|
let mcp = tool_error_contract_from_failure(&failure, locale);
|
||||||
|
|
||||||
|
assert_eq!(admin["code"], code.as_str());
|
||||||
|
assert_eq!(admin["retryability"], "manual_reconcile");
|
||||||
|
assert_eq!(admin["outcome_certainty"], "outcome_unknown");
|
||||||
|
assert_eq!(admin["retryability"], mcp.retryability);
|
||||||
|
assert_eq!(admin["outcome_certainty"], mcp.outcome_certainty);
|
||||||
|
assert_eq!(mcp.error_code, code.as_str());
|
||||||
|
assert!(!mcp.recoverable);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
mod integration {
|
mod integration {
|
||||||
mod catalog_access;
|
mod catalog_access;
|
||||||
mod common;
|
mod common;
|
||||||
|
mod execution_certainty;
|
||||||
mod execution_stages;
|
mod execution_stages;
|
||||||
mod jsonrpc_correlation;
|
mod jsonrpc_correlation;
|
||||||
mod request_context;
|
mod request_context;
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
use std::sync::{
|
||||||
|
Arc,
|
||||||
|
atomic::{AtomicUsize, Ordering},
|
||||||
|
};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use axum::{Router, http::StatusCode, routing::post};
|
||||||
|
use crank_core::{
|
||||||
|
ExecutionErrorCode, InvocationSource, OutcomeCertainty, PlatformApiKeyScope, Retryability,
|
||||||
|
WorkspaceId,
|
||||||
|
};
|
||||||
|
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
use super::common::{
|
||||||
|
agent_mcp_url, build_test_app, create_platform_api_key, initialize_session,
|
||||||
|
post_jsonrpc_response, publish_agent_for_operation, spawn_mcp_server, test_operation,
|
||||||
|
test_registry, test_workspace_id,
|
||||||
|
};
|
||||||
|
|
||||||
|
const REQUEST_ID: &str = "req_mcp_execution_certainty";
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ambiguous_post_failure_is_reported_and_persisted_as_unknown() {
|
||||||
|
let registry = test_registry().await;
|
||||||
|
let upstream_calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let upstream_base_url = spawn_503_upstream(Arc::clone(&upstream_calls)).await;
|
||||||
|
let operation = test_operation(&upstream_base_url, "mcp_execution_certainty");
|
||||||
|
|
||||||
|
registry
|
||||||
|
.create_operation(&test_workspace_id(), &operation, Some("test"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
registry
|
||||||
|
.publish_operation(PublishRequest {
|
||||||
|
workspace_id: &test_workspace_id(),
|
||||||
|
operation_id: &operation.id,
|
||||||
|
version: 1,
|
||||||
|
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
|
||||||
|
published_by: Some("test"),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
publish_agent_for_operation(®istry, &operation, "mcp-certainty-agent").await;
|
||||||
|
let api_key = create_platform_api_key(
|
||||||
|
®istry,
|
||||||
|
"mcp-certainty-agent",
|
||||||
|
"mcp-certainty-key",
|
||||||
|
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let app = build_test_app(registry.clone(), Duration::ZERO, None);
|
||||||
|
let base_url = spawn_mcp_server(app).await;
|
||||||
|
let client = reqwest::Client::new();
|
||||||
|
let mcp_url = agent_mcp_url(&base_url, "mcp-certainty-agent");
|
||||||
|
let session_id = initialize_session(&client, &mcp_url, &api_key).await;
|
||||||
|
|
||||||
|
let response = post_jsonrpc_response(
|
||||||
|
&client,
|
||||||
|
&mcp_url,
|
||||||
|
&api_key,
|
||||||
|
Some(&session_id),
|
||||||
|
Some(REQUEST_ID),
|
||||||
|
json!({
|
||||||
|
"jsonrpc": "2.0",
|
||||||
|
"id": 3,
|
||||||
|
"method": "tools/call",
|
||||||
|
"params": {
|
||||||
|
"name": operation.name,
|
||||||
|
"arguments": { "email": "user@example.com" }
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let response: Value = response.json().await.unwrap();
|
||||||
|
let tool_result = &response["result"];
|
||||||
|
let error = &tool_result["structuredContent"]["error"];
|
||||||
|
assert_eq!(tool_result["isError"], true);
|
||||||
|
assert_eq!(error["code"], "upstream_server_error");
|
||||||
|
assert_eq!(error["upstream_status"], 503);
|
||||||
|
assert_eq!(error["retryability"], "manual_reconcile");
|
||||||
|
assert_eq!(error["outcome_certainty"], "outcome_unknown");
|
||||||
|
assert_eq!(error["recoverable"], false);
|
||||||
|
assert_eq!(upstream_calls.load(Ordering::SeqCst), 1);
|
||||||
|
|
||||||
|
let logs = registry
|
||||||
|
.list_invocation_logs(ListInvocationLogsQuery {
|
||||||
|
workspace_id: &WorkspaceId::new("ws_default"),
|
||||||
|
level: None,
|
||||||
|
status: None,
|
||||||
|
outcome_group: None,
|
||||||
|
search_text: None,
|
||||||
|
source: Some(InvocationSource::AgentToolCall),
|
||||||
|
operation_id: Some(&operation.id),
|
||||||
|
agent_id: None,
|
||||||
|
created_after: None,
|
||||||
|
created_before: None,
|
||||||
|
cursor_created_at: None,
|
||||||
|
cursor_id: None,
|
||||||
|
limit: 10,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let invocation = logs
|
||||||
|
.iter()
|
||||||
|
.find(|record| record.log.request_id.as_deref() == Some(REQUEST_ID))
|
||||||
|
.expect("Invocation History entry for this MCP request");
|
||||||
|
assert_eq!(invocation.log.tool_name, operation.name);
|
||||||
|
assert_eq!(invocation.log.operation_version, Some(1));
|
||||||
|
assert_eq!(invocation.log.status_code, Some(503));
|
||||||
|
assert_eq!(
|
||||||
|
invocation.log.execution_error_code,
|
||||||
|
Some(ExecutionErrorCode::UpstreamServerError)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
invocation.log.retryability,
|
||||||
|
Some(Retryability::ManualReconcile)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
invocation.log.outcome_certainty,
|
||||||
|
Some(OutcomeCertainty::OutcomeUnknown)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn spawn_503_upstream(calls: Arc<AtomicUsize>) -> String {
|
||||||
|
let app = Router::new().route(
|
||||||
|
"/crm/leads",
|
||||||
|
post(move || {
|
||||||
|
let calls = Arc::clone(&calls);
|
||||||
|
async move {
|
||||||
|
calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
(StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable")
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
format!("http://{address}")
|
||||||
|
}
|
||||||
@@ -500,11 +500,16 @@ impl ExecutionFailure {
|
|||||||
self.code,
|
self.code,
|
||||||
ExecutionErrorCode::UpstreamTimeout
|
ExecutionErrorCode::UpstreamTimeout
|
||||||
| ExecutionErrorCode::UpstreamTransportError
|
| ExecutionErrorCode::UpstreamTransportError
|
||||||
|
| ExecutionErrorCode::UpstreamRateLimited
|
||||||
|
| ExecutionErrorCode::UpstreamServerError
|
||||||
|
| ExecutionErrorCode::UpstreamResponseTooLarge
|
||||||
|
| ExecutionErrorCode::UpstreamStatusError
|
||||||
| ExecutionErrorCode::IdempotencyOutcomeUnknown
|
| ExecutionErrorCode::IdempotencyOutcomeUnknown
|
||||||
| ExecutionErrorCode::PersistenceUnavailable
|
| ExecutionErrorCode::PersistenceUnavailable
|
||||||
) {
|
) {
|
||||||
self.retryability = Retryability::ManualReconcile;
|
self.retryability = Retryability::ManualReconcile;
|
||||||
self.outcome_certainty = OutcomeCertainty::OutcomeUnknown;
|
self.outcome_certainty = OutcomeCertainty::OutcomeUnknown;
|
||||||
|
self.retry_after_ms = None;
|
||||||
}
|
}
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -265,16 +265,111 @@ impl PostgresRegistry {
|
|||||||
&self,
|
&self,
|
||||||
request: SaveAgentCatalogConfigRequest<'_>,
|
request: SaveAgentCatalogConfigRequest<'_>,
|
||||||
) -> Result<(), RegistryError> {
|
) -> Result<(), RegistryError> {
|
||||||
|
self.save_agent_catalog_config_inner(request, false)
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn save_agent_catalog_config_editable(
|
||||||
|
&self,
|
||||||
|
request: SaveAgentCatalogConfigRequest<'_>,
|
||||||
|
) -> Result<u32, RegistryError> {
|
||||||
|
self.save_agent_catalog_config_inner(request, true).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn save_agent_catalog_config_inner(
|
||||||
|
&self,
|
||||||
|
request: SaveAgentCatalogConfigRequest<'_>,
|
||||||
|
allow_fork: bool,
|
||||||
|
) -> Result<u32, RegistryError> {
|
||||||
let mut tx = self.pool.begin().await?;
|
let mut tx = self.pool.begin().await?;
|
||||||
Self::lock_agent_and_validate_expected_state(
|
let agent_row = Self::lock_agent_and_validate_expected_state(
|
||||||
&mut tx,
|
&mut tx,
|
||||||
request.workspace_id,
|
request.workspace_id,
|
||||||
request.agent_id,
|
request.agent_id,
|
||||||
request.expected_state,
|
request.expected_state,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Self::reject_published_agent_version(&mut tx, request.agent_id, request.agent_version)
|
|
||||||
.await?;
|
if allow_fork {
|
||||||
|
let status =
|
||||||
|
deserialize_enum_text::<AgentStatus>(agent_row.try_get("status")?, "status")?;
|
||||||
|
if status == AgentStatus::Archived {
|
||||||
|
return Err(RegistryError::InvalidAgentTransition {
|
||||||
|
agent_id: request.agent_id.as_str().to_owned(),
|
||||||
|
from: "archived".to_owned(),
|
||||||
|
action: "edit",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let current_draft_version = from_db_version(
|
||||||
|
agent_row.try_get("current_draft_version")?,
|
||||||
|
"current_draft_version",
|
||||||
|
)?;
|
||||||
|
if request.agent_version != current_draft_version {
|
||||||
|
return Err(RegistryError::InvalidAgentVersionSequence {
|
||||||
|
agent_id: request.agent_id.as_str().to_owned(),
|
||||||
|
expected: current_draft_version,
|
||||||
|
actual: request.agent_version,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let latest_published_version = agent_row
|
||||||
|
.try_get::<Option<i32>, _>("latest_published_version")?
|
||||||
|
.map(|value| from_db_version(value, "latest_published_version"))
|
||||||
|
.transpose()?;
|
||||||
|
|
||||||
|
if latest_published_version == Some(current_draft_version) {
|
||||||
|
let instructions = sqlx::query(
|
||||||
|
"select instructions_json
|
||||||
|
from agent_versions
|
||||||
|
where agent_id = $1 and version = $2",
|
||||||
|
)
|
||||||
|
.bind(request.agent_id.as_str())
|
||||||
|
.bind(to_db_version(current_draft_version))
|
||||||
|
.fetch_optional(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.ok_or_else(|| RegistryError::AgentNotFound {
|
||||||
|
agent_id: request.agent_id.as_str().to_owned(),
|
||||||
|
})?
|
||||||
|
.try_get::<Value, _>("instructions_json")?;
|
||||||
|
|
||||||
|
let now = time::OffsetDateTime::now_utc();
|
||||||
|
let draft = AgentVersion {
|
||||||
|
agent_id: request.agent_id.clone(),
|
||||||
|
version: current_draft_version + 1,
|
||||||
|
status: AgentStatus::Draft,
|
||||||
|
instructions,
|
||||||
|
tool_selection_policy: request.tool_selection_policy.clone(),
|
||||||
|
created_at: now,
|
||||||
|
};
|
||||||
|
insert_agent_version_row(&mut tx, &draft).await?;
|
||||||
|
replace_agent_bindings_rows(
|
||||||
|
&mut tx,
|
||||||
|
request.workspace_id,
|
||||||
|
request.agent_id,
|
||||||
|
draft.version,
|
||||||
|
request.bindings,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
sqlx::query(
|
||||||
|
"update agents
|
||||||
|
set current_draft_version = $3,
|
||||||
|
updated_at = $4::timestamptz
|
||||||
|
where workspace_id = $1 and id = $2",
|
||||||
|
)
|
||||||
|
.bind(request.workspace_id.as_str())
|
||||||
|
.bind(request.agent_id.as_str())
|
||||||
|
.bind(to_db_version(draft.version))
|
||||||
|
.bind(now)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
return Ok(draft.version);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Self::reject_published_agent_version(&mut tx, request.agent_id, request.agent_version)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
let updated = sqlx::query(
|
let updated = sqlx::query(
|
||||||
"update agent_versions
|
"update agent_versions
|
||||||
set tool_selection_policy_json = $3
|
set tool_selection_policy_json = $3
|
||||||
@@ -309,7 +404,7 @@ impl PostgresRegistry {
|
|||||||
.execute(&mut *tx)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
tx.commit().await?;
|
tx.commit().await?;
|
||||||
Ok(())
|
Ok(request.agent_version)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn lock_agent_and_validate_expected_state(
|
async fn lock_agent_and_validate_expected_state(
|
||||||
@@ -317,7 +412,7 @@ impl PostgresRegistry {
|
|||||||
workspace_id: &WorkspaceId,
|
workspace_id: &WorkspaceId,
|
||||||
agent_id: &AgentId,
|
agent_id: &AgentId,
|
||||||
expected_state: Option<&AgentStateExpectation>,
|
expected_state: Option<&AgentStateExpectation>,
|
||||||
) -> Result<(), RegistryError> {
|
) -> Result<sqlx::postgres::PgRow, RegistryError> {
|
||||||
let row = sqlx::query(
|
let row = sqlx::query(
|
||||||
"select
|
"select
|
||||||
status,
|
status,
|
||||||
@@ -364,7 +459,7 @@ impl PostgresRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(row)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reject_published_agent_version(
|
async fn reject_published_agent_version(
|
||||||
|
|||||||
@@ -38,10 +38,13 @@ fn normalize_runtime_error_with_effect(
|
|||||||
) -> ExecutionFailure {
|
) -> ExecutionFailure {
|
||||||
let mut failure = ExecutionFailure::new(error_code(error), correlation.clone());
|
let mut failure = ExecutionFailure::new(error_code(error), correlation.clone());
|
||||||
match error {
|
match error {
|
||||||
RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
RuntimeError::ProtocolAdapter(
|
||||||
status, ..
|
adapter @ ProtocolAdapterError::UnexpectedStatus { status, .. },
|
||||||
}) => {
|
) => {
|
||||||
failure = failure.with_upstream_status(*status);
|
failure = failure.with_upstream_status(*status);
|
||||||
|
if may_mutate && ambiguous_after_dispatch(adapter) {
|
||||||
|
failure = failure.with_dispatch_uncertainty();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
RuntimeError::ProtocolAdapter(adapter)
|
RuntimeError::ProtocolAdapter(adapter)
|
||||||
if may_mutate && ambiguous_after_dispatch(adapter) =>
|
if may_mutate && ambiguous_after_dispatch(adapter) =>
|
||||||
|
|||||||
@@ -0,0 +1,244 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use crank_core::{
|
||||||
|
CorrelationContext, DispatchEvidence, ExecutionErrorCode, ExecutionFailure, HttpMethod,
|
||||||
|
OutcomeCertainty, Protocol, ProtocolAdapterError, RestTarget, Retryability, Target,
|
||||||
|
ToolDescription,
|
||||||
|
};
|
||||||
|
use crank_mapping::MappingSet;
|
||||||
|
use crank_runtime::{RuntimeError, RuntimeOperation, normalize_runtime_error_for_operation};
|
||||||
|
use crank_schema::{Schema, SchemaKind};
|
||||||
|
|
||||||
|
const MUTATING_METHODS: [HttpMethod; 4] = [
|
||||||
|
HttpMethod::Post,
|
||||||
|
HttpMethod::Put,
|
||||||
|
HttpMethod::Patch,
|
||||||
|
HttpMethod::Delete,
|
||||||
|
];
|
||||||
|
const READ_METHODS: [HttpMethod; 2] = [HttpMethod::Get, HttpMethod::Head];
|
||||||
|
const UNCERTAIN_STATUSES: [u16; 4] = [408, 429, 500, 503];
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ambiguous_upstream_failures_are_unknown_only_for_dispatched_mutations() {
|
||||||
|
let mut failures = Vec::new();
|
||||||
|
for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
|
||||||
|
for dispatch in [
|
||||||
|
DispatchEvidence::MayHaveDispatched,
|
||||||
|
DispatchEvidence::NotDispatched,
|
||||||
|
] {
|
||||||
|
for status in UNCERTAIN_STATUSES {
|
||||||
|
let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
||||||
|
status,
|
||||||
|
dispatch,
|
||||||
|
});
|
||||||
|
if let Err(failure) = assert_normalization(
|
||||||
|
method,
|
||||||
|
&error,
|
||||||
|
is_ambiguous_mutation(method, dispatch),
|
||||||
|
Some(status),
|
||||||
|
format!("{method:?} status {status} {dispatch:?}"),
|
||||||
|
) {
|
||||||
|
failures.push(failure);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for adapter in [
|
||||||
|
ProtocolAdapterError::ResponseTooLarge { dispatch },
|
||||||
|
ProtocolAdapterError::InvalidResponse { dispatch },
|
||||||
|
] {
|
||||||
|
let error = RuntimeError::ProtocolAdapter(adapter);
|
||||||
|
if let Err(failure) = assert_normalization(
|
||||||
|
method,
|
||||||
|
&error,
|
||||||
|
is_ambiguous_mutation(method, dispatch),
|
||||||
|
None,
|
||||||
|
format!("{method:?} {error:?}"),
|
||||||
|
) {
|
||||||
|
failures.push(failure);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
failures.is_empty(),
|
||||||
|
"unexpected normalization rows:\n{}",
|
||||||
|
failures.join("\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nonambiguous_statuses_keep_their_existing_descriptor() {
|
||||||
|
let mut failures = Vec::new();
|
||||||
|
for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
|
||||||
|
for status in [401, 404] {
|
||||||
|
let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
||||||
|
status,
|
||||||
|
dispatch: DispatchEvidence::MayHaveDispatched,
|
||||||
|
});
|
||||||
|
if let Err(failure) = assert_normalization(
|
||||||
|
method,
|
||||||
|
&error,
|
||||||
|
false,
|
||||||
|
Some(status),
|
||||||
|
format!("{method:?} nonambiguous status {status}"),
|
||||||
|
) {
|
||||||
|
failures.push(failure);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
failures.is_empty(),
|
||||||
|
"unexpected normalization rows:\n{}",
|
||||||
|
failures.join("\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dispatch_uncertainty_allowlist_covers_uncertain_upstream_codes_and_drops_retry_delay() {
|
||||||
|
let uncertain_codes = [
|
||||||
|
ExecutionErrorCode::UpstreamRateLimited,
|
||||||
|
ExecutionErrorCode::UpstreamServerError,
|
||||||
|
ExecutionErrorCode::UpstreamResponseTooLarge,
|
||||||
|
ExecutionErrorCode::UpstreamStatusError,
|
||||||
|
];
|
||||||
|
let mut failures = Vec::new();
|
||||||
|
for code in uncertain_codes {
|
||||||
|
let failure = ExecutionFailure::new(code, CorrelationContext::generate());
|
||||||
|
let failure = if matches!(code.retryability(), Retryability::AfterDelay) {
|
||||||
|
failure
|
||||||
|
.try_with_retry_after_ms(1_000)
|
||||||
|
.expect("retry delay is valid for the original descriptor")
|
||||||
|
} else {
|
||||||
|
failure
|
||||||
|
}
|
||||||
|
.with_dispatch_uncertainty();
|
||||||
|
|
||||||
|
if failure.error_code() != code
|
||||||
|
|| failure.retryability() != Retryability::ManualReconcile
|
||||||
|
|| failure.outcome_certainty() != OutcomeCertainty::OutcomeUnknown
|
||||||
|
|| failure.retry_after_ms().is_some()
|
||||||
|
{
|
||||||
|
failures.push(format!(
|
||||||
|
"{code:?}: actual code={:?}, retryability={:?}, certainty={:?}, retry_after_ms={:?}",
|
||||||
|
failure.error_code(),
|
||||||
|
failure.retryability(),
|
||||||
|
failure.outcome_certainty(),
|
||||||
|
failure.retry_after_ms(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
failures.is_empty(),
|
||||||
|
"uncertain code allowlist failures:\n{}",
|
||||||
|
failures.join("\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_normalization(
|
||||||
|
method: HttpMethod,
|
||||||
|
error: &RuntimeError,
|
||||||
|
unknown: bool,
|
||||||
|
status: Option<u16>,
|
||||||
|
context: String,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let correlation = CorrelationContext::generate();
|
||||||
|
let failure = normalize_runtime_error_for_operation(error, &correlation, &operation(method));
|
||||||
|
let code = expected_code(error);
|
||||||
|
let expected = ExecutionFailure::new(code, correlation.clone());
|
||||||
|
let expected_retryability = if unknown {
|
||||||
|
Retryability::ManualReconcile
|
||||||
|
} else {
|
||||||
|
expected.retryability()
|
||||||
|
};
|
||||||
|
let expected_certainty = if unknown {
|
||||||
|
OutcomeCertainty::OutcomeUnknown
|
||||||
|
} else {
|
||||||
|
expected.outcome_certainty()
|
||||||
|
};
|
||||||
|
|
||||||
|
if failure.error_code() != code
|
||||||
|
|| failure.retryability() != expected_retryability
|
||||||
|
|| failure.outcome_certainty() != expected_certainty
|
||||||
|
|| failure.upstream_status() != status
|
||||||
|
|| failure.retry_after_ms().is_some()
|
||||||
|
{
|
||||||
|
return Err(format!(
|
||||||
|
"{context}: code={:?} (expected {code:?}), retryability={:?} (expected {expected_retryability:?}), certainty={:?} (expected {expected_certainty:?}), upstream_status={:?} (expected {status:?}), retry_after_ms={:?}",
|
||||||
|
failure.error_code(),
|
||||||
|
failure.retryability(),
|
||||||
|
failure.outcome_certainty(),
|
||||||
|
failure.upstream_status(),
|
||||||
|
failure.retry_after_ms(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expected_code(error: &RuntimeError) -> ExecutionErrorCode {
|
||||||
|
match error {
|
||||||
|
RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
||||||
|
status, ..
|
||||||
|
}) => match status {
|
||||||
|
401 | 403 => ExecutionErrorCode::UpstreamAuthError,
|
||||||
|
404 => ExecutionErrorCode::UpstreamNotFound,
|
||||||
|
408 | 429 => ExecutionErrorCode::UpstreamRateLimited,
|
||||||
|
500..=599 => ExecutionErrorCode::UpstreamServerError,
|
||||||
|
_ => ExecutionErrorCode::UpstreamStatusError,
|
||||||
|
},
|
||||||
|
RuntimeError::ProtocolAdapter(ProtocolAdapterError::ResponseTooLarge { .. }) => {
|
||||||
|
ExecutionErrorCode::UpstreamResponseTooLarge
|
||||||
|
}
|
||||||
|
RuntimeError::ProtocolAdapter(ProtocolAdapterError::InvalidResponse { .. }) => {
|
||||||
|
ExecutionErrorCode::UpstreamStatusError
|
||||||
|
}
|
||||||
|
other => panic!("unexpected test fixture: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_ambiguous_mutation(method: HttpMethod, dispatch: DispatchEvidence) -> bool {
|
||||||
|
MUTATING_METHODS.contains(&method) && dispatch == DispatchEvidence::MayHaveDispatched
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operation(method: HttpMethod) -> RuntimeOperation {
|
||||||
|
RuntimeOperation {
|
||||||
|
operation_id: crank_core::OperationId::new("certainty_test"),
|
||||||
|
operation_version: 1,
|
||||||
|
tool_name: "certainty_test".to_owned(),
|
||||||
|
protocol: Protocol::Rest,
|
||||||
|
target: Target::Rest(RestTarget {
|
||||||
|
base_url: "https://example.invalid".to_owned(),
|
||||||
|
method,
|
||||||
|
path_template: "/items".to_owned(),
|
||||||
|
static_headers: BTreeMap::new(),
|
||||||
|
request_contract: None,
|
||||||
|
response_contract: Vec::new(),
|
||||||
|
multipart_field_mime_types: BTreeMap::new(),
|
||||||
|
}),
|
||||||
|
input_schema: empty_schema(),
|
||||||
|
output_schema: empty_schema(),
|
||||||
|
input_mapping: MappingSet::default(),
|
||||||
|
output_mapping: MappingSet::default(),
|
||||||
|
execution_config: crank_core::ExecutionConfig::default(),
|
||||||
|
tool_description: ToolDescription {
|
||||||
|
title: "Certainty test".to_owned(),
|
||||||
|
description: "Execution certainty fixture".to_owned(),
|
||||||
|
tags: Vec::new(),
|
||||||
|
examples: Vec::new(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn empty_schema() -> Schema {
|
||||||
|
Schema {
|
||||||
|
kind: SchemaKind::Object,
|
||||||
|
format: None,
|
||||||
|
description: None,
|
||||||
|
required: true,
|
||||||
|
nullable: false,
|
||||||
|
default_value: None,
|
||||||
|
fields: BTreeMap::new(),
|
||||||
|
items: None,
|
||||||
|
enum_values: Vec::new(),
|
||||||
|
variants: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -30,6 +30,8 @@ Crank превращает REST API endpoint-ы в MCP-инструменты,
|
|||||||
- [Capability Baseline](./capability-baseline/manifest.json) — versioned SHA-256
|
- [Capability Baseline](./capability-baseline/manifest.json) — versioned SHA-256
|
||||||
snapshot фактических UI, Admin API и MCP flows, taxonomy, checklist и
|
snapshot фактических UI, Admin API и MCP flows, taxonomy, checklist и
|
||||||
sanitized evidence results.
|
sanitized evidence results.
|
||||||
|
- [Evidence разработки](./development-evidence.md) — clean baseline, проверенная
|
||||||
|
revision реализации и отдельная запись текущего completion evidence.
|
||||||
- [Schema Capability Baseline](./schemas/capability-baseline.schema.json) —
|
- [Schema Capability Baseline](./schemas/capability-baseline.schema.json) —
|
||||||
fail-closed manifest contract.
|
fail-closed manifest contract.
|
||||||
- [Настройки окружения](./runtime-config.md)
|
- [Настройки окружения](./runtime-config.md)
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
# Evidence разработки
|
||||||
|
|
||||||
|
Этот порядок помогает восстановить состояние кода и результат каждого
|
||||||
|
проверяемого этапа. Он не заменяет [Capability Baseline](./capability-baseline/manifest.json):
|
||||||
|
baseline хранит версионированный снимок capability Inventory и исторические
|
||||||
|
результаты, а completion evidence указывает, что именно проверено для текущего
|
||||||
|
этапа или завершения.
|
||||||
|
|
||||||
|
## Ревизии этапа
|
||||||
|
|
||||||
|
До начала этапа запишите полный Git commit `baseline_commit` и убедитесь, что
|
||||||
|
`git status --short` пуст. Это проверка tracked, staged и обычных untracked
|
||||||
|
изменений; игнорируемые локальные файлы не входят в Git baseline. Не записывайте
|
||||||
|
`baseline + uncommitted worktree` как revision.
|
||||||
|
|
||||||
|
После изменений сохраните проверенный implementation revision отдельным полным
|
||||||
|
commit hash. Проверки, которые подтверждают завершение этапа, должны относиться
|
||||||
|
к этому commit при чистом worktree. Укажите использованные конфигурацию и
|
||||||
|
класс окружения: одинаковый commit сам по себе не описывает их.
|
||||||
|
|
||||||
|
Если нужен RED до коммита, зафиксируйте его как промежуточное наблюдение: укажите
|
||||||
|
`baseline_commit`, SHA-256 сохранённого diff/artifact, включающего изменения
|
||||||
|
(в том числе untracked test files), команду и её фактический результат. Не
|
||||||
|
представляйте baseline вместе с незакоммиченным diff как implementation revision.
|
||||||
|
После RED повторите нужные проверки на отдельном чистом implementation revision.
|
||||||
|
|
||||||
|
## Запись completion evidence
|
||||||
|
|
||||||
|
Для текущего завершения используйте отдельный versioned index и sanitized
|
||||||
|
evidence-файлы с устойчивыми repository-relative ссылками. Запись должна позволять
|
||||||
|
повторно определить проверенное состояние и результат, не сохраняя секреты,
|
||||||
|
локальные абсолютные пути или произвольные payloads. Для каждой проверки укажите:
|
||||||
|
|
||||||
|
- идентификатор этапа/запуска и полный `baseline_commit`;
|
||||||
|
- полный `implementation_revision` проверенного commit;
|
||||||
|
- конфигурацию/toolchain и класс окружения;
|
||||||
|
- точную команду, exit code и фактические counts `passed`, `failed`, `skipped`
|
||||||
|
(или эквивалентный summary для команды без тестового счётчика);
|
||||||
|
- repository-relative путь к sanitised log/report и его SHA-256.
|
||||||
|
|
||||||
|
Если результаты не относятся к одной и той же ревизии, конфигурации или
|
||||||
|
окружению, записывайте их раздельно. Не выводите PASS из exit code без проверки
|
||||||
|
счётчиков и пропусков; сохраняйте fail, flaky, skipped, not-run и manual-only
|
||||||
|
как наблюдались. Новое completion evidence не переписывает historical baseline
|
||||||
|
или его manual failures.
|
||||||
|
|
||||||
|
Изменение canonical Capability Inventory status/evidence по-прежнему требует
|
||||||
|
обновить versioned `docs/capability-baseline/manifest.json`, sanitized
|
||||||
|
`docs/capability-baseline/results.json`, `baseline_version` и SHA-256 согласно
|
||||||
|
его schema и проверкам. Отдельный completion index не ослабляет и не обходит
|
||||||
|
этот gate.
|
||||||
|
|
||||||
|
Компактный шаблон записи:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
id: <stage-or-run-id>
|
||||||
|
baseline_commit: <full-git-commit>
|
||||||
|
implementation_revision: <full-git-commit>
|
||||||
|
environment:
|
||||||
|
class: <environment-class>
|
||||||
|
config: <tracked-config-or-versioned-reference>
|
||||||
|
command: <exact-command>
|
||||||
|
exit_code: <actual-integer>
|
||||||
|
counts:
|
||||||
|
passed: <actual-count>
|
||||||
|
failed: <actual-count>
|
||||||
|
skipped: <actual-count>
|
||||||
|
evidence:
|
||||||
|
path: <repository-relative-sanitized-report>
|
||||||
|
sha256: <sha256-of-report>
|
||||||
|
```
|
||||||
|
|
||||||
|
Для промежуточного RED вместо `implementation_revision` укажите
|
||||||
|
`diff_sha256` и ссылку на сохранённый diff/artifact; не выдавайте его за
|
||||||
|
проверенный implementation commit.
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { chromium } = require(process.env.CRANK_PROBE_PLAYWRIGHT || require('node:path').resolve(__dirname, '../../../apps/ui/node_modules/@playwright/test'));
|
||||||
|
(async () => {
|
||||||
|
const browser = await chromium.launch();
|
||||||
|
try {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.goto('http://127.0.0.1:3300/login');
|
||||||
|
await page.locator('#email').fill('owner@crank.local');
|
||||||
|
await page.locator('#password').fill('change-me-admin-password');
|
||||||
|
await page.locator('.btn-signin').click();
|
||||||
|
await page.waitForURL('http://127.0.0.1:3300/');
|
||||||
|
const result = await page.evaluate(async () => {
|
||||||
|
const api = window.CrankApi;
|
||||||
|
const session = await api.getSession();
|
||||||
|
const csrf = await api.refreshSessionCsrf();
|
||||||
|
session.csrf_token = csrf.csrf_token; window.CrankAuth.replaceSession(session);
|
||||||
|
const ws = session.current_workspace_id || session.memberships[0].workspace.id;
|
||||||
|
const agents = await api.listAgents(ws);
|
||||||
|
const list = Array.isArray(agents) ? agents : (agents.agents || agents.items);
|
||||||
|
const source = list.find(a => a.latest_published_version != null);
|
||||||
|
if (!source) throw new Error('No published fixture Agent');
|
||||||
|
const sid = source.agent_id || source.id;
|
||||||
|
const version = await fetch('/api/admin/workspaces/'+ws+'/agents/'+sid+'/versions/'+source.latest_published_version).then(r=>r.json());
|
||||||
|
const bindings = version.bindings.map(b => ({operation_id:b.operation_id,operation_version:b.operation_version,tool_name:b.tool_name,tool_title:b.tool_title,tool_description_override:b.tool_description_override,enabled:b.enabled}));
|
||||||
|
const created = await api.createAgent(ws,{slug:'retro-probe-'+Date.now(),display_name:'Retro probe',description:'Synthetic regression probe',instructions:{},tool_selection_policy:{}});
|
||||||
|
const id = created.agent_id || created.id;
|
||||||
|
await api.saveAgentBindings(ws,id,bindings);
|
||||||
|
await api.publishAgent(ws,id,{version:1});
|
||||||
|
const before=await api.getAgent(ws,id);
|
||||||
|
const publishedBefore=await fetch('/api/admin/workspaces/'+ws+'/agents/'+id+'/versions/1').then(r=>r.json());
|
||||||
|
let errorCode=null; let saved=null;
|
||||||
|
try { saved=await api.saveAgentBindings(ws,id,bindings); } catch(e) { errorCode=e.errorCode || e.code || e.message; }
|
||||||
|
const after=await api.getAgent(ws,id);
|
||||||
|
const publishedAfter=await fetch('/api/admin/workspaces/'+ws+'/agents/'+id+'/versions/1').then(r=>r.json());
|
||||||
|
return {before_draft:before.current_draft_version,after_draft:after.current_draft_version,saved_version:saved && saved.version,published_unchanged:JSON.stringify(publishedBefore)===JSON.stringify(publishedAfter),error:errorCode};
|
||||||
|
});
|
||||||
|
console.log(JSON.stringify(result));
|
||||||
|
assert.equal(result.error,null); assert.equal(result.before_draft,1); assert.equal(result.after_draft,2); assert.equal(result.saved_version,2); assert.equal(result.published_unchanged,true);
|
||||||
|
} finally { await browser.close(); }
|
||||||
|
})().catch(e=>{console.error(e.message);process.exitCode=1;});
|
||||||
@@ -19,6 +19,12 @@ declare -A FILE_LIMITS=(
|
|||||||
)
|
)
|
||||||
|
|
||||||
status=0
|
status=0
|
||||||
|
source_roots=()
|
||||||
|
for source_dir in apps crates; do
|
||||||
|
if [[ -d "$ROOT_DIR/$source_dir" ]]; then
|
||||||
|
source_roots+=("$ROOT_DIR/$source_dir")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
echo "Rust code health: checking file size limits"
|
echo "Rust code health: checking file size limits"
|
||||||
|
|
||||||
@@ -32,10 +38,11 @@ while IFS= read -r file; do
|
|||||||
status=1
|
status=1
|
||||||
fi
|
fi
|
||||||
done < <(
|
done < <(
|
||||||
find "$ROOT_DIR" \
|
for source_root in "${source_roots[@]}"; do
|
||||||
-path "$ROOT_DIR/target" -prune -o \
|
find "$source_root" \
|
||||||
-path "$ROOT_DIR/apps/ui/node_modules" -prune -o \
|
-type d \( -name target -o -name node_modules \) -prune -o \
|
||||||
-name '*.rs' -print | sort
|
-type f -name '*.rs' -print
|
||||||
|
done | sort
|
||||||
)
|
)
|
||||||
|
|
||||||
echo "Rust code health: checking large inline test modules"
|
echo "Rust code health: checking large inline test modules"
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ class RustCodeHealthCheckTests(unittest.TestCase):
|
|||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
self.assertIn("checking file size limits", result.stdout)
|
self.assertIn("checking file size limits", result.stdout)
|
||||||
|
|
||||||
def test_fails_for_new_large_rust_file(self) -> None:
|
def test_fails_for_new_untracked_large_rust_file_in_source_scope(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root = Path(tmp)
|
root = Path(tmp)
|
||||||
self.prepare_root(root)
|
self.prepare_root(root)
|
||||||
@@ -55,6 +55,46 @@ class RustCodeHealthCheckTests(unittest.TestCase):
|
|||||||
self.assertNotEqual(result.returncode, 0)
|
self.assertNotEqual(result.returncode, 0)
|
||||||
self.assertIn("apps/demo/src/large.rs has 21 lines", result.stderr)
|
self.assertIn("apps/demo/src/large.rs has 21 lines", result.stderr)
|
||||||
|
|
||||||
|
def test_ignores_large_rust_in_nested_bmad_worktree(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
self.prepare_root(root)
|
||||||
|
worktree = (
|
||||||
|
root
|
||||||
|
/ ".bmad-loop"
|
||||||
|
/ "runs"
|
||||||
|
/ "epic-1"
|
||||||
|
/ "worktrees"
|
||||||
|
/ "fixture"
|
||||||
|
)
|
||||||
|
worktree.mkdir(parents=True)
|
||||||
|
(worktree / "large.rs").write_text(
|
||||||
|
"\n".join("// line" for _ in range(21)) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.run_checker(root)
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
|
||||||
|
def test_ignores_rust_under_nested_target_and_node_modules(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
self.prepare_root(root)
|
||||||
|
generated_dirs = (
|
||||||
|
root / "apps" / "demo" / "target" / "generated",
|
||||||
|
root / "apps" / "demo" / "src" / "node_modules" / "fixture",
|
||||||
|
root / "crates" / "demo" / "node_modules" / "fixture",
|
||||||
|
)
|
||||||
|
large_source = "\n".join("// generated" for _ in range(21)) + "\n"
|
||||||
|
for directory in generated_dirs:
|
||||||
|
directory.mkdir(parents=True)
|
||||||
|
(directory / "large.rs").write_text(large_source, encoding="utf-8")
|
||||||
|
|
||||||
|
result = self.run_checker(root)
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
|
||||||
def test_fails_for_large_inline_test_module(self) -> None:
|
def test_fails_for_large_inline_test_module(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root = Path(tmp)
|
root = Path(tmp)
|
||||||
|
|||||||
Reference in new issue
Block a user