feat: complete Epic 1 production foundation

This commit is contained in:
2026-08-25 01:24:11 +03:00
parent 767428436d
commit 182bde8ac0
298 changed files with 35719 additions and 5299 deletions
+37 -1
View File
@@ -1,11 +1,21 @@
use std::process::Command;
use crank_registry::{
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, MigrationAuthority, PostgresRegistry,
};
use crank_runtime::SecretCrypto;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
const REGISTERED_MASTER_KEY: &str = "registered-master-key-CANARY_SECRET_VALUE-00000000";
const WRONG_MASTER_KEY: &str = "wrong-master-key-CANARY_SECRET_VALUE-0000000000000";
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_mcp-server"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.env("CRANK_MASTER_KEY", "master");
command.env("CRANK_MASTER_KEY", TEST_MASTER_KEY);
for (name, value) in entries {
command.env(name, value);
}
@@ -63,3 +73,29 @@ async fn fresh_database_startup_is_read_only() {
.unwrap();
assert!(!present);
}
#[tokio::test]
async fn master_key_mismatch_blocks_startup_with_safe_diagnostic() {
let database_url = crank_test_support::postgres_schema_url("mcp_master_key_mismatch").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::parse("2026-08-21T00:00:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
let stderr = run_with(&[
("CRANK_DATABASE_URL", &database_url),
("CRANK_MASTER_KEY", WRONG_MASTER_KEY),
]);
assert!(stderr.contains("master_key_identity_mismatch"), "{stderr}");
assert!(!stderr.contains("registered-master-key"));
assert!(!stderr.contains("wrong-master-key"));
}
+5
View File
@@ -114,11 +114,16 @@ async fn preserves_mcp_result_when_postgres_rejects_invocation_history() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &WorkspaceId::new("ws_default"),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: None,
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -23,14 +23,14 @@ use crank_core::{
ApprovalRequestId, ApprovalRequestStatus, ExecutionConfig, HttpMethod, InvocationSource,
Operation, OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
OperationApprovalRiskLevel, OperationId, OperationStatus, PlatformApiKey, PlatformApiKeyId,
PlatformApiKeyScope, PlatformApiKeyStatus, Protocol, RestTarget, Target, ToolDescription,
WorkspaceId,
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, Protocol, RestTarget, Target,
ToolDescription, WorkspaceId,
};
use crank_mapping::{MappingRule, MappingSet};
use crank_registry::{
CreateAgentRequest, CreateApprovalRequest, CreatePlatformApiKeyRequest,
ListInvocationLogsQuery, PostgresRegistry, PublishAgentRequest, PublishRequest,
SaveAgentBindingsRequest,
CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest,
CreatePlatformApiKeyRequest, ListInvocationLogsQuery, PostgresRegistry, PublishAgentRequest,
PublishRequest, SaveAgentBindingsRequest,
};
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, RuntimeExecutor,
@@ -141,7 +141,7 @@ fn build_test_app_with_store(
registry,
refresh_interval,
public_base_url,
SecretCrypto::new("test-master-key").unwrap(),
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
@@ -153,6 +153,17 @@ fn build_test_app_with_store(
)
}
async fn assert_revoked_mcp_request_is_denied(
response: reqwest::Response,
api_key: &str,
key_id: &PlatformApiKeyId,
) {
assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED);
let denied_body = response.text().await.unwrap();
assert!(!denied_body.contains(api_key));
assert!(!denied_body.contains(key_id.as_str()));
}
#[tokio::test]
async fn refreshes_published_tools_without_restart() {
let registry = test_registry().await;
@@ -204,12 +215,33 @@ async fn refreshes_published_tools_without_restart() {
})
.await
.unwrap();
let agent_id = test_agent_id("sales-refresh");
let draft_v2 = AgentVersion {
agent_id: agent_id.clone(),
version: 2,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: Default::default(),
created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
};
registry
.save_agent_bindings(SaveAgentBindingsRequest {
.create_agent_draft_version(CreateAgentDraftVersionRequest {
workspace_id: &test_workspace_id(),
agent_id: &test_agent_id("sales-refresh"),
agent_version: 1,
agent_id: &agent_id,
version: &draft_v2,
bindings: &[binding_for_operation(&operation)],
updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
version: 2,
published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
@@ -236,7 +268,7 @@ async fn refreshes_published_tools_without_restart() {
}
#[tokio::test]
async fn shares_published_catalog_snapshot_across_instances() {
async fn shared_catalog_cache_does_not_serve_stale_snapshot_after_unpublish() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_catalog_shared");
@@ -274,6 +306,7 @@ async fn shares_published_catalog_snapshot_across_instances() {
&test_workspace_id(),
&test_agent_id("sales-shared-catalog"),
&OffsetDateTime::parse("2026-03-26T10:05:00Z", &Rfc3339).unwrap(),
None,
)
.await
.unwrap();
@@ -283,13 +316,15 @@ async fn shares_published_catalog_snapshot_across_instances() {
Duration::from_secs(60),
coordination_store,
);
let tools_b = catalog_b
let error = catalog_b
.list_tools(test_workspace_slug(), "sales-shared-catalog")
.await
.unwrap();
.expect_err("unpublished Agent must not be served from shared cache");
assert_eq!(tools_b.len(), 1);
assert_eq!(tools_b[0].tool_name, operation.name);
assert!(matches!(
error,
crank_registry::RegistryError::PublishedAgentNotFound { .. }
));
}
#[tokio::test]
@@ -456,6 +491,135 @@ async fn rejects_initialize_with_key_from_different_agent() {
assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn revoked_mcp_key_stops_existing_session_without_restart() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_revoked_key");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-revoked-key").await;
let api_key = create_platform_api_key(
&registry,
"sales-revoked-key",
"mcp-revoked-boundary",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-revoked-key");
let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
let before_revoke = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
json!({
"jsonrpc": "2.0",
"id": 2,
"method": "tools/list",
"params": {}
}),
)
.await;
assert_eq!(
before_revoke["result"]["tools"][0]["name"],
"crm_revoked_key"
);
let key_id = registry
.list_platform_api_keys_for_agent(&test_workspace_id(), &test_agent_id("sales-revoked-key"))
.await
.unwrap()
.into_iter()
.find(|record| record.api_key.name == "mcp-revoked-boundary")
.unwrap()
.api_key
.id;
registry
.revoke_platform_api_key_for_agent(
&test_workspace_id(),
&test_agent_id("sales-revoked-key"),
&key_id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
let rejected_initialize = post_jsonrpc_response(
&client,
&mcp_url,
&api_key,
None,
None,
json!({
"jsonrpc": "2.0",
"id": 3,
"method": "initialize",
"params": {
"protocolVersion": "2025-11-25"
}
}),
)
.await;
assert_revoked_mcp_request_is_denied(rejected_initialize, &api_key, &key_id).await;
let rejected_tools_list = post_jsonrpc_response(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
None,
json!({
"jsonrpc": "2.0",
"id": 4,
"method": "tools/list",
"params": {}
}),
)
.await;
assert_revoked_mcp_request_is_denied(rejected_tools_list, &api_key, &key_id).await;
let rejected_tools_call = post_jsonrpc_response(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
None,
json!({
"jsonrpc": "2.0",
"id": 5,
"method": "tools/call",
"params": {
"name": "crm_revoked_key",
"arguments": {}
}
}),
)
.await;
assert_revoked_mcp_request_is_denied(rejected_tools_call, &api_key, &key_id).await;
}
#[tokio::test]
async fn rejects_initialize_without_platform_api_key() {
let registry = test_registry().await;
@@ -1,5 +1,8 @@
use super::*;
mod pending;
mod revocation;
#[tokio::test]
async fn approval_key_lists_and_decides_pending_requests() {
let registry = test_registry().await;
@@ -23,6 +26,8 @@ async fn approval_key_lists_and_decides_pending_requests() {
operation_version: 1,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: json!({"email": "ada@example.com"}),
response_payload: None,
created_at: OffsetDateTime::now_utc(),
@@ -164,11 +169,16 @@ async fn approval_key_lists_and_decides_pending_requests() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: Some(&test_agent_id("sales-human-approval")),
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -203,6 +213,8 @@ async fn approval_key_denies_without_executing_upstream() {
operation_version: 1,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: json!({"email": "deny@example.com"}),
response_payload: None,
created_at: OffsetDateTime::now_utc(),
@@ -265,11 +277,16 @@ async fn approval_key_denies_without_executing_upstream() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: Some(&test_agent_id("sales-human-deny")),
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -277,6 +294,112 @@ async fn approval_key_denies_without_executing_upstream() {
assert!(logs.is_empty());
}
#[tokio::test]
async fn approval_key_allowed_origins_are_enforced_at_approval_boundary() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_origin_guard");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_with_bindings(
&registry,
"sales-origin-guard",
vec![binding_for_operation(&operation)],
)
.await;
let approval = ApprovalRequest {
id: ApprovalRequestId::new("approval_origin_guard_01"),
workspace_id: test_workspace_id(),
agent_id: test_agent_id("sales-origin-guard"),
operation_id: operation.id.clone(),
operation_version: 1,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: json!({"email": "origin@example.com"}),
response_payload: None,
created_at: OffsetDateTime::now_utc(),
expires_at: OffsetDateTime::now_utc() + time::Duration::minutes(5),
decided_at: None,
decided_by_key_id: None,
decision_note: None,
};
registry
.create_approval_request(CreateApprovalRequest {
approval: &approval,
})
.await
.unwrap();
let approval_secret = format!("crk_appr_origin_{}", uuid::Uuid::now_v7().simple());
let approval_key = PlatformApiKey {
id: PlatformApiKeyId::new("pk_approval_origin_guard"),
workspace_id: test_workspace_id(),
agent_id: Some(test_agent_id("sales-origin-guard")),
key_kind: PlatformApiKeyKind::Approval,
name: "approval-origin-guard".to_owned(),
prefix: approval_secret.chars().take(16).collect(),
scopes: vec![PlatformApiKeyScope::ReadPending],
status: PlatformApiKeyStatus::Active,
created_at: OffsetDateTime::now_utc(),
last_used_at: None,
expires_at: None,
allowed_origins: vec!["https://allowed.example.test".to_owned()],
};
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
api_key: &approval_key,
secret_hash: &hash_access_secret(&approval_secret),
})
.await
.unwrap();
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let approvals_url = format!(
"{}/approvals",
agent_mcp_url(&base_url, "sales-origin-guard")
);
let rejected = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_secret}"))
.header(header::ORIGIN, "https://evil.example.test")
.send()
.await
.unwrap();
assert_eq!(rejected.status(), reqwest::StatusCode::FORBIDDEN);
let rejected_body = rejected.text().await.unwrap();
assert!(!rejected_body.contains("allowed.example.test"));
assert!(!rejected_body.contains("evil.example.test"));
assert!(!rejected_body.contains(&approval_secret));
let server_side_client = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_secret}"))
.send()
.await
.unwrap();
assert_eq!(server_side_client.status(), reqwest::StatusCode::OK);
let accepted = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_secret}"))
.header(header::ORIGIN, "https://allowed.example.test")
.send()
.await
.unwrap();
assert_eq!(accepted.status(), reqwest::StatusCode::OK);
}
#[tokio::test]
async fn approval_key_expires_without_executing_upstream() {
let registry = test_registry().await;
@@ -300,6 +423,8 @@ async fn approval_key_expires_without_executing_upstream() {
operation_version: 1,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: json!({"email": "expired@example.com"}),
response_payload: None,
created_at: OffsetDateTime::now_utc() - time::Duration::minutes(10),
@@ -349,11 +474,16 @@ async fn approval_key_expires_without_executing_upstream() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: Some(&test_agent_id("sales-human-expired")),
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -361,116 +491,6 @@ async fn approval_key_expires_without_executing_upstream() {
assert!(logs.is_empty());
}
#[tokio::test]
async fn tool_call_with_approval_policy_creates_pending_request() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let mut operation = test_operation(&upstream_base_url, "crm_requires_human_approval");
operation.execution_config.approval_policy = Some(OperationApprovalPolicy {
required: true,
mode: OperationApprovalMode::Custom,
risk_level: OperationApprovalRiskLevel::Dangerous,
ttl_seconds: 300,
show_payload_preview: true,
payload_preview_mode: OperationApprovalPayloadPreviewMode::MaskedJson,
elicitation_message: None,
});
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-gated").await;
let api_key = create_platform_api_key(
&registry,
"sales-gated",
"mcp-gated",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let approval_key =
create_approval_platform_api_key(&registry, "sales-gated", "approval-gated").await;
let base_url = spawn_mcp_server(build_test_app(
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-gated");
let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
let tool_call = json!({
"jsonrpc": "2.0",
"id": 9,
"method": "tools/call",
"params": {
"name": "crm_requires_human_approval",
"arguments": {
"email": "ada@example.com"
}
}
});
let tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
tool_call.clone(),
)
.await;
assert_eq!(
tool_result["result"]["structuredContent"]["status"],
"approval_required"
);
assert_eq!(tool_result["result"]["isError"], false);
let approval_id = tool_result["result"]["structuredContent"]["approval_id"]
.as_str()
.unwrap();
assert!(approval_id.starts_with("approval_"));
let repeated_tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
tool_call,
)
.await;
assert_eq!(
repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
"deduplicated tools/call must return the persisted approval id",
);
let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
let pending = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(pending["items"].as_array().unwrap().len(), 1);
assert_eq!(pending["items"][0]["approval"]["id"], approval_id);
assert_eq!(
pending["items"][0]["approval"]["request_payload"]["email"],
"ada@example.com"
);
}
#[tokio::test]
async fn approval_http_endpoints_enforce_request_rate_limit() {
let registry = test_registry().await;
@@ -589,6 +609,8 @@ async fn recovery_does_not_repeat_interrupted_mutating_approval() {
operation_version: operation.version,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: json!({"email": "interrupted@example.com"}),
response_payload: None,
created_at: now - time::Duration::minutes(10),
@@ -609,9 +631,12 @@ async fn recovery_does_not_repeat_interrupted_mutating_approval() {
workspace_id: &approval.workspace_id,
agent_id: &approval.agent_id,
approval_id: &approval.id,
operation_id: &approval.operation_id,
operation_version: approval.operation_version,
request_payload: &approval.request_payload,
status: ApprovalRequestStatus::Approved,
decided_at: now - time::Duration::minutes(10),
decided_by_key_id: &approval_key_id,
decided_by_key_id: Some(&approval_key_id),
response_payload: Some(json!({"approve": "yes"})),
decision_note: None,
})
@@ -666,7 +691,7 @@ fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router {
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
SecretCrypto::new("test-master-key").unwrap(),
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
@@ -0,0 +1,151 @@
use super::*;
#[tokio::test]
async fn tool_call_with_approval_policy_creates_pending_request() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let mut operation = test_operation(&upstream_base_url, "crm_requires_human_approval");
operation.execution_config.approval_policy = Some(OperationApprovalPolicy {
required: true,
mode: OperationApprovalMode::Custom,
risk_level: OperationApprovalRiskLevel::Dangerous,
ttl_seconds: 300,
show_payload_preview: true,
payload_preview_mode: OperationApprovalPayloadPreviewMode::MaskedJson,
elicitation_message: None,
});
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-gated").await;
let api_key = create_platform_api_key(
&registry,
"sales-gated",
"mcp-gated",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let approval_key =
create_approval_platform_api_key(&registry, "sales-gated", "approval-gated").await;
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-gated");
let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
let tool_call = json!({
"jsonrpc": "2.0",
"id": 9,
"method": "tools/call",
"params": {
"name": "crm_requires_human_approval",
"arguments": {
"email": "ada@example.com",
"api_key": "history-secret-canary"
}
}
});
let tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
tool_call.clone(),
)
.await;
assert_eq!(
tool_result["result"]["structuredContent"]["status"],
"approval_required"
);
assert_eq!(tool_result["result"]["isError"], false);
let approval_id = tool_result["result"]["structuredContent"]["approval_id"]
.as_str()
.unwrap();
assert!(approval_id.starts_with("approval_"));
let repeated_tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
tool_call,
)
.await;
assert_eq!(
repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
"deduplicated tools/call must return the persisted approval id",
);
let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
let pending = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(pending["items"].as_array().unwrap().len(), 1);
assert_eq!(pending["items"][0]["approval"]["id"], approval_id);
assert_eq!(
pending["items"][0]["approval"]["request_payload"]["email"],
"ada@example.com"
);
let pending_preview =
serde_json::to_string(&pending["items"][0]["approval"]["request_payload"]).unwrap();
assert!(!pending_preview.contains("history-secret-canary"));
assert!(pending_preview.contains("[REDACTED]"));
let logs = registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: Some(&test_agent_id("sales-gated")),
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
.unwrap();
assert!(!logs.is_empty());
let history_preview = serde_json::to_string(
&logs
.iter()
.map(|record| {
json!({
"request": record.log.request_preview,
"response": record.log.response_preview,
})
})
.collect::<Vec<_>>(),
)
.unwrap();
assert!(!history_preview.contains("history-secret-canary"));
assert!(!history_preview.contains("ada@example.com"));
assert!(history_preview.contains("approval_required"));
}
@@ -0,0 +1,137 @@
use super::*;
#[tokio::test]
async fn revoked_approval_key_stops_list_approve_and_deny_without_restart() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_approval_revoked");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_with_bindings(
&registry,
"sales-approval-revoked",
vec![binding_for_operation(&operation)],
)
.await;
let approval = ApprovalRequest {
id: ApprovalRequestId::new("approval_mcp_revoked_01"),
workspace_id: test_workspace_id(),
agent_id: test_agent_id("sales-approval-revoked"),
operation_id: operation.id.clone(),
operation_version: 1,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: json!({"email": "revoked@example.com"}),
response_payload: None,
created_at: OffsetDateTime::now_utc(),
expires_at: OffsetDateTime::now_utc() + time::Duration::minutes(5),
decided_at: None,
decided_by_key_id: None,
decision_note: None,
};
registry
.create_approval_request(CreateApprovalRequest {
approval: &approval,
})
.await
.unwrap();
let approval_key = create_approval_platform_api_key(
&registry,
"sales-approval-revoked",
"approval-revoked-http",
)
.await;
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let approvals_url = format!(
"{}/approvals",
agent_mcp_url(&base_url, "sales-approval-revoked")
);
let approve_url = format!(
"{}/approvals/{}/approve",
agent_mcp_url(&base_url, "sales-approval-revoked"),
approval.id
);
let deny_url = format!(
"{}/approvals/{}/deny",
agent_mcp_url(&base_url, "sales-approval-revoked"),
approval.id
);
let before_revoke = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap();
assert_eq!(before_revoke.status(), reqwest::StatusCode::OK);
let approval_key_id = registry
.list_platform_api_keys_for_agent(
&test_workspace_id(),
&test_agent_id("sales-approval-revoked"),
)
.await
.unwrap()
.into_iter()
.find(|record| record.api_key.name == "approval-revoked-http")
.unwrap()
.api_key
.id;
registry
.revoke_platform_api_key_for_agent(
&test_workspace_id(),
&test_agent_id("sales-approval-revoked"),
&approval_key_id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
let denied_list = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap();
assert_eq!(denied_list.status(), reqwest::StatusCode::UNAUTHORIZED);
let denied_approve = client
.post(&approve_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.json(&json!({ "approve": "yes" }))
.send()
.await
.unwrap();
assert_eq!(denied_approve.status(), reqwest::StatusCode::UNAUTHORIZED);
let denied_deny = client
.post(&deny_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.json(&json!({ "approve": "no" }))
.send()
.await
.unwrap();
assert_eq!(denied_deny.status(), reqwest::StatusCode::UNAUTHORIZED);
let current = registry
.get_approval_request_for_agent(
&test_workspace_id(),
&test_agent_id("sales-approval-revoked"),
&approval.id,
)
.await
.unwrap()
.unwrap();
assert_eq!(current.approval.status, ApprovalRequestStatus::Pending);
}
+2 -1
View File
@@ -134,7 +134,7 @@ fn build_test_app_with_store(
registry,
refresh_interval,
public_base_url,
SecretCrypto::new("test-master-key").unwrap(),
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
@@ -417,6 +417,7 @@ pub(super) async fn publish_agent_with_policy(
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
@@ -276,11 +276,16 @@ async fn exports_real_tool_stages_without_sensitive_data() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -3,7 +3,8 @@ use super::common::*;
use std::time::Duration;
use crank_core::{
PlatformApiKeyScope, ToolAccessMode, ToolGroup, ToolSearchSettings, ToolSelectionPolicy,
AgentId, PlatformApiKeyScope, ToolAccessMode, ToolGroup, ToolSearchSettings,
ToolSelectionPolicy,
};
use crank_registry::PublishRequest;
use serde_json::json;
@@ -108,10 +109,16 @@ async fn search_mode_discovers_and_calls_tools_through_meta_tools() {
search["result"]["structuredContent"]["tools"][0]["name"],
"create_invoice"
);
assert_eq!(
search["result"]["structuredContent"]["catalog_revision"],
"agent-version-1"
assert!(
search["result"]["structuredContent"]["catalog_revision"]
.as_str()
.unwrap()
.starts_with("agent-version-1-catalog-revision-")
);
let catalog_revision = search["result"]["structuredContent"]["catalog_revision"]
.as_str()
.unwrap()
.to_owned();
let stale_call = post_jsonrpc(
&client,
@@ -131,7 +138,7 @@ async fn search_mode_discovers_and_calls_tools_through_meta_tools() {
assert_eq!(stale_call["result"]["isError"], true);
assert_eq!(
stale_call["result"]["structuredContent"]["error"]["code"],
"catalog_revision_changed"
"agent_catalog_result_stale"
);
let call = post_jsonrpc(
@@ -144,7 +151,7 @@ async fn search_mode_discovers_and_calls_tools_through_meta_tools() {
"params":{"name":"call_tool","arguments":{
"name":"create_invoice",
"arguments":{"email":"user@example.com"},
"catalog_revision":"agent-version-1"
"catalog_revision": catalog_revision
}}
}),
)
@@ -152,3 +159,116 @@ async fn search_mode_discovers_and_calls_tools_through_meta_tools() {
assert_eq!(call["result"]["isError"], false);
assert_eq!(call["result"]["structuredContent"]["id"], "lead_123");
}
#[tokio::test]
async fn stale_search_result_rejects_tool_call() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let invoice = test_operation(&upstream_base_url, "stale_invoice_tool");
registry
.create_operation(&test_workspace_id(), &invoice, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &invoice.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_with_policy(
&registry,
"stale-search-agent",
vec![binding_for_operation(&invoice)],
ToolSelectionPolicy {
mode: ToolAccessMode::Search,
groups: vec![ToolGroup {
id: "finance".to_owned(),
name: "Finance".to_owned(),
description: "Invoices and payments".to_owned(),
tool_names: vec![invoice.name.clone()],
}],
search: ToolSearchSettings { max_results: 5 },
},
)
.await;
let agent_id = AgentId::new("agent_stale-search-agent");
let api_key = create_platform_api_key(
&registry,
"stale-search-agent",
"mcp-stale-search",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "stale-search-agent");
let session = initialize_session(&client, &mcp_url, &api_key).await;
let search = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&session),
json!({
"jsonrpc":"2.0","id":3,"method":"tools/call",
"params":{"name":"search_tools","arguments":{"query":"invoice","group_ids":["finance"]}}
}),
)
.await;
let stale_revision = search["result"]["structuredContent"]["catalog_revision"]
.as_str()
.unwrap()
.to_owned();
registry
.unpublish_agent(
&test_workspace_id(),
&agent_id,
&OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
None,
)
.await
.unwrap();
registry
.publish_agent(crank_registry::PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let stale_call = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&session),
json!({
"jsonrpc":"2.0","id":4,"method":"tools/call",
"params":{"name":"call_tool","arguments":{
"name":"stale_invoice_tool",
"arguments":{"email":"user@example.com"},
"catalog_revision": stale_revision
}}
}),
)
.await;
assert_eq!(stale_call["result"]["isError"], true);
assert_eq!(
stale_call["result"]["structuredContent"]["error"]["code"],
"agent_catalog_result_stale"
);
}
@@ -140,7 +140,7 @@ fn build_test_app_with_store(
registry,
refresh_interval,
public_base_url,
SecretCrypto::new("test-master-key").unwrap(),
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
@@ -234,11 +234,16 @@ async fn initializes_lists_and_calls_published_tool_via_mcp() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(crank_core::InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -333,11 +338,16 @@ async fn preserves_request_id_for_tool_call_invocations() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(crank_core::InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -432,11 +442,16 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(crank_core::InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
+321
View File
@@ -0,0 +1,321 @@
#[path = "integration/common.rs"]
mod common;
use std::time::Duration;
use crank_core::{
ExecutionStage, InvocationLevel, InvocationLog, InvocationLogId, InvocationSource,
InvocationStatus, OutcomeCertainty, PlatformApiKeyScope, ProductEventKind, Retryability,
};
use crank_registry::{
CreateInvocationLogRequest, InvocationHistoryWriteOutcome, ListInvocationLogsQuery,
ListProductEventsQuery, PublishRequest,
};
use serde_json::json;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use common::{
agent_mcp_url, build_test_app, create_platform_api_key, initialize_session, post_jsonrpc,
publish_agent_for_operation, spawn_mcp_server, spawn_upstream_server, test_operation,
test_registry, test_workspace_id,
};
#[tokio::test]
async fn onboarding_completes_only_after_exact_key_successful_public_tool_call() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "onboarding_first_call");
let mut operation_draft = operation.clone();
operation_draft.status = crank_core::OperationStatus::Draft;
operation_draft.published_at = None;
registry
.create_operation(
&test_workspace_id(),
&operation_draft,
Some("onboarding-test"),
)
.await
.unwrap();
let admin_test = InvocationLog {
id: InvocationLogId::new("log_onboarding_admin_test"),
workspace_id: test_workspace_id(),
agent_id: None,
platform_api_key_id: None,
operation_id: operation.id.clone(),
operation_version: Some(1),
source: InvocationSource::AdminTestRun,
level: InvocationLevel::Info,
status: InvocationStatus::Ok,
tool_name: operation.name.clone(),
message: "admin test succeeded".to_owned(),
request_id: Some("req_onboarding_admin_test".to_owned()),
trace_id: Some("4bf92f3577b34da6a3ce929d0e0e4736".to_owned()),
status_code: Some(200),
duration_ms: 1,
error_kind: None,
execution_stage: Some(ExecutionStage::Runtime),
execution_error_code: None,
retryability: Some(Retryability::Never),
outcome_certainty: Some(OutcomeCertainty::Certain),
request_preview: json!({}),
response_preview: json!({"ok": true}),
created_at: OffsetDateTime::parse("2026-08-23T07:59:00Z", &Rfc3339).unwrap(),
};
assert_eq!(
registry
.create_invocation_log(CreateInvocationLogRequest { log: &admin_test })
.await,
InvocationHistoryWriteOutcome::Recorded
);
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-08-23T08:00:00Z", &Rfc3339).unwrap(),
published_by: Some("onboarding-test"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "onboarding-agent").await;
let selected_key_name = "onboarding-raw-key-canary";
let selected_key_id = format!("pk_{selected_key_name}");
let selected_key = create_platform_api_key(
&registry,
"onboarding-agent",
selected_key_name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let other_key = create_platform_api_key(
&registry,
"onboarding-agent",
"onboarding-other",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let pre_call_projection = registry
.get_onboarding_projection(&test_workspace_id())
.await
.unwrap();
assert!(
pre_call_projection
.step(crank_core::OnboardingStepId::PublishOperation)
.is_some_and(|step| step.completed),
"fixture must expose a Published Operation: {pre_call_projection:#?}"
);
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
))
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "onboarding-agent");
let selected_session = initialize_session(&client, &mcp_url, &selected_key).await;
let listed = post_jsonrpc(
&client,
&mcp_url,
&selected_key,
Some(&selected_session),
json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}),
)
.await;
assert_eq!(listed["result"]["tools"][0]["name"], operation.name);
assert!(
exact_key_successes(&registry, &operation.id, &selected_key_id)
.await
.is_empty(),
"initialize and tools/list must not complete onboarding"
);
let failed = post_jsonrpc(
&client,
&mcp_url,
&selected_key,
Some(&selected_session),
json!({
"jsonrpc":"2.0",
"id":3,
"method":"tools/call",
"params":{"name":operation.name,"arguments":{}}
}),
)
.await;
assert_eq!(failed["result"]["isError"], true);
assert!(
exact_key_successes(&registry, &operation.id, &selected_key_id)
.await
.is_empty(),
"a failed tools/call must not complete onboarding"
);
let other_session = initialize_session(&client, &mcp_url, &other_key).await;
let other_success = post_jsonrpc(
&client,
&mcp_url,
&other_key,
Some(&other_session),
json!({
"jsonrpc":"2.0",
"id":4,
"method":"tools/call",
"params":{
"name":operation.name,
"arguments":{"email":"other@example.com"}
}
}),
)
.await;
assert_eq!(other_success["result"]["isError"], false);
assert!(
exact_key_successes(&registry, &operation.id, &selected_key_id)
.await
.is_empty(),
"a successful tools/call made with another key must not complete onboarding"
);
let selected_success = post_jsonrpc(
&client,
&mcp_url,
&selected_key,
Some(&selected_session),
json!({
"jsonrpc":"2.0",
"id":5,
"method":"tools/call",
"params":{
"name":operation.name,
"arguments":{"email":"selected@example.com"}
}
}),
)
.await;
assert_eq!(selected_success["result"]["isError"], false);
let exact_successes = exact_key_successes(&registry, &operation.id, &selected_key_id).await;
assert_eq!(
exact_successes.len(),
1,
"onboarding must complete from exactly one successful public tools/call made with the selected key"
);
assert_eq!(
exact_successes[0]["platform_api_key_id"], selected_key_id,
"successful invocation evidence must retain the typed key identity"
);
assert!(
exact_successes[0]["request_id"]
.as_str()
.is_some_and(|id| !id.is_empty())
);
assert!(
exact_successes[0]["trace_id"]
.as_str()
.is_some_and(|id| id.len() == 32)
);
// A direct MCP call can precede the first onboarding snapshot. The later
// server-owned eligibility write must repair the completion event without
// replacing the invocation timestamp that anchored the exact lineage.
let projection = registry
.ensure_onboarding_eligibility(&test_workspace_id(), OffsetDateTime::now_utc())
.await
.unwrap();
let operation_summary = registry
.get_operation_summary(&test_workspace_id(), &operation.id)
.await
.unwrap();
assert!(
projection.completed,
"projection did not complete: {projection:#?}; operation: {operation_summary:#?}"
);
let completion_events = registry
.list_product_events(ListProductEventsQuery {
workspace_id: &test_workspace_id(),
kind: Some(ProductEventKind::OnboardingCompleted),
created_after: OffsetDateTime::UNIX_EPOCH,
created_before: OffsetDateTime::now_utc() + time::Duration::minutes(1),
limit: 10,
})
.await
.unwrap();
assert_eq!(completion_events.len(), 1);
assert_eq!(
completion_events[0].event.occurred_at,
projection.first_call_at.unwrap()
);
let persisted_evidence = invocation_evidence(&registry, &operation.id).await;
assert!(!persisted_evidence.is_empty());
for evidence in persisted_evidence {
let serialized = serde_json::to_string(&evidence).unwrap();
assert!(
!serialized.contains(&selected_key),
"raw Bearer canary must never appear in serialized invocation evidence"
);
assert!(
!serialized.contains(&other_key),
"another raw Bearer secret must never appear in serialized invocation evidence"
);
for field in ["message", "request_preview", "response_preview"] {
let persisted_field = serde_json::to_string(&evidence[field]).unwrap();
assert!(
!persisted_field.contains(&selected_key),
"raw Bearer canary leaked into persisted {field}"
);
assert!(
!persisted_field.contains(&other_key),
"another raw Bearer secret leaked into persisted {field}"
);
}
}
}
async fn exact_key_successes(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
selected_key_id: &str,
) -> Vec<serde_json::Value> {
invocation_evidence(registry, operation_id)
.await
.into_iter()
.filter(|log| {
log.get("platform_api_key_id")
.and_then(serde_json::Value::as_str)
== Some(selected_key_id)
&& log.get("status").and_then(serde_json::Value::as_str) == Some("ok")
})
.collect()
}
async fn invocation_evidence(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
) -> Vec<serde_json::Value> {
registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(operation_id),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 100,
})
.await
.unwrap()
.into_iter()
.map(|record| serde_json::to_value(record.log).unwrap())
.collect()
}