feat: complete Epic 1 production foundation

This commit is contained in:
2026-08-25 01:24:11 +03:00
parent 767428436d
commit 182bde8ac0
298 changed files with 35719 additions and 5299 deletions
@@ -23,8 +23,9 @@ use crank_registry::{
CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, DescriptorKind,
DescriptorMetadata, OperationSampleMetadata, PlatformApiKeyRecord, PostgresRegistry,
PublishAgentRequest, PublishRequest, RegistryError, RegistryOperation, SampleKind,
SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest,
WorkspaceRecord, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
SaveAgentCatalogConfigRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest,
SaveSampleMetadataRequest, UsageBucket, UsageOutcomeGroup, UsageQuery, WorkspaceRecord,
YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
};
fn test_workspace_id() -> WorkspaceId {
@@ -145,6 +146,7 @@ async fn manages_published_agent_tool_reads() {
version: version.version,
published_at: &timestamp("2026-03-25T12:11:00Z"),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
@@ -170,6 +172,333 @@ async fn manages_published_agent_tool_reads() {
database.cleanup().await;
}
#[tokio::test]
async fn published_agent_snapshot_remains_immutable_after_draft_edit() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_agent_immutable_01", 1, OperationStatus::Draft);
let agent = test_agent("agent_immutable_01", AgentStatus::Draft);
let version = test_agent_version(&agent.id, 1, AgentStatus::Draft);
let published_binding = AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: operation.version,
tool_name: "create_lead_immutable".to_owned(),
tool_title: "Create lead".to_owned(),
tool_description_override: Some("Published immutable binding".to_owned()),
enabled: true,
};
registry
.create_operation(&test_workspace_id(), &operation, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: operation.version,
published_at: &timestamp("2026-03-25T12:10:00Z"),
published_by: Some("alice"),
})
.await
.unwrap();
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &version,
bindings: std::slice::from_ref(&published_binding),
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent.id,
version: version.version,
published_at: &timestamp("2026-03-25T12:11:00Z"),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let before = registry
.get_published_agent_tools_by_slug("default", &agent.slug)
.await
.unwrap();
assert_eq!(before.len(), 1);
assert_eq!(before[0].tool_name, published_binding.tool_name);
registry
.save_agent_catalog_config(SaveAgentCatalogConfigRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent.id,
agent_version: version.version,
bindings: &[],
tool_selection_policy: &Default::default(),
expected_state: None,
})
.await
.expect_err("editing after publish must not mutate the published Agent Version");
let after = registry
.get_published_agent_tools_by_slug("default", &agent.slug)
.await
.unwrap();
assert_eq!(after, before);
database.cleanup().await;
}
#[tokio::test]
async fn stale_agent_revision_rejects_mutation() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_agent_stale_01", 1, OperationStatus::Draft);
let agent = test_agent("agent_stale_01", AgentStatus::Draft);
let version = test_agent_version(&agent.id, 1, AgentStatus::Draft);
let binding = AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: operation.version,
tool_name: "create_lead_stale".to_owned(),
tool_title: "Create lead".to_owned(),
tool_description_override: None,
enabled: true,
};
registry
.create_operation(&test_workspace_id(), &operation, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: operation.version,
published_at: &timestamp("2026-03-25T12:10:00Z"),
published_by: Some("alice"),
})
.await
.unwrap();
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &version,
bindings: std::slice::from_ref(&binding),
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent.id,
version: version.version,
published_at: &timestamp("2026-03-25T12:11:00Z"),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let stale_save = registry
.save_agent_catalog_config(SaveAgentCatalogConfigRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent.id,
agent_version: version.version,
bindings: &[],
tool_selection_policy: &Default::default(),
expected_state: None,
})
.await;
assert!(
stale_save.is_err(),
"stale write against already-published Agent Version must be rejected"
);
database.cleanup().await;
}
#[tokio::test]
async fn published_agent_catalog_revision_is_durable_and_monotonic() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_agent_revision_01", 1, OperationStatus::Draft);
let agent = test_agent("agent_revision_01", AgentStatus::Draft);
let version = test_agent_version(&agent.id, 1, AgentStatus::Draft);
let binding = AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: operation.version,
tool_name: "create_lead_revision".to_owned(),
tool_title: "Create lead".to_owned(),
tool_description_override: None,
enabled: true,
};
registry
.create_operation(&test_workspace_id(), &operation, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: operation.version,
published_at: &timestamp("2026-03-25T12:10:00Z"),
published_by: Some("alice"),
})
.await
.unwrap();
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &version,
bindings: std::slice::from_ref(&binding),
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent.id,
version: version.version,
published_at: &timestamp("2026-03-25T12:11:00Z"),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let first = registry
.get_published_agent_catalog_by_slug("default", &agent.slug)
.await
.unwrap()
.catalog_revision;
registry
.unpublish_agent(
&test_workspace_id(),
&agent.id,
&timestamp("2026-03-25T12:12:00Z"),
None,
)
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent.id,
version: version.version,
published_at: &timestamp("2026-03-25T12:13:00Z"),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let second = registry
.get_published_agent_catalog_by_slug("default", &agent.slug)
.await
.unwrap()
.catalog_revision;
assert_ne!(
first, second,
"unpublish/re-publish of the same Agent Version must invalidate stale catalog search results"
);
database.cleanup().await;
}
#[tokio::test]
async fn database_rejects_direct_published_agent_snapshot_mutation() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_agent_db_guard_01", 1, OperationStatus::Draft);
let agent = test_agent("agent_db_guard_01", AgentStatus::Draft);
let version = test_agent_version(&agent.id, 1, AgentStatus::Draft);
let binding = AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: operation.version,
tool_name: "create_lead_db_guard".to_owned(),
tool_title: "Create lead".to_owned(),
tool_description_override: None,
enabled: true,
};
registry
.create_operation(&test_workspace_id(), &operation, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: operation.version,
published_at: &timestamp("2026-03-25T12:10:00Z"),
published_by: Some("alice"),
})
.await
.unwrap();
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &version,
bindings: std::slice::from_ref(&binding),
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent.id,
version: version.version,
published_at: &timestamp("2026-03-25T12:11:00Z"),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let policy_update = sqlx::query(
"update agent_versions
set tool_selection_policy_json = '{\"changed\":true}'::jsonb
where agent_id = $1 and version = 1",
)
.bind(agent.id.as_str())
.execute(registry.pool())
.await;
assert!(
policy_update.is_err(),
"database trigger must reject direct Published Agent Version mutation"
);
let binding_delete = sqlx::query(
"delete from agent_operation_bindings
where agent_id = $1 and agent_version = 1",
)
.bind(agent.id.as_str())
.execute(registry.pool())
.await;
assert!(
binding_delete.is_err(),
"database trigger must reject direct Published Agent binding deletion"
);
let pointer_rewind = sqlx::query(
"update published_agents
set catalog_revision = catalog_revision
where agent_id = $1",
)
.bind(agent.id.as_str())
.execute(registry.pool())
.await;
assert!(
pointer_rewind.is_err(),
"database trigger must reject non-increasing catalog revision"
);
database.cleanup().await;
}
#[tokio::test]
async fn manages_operation_usage_and_agent_ref_reads() {
let database = TestDatabase::new().await;
@@ -225,6 +554,7 @@ async fn manages_operation_usage_and_agent_ref_reads() {
version: version.version,
published_at: &timestamp("2026-03-25T12:11:00Z"),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
@@ -243,6 +573,27 @@ async fn manages_operation_usage_and_agent_ref_reads() {
.await,
crank_registry::InvocationHistoryWriteOutcome::Recorded
);
let stored = registry
.get_invocation_log(
&test_workspace_id(),
&crank_core::InvocationLogId::new("log_usage_ok"),
)
.await
.unwrap()
.expect("typed invocation history");
assert_eq!(stored.log.operation_version, Some(1));
assert_eq!(
stored.log.execution_stage,
Some(crank_core::ExecutionStage::Runtime)
);
assert_eq!(
stored.log.retryability,
Some(crank_core::Retryability::Never)
);
assert_eq!(
stored.log.outcome_certainty,
Some(crank_core::OutcomeCertainty::Certain)
);
assert_eq!(
registry
.create_invocation_log(CreateInvocationLogRequest {
@@ -286,3 +637,214 @@ async fn manages_operation_usage_and_agent_ref_reads() {
database.cleanup().await;
}
#[tokio::test]
async fn invocation_history_filters_cursor_and_usage_outcomes_are_bounded() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_history_01", 1, OperationStatus::Draft);
let agent = test_agent("agent_history_01", AgentStatus::Draft);
registry
.create_operation(&test_workspace_id(), &operation, None)
.await
.unwrap();
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &test_agent_version(&agent.id, 1, AgentStatus::Draft),
bindings: &[],
})
.await
.unwrap();
let mut success = test_invocation_log(
"log_history_success",
&operation.id,
Some(agent.id.clone()),
crank_core::InvocationStatus::Ok,
50,
"2026-03-25T12:00:00Z",
);
success.execution_error_code = None;
success.message = "=formula must remain data".to_owned();
let mut upstream = test_invocation_log(
"log_history_upstream",
&operation.id,
Some(agent.id.clone()),
crank_core::InvocationStatus::Error,
150,
"2026-03-25T12:01:00Z",
);
upstream.execution_error_code = Some(crank_core::ExecutionErrorCode::UpstreamTimeout);
let mut client = test_invocation_log(
"log_history_client",
&operation.id,
Some(agent.id.clone()),
crank_core::InvocationStatus::Error,
250,
"2026-03-25T12:02:00Z",
);
client.execution_error_code = Some(crank_core::ExecutionErrorCode::InputSchemaInvalid);
let mut schema = test_invocation_log(
"log_history_schema",
&operation.id,
Some(agent.id.clone()),
crank_core::InvocationStatus::Error,
350,
"2026-03-25T12:03:00Z",
);
schema.execution_error_code = Some(crank_core::ExecutionErrorCode::OutputSchemaInvalid);
let mut crank = test_invocation_log(
"log_history_crank",
&operation.id,
Some(agent.id.clone()),
crank_core::InvocationStatus::Error,
450,
"2026-03-25T12:04:00Z",
);
crank.execution_error_code = Some(crank_core::ExecutionErrorCode::RuntimeInternal);
for log in [&success, &upstream, &client, &schema, &crank] {
assert_eq!(
registry
.create_invocation_log(CreateInvocationLogRequest { log })
.await,
crank_registry::InvocationHistoryWriteOutcome::Recorded
);
}
let first_page = registry
.list_invocation_logs(crank_registry::ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: Some(crank_core::InvocationStatus::Error),
outcome_group: None,
search_text: None,
source: None,
operation_id: None,
agent_id: None,
created_after: Some("2026-03-25T12:00:00Z"),
created_before: Some("2026-03-25T12:04:00Z"),
cursor_created_at: None,
cursor_id: None,
limit: 2,
})
.await
.unwrap();
assert_eq!(first_page.len(), 2);
assert_eq!(first_page[0].log.id.as_str(), "log_history_schema");
assert_eq!(first_page[1].log.id.as_str(), "log_history_client");
let second_page = registry
.list_invocation_logs(crank_registry::ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: Some(crank_core::InvocationStatus::Error),
outcome_group: None,
search_text: None,
source: None,
operation_id: None,
agent_id: None,
created_after: Some("2026-03-25T12:00:00Z"),
created_before: Some("2026-03-25T12:04:00Z"),
cursor_created_at: Some("2026-03-25T12:02:00Z"),
cursor_id: Some(&crank_core::InvocationLogId::new("log_history_client")),
limit: 2,
})
.await
.unwrap();
assert_eq!(second_page.len(), 1);
assert_eq!(second_page[0].log.id.as_str(), "log_history_upstream");
let upstream_only = registry
.list_invocation_logs(crank_registry::ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: Some(UsageOutcomeGroup::Upstream),
search_text: None,
source: None,
operation_id: None,
agent_id: None,
created_after: Some("2026-03-25T12:00:00Z"),
created_before: Some("2026-03-25T12:05:00Z"),
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
.unwrap();
assert_eq!(upstream_only.len(), 1);
assert_eq!(upstream_only[0].log.id.as_str(), "log_history_upstream");
let usage = registry
.list_usage_outcomes(UsageQuery {
workspace_id: &test_workspace_id(),
period: crank_core::UsagePeriod::Last24Hours,
source: None,
created_after: "2026-03-25T12:00:00Z",
created_before: "2026-03-25T12:05:00Z",
bucket: UsageBucket::Hour,
})
.await
.unwrap();
let by_group = usage
.iter()
.map(|item| (item.group, item.calls_total))
.collect::<BTreeMap<_, _>>();
assert_eq!(by_group.get(&UsageOutcomeGroup::Success), Some(&1));
assert_eq!(by_group.get(&UsageOutcomeGroup::Upstream), Some(&1));
assert_eq!(by_group.get(&UsageOutcomeGroup::Client), Some(&1));
assert_eq!(by_group.get(&UsageOutcomeGroup::Schema), Some(&1));
assert_eq!(by_group.get(&UsageOutcomeGroup::Crank), Some(&1));
let half_open = registry
.summarize_usage(UsageQuery {
workspace_id: &test_workspace_id(),
period: crank_core::UsagePeriod::Last24Hours,
source: None,
created_after: "2026-03-25T12:00:00Z",
created_before: "2026-03-25T12:04:00Z",
bucket: UsageBucket::Hour,
})
.await
.unwrap();
assert_eq!(half_open.rollup.calls_total, 4);
let retention = registry
.delete_invocation_logs_before(timestamp("2026-03-25T12:02:00Z"))
.await
.unwrap();
assert_eq!(retention.deleted_records, 2);
assert_eq!(
retention.status,
crank_registry::InvocationRetentionStatus::Completed
);
assert_eq!(retention.policy.preserved_usage_window_days, 90);
let retained = registry
.list_invocation_logs(crank_registry::ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: None,
operation_id: None,
agent_id: None,
created_after: Some("2026-03-25T12:00:00Z"),
created_before: Some("2026-03-25T12:05:00Z"),
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
.unwrap();
assert_eq!(retained.len(), 3);
assert_eq!(
retained.last().unwrap().log.id.as_str(),
"log_history_client"
);
database.cleanup().await;
}
@@ -0,0 +1,314 @@
use std::sync::Arc;
use crank_core::{
AgentId, AgentOperationBinding, AgentStatus, ApprovalRequest, ApprovalRequestId,
ApprovalRequestStatus, OperationApprovalRiskLevel, OperationId, PlatformApiKey,
PlatformApiKeyId, PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, WorkspaceId,
};
use crank_registry::{
CreateAgentRequest, CreateApprovalRequest, CreatePlatformApiKeyRequest, DecideApprovalRequest,
FinishApprovalRequest,
};
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use super::common::{TestDatabase, test_agent, test_agent_version, test_operation};
fn timestamp(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).unwrap()
}
fn approval_request(id: &str, payload: Value) -> ApprovalRequest {
ApprovalRequest {
id: ApprovalRequestId::new(id),
workspace_id: WorkspaceId::new("ws_default"),
agent_id: AgentId::new("agent_approval_atomic"),
operation_id: OperationId::new("operation_approval_atomic"),
operation_version: 1,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: payload,
response_payload: None,
created_at: timestamp("2026-03-25T12:01:00Z"),
expires_at: timestamp("2027-03-25T12:06:00Z"),
decided_at: None,
decided_by_key_id: None,
decision_note: None,
}
}
fn approval_key() -> PlatformApiKey {
PlatformApiKey {
id: PlatformApiKeyId::new("approval_atomic_key"),
workspace_id: WorkspaceId::new("ws_default"),
agent_id: Some(AgentId::new("agent_approval_atomic")),
key_kind: PlatformApiKeyKind::Approval,
name: "approval atomic key".to_owned(),
prefix: "crk_appr".to_owned(),
scopes: vec![PlatformApiKeyScope::Approve, PlatformApiKeyScope::Deny],
status: PlatformApiKeyStatus::Active,
created_at: timestamp("2026-03-25T12:00:00Z"),
last_used_at: None,
expires_at: None,
allowed_origins: Vec::new(),
}
}
async fn seed_approval_graph(registry: &crank_registry::PostgresRegistry) {
let workspace_id = WorkspaceId::new("ws_default");
let operation = test_operation(
"operation_approval_atomic",
1,
crank_core::OperationStatus::Published,
);
registry
.create_operation(&workspace_id, &operation, None)
.await
.unwrap();
let agent = test_agent("agent_approval_atomic", AgentStatus::Draft);
let version = test_agent_version(&agent.id, 1, AgentStatus::Draft);
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &version,
bindings: &[AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: "dangerous_write".to_owned(),
tool_title: "Dangerous write".to_owned(),
tool_description_override: None,
enabled: true,
}],
})
.await
.unwrap();
let key = approval_key();
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
api_key: &key,
secret_hash: "approval-key-secret-hash",
})
.await
.unwrap();
}
#[tokio::test]
async fn concurrent_equal_scope_creates_one_pending_approval_without_losing_raw_execution_payload()
{
let database = TestDatabase::new().await;
let registry = Arc::new(database.registry().await);
seed_approval_graph(&registry).await;
let mut tasks = Vec::new();
for index in 0..32 {
let registry = Arc::clone(&registry);
tasks.push(tokio::spawn(async move {
let mut approval = approval_request(
&format!("approval_concurrent_{index}"),
json!({
"email": "customer@example.com",
"token": "SECRET_APPROVAL_CANARY",
"_crank_confirmation_token": format!("ct_{index}")
}),
);
approval.created_at += time::Duration::milliseconds(index);
registry
.create_approval_request(CreateApprovalRequest {
approval: &approval,
})
.await
.unwrap()
}));
}
let mut ids = std::collections::BTreeSet::new();
for task in tasks {
ids.insert(task.await.unwrap().approval.id.as_str().to_owned());
}
assert_eq!(
ids.len(),
1,
"all callers must receive one canonical pending approval"
);
let pending = registry
.list_pending_approval_requests_for_agent(
&WorkspaceId::new("ws_default"),
&AgentId::new("agent_approval_atomic"),
)
.await
.unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(
pending[0].approval.request_payload["token"], "SECRET_APPROVAL_CANARY",
"registry keeps the raw payload for the eventual approved execution"
);
assert!(
pending[0].approval.request_payload["_crank_confirmation_token"]
.as_str()
.is_some_and(|value| value.starts_with("ct_")),
"only projections may redact control data; execution input remains intact"
);
let safe_preview =
crank_core::sanitize_invocation_preview(&pending[0].approval.request_payload);
let preview = safe_preview.to_string();
assert!(!preview.contains("SECRET_APPROVAL_CANARY"));
database.cleanup().await;
}
#[tokio::test]
async fn nested_business_control_like_fields_are_part_of_approval_fingerprint() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
seed_approval_graph(&registry).await;
let first = approval_request(
"approval_nested_control_a",
json!({
"line": {
"_crank_approval_id": "business-a"
},
"_crank_confirmation_token": "transport-a"
}),
);
registry
.create_approval_request(CreateApprovalRequest { approval: &first })
.await
.unwrap();
let second = approval_request(
"approval_nested_control_b",
json!({
"line": {
"_crank_approval_id": "business-b"
},
"_crank_confirmation_token": "transport-b"
}),
);
let created_second = registry
.create_approval_request(CreateApprovalRequest { approval: &second })
.await
.unwrap();
assert_eq!(created_second.approval.id, second.id);
let pending = registry
.list_pending_approval_requests_for_agent(
&WorkspaceId::new("ws_default"),
&AgentId::new("agent_approval_atomic"),
)
.await
.unwrap();
assert_eq!(pending.len(), 2);
database.cleanup().await;
}
#[tokio::test]
async fn decision_claim_finish_and_replay_are_single_winner_transitions() {
let database = TestDatabase::new().await;
let registry = Arc::new(database.registry().await);
seed_approval_graph(&registry).await;
let approval = approval_request(
"approval_single_winner",
json!({"email":"lead@example.com"}),
);
let created = registry
.create_approval_request(CreateApprovalRequest {
approval: &approval,
})
.await
.unwrap();
assert_eq!(created.approval.status, ApprovalRequestStatus::Pending);
let mut decisions = Vec::new();
for index in 0..16 {
let registry = Arc::clone(&registry);
let operation_id = approval.operation_id.clone();
let operation_version = approval.operation_version;
let request_payload = approval.request_payload.clone();
decisions.push(tokio::spawn(async move {
let key_id = PlatformApiKeyId::new("approval_atomic_key");
registry
.decide_approval_request(DecideApprovalRequest {
workspace_id: &WorkspaceId::new("ws_default"),
agent_id: &AgentId::new("agent_approval_atomic"),
approval_id: &ApprovalRequestId::new("approval_single_winner"),
operation_id: &operation_id,
operation_version,
request_payload: &request_payload,
status: if index % 2 == 0 {
ApprovalRequestStatus::Approved
} else {
ApprovalRequestStatus::Denied
},
decided_at: timestamp("2026-03-25T12:02:00Z"),
decided_by_key_id: Some(&key_id),
response_payload: Some(json!({ "decision": index })),
decision_note: Some("race decision"),
})
.await
.unwrap()
}));
}
let mut winners = 0;
for decision in decisions {
if decision.await.unwrap().is_some() {
winners += 1;
}
}
assert_eq!(winners, 1);
let mut claims = Vec::new();
for _ in 0..16 {
let registry = Arc::clone(&registry);
claims.push(tokio::spawn(async move {
registry
.claim_approval_request(
&WorkspaceId::new("ws_default"),
&AgentId::new("agent_approval_atomic"),
&ApprovalRequestId::new("approval_single_winner"),
timestamp("2026-03-25T12:02:01Z"),
)
.await
.unwrap()
}));
}
let mut claim_winners = 0;
for claim in claims {
if claim.await.unwrap().is_some() {
claim_winners += 1;
}
}
assert_eq!(claim_winners, 1);
let first_finish = registry
.finish_approval_request(FinishApprovalRequest {
workspace_id: &WorkspaceId::new("ws_default"),
agent_id: &AgentId::new("agent_approval_atomic"),
approval_id: &ApprovalRequestId::new("approval_single_winner"),
status: ApprovalRequestStatus::Completed,
response_payload: Some(json!({"ok":true})),
decision_note: None,
})
.await
.unwrap();
assert!(first_finish.is_some());
let replay_finish = registry
.finish_approval_request(FinishApprovalRequest {
workspace_id: &WorkspaceId::new("ws_default"),
agent_id: &AgentId::new("agent_approval_atomic"),
approval_id: &ApprovalRequestId::new("approval_single_winner"),
status: ApprovalRequestStatus::Completed,
response_payload: Some(json!({"ok":"replay"})),
decision_note: None,
})
.await
.unwrap();
assert!(replay_finish.is_none());
database.cleanup().await;
}
@@ -21,11 +21,12 @@ use crank_registry::{
CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, DescriptorKind,
DescriptorMetadata, OperationSampleMetadata, PlatformApiKeyRecord, PostgresRegistry,
PublishAgentRequest, PublishRequest, RegistryError, RegistryOperation, SampleKind,
SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest,
WorkspaceRecord, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
SaveAgentCatalogConfigRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest,
SaveSampleMetadataRequest, WorkspaceRecord, YamlImportJobCompletion, YamlImportJobId,
YamlImportJobStatus,
};
fn test_workspace_id() -> WorkspaceId {
pub(super) fn test_workspace_id() -> WorkspaceId {
WorkspaceId::new("ws_default")
}
@@ -209,7 +210,9 @@ pub(super) fn test_invocation_log(
id: crank_core::InvocationLogId::new(id),
workspace_id: test_workspace_id(),
agent_id,
platform_api_key_id: None,
operation_id: operation_id.clone(),
operation_version: Some(1),
source: crank_core::InvocationSource::AgentToolCall,
level: crank_core::InvocationLevel::Info,
status,
@@ -220,6 +223,11 @@ pub(super) fn test_invocation_log(
status_code: Some(200),
duration_ms,
error_kind: None,
execution_stage: Some(crank_core::ExecutionStage::Runtime),
execution_error_code: (status == crank_core::InvocationStatus::Error)
.then_some(crank_core::ExecutionErrorCode::RuntimeInternal),
retryability: Some(crank_core::Retryability::Never),
outcome_certainty: Some(crank_core::OutcomeCertainty::Certain),
request_preview: json!({"input":"value"}),
response_preview: json!({"ok":true}),
created_at: timestamp(created_at),
@@ -268,6 +276,18 @@ impl TestDatabase {
PostgresRegistry::connect(&database_url).await.unwrap()
}
pub(super) async fn raw_pool(&self) -> PgPool {
let database_url = format!(
"{}?options=-csearch_path%3D{}",
self.database_url, self.schema
);
PgPoolOptions::new()
.max_connections(1)
.connect(&database_url)
.await
.unwrap()
}
pub(super) async fn cleanup(&self) {
self.admin_pool
.execute(sqlx::query(sqlx::AssertSqlSafe(format!(
@@ -0,0 +1,296 @@
use super::common::*;
use std::time::{Duration, Instant};
use crank_core::{
PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind, PlatformApiKeyScope,
PlatformApiKeyStatus, Secret, SecretId, SecretKind, SecretStatus, Workspace, WorkspaceId,
WorkspaceStatus,
};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use crank_registry::{
CreatePlatformApiKeyRequest, CreateSecretRequest, CreateWorkspaceRequest,
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, RegistryError,
};
fn timestamp(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).unwrap()
}
fn workspace() -> Workspace {
Workspace {
id: WorkspaceId::new("ws_credential_touch"),
slug: "credential-touch".to_owned(),
display_name: "Credential Touch".to_owned(),
status: WorkspaceStatus::Active,
settings: json!({}),
created_at: timestamp("2026-08-24T12:00:00Z"),
updated_at: timestamp("2026-08-24T12:00:00Z"),
}
}
async fn wait_for_row_lock(inspector: &PgPool, holder_pid: i32, query_marker: &str) {
let deadline = Instant::now() + Duration::from_secs(5);
loop {
let blocked: i64 = sqlx::query_scalar(
"select count(*)::bigint
from pg_stat_activity activity
where $1 = any(pg_blocking_pids(activity.pid))
and activity.wait_event_type = 'Lock'
and activity.query like '%' || $2 || '%'",
)
.bind(holder_pid)
.bind(query_marker)
.fetch_one(inspector)
.await
.unwrap();
if blocked > 0 {
return;
}
if Instant::now() >= deadline {
let diagnostics = sqlx::query(
"select pid, state, wait_event_type, wait_event, query
from pg_stat_activity
where $1 = any(pg_blocking_pids(pid))
or pid = $1",
)
.bind(holder_pid)
.fetch_all(inspector)
.await
.unwrap()
.into_iter()
.map(|row| {
format!(
"pid={}, state={}, wait_event_type={:?}, wait_event={:?}, query={}",
row.get::<i32, _>("pid"),
row.get::<String, _>("state"),
row.get::<Option<String>, _>("wait_event_type"),
row.get::<Option<String>, _>("wait_event"),
row.get::<String, _>("query"),
)
})
.collect::<Vec<_>>();
panic!(
"touch query did not block on backend {holder_pid} within 5 seconds; \
pg_stat_activity: {diagnostics:#?}"
);
}
tokio::task::yield_now().await;
}
}
#[tokio::test]
async fn touch_platform_api_key_rejects_revoke_that_won_row_lock() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace = workspace();
let key = PlatformApiKey {
id: PlatformApiKeyId::new("key_touch_race"),
workspace_id: workspace.id.clone(),
agent_id: None,
key_kind: PlatformApiKeyKind::McpClient,
name: "Race key".to_owned(),
prefix: "crk_live".to_owned(),
scopes: vec![PlatformApiKeyScope::Read],
status: PlatformApiKeyStatus::Active,
created_at: timestamp("2026-08-24T12:00:00Z"),
last_used_at: None,
expires_at: None,
allowed_origins: Vec::new(),
};
registry
.create_workspace(CreateWorkspaceRequest {
workspace: &workspace,
})
.await
.unwrap();
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
api_key: &key,
secret_hash: "touch-race-secret-hash",
})
.await
.unwrap();
let lock_pool = database.raw_pool().await;
let inspector = database.raw_pool().await;
let mut lock_tx = lock_pool.begin().await.unwrap();
let holder_pid: i32 = sqlx::query_scalar("select pg_backend_pid()")
.fetch_one(&mut *lock_tx)
.await
.unwrap();
sqlx::query(
"select id
from platform_api_keys
where workspace_id = $1 and id = $2
for update",
)
.bind(workspace.id.as_str())
.bind(key.id.as_str())
.execute(&mut *lock_tx)
.await
.unwrap();
let touch_registry = registry.clone();
let touch_workspace_id = workspace.id.clone();
let touch_key_id = key.id.clone();
let touch = tokio::spawn(async move {
touch_registry
.touch_platform_api_key(
&touch_workspace_id,
&touch_key_id,
&timestamp("2026-08-24T12:05:00Z"),
)
.await
});
wait_for_row_lock(&inspector, holder_pid, "from platform_api_keys").await;
sqlx::query(
"update platform_api_keys
set status = 'revoked'
where workspace_id = $1 and id = $2",
)
.bind(workspace.id.as_str())
.bind(key.id.as_str())
.execute(&mut *lock_tx)
.await
.unwrap();
lock_tx.commit().await.unwrap();
let touch_result = touch.await.unwrap();
assert!(matches!(
touch_result,
Err(RegistryError::PlatformApiKeyInactive { key_id }) if key_id == key.id.as_str()
));
let last_used_at: Option<OffsetDateTime> = sqlx::query_scalar(
"select last_used_at
from platform_api_keys
where workspace_id = $1 and id = $2",
)
.bind(workspace.id.as_str())
.bind(key.id.as_str())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(last_used_at, None);
database.cleanup().await;
}
#[tokio::test]
async fn touch_secret_rejects_disable_that_won_row_lock() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace = workspace();
let secret = Secret {
id: SecretId::new("secret_touch_race"),
workspace_id: workspace.id.clone(),
name: "Race secret".to_owned(),
kind: SecretKind::Token,
status: SecretStatus::Active,
current_version: 1,
created_at: timestamp("2026-08-24T12:00:00Z"),
updated_at: timestamp("2026-08-24T12:00:00Z"),
last_used_at: None,
};
registry
.create_workspace(CreateWorkspaceRequest {
workspace: &workspace,
})
.await
.unwrap();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &timestamp("2026-08-24T12:00:00Z"),
})
.await
.unwrap();
registry
.create_secret(CreateSecretRequest {
secret: &secret,
ciphertext: "touch-race-ciphertext",
key_version: "test-key-v1",
master_key_epoch: 1,
created_by: None,
})
.await
.unwrap();
let lock_pool = database.raw_pool().await;
let inspector = database.raw_pool().await;
let mut lock_tx = lock_pool.begin().await.unwrap();
let holder_pid: i32 = sqlx::query_scalar("select pg_backend_pid()")
.fetch_one(&mut *lock_tx)
.await
.unwrap();
sqlx::query(
"select id
from secrets
where workspace_id = $1 and id = $2
for update",
)
.bind(workspace.id.as_str())
.bind(secret.id.as_str())
.execute(&mut *lock_tx)
.await
.unwrap();
let touch_registry = registry.clone();
let touch_workspace_id = workspace.id.clone();
let touch_secret_id = secret.id.clone();
let touch = tokio::spawn(async move {
touch_registry
.touch_secret(
&touch_workspace_id,
&touch_secret_id,
&timestamp("2026-08-24T12:05:00Z"),
)
.await
});
wait_for_row_lock(&inspector, holder_pid, "from secrets").await;
sqlx::query(
"update secrets
set status = 'disabled'
where workspace_id = $1 and id = $2",
)
.bind(workspace.id.as_str())
.bind(secret.id.as_str())
.execute(&mut *lock_tx)
.await
.unwrap();
lock_tx.commit().await.unwrap();
let touch_result = touch.await.unwrap();
assert!(matches!(
touch_result,
Err(RegistryError::SecretInactive { secret_id }) if secret_id == secret.id.as_str()
));
let last_used_at: Option<OffsetDateTime> = sqlx::query_scalar(
"select last_used_at
from secrets
where workspace_id = $1 and id = $2",
)
.bind(workspace.id.as_str())
.bind(secret.id.as_str())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(last_used_at, None);
database.cleanup().await;
}
@@ -0,0 +1,526 @@
use crank_core::{Secret, SecretId, SecretKind, SecretStatus, WorkspaceId};
use crank_registry::{
CreateSecretRequest, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, RegistryError,
RotateSecretRequest,
};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use super::common::TestDatabase;
fn timestamp(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).unwrap()
}
#[tokio::test]
async fn registers_and_verifies_active_master_key_identity() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let now = timestamp("2026-08-21T00:00:00Z");
let fingerprint = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
let registered = registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint,
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
let verified = registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint,
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
assert_eq!(registered.epoch, 1);
assert_eq!(registered.status, "active");
assert_eq!(registered, verified);
database.cleanup().await;
}
#[tokio::test]
async fn rejects_different_active_master_key_identity() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let now = timestamp("2026-08-21T00:00:00Z");
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
let error = registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210",
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap_err();
assert!(matches!(
error,
RegistryError::MasterKeyIdentityMismatch { epoch: 1 }
));
assert!(!error.to_string().contains("fedcba"));
assert!(!error.to_string().contains("012345"));
database.cleanup().await;
}
#[tokio::test]
async fn master_key_rotation_is_resumable_and_promotes_atomically() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let now = timestamp("2026-08-21T00:00:00Z");
let later = timestamp("2026-08-21T00:01:00Z");
let current_fingerprint = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
let target_fingerprint = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210";
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: current_fingerprint,
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
insert_secret_version(&registry, "secret_a", "cipher-a", &now).await;
insert_secret_version(&registry, "secret_b", "cipher-b", &now).await;
let snapshot = registry
.list_secret_versions_for_master_key_epoch(1)
.await
.unwrap();
assert_eq!(snapshot.len(), 2);
let preflight_status = registry.master_key_rotation_status().await.unwrap();
assert_eq!(preflight_status.active_identity.unwrap().epoch, 1);
assert!(preflight_status.rotations.is_empty());
assert!(
registry
.list_secret_versions_for_master_key_epoch(1)
.await
.unwrap()
.iter()
.all(|version| version.target_ciphertext.is_none())
);
let rotation = registry
.begin_master_key_rotation(1, 2, target_fingerprint, Some("offline-backup-ref"), &now)
.await
.unwrap();
assert_eq!(rotation.state, "running");
assert_eq!(rotation.total_secret_versions, 2);
registry
.stage_master_key_rotation_ciphertext(
&rotation.id,
&SecretId::new("secret_a"),
1,
1,
"target-cipher-a",
"v2",
2,
&later,
)
.await
.unwrap();
let resumed = registry
.begin_master_key_rotation(1, 2, target_fingerprint, Some("offline-backup-ref"), &later)
.await
.unwrap();
assert_eq!(resumed.processed_secret_versions, 1);
registry
.stage_master_key_rotation_ciphertext(
&rotation.id,
&SecretId::new("secret_b"),
1,
1,
"target-cipher-b",
"v2",
2,
&later,
)
.await
.unwrap();
let ready = registry
.finish_master_key_rotation_batches(&rotation.id, &later)
.await
.unwrap();
assert_eq!(ready.state, "verifying");
registry
.verify_master_key_rotation(&rotation.id, 2, &later)
.await
.unwrap();
registry
.promote_master_key_rotation(
&rotation.id,
MasterKeyIdentityCandidate {
epoch: 2,
fingerprint: target_fingerprint,
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &later,
},
&later,
)
.await
.unwrap();
let status = registry.master_key_rotation_status().await.unwrap();
let active = status.active_identity.unwrap();
assert_eq!(active.epoch, 2);
assert_eq!(active.fingerprint, target_fingerprint);
assert_eq!(status.rotations[0].state, "promoted");
assert!(
registry
.list_secret_versions_for_master_key_epoch(1)
.await
.unwrap()
.is_empty()
);
let promoted = registry
.list_secret_versions_for_master_key_epoch(2)
.await
.unwrap();
assert_eq!(promoted.len(), 2);
assert!(
promoted
.iter()
.all(|version| version.target_ciphertext.is_none())
);
database.cleanup().await;
}
#[tokio::test]
async fn abort_preserves_current_epoch_and_clears_target_ciphertext() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let now = timestamp("2026-08-21T00:00:00Z");
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
insert_secret_version(&registry, "secret_abort", "cipher-a", &now).await;
let rotation = registry
.begin_master_key_rotation(
1,
2,
"fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210",
None,
&now,
)
.await
.unwrap();
registry
.stage_master_key_rotation_ciphertext(
&rotation.id,
&SecretId::new("secret_abort"),
1,
1,
"target-cipher-a",
"v2",
2,
&now,
)
.await
.unwrap();
registry
.abort_master_key_rotation(&rotation.id, &now)
.await
.unwrap();
let status = registry.master_key_rotation_status().await.unwrap();
assert_eq!(status.active_identity.unwrap().epoch, 1);
assert_eq!(status.rotations[0].state, "aborted");
let versions = registry
.list_secret_versions_for_master_key_epoch(1)
.await
.unwrap();
assert_eq!(versions.len(), 1);
assert!(versions[0].target_ciphertext.is_none());
database.cleanup().await;
}
#[tokio::test]
async fn aborted_rotation_can_be_rerun_for_same_source_epoch() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let now = timestamp("2026-08-21T00:00:00Z");
let target_fingerprint = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210";
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
insert_secret_version(&registry, "secret_retry", "cipher-a", &now).await;
let rotation = registry
.begin_master_key_rotation(1, 2, target_fingerprint, None, &now)
.await
.unwrap();
registry
.stage_master_key_rotation_ciphertext(
&rotation.id,
&SecretId::new("secret_retry"),
1,
1,
"target-cipher-a",
"v2",
2,
&now,
)
.await
.unwrap();
registry
.abort_master_key_rotation(&rotation.id, &now)
.await
.unwrap();
let rerun = registry
.begin_master_key_rotation(1, 2, target_fingerprint, Some("second-attempt"), &now)
.await
.unwrap();
assert_eq!(rerun.id, rotation.id);
assert_eq!(rerun.state, "running");
assert_eq!(rerun.processed_secret_versions, 0);
assert_eq!(rerun.backup_ref.as_deref(), Some("second-attempt"));
assert!(
registry
.list_secret_versions_for_master_key_epoch(1)
.await
.unwrap()
.iter()
.all(|version| version.target_ciphertext.is_none())
);
database.cleanup().await;
}
#[tokio::test]
async fn active_rotation_rejects_new_secret_writes() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let now = timestamp("2026-08-21T00:00:00Z");
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
insert_secret_version(&registry, "existing_secret", "cipher-a", &now).await;
registry
.begin_master_key_rotation(
1,
2,
"fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210",
None,
&now,
)
.await
.unwrap();
let create_error = registry
.create_secret(CreateSecretRequest {
secret: &test_secret("new_secret", &now),
ciphertext: "cipher-new",
key_version: "v2",
master_key_epoch: 1,
created_by: None,
})
.await
.unwrap_err();
assert!(matches!(
create_error,
RegistryError::MasterKeyRotationInProgress
));
let rotate_error = registry
.rotate_secret(RotateSecretRequest {
workspace_id: &WorkspaceId::new("ws_default"),
secret_id: &SecretId::new("existing_secret"),
ciphertext: "cipher-rotated",
key_version: "v2",
master_key_epoch: 1,
created_at: &now,
updated_at: &now,
created_by: None,
})
.await
.unwrap_err();
assert!(matches!(
rotate_error,
RegistryError::MasterKeyRotationInProgress
));
let delete_error = registry
.delete_secret(
&WorkspaceId::new("ws_default"),
&SecretId::new("existing_secret"),
)
.await
.unwrap_err();
assert!(matches!(
delete_error,
RegistryError::MasterKeyRotationInProgress
));
database.cleanup().await;
}
#[tokio::test]
async fn stale_source_epoch_writer_is_rejected_after_promotion() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let now = timestamp("2026-08-21T00:00:00Z");
let source_fingerprint = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
let target_fingerprint = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210";
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: source_fingerprint,
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
insert_secret_version(&registry, "promoted_secret", "cipher-a", &now).await;
let rotation = registry
.begin_master_key_rotation(1, 2, target_fingerprint, None, &now)
.await
.unwrap();
registry
.stage_master_key_rotation_ciphertext(
&rotation.id,
&SecretId::new("promoted_secret"),
1,
1,
"target-cipher-a",
"v2",
2,
&now,
)
.await
.unwrap();
registry
.finish_master_key_rotation_batches(&rotation.id, &now)
.await
.unwrap();
registry
.verify_master_key_rotation(&rotation.id, 1, &now)
.await
.unwrap();
registry
.promote_master_key_rotation(
&rotation.id,
MasterKeyIdentityCandidate {
epoch: 2,
fingerprint: target_fingerprint,
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
},
&now,
)
.await
.unwrap();
let create_error = registry
.create_secret(CreateSecretRequest {
secret: &test_secret("stale_new_secret", &now),
ciphertext: "cipher-new",
key_version: "v2",
master_key_epoch: 1,
created_by: None,
})
.await
.unwrap_err();
assert!(matches!(
create_error,
RegistryError::MasterKeyIdentityMismatch { epoch: 2 }
));
let rotate_error = registry
.rotate_secret(RotateSecretRequest {
workspace_id: &WorkspaceId::new("ws_default"),
secret_id: &SecretId::new("promoted_secret"),
ciphertext: "cipher-rotated",
key_version: "v2",
master_key_epoch: 1,
created_at: &now,
updated_at: &now,
created_by: None,
})
.await
.unwrap_err();
assert!(matches!(
rotate_error,
RegistryError::MasterKeyIdentityMismatch { epoch: 2 }
));
database.cleanup().await;
}
async fn insert_secret_version(
registry: &crank_registry::PostgresRegistry,
id: &str,
ciphertext: &str,
now: &OffsetDateTime,
) {
registry
.create_secret(CreateSecretRequest {
secret: &test_secret(id, now),
ciphertext,
key_version: "v2",
master_key_epoch: 1,
created_by: None,
})
.await
.unwrap();
}
fn test_secret(id: &str, now: &OffsetDateTime) -> Secret {
Secret {
id: SecretId::new(id),
workspace_id: WorkspaceId::new("ws_default"),
name: id.to_owned(),
kind: SecretKind::Token,
status: SecretStatus::Active,
current_version: 1,
created_at: *now,
updated_at: *now,
last_used_at: None,
}
}
@@ -1,47 +1,62 @@
use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry};
use sqlx::Row;
mod rollback;
static EVENT_TRIGGER_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
fn legacy_preservation_row_expr(table: &str) -> &'static str {
match table {
"invocation_logs" => {
"to_jsonb(t) - array['trace_id','operation_version','execution_stage','execution_error_code','retryability','outcome_certainty','platform_api_key_id']"
}
"operation_versions" => {
"to_jsonb(t) - array['name','display_name','category','protocol','security_level','snapshot_provenance','snapshot_observed_at','published_at','published_by']"
}
"approval_requests" => "to_jsonb(t) - array['request_id','trace_id']",
"agents" => "to_jsonb(t) - array['catalog_revision']",
_ => "to_jsonb(t)",
}
}
#[tokio::test]
async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
let database_url = crank_test_support::postgres_schema_url("test_core_migration").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let (first, second) = tokio::join!(
MigrationAuthority::apply(&pool),
MigrationAuthority::apply(&pool),
);
first.expect("first controlled runner must apply the sequence");
second.expect("second controlled runner must observe the applied sequence");
let first = PostgresRegistry::connect(&database_url)
.await
.expect("service startup must verify the migrated schema");
let rows =
sqlx::query("select version, name, checksum from __crank_migrations order by version")
.fetch_all(first.pool())
.await
.expect("migration ledger must be readable");
assert_eq!(rows.len(), 3);
assert_eq!(rows[0].get::<i64, _>("version"), 1);
assert_eq!(rows[0].get::<String, _>("name"), "community-baseline-v1");
assert_eq!(
rows[0].get::<String, _>("checksum"),
"crank-community-baseline-v1"
);
assert_eq!(rows[1].get::<i64, _>("version"), 2);
assert_eq!(rows[1].get::<String, _>("name"), "legacy-consolidation-v2");
assert_eq!(rows[1].get::<String, _>("checksum").len(), 64);
assert_eq!(rows[2].get::<i64, _>("version"), 3);
assert_eq!(
rows[2].get::<String, _>("name"),
"request-trace-identity-v3"
);
assert_eq!(rows[2].get::<String, _>("checksum").len(), 64);
let expected = [
(1, "community-baseline-v1"),
(2, "legacy-consolidation-v2"),
(3, "request-trace-identity-v3"),
(4, "operation-lifecycle-v4"),
(5, "execution-outcome-v5"),
(6, "platform-key-name-reuse-v6"),
(7, "master-key-identity-v7"),
(8, "admin-auth-lifecycle-v8"),
(9, "agent-catalog-lifecycle-v9"),
(10, "approval-side-effects-v10"),
(11, "onboarding-product-events-v11"),
];
assert_eq!(rows.len(), expected.len());
for (row, (version, name)) in rows.iter().zip(expected) {
assert_eq!(row.get::<i64, _>("version"), version);
assert_eq!(row.get::<String, _>("name"), name);
let checksum = row.get::<String, _>("checksum");
assert!(checksum == "crank-community-baseline-v1" || checksum.len() == 64);
}
let approval_columns = sqlx::query(
"select column_name
from information_schema.columns
@@ -53,22 +68,41 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
.await
.expect("approval schema must be readable");
assert_eq!(approval_columns.len(), 3);
let onboarding_relations = sqlx::query(
"select table_name
from information_schema.tables
where table_schema = current_schema()
and table_name in ('product_events', 'product_event_daily_rollups', 'onboarding_selections')
order by table_name",
)
.fetch_all(first.pool())
.await
.expect("V11 onboarding relations must be readable");
assert_eq!(onboarding_relations.len(), 3);
let key_scope_column: bool = sqlx::query_scalar(
"select exists (
select 1 from information_schema.columns
where table_schema = current_schema()
and table_name = 'invocation_logs'
and column_name = 'platform_api_key_id'
)",
)
.fetch_one(first.pool())
.await
.expect("V11 key provenance column must be readable");
assert!(key_scope_column);
assert_eq!(
MigrationAuthority::preflight(first.pool()).await.unwrap(),
MigrationPreflight::Current { version: 3 }
MigrationPreflight::Current { version: 11 }
);
}
#[tokio::test]
async fn service_connect_is_read_only_on_fresh_database() {
let database_url = crank_test_support::postgres_schema_url("test_read_only_startup").await;
let error = PostgresRegistry::connect(&database_url)
.await
.expect_err("fresh schema must require the controlled migration command");
assert!(error.to_string().contains("schema_missing"));
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let ledger = sqlx::query("select to_regclass('__crank_migrations')::text as name")
.fetch_one(&pool)
@@ -81,7 +115,6 @@ async fn service_connect_is_read_only_on_fresh_database() {
"startup compatibility check must not create DDL"
);
}
#[tokio::test]
async fn changed_checksum_fails_closed_without_repair() {
let database_url = crank_test_support::postgres_schema_url("test_changed_checksum").await;
@@ -91,7 +124,6 @@ async fn changed_checksum_fails_closed_without_repair() {
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::apply(&pool)
.await
.expect_err("published checksum mismatch must fail closed");
@@ -106,12 +138,12 @@ async fn changed_checksum_fails_closed_without_repair() {
"authority must not rewrite corrupt history"
);
}
#[tokio::test]
async fn legacy_core_baseline_is_consolidated_without_data_loss() {
let database_url = crank_test_support::postgres_schema_url("test_legacy_core").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
remove_v4_schema(&pool).await;
sqlx::query(
"insert into workspaces (id, slug, display_name, status, settings_json, created_at, updated_at)
values ('ws_preserved', 'preserved', 'Preserved', 'active', '{}'::jsonb, now(), now())",
@@ -164,11 +196,7 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
];
let mut before = Vec::new();
for table in tables {
let row = if table == "invocation_logs" {
"to_jsonb(t) - 'trace_id'"
} else {
"to_jsonb(t)"
};
let row = legacy_preservation_row_expr(table);
let sql = format!("select jsonb_agg({row} order by {row}::text)::text from {table} t");
before.push(
sqlx::query_scalar::<_, Option<String>>(sqlx::AssertSqlSafe(sql))
@@ -184,12 +212,11 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
.execute(&pool)
.await
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 1,
target: 3,
target: 11,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -201,11 +228,7 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
assert_eq!(display_name, "Preserved");
let mut after = Vec::new();
for table in tables {
let row = if table == "invocation_logs" {
"to_jsonb(t) - 'trace_id'"
} else {
"to_jsonb(t)"
};
let row = legacy_preservation_row_expr(table);
let sql = format!("select jsonb_agg({row} order by {row}::text)::text from {table} t");
after.push(
sqlx::query_scalar::<_, Option<String>>(sqlx::AssertSqlSafe(sql))
@@ -216,7 +239,6 @@ async fn legacy_core_baseline_is_consolidated_without_data_loss() {
}
assert_eq!(before, after, "brownfield rows must remain byte-equivalent");
}
#[tokio::test]
async fn legacy_mcp_sessions_survive_consolidation() {
let database_url = crank_test_support::postgres_schema_url("test_legacy_mcp").await;
@@ -232,12 +254,12 @@ async fn legacy_mcp_sessions_survive_consolidation() {
.execute(&pool)
.await
.unwrap();
remove_v4_schema(&pool).await;
remove_v3_schema(&pool).await;
sqlx::query("drop table __crank_migrations, __crank_migration_legacy_audit")
.execute(&pool)
.await
.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let count = sqlx::query("select count(*)::bigint as count from mcp_transport_sessions where id = 'session_preserved'")
.fetch_one(&pool)
@@ -246,7 +268,6 @@ async fn legacy_mcp_sessions_survive_consolidation() {
.get::<i64, _>("count");
assert_eq!(count, 1);
}
#[tokio::test]
async fn repeated_apply_does_not_rewrite_audit_timestamps() {
let database_url = crank_test_support::postgres_schema_url("test_repeat_apply").await;
@@ -259,7 +280,6 @@ async fn repeated_apply_does_not_rewrite_audit_timestamps() {
.into_iter()
.map(|row| row.get::<time::OffsetDateTime, _>("applied_at"))
.collect::<Vec<_>>();
MigrationAuthority::apply(&pool).await.unwrap();
let after = sqlx::query("select applied_at from __crank_migrations order by version")
.fetch_all(&pool)
@@ -270,13 +290,12 @@ async fn repeated_apply_does_not_rewrite_audit_timestamps() {
.collect::<Vec<_>>();
assert_eq!(before, after);
}
#[tokio::test]
async fn future_sequence_fails_closed() {
let database_url = crank_test_support::postgres_schema_url("test_future_sequence").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
sqlx::query("update __crank_migrations set version = 4 where version = 3")
sqlx::query("update __crank_migrations set version = 12 where version = 11")
.execute(&pool)
.await
.unwrap();
@@ -288,29 +307,28 @@ async fn future_sequence_fails_closed() {
"future_version"
);
}
#[tokio::test]
async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
let database_url = crank_test_support::postgres_schema_url("test_v2_to_v3_identity").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
remove_v4_schema(&pool).await;
sqlx::query("delete from __crank_migrations where version = 3")
.execute(&pool)
.await
.unwrap();
remove_v3_schema(&pool).await;
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 2,
target: 3,
target: 11,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 3 }
MigrationPreflight::Current { version: 11 }
);
let trace_column: bool = sqlx::query_scalar(
"select exists (
@@ -325,12 +343,183 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
.unwrap();
assert!(trace_column);
}
#[tokio::test]
async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
let database_url = crank_test_support::postgres_schema_url("test_v3_to_v4_lifecycle").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
remove_v4_schema(&pool).await;
sqlx::raw_sql(
"insert into operations
(id, workspace_id, name, display_name, category, protocol, security_level, status,
current_draft_version, latest_published_version, created_at, updated_at, published_at)
values ('op_v3_cutover', 'ws_default', 'v3_cutover', 'V3 Cutover', 'general', 'rest',
'standard', 'published', 1, 1, now(), now(), now());
insert into operation_versions
(operation_id, version, status, target_json, input_schema_json, output_schema_json,
input_mapping_json, output_mapping_json, execution_config_json,
tool_description_json, created_at)
values ('op_v3_cutover', 1, 'published', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb,
'{}'::jsonb, '{}'::jsonb, '{}'::jsonb, '{}'::jsonb, now());
insert into published_operations(operation_id, version, published_at, published_by)
values ('op_v3_cutover', 1, now(), 'legacy-owner');",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 3,
target: 11,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
let row = sqlx::query(
"select name, display_name, snapshot_provenance, snapshot_observed_at,
published_at, published_by
from operation_versions where operation_id = 'op_v3_cutover' and version = 1",
)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(row.get::<String, _>("name"), "v3_cutover");
assert_eq!(row.get::<String, _>("display_name"), "V3 Cutover");
assert_eq!(
row.get::<String, _>("snapshot_provenance"),
"legacy_observed"
);
assert!(
row.try_get::<time::OffsetDateTime, _>("snapshot_observed_at")
.is_ok()
);
assert!(
row.try_get::<time::OffsetDateTime, _>("published_at")
.is_ok()
);
assert_eq!(
row.try_get::<Option<String>, _>("published_by").unwrap(),
Some("legacy-owner".to_owned())
);
}
#[tokio::test]
async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
let database_url = crank_test_support::postgres_schema_url("test_v4_to_v5_outcome").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
remove_v5_schema(&pool).await;
sqlx::query(
"insert into operations
(id, workspace_id, name, display_name, protocol, status, created_at, updated_at)
values ('op_v4_history', 'ws_default', 'v4_history', 'V4 History', 'rest',
'draft', now(), now())",
)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"insert into invocation_logs
(id, workspace_id, operation_id, source, level, status, tool_name, message,
duration_ms, request_preview_json, response_preview_json, created_at)
values ('legacy_v4_log', 'ws_default', 'op_v4_history', 'admin', 'info',
'success', 'legacy', 'safe', 1, '{}'::jsonb, '{}'::jsonb, now())",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 4,
target: 11,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
let row = sqlx::query(
"select operation_version, execution_stage, execution_error_code,
retryability, outcome_certainty
from invocation_logs where id = 'legacy_v4_log'",
)
.fetch_one(&pool)
.await
.unwrap();
for column in [
"operation_version",
"execution_stage",
"execution_error_code",
"retryability",
"outcome_certainty",
] {
assert!(
row.try_get::<Option<String>, _>(column)
.is_ok_and(|value| value.is_none())
|| row
.try_get::<Option<i32>, _>(column)
.is_ok_and(|value| value.is_none())
);
}
}
#[tokio::test]
async fn failed_operation_lifecycle_migration_rolls_back_schema_and_ledger() {
let _event_trigger_guard = EVENT_TRIGGER_TEST_LOCK.lock().await;
let database_url =
crank_test_support::postgres_schema_url("test_operation_lifecycle_rollback").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
remove_v4_schema(&pool).await;
let schema: String = sqlx::query_scalar("select current_schema()")
.fetch_one(&pool)
.await
.unwrap();
let failure_trigger = format!(
"create function reject_story17_v4() returns event_trigger language plpgsql as $$
begin
if current_schema() = '{schema}' and current_query() like '%operation_versions_immutable_guard%' then
raise exception 'injected v4 ddl failure';
end if;
end $$;
create event trigger reject_story17_v4 on ddl_command_start
execute function reject_story17_v4();"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(failure_trigger))
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
sqlx::raw_sql(
"drop event trigger reject_story17_v4;
drop function reject_story17_v4();",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(error.code(), "apply_failed");
assert_eq!(error.version(), Some(4));
let lifecycle_column: bool = sqlx::query_scalar(
"select exists (
select 1 from information_schema.columns
where table_schema = current_schema()
and table_name = 'operation_versions'
and column_name = 'snapshot_provenance'
)",
)
.fetch_one(&pool)
.await
.unwrap();
assert!(!lifecycle_column);
let ledger_v4: i64 =
sqlx::query_scalar("select count(*) from __crank_migrations where version = 4")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(ledger_v4, 0);
}
#[tokio::test]
async fn v2_with_partial_v3_objects_fails_before_apply() {
let database_url = crank_test_support::postgres_schema_url("test_v2_partial_v3").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
remove_v4_schema(&pool).await;
sqlx::query("delete from __crank_migrations where version = 3")
.execute(&pool)
.await
@@ -343,12 +532,10 @@ async fn v2_with_partial_v3_objects_fails_before_apply() {
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "partial_sequence");
assert_eq!(error.stage(), "preflight.schema");
}
#[tokio::test]
async fn v3_rejects_same_named_constraint_or_index_with_wrong_definition() {
let database_url = crank_test_support::postgres_schema_url("test_v3_named_drift").await;
@@ -369,7 +556,6 @@ async fn v3_rejects_same_named_constraint_or_index_with_wrong_definition() {
.code(),
"partial_sequence"
);
sqlx::raw_sql(
"alter table invocation_logs drop constraint invocation_logs_trace_id_format_check;
alter table invocation_logs add constraint invocation_logs_trace_id_format_check
@@ -393,7 +579,78 @@ async fn v3_rejects_same_named_constraint_or_index_with_wrong_definition() {
"partial_sequence"
);
}
#[tokio::test]
async fn v5_rejects_same_named_execution_code_constraint_with_wrong_definition() {
let database_url =
crank_test_support::postgres_schema_url("test_v5_execution_code_named_drift").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
sqlx::raw_sql(
"alter table invocation_logs
drop constraint invocation_logs_execution_error_code_check;
alter table invocation_logs
add constraint invocation_logs_execution_error_code_check check (true) not valid;",
)
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "partial_sequence");
assert_eq!(error.stage(), "preflight.schema");
}
#[tokio::test]
async fn v11_rejects_same_named_product_event_contract_drift() {
let database_url =
crank_test_support::postgres_schema_url("test_v11_product_event_named_drift").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
sqlx::raw_sql(
"alter table product_events drop constraint product_events_name_check;
alter table product_events add constraint product_events_name_check check (true) not valid;",
)
.execute(&pool)
.await
.unwrap();
let constraint_error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(constraint_error.code(), "partial_sequence");
assert_eq!(constraint_error.stage(), "preflight.schema");
sqlx::raw_sql(
"alter table product_events drop constraint product_events_name_check;
alter table product_events add constraint product_events_name_check check (event_name in (
'onboarding_eligible', 'onboarding_started', 'onboarding_resumed',
'onboarding_dismissed', 'onboarding_abandoned', 'onboarding_completed'
));
drop index product_events_workspace_occurred_idx;
create index product_events_workspace_occurred_idx
on product_events(occurred_at, workspace_id, id);",
)
.execute(&pool)
.await
.unwrap();
let index_error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(index_error.code(), "partial_sequence");
assert_eq!(index_error.stage(), "preflight.schema");
sqlx::raw_sql(
"drop index product_events_workspace_occurred_idx;
create index product_events_workspace_occurred_idx
on product_events(workspace_id, occurred_at, id);
create or replace function crank_reject_product_event_mutation()
returns trigger language plpgsql as $$
begin
raise exception 'ProductEvent is append-only' using errcode = '23514';
end;
$$;",
)
.execute(&pool)
.await
.unwrap();
let function_error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(function_error.code(), "partial_sequence");
assert_eq!(function_error.stage(), "preflight.schema");
}
#[tokio::test]
async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
let database_url = crank_test_support::postgres_schema_url("test_v3_legacy_request_id").await;
@@ -419,19 +676,18 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
.execute(&pool)
.await
.unwrap();
remove_v4_schema(&pool).await;
sqlx::query("delete from __crank_migrations where version = 3")
.execute(&pool)
.await
.unwrap();
remove_v3_schema(&pool).await;
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 3 }
MigrationPreflight::Current { version: 11 }
);
}
async fn remove_v3_schema(pool: &sqlx::PgPool) {
sqlx::raw_sql(
"drop index if exists invocation_logs_workspace_request_id_idx;
@@ -441,7 +697,147 @@ async fn remove_v3_schema(pool: &sqlx::PgPool) {
.await
.unwrap();
}
async fn remove_v4_schema(pool: &sqlx::PgPool) {
remove_v5_schema(pool).await;
sqlx::raw_sql(
"drop trigger if exists operation_versions_immutable_guard on operation_versions;
drop function if exists crank_guard_operation_version_immutable();
drop trigger if exists published_operations_monotonic_guard on published_operations;
drop function if exists crank_guard_published_operation_pointer();
drop trigger if exists operations_latest_pointer_monotonic_guard on operations;
drop function if exists crank_guard_operation_latest_pointer();
alter table operation_versions
drop constraint if exists operation_versions_snapshot_provenance_check,
drop column if exists name,
drop column if exists display_name,
drop column if exists category,
drop column if exists protocol,
drop column if exists security_level,
drop column if exists snapshot_provenance,
drop column if exists snapshot_observed_at,
drop column if exists published_at,
drop column if exists published_by;
delete from __crank_migrations where version = 4;",
)
.execute(pool)
.await
.unwrap();
}
async fn remove_v5_schema(pool: &sqlx::PgPool) {
remove_v6_schema(pool).await;
sqlx::raw_sql(
"drop index if exists invocation_logs_workspace_operation_version_idx;
alter table invocation_logs
drop column if exists operation_version,
drop column if exists execution_stage,
drop column if exists execution_error_code,
drop column if exists retryability,
drop column if exists outcome_certainty;
delete from __crank_migrations where version = 5;",
)
.execute(pool)
.await
.unwrap();
}
async fn remove_v6_schema(pool: &sqlx::PgPool) {
remove_v7_schema(pool).await;
sqlx::raw_sql(
"drop index if exists platform_api_keys_workspace_name_active_idx;
create unique index if not exists platform_api_keys_workspace_name_idx on platform_api_keys(workspace_id, name);
delete from __crank_migrations where version = 6;",
)
.execute(pool)
.await
.unwrap();
}
async fn remove_v7_schema(pool: &sqlx::PgPool) {
remove_v8_schema(pool).await;
sqlx::raw_sql(
"alter table secret_versions drop constraint if exists secret_versions_target_all_or_none_check,
drop constraint if exists secret_versions_target_epoch_check,
drop constraint if exists secret_versions_master_key_epoch_check,
drop column if exists target_master_key_epoch, drop column if exists target_key_version,
drop column if exists target_ciphertext, drop column if exists master_key_epoch;
drop table if exists master_key_rotations;
drop table if exists master_key_identities;
delete from __crank_migrations where version = 7;",
)
.execute(pool)
.await
.unwrap();
}
async fn remove_v8_schema(pool: &sqlx::PgPool) {
remove_v9_schema(pool).await;
sqlx::raw_sql(
"drop table if exists admin_security_audit_events;
drop table if exists admin_login_backoff;
drop table if exists admin_bootstrap_contracts;
alter table user_sessions
drop constraint if exists user_sessions_csrf_hash_check,
drop column if exists csrf_hash,
drop column if exists revoked_at;
delete from __crank_migrations where version = 8;",
)
.execute(pool)
.await
.unwrap();
}
async fn remove_v9_schema(pool: &sqlx::PgPool) {
remove_v10_schema(pool).await;
sqlx::raw_sql(
"drop trigger if exists published_agents_monotonic_guard on published_agents;
drop function if exists crank_reject_published_agent_pointer_rewind();
drop trigger if exists agent_operation_bindings_immutable_guard on agent_operation_bindings;
drop function if exists crank_reject_published_agent_binding_mutation();
drop trigger if exists agent_versions_immutable_guard on agent_versions;
drop function if exists crank_reject_published_agent_version_mutation();
alter table published_agents
drop constraint if exists published_agents_catalog_revision_check,
drop column if exists catalog_revision;
alter table agents
drop constraint if exists agents_catalog_revision_check,
drop column if exists catalog_revision;
delete from __crank_migrations where version = 9;",
)
.execute(pool)
.await
.unwrap();
}
async fn remove_v10_schema(pool: &sqlx::PgPool) {
remove_v11_schema(pool).await;
sqlx::raw_sql(
"drop index if exists approval_requests_pending_scope_fingerprint_idx;
drop index if exists approval_requests_workspace_request_trace_idx;
alter table approval_requests drop constraint if exists approval_requests_request_id_check;
alter table approval_requests drop constraint if exists approval_requests_trace_id_check;
alter table approval_requests drop column if exists request_id;
alter table approval_requests drop column if exists trace_id;
create unique index if not exists approval_requests_pending_fingerprint_idx
on approval_requests(agent_id, operation_id, operation_version, request_fingerprint)
where status = 'pending' and request_fingerprint is not null;
delete from __crank_migrations where version = 10;",
)
.execute(pool)
.await
.unwrap();
}
async fn remove_v11_schema(pool: &sqlx::PgPool) {
sqlx::raw_sql(
"drop table if exists onboarding_selections;
drop trigger if exists product_events_append_only_guard on product_events;
drop function if exists crank_reject_product_event_mutation();
drop table if exists product_event_daily_rollups;
drop table if exists product_events;
drop index if exists invocation_logs_workspace_agent_key_success_idx;
alter table invocation_logs drop constraint if exists invocation_logs_platform_key_scope_fk;
alter table invocation_logs drop column if exists platform_api_key_id;
alter table platform_api_keys drop constraint if exists platform_api_keys_workspace_agent_id_unique;
delete from __crank_migrations where version = 11;",
)
.execute(pool)
.await
.unwrap();
}
#[tokio::test]
async fn partial_sequence_fails_closed() {
let database_url = crank_test_support::postgres_schema_url("test_partial_sequence").await;
@@ -459,7 +855,6 @@ async fn partial_sequence_fails_closed() {
"partial_sequence"
);
}
#[tokio::test]
async fn missing_relation_with_current_ledger_fails_closed() {
let database_url = crank_test_support::postgres_schema_url("test_missing_relation").await;
@@ -473,7 +868,6 @@ async fn missing_relation_with_current_ledger_fails_closed() {
assert_eq!(error.code(), "partial_sequence");
assert_eq!(error.stage(), "preflight.schema");
}
#[tokio::test]
async fn unregistered_legacy_extension_provenance_is_rejected() {
let database_url = crank_test_support::postgres_schema_url("test_legacy_extension").await;
@@ -492,11 +886,9 @@ async fn unregistered_legacy_extension_provenance_is_rejected() {
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
assert_eq!(error.code(), "legacy_conflict");
}
#[tokio::test]
async fn any_owned_relation_without_core_ledger_is_partial() {
let database_url = crank_test_support::postgres_schema_url("test_owned_partial").await;
@@ -508,7 +900,6 @@ async fn any_owned_relation_without_core_ledger_is_partial() {
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "partial_sequence");
}
#[tokio::test]
async fn current_ledger_with_structural_drift_fails_closed() {
let database_url = crank_test_support::postgres_schema_url("test_structural_drift").await;
@@ -522,7 +913,6 @@ async fn current_ledger_with_structural_drift_fails_closed() {
assert_eq!(error.code(), "partial_sequence");
assert_eq!(error.stage(), "preflight.schema");
}
#[tokio::test]
async fn tampered_legacy_audit_fails_closed() {
let database_url = crank_test_support::postgres_schema_url("test_tampered_audit").await;
@@ -537,173 +927,3 @@ async fn tampered_legacy_audit_fails_closed() {
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
assert_eq!(error.code(), "legacy_conflict");
}
#[tokio::test]
async fn failed_consolidation_rolls_back_all_changes() {
let _event_trigger_guard = EVENT_TRIGGER_TEST_LOCK.lock().await;
let database_url = crank_test_support::postgres_schema_url("test_apply_rollback").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
sqlx::query(
"insert into workspaces (id, slug, display_name, status, settings_json, created_at, updated_at)
values ('rollback_preserved', 'rollback-preserved', 'Rollback Preserved', 'active', '{}'::jsonb, now(), now())",
)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"drop table __crank_migrations, __crank_migration_legacy_audit,
__crank_mcp_migrations, mcp_transport_sessions, __crank_ext_migrations",
)
.execute(&pool)
.await
.unwrap();
let schema: String = sqlx::query_scalar("select current_schema()")
.fetch_one(&pool)
.await
.unwrap();
let failure_trigger = format!(
"create function reject_story14_v2() returns event_trigger language plpgsql as $$
begin
if current_schema() = '{schema}' and current_query() like '%__crank_migrations%' then
raise exception 'injected v2 ddl failure';
end if;
end $$;
create event trigger reject_story14_v2 on ddl_command_start
execute function reject_story14_v2();"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(failure_trigger))
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
sqlx::raw_sql(
"drop event trigger reject_story14_v2;
drop function reject_story14_v2();",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(error.code(), "apply_failed");
for relation in [
"__crank_migrations",
"__crank_migration_legacy_audit",
"__crank_mcp_migrations",
"mcp_transport_sessions",
"__crank_ext_migrations",
] {
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), $1)) is not null",
)
.bind(relation)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present, "{relation} must roll back with failed v2 DDL");
}
let preserved: String =
sqlx::query_scalar("select display_name from workspaces where id = 'rollback_preserved'")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(preserved, "Rollback Preserved");
}
#[tokio::test]
async fn failed_request_trace_identity_migration_rolls_back_all_changes() {
let _event_trigger_guard = EVENT_TRIGGER_TEST_LOCK.lock().await;
let database_url =
crank_test_support::postgres_schema_url("test_trace_identity_rollback").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
sqlx::raw_sql(
"insert into operations
(id, workspace_id, name, display_name, protocol, status, created_at, updated_at)
values ('op_trace_rollback', 'ws_default', 'trace-rollback', 'Trace rollback',
'rest', 'draft', now(), now());
insert into invocation_logs
(id, workspace_id, operation_id, source, level, status, tool_name, message,
duration_ms, request_preview_json, response_preview_json, created_at)
values ('trace_rollback_preserved', 'ws_default', 'op_trace_rollback', 'admin',
'info', 'success', 'trace_rollback', 'safe preserved row', 1,
'{}'::jsonb, '{}'::jsonb, now());",
)
.execute(&pool)
.await
.unwrap();
sqlx::query("delete from __crank_migrations where version = 3")
.execute(&pool)
.await
.unwrap();
remove_v3_schema(&pool).await;
let schema: String = sqlx::query_scalar("select current_schema()")
.fetch_one(&pool)
.await
.unwrap();
let failure_trigger = format!(
"create function reject_story15_v3() returns event_trigger language plpgsql as $$
begin
if current_schema() = '{schema}' and current_query() like '%invocation_logs_workspace_trace_id_idx%' then
raise exception 'injected v3 ddl failure';
end if;
end $$;
create event trigger reject_story15_v3 on ddl_command_start
execute function reject_story15_v3();"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(failure_trigger))
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
sqlx::raw_sql(
"drop event trigger reject_story15_v3;
drop function reject_story15_v3();",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(error.code(), "apply_failed");
assert_eq!(error.version(), Some(3));
let trace_column: bool = sqlx::query_scalar(
"select exists (
select 1 from information_schema.columns
where table_schema = current_schema()
and table_name = 'invocation_logs'
and column_name = 'trace_id'
)",
)
.fetch_one(&pool)
.await
.unwrap();
assert!(!trace_column, "trace_id column must roll back with v3");
for index in [
"invocation_logs_workspace_request_id_idx",
"invocation_logs_workspace_trace_id_idx",
] {
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), $1)) is not null",
)
.bind(index)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present, "{index} must roll back with failed v3 DDL");
}
let ledger_v3: i64 =
sqlx::query_scalar("select count(*) from __crank_migrations where version = 3")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(ledger_v3, 0, "failed v3 must not be recorded as applied");
let preserved: String = sqlx::query_scalar(
"select message from invocation_logs where id = 'trace_rollback_preserved'",
)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(preserved, "safe preserved row");
}
@@ -0,0 +1,168 @@
use super::*;
#[tokio::test]
async fn failed_consolidation_rolls_back_all_changes() {
let _event_trigger_guard = EVENT_TRIGGER_TEST_LOCK.lock().await;
let database_url = crank_test_support::postgres_schema_url("test_apply_rollback").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
sqlx::query(
"insert into workspaces (id, slug, display_name, status, settings_json, created_at, updated_at)
values ('rollback_preserved', 'rollback-preserved', 'Rollback Preserved', 'active', '{}'::jsonb, now(), now())",
)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"drop table __crank_migrations, __crank_migration_legacy_audit,
__crank_mcp_migrations, mcp_transport_sessions, __crank_ext_migrations",
)
.execute(&pool)
.await
.unwrap();
let schema: String = sqlx::query_scalar("select current_schema()")
.fetch_one(&pool)
.await
.unwrap();
let failure_trigger = format!(
"create function reject_story14_v2() returns event_trigger language plpgsql as $$
begin
if current_schema() = '{schema}' and current_query() like '%__crank_migrations%' then
raise exception 'injected v2 ddl failure';
end if;
end $$;
create event trigger reject_story14_v2 on ddl_command_start
execute function reject_story14_v2();"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(failure_trigger))
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
sqlx::raw_sql(
"drop event trigger reject_story14_v2;
drop function reject_story14_v2();",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(error.code(), "apply_failed");
for relation in [
"__crank_migrations",
"__crank_migration_legacy_audit",
"__crank_mcp_migrations",
"mcp_transport_sessions",
"__crank_ext_migrations",
] {
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), $1)) is not null",
)
.bind(relation)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present, "{relation} must roll back with failed v2 DDL");
}
let preserved: String =
sqlx::query_scalar("select display_name from workspaces where id = 'rollback_preserved'")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(preserved, "Rollback Preserved");
}
#[tokio::test]
async fn failed_request_trace_identity_migration_rolls_back_all_changes() {
let _event_trigger_guard = EVENT_TRIGGER_TEST_LOCK.lock().await;
let database_url =
crank_test_support::postgres_schema_url("test_trace_identity_rollback").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
sqlx::raw_sql(
"insert into operations
(id, workspace_id, name, display_name, protocol, status, created_at, updated_at)
values ('op_trace_rollback', 'ws_default', 'trace-rollback', 'Trace rollback',
'rest', 'draft', now(), now());
insert into invocation_logs
(id, workspace_id, operation_id, source, level, status, tool_name, message,
duration_ms, request_preview_json, response_preview_json, created_at)
values ('trace_rollback_preserved', 'ws_default', 'op_trace_rollback', 'admin',
'info', 'success', 'trace_rollback', 'safe preserved row', 1,
'{}'::jsonb, '{}'::jsonb, now());",
)
.execute(&pool)
.await
.unwrap();
remove_v4_schema(&pool).await;
sqlx::query("delete from __crank_migrations where version = 3")
.execute(&pool)
.await
.unwrap();
remove_v3_schema(&pool).await;
let schema: String = sqlx::query_scalar("select current_schema()")
.fetch_one(&pool)
.await
.unwrap();
let failure_trigger = format!(
"create function reject_story15_v3() returns event_trigger language plpgsql as $$
begin
if current_schema() = '{schema}' and current_query() like '%invocation_logs_workspace_trace_id_idx%' then
raise exception 'injected v3 ddl failure';
end if;
end $$;
create event trigger reject_story15_v3 on ddl_command_start
execute function reject_story15_v3();"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(failure_trigger))
.execute(&pool)
.await
.unwrap();
let error = MigrationAuthority::apply(&pool).await.unwrap_err();
sqlx::raw_sql(
"drop event trigger reject_story15_v3;
drop function reject_story15_v3();",
)
.execute(&pool)
.await
.unwrap();
assert_eq!(error.code(), "apply_failed");
assert_eq!(error.version(), Some(3));
let trace_column: bool = sqlx::query_scalar(
"select exists (
select 1 from information_schema.columns
where table_schema = current_schema()
and table_name = 'invocation_logs'
and column_name = 'trace_id'
)",
)
.fetch_one(&pool)
.await
.unwrap();
assert!(!trace_column, "trace_id column must roll back with v3");
for index in [
"invocation_logs_workspace_request_id_idx",
"invocation_logs_workspace_trace_id_idx",
] {
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), $1)) is not null",
)
.bind(index)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present, "{index} must roll back with failed v3 DDL");
}
let ledger_v3: i64 =
sqlx::query_scalar("select count(*) from __crank_migrations where version = 3")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(ledger_v3, 0, "failed v3 must not be recorded as applied");
let preserved: String = sqlx::query_scalar(
"select message from invocation_logs where id = 'trace_rollback_preserved'",
)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(preserved, "safe preserved row");
}
@@ -1,8 +1,10 @@
use crank_registry::{
CreateInvocationLogRequest, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
ListInvocationLogsQuery, UsageBucket, UsageQuery,
};
use sqlx::Row;
use super::common::{TestDatabase, test_invocation_log};
use super::common::{TestDatabase, test_invocation_log, test_operation, test_workspace_id};
#[tokio::test]
async fn invocation_history_write_returns_typed_loss_without_error_details() {
@@ -29,3 +31,115 @@ async fn invocation_history_write_returns_typed_loss_without_error_details() {
);
database.cleanup().await;
}
#[tokio::test]
#[ignore = "production-size query-plan evidence: inserts 1,000,000 invocation_logs rows"]
async fn production_size_invocation_history_queries_stay_bounded() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let pool = database.raw_pool().await;
let operation = test_operation("op_history_scale", 1, crank_core::OperationStatus::Draft);
registry
.create_operation(&test_workspace_id(), &operation, None)
.await
.unwrap();
sqlx::query(
r#"
insert into invocation_logs (
id, workspace_id, operation_id, operation_version, source, level, status,
tool_name, message, request_id, trace_id, status_code, duration_ms,
error_kind, execution_stage, execution_error_code, retryability, outcome_certainty,
request_preview_json, response_preview_json, created_at
)
select
'log_scale_' || series::text,
'ws_default',
'op_history_scale',
1,
'admin_test_run',
'info',
case when series % 10 = 0 then 'error' else 'ok' end,
'scale_tool',
'scale invocation',
'018f0000-0000-7000-8000-' || lpad(series::text, 12, '0'),
'0af7651916cd43dd8448eb211c80319c',
case when series % 10 = 0 then 500 else 200 end,
20 + (series % 250),
null,
'runtime',
case when series % 10 = 0 then 'runtime_internal' else null end,
'never',
'certain',
'{"input":"bounded"}'::jsonb,
'{"ok":true}'::jsonb,
'2026-03-25T00:00:00Z'::timestamptz + (series || ' seconds')::interval
from generate_series(1, 1000000) as series
"#,
)
.execute(&pool)
.await
.unwrap();
let page = registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: None,
operation_id: Some(&operation.id),
agent_id: None,
created_after: Some("2026-03-25T00:00:00Z"),
created_before: Some("2026-04-06T00:00:00Z"),
cursor_created_at: None,
cursor_id: None,
limit: 101,
})
.await
.unwrap();
assert_eq!(page.len(), 101);
let summary = registry
.summarize_usage(UsageQuery {
workspace_id: &test_workspace_id(),
period: crank_core::UsagePeriod::Last7Days,
source: None,
created_after: "2026-03-25T00:00:00Z",
created_before: "2026-04-06T00:00:00Z",
bucket: UsageBucket::Day,
})
.await
.unwrap();
assert_eq!(summary.rollup.calls_total, 1_000_000);
let explain = sqlx::query(
r#"
explain
select id
from invocation_logs
where workspace_id = 'ws_default'
and operation_id = 'op_history_scale'
and created_at >= '2026-03-25T00:00:00Z'::timestamptz
and created_at < '2026-04-06T00:00:00Z'::timestamptz
order by created_at desc, id desc
limit 101
"#,
)
.fetch_all(&pool)
.await
.unwrap()
.into_iter()
.map(|row| row.get::<String, _>(0))
.collect::<Vec<_>>()
.join("\n");
assert!(
explain.contains("Index Scan") || explain.contains("Bitmap Index Scan"),
"{explain}"
);
assert!(!explain.contains("Seq Scan"), "{explain}");
database.cleanup().await;
}
@@ -0,0 +1,844 @@
#![allow(dead_code, unused_imports)]
use super::common::*;
use crank_core::{
AgentOperationBinding, AgentStatus, InvocationSource, InvocationStatus, OnboardingStepId,
OperationStatus, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind, PlatformApiKeyScope,
PlatformApiKeyStatus, ProductEventId, Workspace, WorkspaceId,
};
use crank_registry::{
CreateAgentRequest, CreateInvocationLogRequest, CreatePlatformApiKeyRequest,
CreateWorkspaceRequest, OnboardingPresentationMilestone, PublishAgentRequest, PublishRequest,
RecordOnboardingMilestoneRequest,
};
use serde_json::json;
use sqlx::Row;
use time::{Duration, OffsetDateTime};
#[path = "onboarding_lifecycle.rs"]
mod onboarding_lifecycle;
fn truncate_to_micros(value: OffsetDateTime) -> OffsetDateTime {
value
.replace_nanosecond((value.nanosecond() / 1_000) * 1_000)
.unwrap()
}
#[tokio::test]
async fn migration_exposes_product_events_and_exact_invocation_key_scope() {
let database = TestDatabase::new().await;
let _registry = database.registry().await;
let pool = database.raw_pool().await;
let product_events = relation_exists(&pool, "product_events").await;
let product_rollups = relation_exists(&pool, "product_event_daily_rollups").await;
let key_scope = column_exists(&pool, "invocation_logs", "platform_api_key_id").await;
assert!(product_events, "V11 must own immutable local ProductEvents");
assert!(
product_rollups,
"V11 must preserve bounded UTC product-metric denominators"
);
assert!(
key_scope,
"successful MCP evidence must retain the exact non-secret key identity"
);
database.cleanup().await;
}
#[tokio::test]
async fn product_event_idempotency_is_scoped_to_workspace() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let pool = database.raw_pool().await;
create_workspace(&registry, "ws_onboarding_other", "onboarding-other").await;
insert_product_event(
&pool,
"pe_start_default",
"ws_default",
"onboarding_started",
"eligible:first-login",
)
.await
.unwrap();
let replay = insert_product_event(
&pool,
"pe_start_replay",
"ws_default",
"onboarding_started",
"eligible:first-login",
)
.await;
assert!(
replay.is_err(),
"same workspace/idempotency key must not double-count a milestone"
);
insert_product_event(
&pool,
"pe_start_other",
"ws_onboarding_other",
"onboarding_started",
"eligible:first-login",
)
.await
.expect("an independent workspace may use the same local idempotency key");
let count: i64 =
sqlx::query_scalar("select count(*) from product_events where idempotency_key = $1")
.bind("eligible:first-login")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(count, 2);
database.cleanup().await;
}
#[tokio::test]
async fn exact_key_success_is_authoritative_and_revocation_regresses_progress() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let pool = database.raw_pool().await;
let (operation_id, agent_id, key_id) = published_path(&registry).await;
let log = test_invocation_log(
"log_onboarding_first_call",
&operation_id,
Some(agent_id.clone()),
InvocationStatus::Ok,
25,
"2026-03-25T12:15:00Z",
);
assert_eq!(
registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.await,
crank_registry::InvocationHistoryWriteOutcome::Recorded
);
sqlx::query("update invocation_logs set platform_api_key_id = $1 where id = $2")
.bind(key_id.as_str())
.bind(log.id.as_str())
.execute(&pool)
.await
.unwrap();
assert!(
authoritative_first_call(&pool, "ws_default", agent_id.as_str(), key_id.as_str()).await,
"an active exact-scope key and real successful Agent tool invocation complete first value"
);
registry
.revoke_platform_api_key_for_agent(
&WorkspaceId::new("ws_default"),
&agent_id,
&key_id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
assert!(
!authoritative_first_call(&pool, "ws_default", agent_id.as_str(), key_id.as_str()).await,
"revocation must return connection/onboarding to an actionable state"
);
database.cleanup().await;
}
#[tokio::test]
async fn invocation_key_scope_rejects_cross_workspace_evidence() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let pool = database.raw_pool().await;
assert!(
column_exists(&pool, "invocation_logs", "platform_api_key_id").await,
"cross-workspace rejection requires the exact-key evidence column"
);
let (operation_id, agent_id, _key_id) = published_path(&registry).await;
create_workspace(&registry, "ws_onboarding_foreign", "onboarding-foreign").await;
let foreign_key = PlatformApiKey {
id: PlatformApiKeyId::new("pk_onboarding_foreign"),
workspace_id: WorkspaceId::new("ws_onboarding_foreign"),
agent_id: None,
key_kind: PlatformApiKeyKind::McpClient,
name: "foreign-onboarding-key".to_owned(),
prefix: "crk_foreign".to_owned(),
scopes: vec![PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
status: PlatformApiKeyStatus::Active,
created_at: OffsetDateTime::now_utc(),
last_used_at: None,
expires_at: None,
allowed_origins: Vec::new(),
};
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
api_key: &foreign_key,
secret_hash: "foreign-secret-hash",
})
.await
.unwrap();
let log = test_invocation_log(
"log_onboarding_cross_workspace",
&operation_id,
Some(agent_id),
InvocationStatus::Ok,
25,
"2026-03-25T12:16:00Z",
);
assert_eq!(
registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.await,
crank_registry::InvocationHistoryWriteOutcome::Recorded
);
let poisoned = sqlx::query(
"update invocation_logs set platform_api_key_id = $1 where workspace_id = $2 and id = $3",
)
.bind(foreign_key.id.as_str())
.bind("ws_default")
.bind(log.id.as_str())
.execute(&pool)
.await;
assert!(
poisoned.is_err(),
"database constraints must prevent foreign-workspace key evidence"
);
database.cleanup().await;
}
#[tokio::test]
async fn projection_moves_from_empty_to_exact_terminal_success_and_regresses_on_revoke() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let empty = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(!empty.completed);
assert_eq!(
empty.steps.iter().map(|step| step.id).collect::<Vec<_>>(),
OnboardingStepId::ORDERED
);
let (operation_id, agent_id, key_id) = published_path(&registry).await;
let mut test_log = test_invocation_log(
"log_onboarding_admin_test",
&operation_id,
None,
InvocationStatus::Ok,
20,
"2026-03-25T12:14:00Z",
);
test_log.source = InvocationSource::AdminTestRun;
registry
.create_invocation_log(CreateInvocationLogRequest { log: &test_log })
.await;
registry
.touch_platform_api_key(&workspace_id, &key_id, &OffsetDateTime::now_utc())
.await
.unwrap();
let mut pending = test_invocation_log(
"log_onboarding_approval_pending",
&operation_id,
Some(agent_id.clone()),
InvocationStatus::Ok,
10,
"2026-03-25T12:15:00Z",
);
pending.platform_api_key_id = Some(key_id.clone());
pending.execution_stage = Some(crank_core::ExecutionStage::MandatoryPersistence);
pending.created_at = OffsetDateTime::now_utc();
registry
.create_invocation_log(CreateInvocationLogRequest { log: &pending })
.await;
let not_terminal = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(!not_terminal.completed);
assert!(
!not_terminal
.step(OnboardingStepId::FirstCall)
.unwrap()
.completed
);
let mut success = test_invocation_log(
"log_onboarding_terminal_success",
&operation_id,
Some(agent_id.clone()),
InvocationStatus::Ok,
25,
"2026-03-25T12:16:00Z",
);
success.platform_api_key_id = Some(key_id.clone());
success.created_at = truncate_to_micros(OffsetDateTime::now_utc());
registry
.create_invocation_log(CreateInvocationLogRequest { log: &success })
.await;
let complete = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(complete.completed);
assert_eq!(complete.platform_api_key_id, Some(key_id.clone()));
assert_eq!(complete.first_call_log_id, Some(success.id));
assert_eq!(
complete.first_call_tool_name.as_deref(),
Some("create_lead")
);
assert_eq!(complete.first_call_at, Some(success.created_at));
// A different active key/path must not replace the exact path that produced
// the terminal onboarding call when that exact key is later revoked.
let (_other_agent_id, _other_key_id) =
publish_additional_agent_path(&registry, &operation_id).await;
registry
.revoke_platform_api_key_for_agent(
&workspace_id,
&agent_id,
&key_id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
let regressed = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(!regressed.completed);
assert!(!regressed.step(OnboardingStepId::Key).unwrap().completed);
assert!(
!regressed
.step(OnboardingStepId::FirstCall)
.unwrap()
.completed
);
database.cleanup().await;
}
#[tokio::test]
async fn current_agent_publication_and_key_expiry_invalidate_old_first_call_evidence() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let pool = database.raw_pool().await;
let workspace_id = WorkspaceId::new("ws_default");
let (operation_id, agent_id, key_id) = published_path(&registry).await;
record_onboarding_test_and_call(
&registry,
&operation_id,
&agent_id,
&key_id,
"publication_before_republish",
OffsetDateTime::now_utc(),
)
.await;
let completed = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(completed.completed);
let unpublished_at = OffsetDateTime::now_utc() + Duration::seconds(1);
registry
.unpublish_agent(&workspace_id, &agent_id, &unpublished_at, None)
.await
.unwrap();
let unpublished = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(!unpublished.step(OnboardingStepId::Agent).unwrap().completed);
let republished_at = unpublished_at + Duration::seconds(1);
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id,
agent_id: &agent_id,
version: 1,
published_at: &republished_at,
published_by: Some("onboarding-republish-test"),
expected_state: None,
})
.await
.unwrap();
let republished = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(
!republished.step(OnboardingStepId::Agent).unwrap().completed,
"republishing must not silently replace the exact catalog revision that completed onboarding"
);
assert!(
!republished
.step(OnboardingStepId::FirstCall)
.unwrap()
.completed,
"a call from an older Agent publication must not complete the current catalog revision"
);
assert_ne!(republished.revision, completed.revision);
let reset = registry
.reset_onboarding_selection(
&workspace_id,
republished.revision,
republished_at + Duration::milliseconds(1),
)
.await
.unwrap();
assert!(reset.step(OnboardingStepId::Agent).unwrap().completed);
assert!(reset.step(OnboardingStepId::Key).unwrap().completed);
record_onboarding_test_and_call(
&registry,
&operation_id,
&agent_id,
&key_id,
"publication_after_republish",
republished_at + Duration::seconds(1),
)
.await;
let completed_again = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(completed_again.completed);
sqlx::query(
"update platform_api_keys set expires_at = now() - interval '1 minute' where id = $1",
)
.bind(key_id.as_str())
.execute(&pool)
.await
.unwrap();
let expired = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(!expired.step(OnboardingStepId::Key).unwrap().completed);
assert!(!expired.step(OnboardingStepId::FirstCall).unwrap().completed);
assert_ne!(expired.revision, completed_again.revision);
database.cleanup().await;
}
#[tokio::test]
async fn archiving_selected_operation_or_agent_regresses_authoritative_projection() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let (operation_id, agent_id, key_id) = published_path(&registry).await;
record_onboarding_test_and_call(
&registry,
&operation_id,
&agent_id,
&key_id,
"archive_agent_path",
OffsetDateTime::now_utc(),
)
.await;
assert!(
registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap()
.completed
);
// A second healthy path must not replace the exact lineage that produced
// the authoritative first call when that lineage is later archived.
let (alternative_operation_id, _, _) = publish_named_path(
&registry,
"op_onboarding_alternative",
"agent_onboarding_alternative",
"pk_onboarding_alternative",
)
.await;
registry
.archive_agent(
&workspace_id,
&agent_id,
&(OffsetDateTime::now_utc() + Duration::seconds(1)),
None,
)
.await
.unwrap();
let archived_agent = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert_eq!(archived_agent.operation_id, Some(operation_id.clone()));
assert_eq!(archived_agent.agent_id, Some(agent_id));
assert_ne!(archived_agent.operation_id, Some(alternative_operation_id));
assert!(
!archived_agent
.step(OnboardingStepId::Agent)
.unwrap()
.completed
);
assert!(
!archived_agent
.step(OnboardingStepId::FirstCall)
.unwrap()
.completed
);
registry
.archive_operation(
&workspace_id,
&operation_id,
&(OffsetDateTime::now_utc() + Duration::seconds(2)),
)
.await
.unwrap();
let archived_operation = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(
!archived_operation
.step(OnboardingStepId::Operation)
.unwrap()
.completed
);
database.cleanup().await;
}
async fn published_path(
registry: &crank_registry::PostgresRegistry,
) -> (
crank_core::OperationId,
crank_core::AgentId,
PlatformApiKeyId,
) {
publish_named_path(
registry,
"op_onboarding",
"agent_onboarding",
"pk_onboarding_exact",
)
.await
}
async fn publish_named_path(
registry: &crank_registry::PostgresRegistry,
operation_id: &str,
agent_id: &str,
key_id: &str,
) -> (
crank_core::OperationId,
crank_core::AgentId,
PlatformApiKeyId,
) {
let workspace_id = WorkspaceId::new("ws_default");
let operation = test_operation(operation_id, 1, OperationStatus::Draft);
registry
.create_operation(&workspace_id, &operation, None)
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &workspace_id,
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("onboarding-test"),
})
.await
.unwrap();
let agent = test_agent(agent_id, AgentStatus::Draft);
let version = test_agent_version(&agent.id, 1, AgentStatus::Draft);
let bindings = vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: "create_lead".to_owned(),
tool_title: "Create lead".to_owned(),
tool_description_override: None,
enabled: true,
}];
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &version,
bindings: &bindings,
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id,
agent_id: &agent.id,
version: 1,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("onboarding-test"),
expected_state: None,
})
.await
.unwrap();
let key_id = PlatformApiKeyId::new(key_id);
let key = PlatformApiKey {
id: key_id.clone(),
workspace_id,
agent_id: Some(agent.id.clone()),
key_kind: PlatformApiKeyKind::McpClient,
name: format!("onboarding-{key_id}-key"),
prefix: format!("crk_{}", key_id.as_str()),
scopes: vec![PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
status: PlatformApiKeyStatus::Active,
created_at: OffsetDateTime::now_utc(),
last_used_at: None,
expires_at: None,
allowed_origins: Vec::new(),
};
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
api_key: &key,
secret_hash: "exact-secret-hash",
})
.await
.unwrap();
(operation.id, agent.id, key_id)
}
async fn record_onboarding_test_and_call(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
agent_id: &crank_core::AgentId,
key_id: &PlatformApiKeyId,
suffix: &str,
occurred_at: OffsetDateTime,
) {
let mut test_log = test_invocation_log(
&format!("log_onboarding_test_{suffix}"),
operation_id,
None,
InvocationStatus::Ok,
20,
"2026-03-25T12:14:00Z",
);
test_log.source = InvocationSource::AdminTestRun;
test_log.created_at = truncate_to_micros(occurred_at);
registry
.create_invocation_log(CreateInvocationLogRequest { log: &test_log })
.await;
registry
.touch_platform_api_key(
&WorkspaceId::new("ws_default"),
key_id,
&truncate_to_micros(occurred_at),
)
.await
.unwrap();
record_onboarding_call(
registry,
operation_id,
agent_id,
key_id,
suffix,
occurred_at,
)
.await;
}
async fn record_onboarding_call(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
agent_id: &crank_core::AgentId,
key_id: &PlatformApiKeyId,
suffix: &str,
occurred_at: OffsetDateTime,
) {
let mut success = test_invocation_log(
&format!("log_onboarding_call_{suffix}"),
operation_id,
Some(agent_id.clone()),
InvocationStatus::Ok,
25,
"2026-03-25T12:16:00Z",
);
success.platform_api_key_id = Some(key_id.clone());
success.created_at = truncate_to_micros(occurred_at);
registry
.create_invocation_log(CreateInvocationLogRequest { log: &success })
.await;
}
async fn publish_additional_agent_path(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
) -> (crank_core::AgentId, PlatformApiKeyId) {
let workspace_id = WorkspaceId::new("ws_default");
let agent = test_agent("agent_onboarding_later", AgentStatus::Draft);
let version = test_agent_version(&agent.id, 1, AgentStatus::Draft);
let bindings = vec![AgentOperationBinding {
operation_id: operation_id.clone(),
operation_version: 1,
tool_name: "create_lead".to_owned(),
tool_title: "Create lead".to_owned(),
tool_description_override: None,
enabled: true,
}];
registry
.create_agent(CreateAgentRequest {
agent: &agent,
version: &version,
bindings: &bindings,
})
.await
.unwrap();
registry
.publish_agent(PublishAgentRequest {
workspace_id: &workspace_id,
agent_id: &agent.id,
version: 1,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("onboarding-test"),
expected_state: None,
})
.await
.unwrap();
let key_id = PlatformApiKeyId::new("pk_onboarding_later");
let key = PlatformApiKey {
id: key_id.clone(),
workspace_id,
agent_id: Some(agent.id.clone()),
key_kind: PlatformApiKeyKind::McpClient,
name: "onboarding-later-key".to_owned(),
prefix: "crk_later".to_owned(),
scopes: vec![PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
status: PlatformApiKeyStatus::Active,
created_at: OffsetDateTime::now_utc(),
last_used_at: None,
expires_at: None,
allowed_origins: Vec::new(),
};
registry
.create_platform_api_key(CreatePlatformApiKeyRequest {
api_key: &key,
secret_hash: "later-secret-hash",
})
.await
.unwrap();
(agent.id, key_id)
}
async fn authoritative_first_call(
pool: &sqlx::PgPool,
workspace_id: &str,
agent_id: &str,
key_id: &str,
) -> bool {
sqlx::query_scalar(
"select exists (
select 1
from invocation_logs l
join published_agents pa on pa.agent_id = l.agent_id
join platform_api_keys k
on k.id = l.platform_api_key_id
and k.workspace_id = l.workspace_id
and k.agent_id = l.agent_id
where l.workspace_id = $1
and l.agent_id = $2
and l.platform_api_key_id = $3
and l.source = 'agent_tool_call'
and l.status = 'ok'
and k.status = 'active'
and (k.expires_at is null or k.expires_at > now())
)",
)
.bind(workspace_id)
.bind(agent_id)
.bind(key_id)
.fetch_one(pool)
.await
.unwrap()
}
async fn insert_product_event(
pool: &sqlx::PgPool,
id: &str,
workspace_id: &str,
event_name: &str,
idempotency_key: &str,
) -> Result<(), sqlx::Error> {
sqlx::query(
"insert into product_events (
id, workspace_id, event_name, schema_version, occurred_at, idempotency_key,
properties_json
) values ($1, $2, $3, 1, now(), $4, $5)",
)
.bind(id)
.bind(workspace_id)
.bind(event_name)
.bind(idempotency_key)
.bind(json!({"eligible": true}))
.execute(pool)
.await?;
Ok(())
}
async fn relation_exists(pool: &sqlx::PgPool, relation: &str) -> bool {
sqlx::query_scalar(
"select exists (
select 1 from information_schema.tables
where table_schema = current_schema() and table_name = $1
)",
)
.bind(relation)
.fetch_one(pool)
.await
.unwrap()
}
async fn column_exists(pool: &sqlx::PgPool, relation: &str, column: &str) -> bool {
sqlx::query_scalar(
"select exists (
select 1 from information_schema.columns
where table_schema = current_schema() and table_name = $1 and column_name = $2
)",
)
.bind(relation)
.bind(column)
.fetch_one(pool)
.await
.unwrap()
}
async fn create_workspace(registry: &crank_registry::PostgresRegistry, id: &str, slug: &str) {
let now = OffsetDateTime::now_utc();
let workspace = Workspace {
id: WorkspaceId::new(id),
slug: slug.to_owned(),
display_name: slug.to_owned(),
status: crank_core::WorkspaceStatus::Active,
settings: json!({}),
created_at: now,
updated_at: now,
};
registry
.create_workspace(CreateWorkspaceRequest {
workspace: &workspace,
})
.await
.unwrap();
}
@@ -0,0 +1,219 @@
use super::*;
#[tokio::test]
async fn server_owned_lifecycle_events_are_idempotent_and_completion_uses_the_eligible_cohort() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let eligible_at = truncate_to_micros(OffsetDateTime::now_utc() - Duration::minutes(3));
let eligible = registry
.ensure_onboarding_eligibility(&workspace_id, eligible_at)
.await
.unwrap();
assert_eq!(eligible.eligible_since, Some(eligible_at));
let repeated = registry
.ensure_onboarding_eligibility(&workspace_id, OffsetDateTime::now_utc())
.await
.unwrap();
assert_eq!(repeated.eligible_since, Some(eligible_at));
let (operation_id, agent_id, key_id) = published_path(&registry).await;
let mut test_log = test_invocation_log(
"log_onboarding_server_test",
&operation_id,
None,
InvocationStatus::Ok,
20,
"2026-03-25T12:14:00Z",
);
test_log.source = InvocationSource::AdminTestRun;
registry
.create_invocation_log(CreateInvocationLogRequest { log: &test_log })
.await;
registry
.touch_platform_api_key(&workspace_id, &key_id, &OffsetDateTime::now_utc())
.await
.unwrap();
let mut success = test_invocation_log(
"log_onboarding_server_call",
&operation_id,
Some(agent_id),
InvocationStatus::Ok,
25,
"2026-03-25T12:16:00Z",
);
success.platform_api_key_id = Some(key_id);
success.created_at = truncate_to_micros(OffsetDateTime::now_utc());
registry
.create_invocation_log(CreateInvocationLogRequest { log: &success })
.await;
let completed = registry
.ensure_onboarding_completion(&workspace_id, OffsetDateTime::now_utc())
.await
.unwrap();
assert!(completed.completed);
let later_draft = test_operation("op_onboarding_later_draft", 1, OperationStatus::Draft);
registry
.create_operation(&workspace_id, &later_draft, None)
.await
.unwrap();
let after_later_draft = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert_eq!(after_later_draft.operation_id, Some(operation_id));
assert!(after_later_draft.completed);
let events = registry
.list_product_events(crank_registry::ListProductEventsQuery {
workspace_id: &workspace_id,
kind: Some(crank_core::ProductEventKind::OnboardingCompleted),
created_after: OffsetDateTime::UNIX_EPOCH,
created_before: OffsetDateTime::now_utc() + Duration::minutes(1),
limit: 10,
})
.await
.unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].event.eligible_since, Some(eligible_at));
assert_eq!(events[0].event.occurred_at, success.created_at);
registry
.ensure_onboarding_completion(&workspace_id, OffsetDateTime::now_utc())
.await
.unwrap();
let events = registry
.list_product_events(crank_registry::ListProductEventsQuery {
workspace_id: &workspace_id,
kind: Some(crank_core::ProductEventKind::OnboardingCompleted),
created_after: OffsetDateTime::UNIX_EPOCH,
created_before: OffsetDateTime::now_utc() + Duration::minutes(1),
limit: 10,
})
.await
.unwrap();
assert_eq!(events.len(), 1);
database.cleanup().await;
}
#[tokio::test]
async fn first_call_before_first_eligibility_is_repaired_with_the_invocation_timestamp() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let (operation_id, agent_id, key_id) = published_path(&registry).await;
let call_at = truncate_to_micros(OffsetDateTime::now_utc() + Duration::seconds(1));
record_onboarding_test_and_call(
&registry,
&operation_id,
&agent_id,
&key_id,
"before_eligibility",
call_at,
)
.await;
let before_eligibility = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert!(before_eligibility.completed);
assert!(!before_eligibility.was_completed);
let eligible_at = call_at + Duration::seconds(1);
let repaired = registry
.ensure_onboarding_eligibility(&workspace_id, eligible_at)
.await
.unwrap();
assert!(repaired.completed);
assert!(repaired.was_completed);
let events = registry
.list_product_events(crank_registry::ListProductEventsQuery {
workspace_id: &workspace_id,
kind: Some(crank_core::ProductEventKind::OnboardingCompleted),
created_after: OffsetDateTime::UNIX_EPOCH,
created_before: eligible_at + Duration::minutes(1),
limit: 10,
})
.await
.unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].event.occurred_at, call_at);
assert_eq!(events[0].event.eligible_since, Some(eligible_at));
database.cleanup().await;
}
#[tokio::test]
async fn projection_keeps_agent_and_key_on_the_same_terminal_path() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let (operation_id, _earlier_agent_id, _earlier_key_id) = published_path(&registry).await;
let (later_agent_id, later_key_id) =
publish_additional_agent_path(&registry, &operation_id).await;
let mut success = test_invocation_log(
"log_onboarding_later_agent_success",
&operation_id,
Some(later_agent_id.clone()),
InvocationStatus::Ok,
25,
"2026-03-25T12:18:00Z",
);
success.platform_api_key_id = Some(later_key_id.clone());
success.created_at = OffsetDateTime::now_utc();
registry
.create_invocation_log(CreateInvocationLogRequest { log: &success })
.await;
let projection = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
assert_eq!(projection.agent_id, Some(later_agent_id));
assert_eq!(projection.platform_api_key_id, Some(later_key_id));
assert_eq!(projection.first_call_log_id, Some(success.id));
database.cleanup().await;
}
#[tokio::test]
async fn presentation_milestone_is_idempotent_and_revision_guarded() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let initial = registry
.get_onboarding_projection(&workspace_id)
.await
.unwrap();
let event_id = ProductEventId::new("pe_onboarding_eligible");
let request = RecordOnboardingMilestoneRequest {
workspace_id: &workspace_id,
event_id: &event_id,
milestone: OnboardingPresentationMilestone::Eligible,
idempotency_key: "eligible:first-login",
expected_revision: initial.revision,
occurred_at: OffsetDateTime::now_utc(),
eligible_since: Some(OffsetDateTime::now_utc()),
};
let first = registry
.record_onboarding_milestone(request.clone())
.await
.unwrap();
assert!(first.accepted);
let replay = registry
.record_onboarding_milestone(RecordOnboardingMilestoneRequest {
expected_revision: first.projection.revision,
..request.clone()
})
.await
.unwrap();
assert!(!replay.accepted);
assert_eq!(replay.projection.revision, first.projection.revision);
let stale_replay = registry.record_onboarding_milestone(request).await.unwrap();
assert!(!stale_replay.accepted);
assert_eq!(stale_replay.projection.revision, first.projection.revision);
database.cleanup().await;
}
@@ -9,8 +9,9 @@ use crank_core::{
AuthKind, AuthProfile, ConfigExport, ExecutionConfig, ExportMode, GeneratedDraft,
GeneratedDraftStatus, HttpMethod, InvocationLog, MembershipRole, OperationId,
OperationSecurityLevel, OperationStatus, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyScope,
PlatformApiKeyStatus, Protocol, RestTarget, RetryPolicy, Samples, SecretId, Target,
ToolDescription, ToolExample, User, UserId, UserSessionId, WizardState, Workspace, WorkspaceId,
PlatformApiKeyStatus, Protocol, RestTarget, RetryPolicy, Samples, Secret, SecretId, SecretKind,
SecretStatus, Target, ToolDescription, ToolExample, User, UserId, UserSessionId, WizardState,
Workspace, WorkspaceId,
};
use crank_mapping::{MappingRule, MappingSet};
use crank_schema::{Schema, SchemaKind};
@@ -20,11 +21,12 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use crank_registry::{
CreateAgentRequest, CreateInvocationLogRequest, CreatePlatformApiKeyRequest,
CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, DescriptorKind,
DescriptorMetadata, OperationSampleMetadata, PlatformApiKeyRecord, PostgresRegistry,
PublishAgentRequest, PublishRequest, RegistryError, RegistryOperation, SampleKind,
SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest,
WorkspaceRecord, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
CreateSecretRequest, CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest,
DescriptorKind, DescriptorMetadata, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
OperationSampleMetadata, PlatformApiKeyRecord, PostgresRegistry, PublishAgentRequest,
PublishRequest, RegistryError, RegistryOperation, SampleKind, SaveAuthProfileRequest,
SaveDescriptorMetadataRequest, SaveSampleMetadataRequest, WorkspaceRecord,
YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
};
fn test_workspace_id() -> WorkspaceId {
@@ -35,6 +37,39 @@ fn timestamp(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).unwrap()
}
async fn create_test_secret(registry: &PostgresRegistry, id: &SecretId, name: &str) {
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &timestamp("2026-03-25T12:00:00Z"),
})
.await
.unwrap();
let secret = Secret {
id: id.clone(),
workspace_id: test_workspace_id(),
name: name.to_owned(),
kind: SecretKind::Token,
status: SecretStatus::Active,
current_version: 1,
created_at: timestamp("2026-03-25T12:00:00Z"),
updated_at: timestamp("2026-03-25T12:00:00Z"),
last_used_at: None,
};
registry
.create_secret(CreateSecretRequest {
secret: &secret,
ciphertext: "test-ciphertext",
key_version: "test-key-v1",
master_key_epoch: 1,
created_by: None,
})
.await
.unwrap();
}
#[tokio::test]
async fn stores_versions_and_published_operations() {
let database = TestDatabase::new().await;
@@ -137,7 +172,7 @@ async fn rejects_out_of_order_versions() {
assert!(matches!(
error,
RegistryError::InvalidVersionSequence {
RegistryError::OperationStaleVersion {
expected: 2,
actual: 3,
..
@@ -170,7 +205,7 @@ async fn update_operation_draft_persists_optional_json_columns_as_sql_null() {
.unwrap();
let stored = registry
.get_operation_version(&test_workspace_id(), &operation.id, operation.version)
.get_operation_version(&test_workspace_id(), &operation.id, operation.version + 1)
.await
.unwrap()
.unwrap();
@@ -183,6 +218,279 @@ async fn update_operation_draft_persists_optional_json_columns_as_sql_null() {
database.cleanup().await;
}
#[tokio::test]
async fn published_version_is_not_rewritten_by_a_later_save() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_immutable_publish", 1, OperationStatus::Draft);
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &timestamp("2026-03-25T12:10:00Z"),
published_by: Some("alice"),
})
.await
.unwrap();
let published_before = registry
.get_published_operation(&operation.id)
.await
.unwrap()
.unwrap();
let mut changed = operation.clone();
changed.display_name = "MUTATED AFTER PUBLISH".to_owned();
changed.target = Target::Rest(RestTarget {
base_url: "https://mutated.example.com".to_owned(),
method: HttpMethod::Post,
path_template: "/mutated".to_owned(),
static_headers: BTreeMap::new(),
});
changed.updated_at = timestamp("2026-03-25T12:20:00Z");
registry
.update_operation_draft(&test_workspace_id(), &changed)
.await
.unwrap();
let published_after = registry
.get_published_operation(&operation.id)
.await
.unwrap()
.unwrap();
assert_eq!(published_after, published_before);
assert_eq!(
registry
.get_operation_summary(&test_workspace_id(), &operation.id)
.await
.unwrap()
.unwrap()
.current_draft_version,
2,
"saving after publish must append a new Draft revision"
);
database.cleanup().await;
}
#[tokio::test]
async fn database_guard_rejects_published_update_and_parent_cascade_delete() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_db_immutable", 1, OperationStatus::Draft);
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &timestamp("2026-03-25T12:10:00Z"),
published_by: Some("alice"),
})
.await
.unwrap();
let update = sqlx::query(
"update operation_versions set display_name = 'tampered'
where operation_id = $1 and version = 1",
)
.bind(operation.id.as_str())
.execute(registry.pool())
.await;
assert!(update.is_err());
let rewind = sqlx::query("update operations set latest_published_version = null where id = $1")
.bind(operation.id.as_str())
.execute(registry.pool())
.await;
assert!(rewind.is_err());
let pointer_delete = sqlx::query("delete from published_operations where operation_id = $1")
.bind(operation.id.as_str())
.execute(registry.pool())
.await;
assert!(pointer_delete.is_err());
let delete = sqlx::query("delete from operations where id = $1")
.bind(operation.id.as_str())
.execute(registry.pool())
.await;
assert!(delete.is_err());
assert!(
registry
.get_published_operation(&operation.id)
.await
.unwrap()
.is_some()
);
database.cleanup().await;
}
#[tokio::test]
async fn concurrent_saves_from_one_base_have_one_winner_and_no_version_gap() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let operation = test_operation("op_concurrent_save", 1, OperationStatus::Draft);
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
let mut tasks = Vec::new();
for contender in 0..32_u32 {
let registry = registry.clone();
let mut candidate = operation.clone();
candidate.display_name = format!("Contender {contender}");
candidate.updated_at = timestamp("2026-03-25T12:20:00Z");
tasks.push(tokio::spawn(async move {
registry
.update_operation_draft(&test_workspace_id(), &candidate)
.await
}));
}
let mut successes = 0;
let mut stale = 0;
for task in tasks {
match task.await.unwrap() {
Ok(()) => successes += 1,
Err(RegistryError::OperationStaleVersion { .. }) => stale += 1,
Err(error) => panic!("unexpected contender error: {error}"),
}
}
assert_eq!(successes, 1);
assert_eq!(stale, 31);
let versions = registry
.list_operation_versions(&test_workspace_id(), &operation.id)
.await
.unwrap();
assert_eq!(
versions
.iter()
.map(|record| record.version)
.collect::<Vec<_>>(),
vec![1, 2]
);
database.cleanup().await;
}
#[tokio::test]
async fn concurrent_same_name_create_maps_unique_loser_to_typed_conflict() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let first = test_operation("op_same_name_first", 1, OperationStatus::Draft);
let mut second = test_operation("op_same_name_second", 1, OperationStatus::Draft);
second.name = first.name.clone();
let first_registry = registry.clone();
let second_registry = registry.clone();
let workspace = test_workspace_id();
let first_workspace = workspace.clone();
let second_workspace = workspace.clone();
let (first_result, second_result) = tokio::join!(
async move {
first_registry
.create_operation(&first_workspace, &first, Some("alice"))
.await
},
async move {
second_registry
.create_operation(&second_workspace, &second, Some("alice"))
.await
}
);
let results = [first_result, second_result];
assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1);
assert_eq!(
results
.iter()
.filter(|result| matches!(result, Err(RegistryError::OperationAlreadyExists { .. })))
.count(),
1
);
database.cleanup().await;
}
#[tokio::test]
async fn archive_preserves_published_version_and_delete_is_restricted_to_unused_draft() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let published = test_operation("op_archive_preserve", 1, OperationStatus::Draft);
registry
.create_operation(&test_workspace_id(), &published, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &published.id,
version: 1,
published_at: &timestamp("2026-03-25T12:10:00Z"),
published_by: Some("alice"),
})
.await
.unwrap();
registry
.archive_operation(
&test_workspace_id(),
&published.id,
&timestamp("2026-03-25T12:20:00Z"),
)
.await
.unwrap();
registry
.archive_operation(
&test_workspace_id(),
&published.id,
&timestamp("2026-03-25T12:21:00Z"),
)
.await
.unwrap();
assert!(matches!(
registry
.delete_operation(&test_workspace_id(), &published.id)
.await,
Err(RegistryError::OperationDeleteForbidden { .. })
));
assert_eq!(
registry
.get_published_operation(&published.id)
.await
.unwrap()
.unwrap()
.status,
OperationStatus::Published
);
let draft = test_operation("op_delete_unused", 1, OperationStatus::Draft);
registry
.create_operation(&test_workspace_id(), &draft, Some("alice"))
.await
.unwrap();
registry
.delete_operation(&test_workspace_id(), &draft.id)
.await
.unwrap();
assert!(
registry
.get_operation_summary(&test_workspace_id(), &draft.id)
.await
.unwrap()
.is_none()
);
database.cleanup().await;
}
#[tokio::test]
async fn stores_auth_profiles_and_artifact_metadata() {
let database = TestDatabase::new().await;
@@ -193,6 +501,12 @@ async fn stores_auth_profiles_and_artifact_metadata() {
.create_operation(&test_workspace_id(), &operation, None)
.await
.unwrap();
create_test_secret(
&registry,
&SecretId::new("secret_crank_api_key"),
"Crank API key",
)
.await;
let auth_profile = AuthProfile {
id: "auth_crank".into(),
@@ -275,6 +589,8 @@ async fn lists_auth_profiles_referencing_secret() {
let registry = database.registry().await;
let primary_secret_id = SecretId::new("secret_primary");
let secondary_secret_id = SecretId::new("secret_secondary");
create_test_secret(&registry, &primary_secret_id, "Primary secret").await;
create_test_secret(&registry, &secondary_secret_id, "Secondary secret").await;
let profile = AuthProfile {
id: "auth_crank".into(),
workspace_id: test_workspace_id(),
@@ -247,6 +247,7 @@ async fn lists_default_workspace_first_for_community_sessions() {
&user_id,
Some(&legacy_workspace.id),
secret_hash,
None,
&timestamp("2027-03-25T13:00:00Z"),
)
.await
@@ -368,6 +369,7 @@ async fn creates_and_loads_user_sessions_with_typed_expiration() {
&user_id,
Some(&workspace.id),
"secret-hash-01",
None,
&expires_at,
)
.await
@@ -571,6 +573,8 @@ async fn manages_approval_request_lifecycle() {
operation_version: 1,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_id: None,
trace_id: None,
request_payload: json!({"amount": 100}),
response_payload: None,
created_at: timestamp("2026-03-25T12:01:00Z"),
@@ -647,9 +651,12 @@ async fn manages_approval_request_lifecycle() {
workspace_id: &workspace_id,
agent_id: &agent.id,
approval_id: &approval.id,
operation_id: &approval.operation_id,
operation_version: approval.operation_version,
request_payload: &approval.request_payload,
status: ApprovalRequestStatus::Approved,
decided_at: timestamp("2026-03-25T12:02:00Z"),
decided_by_key_id: &approval_key.id,
decided_by_key_id: Some(&approval_key.id),
response_payload: Some(json!({"approve": "yes"})),
decision_note: Some("confirmed in test"),
})
@@ -738,9 +745,12 @@ async fn manages_approval_request_lifecycle() {
workspace_id: &workspace_id,
agent_id: &agent.id,
approval_id: &interrupted_approval.id,
operation_id: &interrupted_approval.operation_id,
operation_version: interrupted_approval.operation_version,
request_payload: &interrupted_approval.request_payload,
status: ApprovalRequestStatus::Approved,
decided_at: timestamp("2026-03-25T12:02:20Z"),
decided_by_key_id: &approval_key.id,
decided_by_key_id: Some(&approval_key.id),
response_payload: Some(json!({"approve": "yes"})),
decision_note: None,
})
@@ -810,9 +820,12 @@ async fn manages_approval_request_lifecycle() {
workspace_id: &workspace_id,
agent_id: &agent.id,
approval_id: &approval.id,
operation_id: &approval.operation_id,
operation_version: approval.operation_version,
request_payload: &approval.request_payload,
status: ApprovalRequestStatus::Denied,
decided_at: timestamp("2026-03-25T12:03:00Z"),
decided_by_key_id: &approval_key.id,
decided_by_key_id: Some(&approval_key.id),
response_payload: Some(json!({"approve": "no"})),
decision_note: None,
})