feat: complete Epic 1 production foundation

This commit is contained in:
2026-08-25 01:24:11 +03:00
parent 767428436d
commit 182bde8ac0
298 changed files with 35719 additions and 5299 deletions
+98 -1
View File
@@ -1,12 +1,20 @@
use std::process::Command;
use crank_registry::{MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresRegistry};
use crank_runtime::SecretCrypto;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
const REGISTERED_MASTER_KEY: &str = "registered-master-key-CANARY_SECRET_VALUE-00000000";
const WRONG_MASTER_KEY: &str = "wrong-master-key-CANARY_SECRET_VALUE-0000000000000";
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.envs([
("CRANK_MASTER_KEY", "master"),
("CRANK_MASTER_KEY", TEST_MASTER_KEY),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
@@ -69,3 +77,92 @@ async fn fresh_database_startup_is_read_only() {
.unwrap();
assert!(!present);
}
#[tokio::test]
async fn master_key_mismatch_blocks_startup_with_safe_diagnostic() {
let database_url = crank_test_support::postgres_schema_url("admin_master_key_mismatch").await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::parse("2026-08-21T00:00:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
let stderr = run_with(&[
("CRANK_DATABASE_URL", &database_url),
("CRANK_MASTER_KEY", WRONG_MASTER_KEY),
]);
assert!(stderr.contains("master_key_identity_mismatch"), "{stderr}");
assert!(!stderr.contains("registered-master-key"));
assert!(!stderr.contains("wrong-master-key"));
}
#[tokio::test]
async fn populated_database_without_identity_rejects_wrong_first_key() {
let database_url =
crank_test_support::postgres_schema_url("admin_master_key_first_registration_wrong").await;
let applied = Command::new(env!("CARGO_BIN_EXE_crank-migrate"))
.arg("apply")
.env("CRANK_DATABASE_URL", &database_url)
.output()
.expect("migration command executes");
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let old_crypto = SecretCrypto::new(REGISTERED_MASTER_KEY).unwrap();
let ciphertext = old_crypto
.encrypt(&serde_json::json!({"token": "CANARY_SECRET_VALUE"}))
.unwrap();
sqlx::query(
"insert into secrets (
id, workspace_id, name, kind, status, current_version, created_at, updated_at
) values (
'legacy_secret', 'ws_default', 'legacy secret', 'token', 'active', 1, now(), now()
)",
)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"insert into secret_versions (
secret_id, version, ciphertext, key_version, master_key_epoch, created_at, created_by
) values (
'legacy_secret', 1, $1, $2, 1, now(), null
)",
)
.bind(ciphertext)
.bind(old_crypto.key_version())
.execute(&pool)
.await
.unwrap();
let stderr = run_with(&[
("CRANK_DATABASE_URL", &database_url),
("CRANK_MASTER_KEY", WRONG_MASTER_KEY),
]);
assert!(stderr.contains("startup_failed"), "{stderr}");
assert!(!stderr.contains("CANARY_SECRET_VALUE"));
let identities: i64 = sqlx::query_scalar("select count(*) from master_key_identities")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(identities, 0);
}
+5
View File
@@ -1,8 +1,13 @@
mod integration {
mod agent_catalog;
mod auth_rate_limit;
mod common;
mod community_access_usage;
mod credential_lifecycle;
mod logs_usage;
mod onboarding;
mod openapi_import;
mod operation_lifecycle;
mod operations_agents;
mod request_context;
mod secrets_import_auth;
@@ -0,0 +1,310 @@
#![allow(dead_code, unused_imports)]
use super::common::*;
use serde_json::{Value, json};
use serial_test::serial;
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn binding_requires_exact_published_operation_version() {
let registry = test_registry().await;
let storage_root = test_storage_root("agent_catalog_binding_requires_published");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let draft_operation = assert_success_json(
client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream_base_url,
"draft_not_bindable_tool",
))
.send()
.await
.unwrap(),
)
.await;
let draft_operation_id = draft_operation["operation_id"].as_str().unwrap().to_owned();
let agent = assert_success_json(
client
.post(format!("{base_url}/agents"))
.json(&json!({
"slug": "strict-catalog-agent",
"display_name": "Strict Catalog Agent",
"description": "Rejects draft bindings",
"instructions": {},
"tool_selection_policy": {}
}))
.send()
.await
.unwrap(),
)
.await;
let agent_id = agent["agent_id"].as_str().unwrap().to_owned();
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!([
{
"operation_id": draft_operation_id,
"operation_version": 1,
"tool_name": "draft_not_bindable_tool",
"tool_title": "Draft should not bind",
"tool_description_override": null,
"enabled": true
}
]))
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
let body = response.json::<Value>().await.unwrap();
assert_eq!(
body["error"]["code"], "agent_binding_not_published",
"binding Draft Operation must fail before publish and not be silently filtered"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn stale_agent_revision_rejects_mutation() {
let registry = test_registry().await;
let storage_root = test_storage_root("agent_catalog_stale_revision");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let operation = assert_success_json(
client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream_base_url,
"stale_agent_tool",
))
.send()
.await
.unwrap(),
)
.await;
let operation_id = operation["operation_id"].as_str().unwrap().to_owned();
assert_success_json(
client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap(),
)
.await;
let agent = assert_success_json(
client
.post(format!("{base_url}/agents"))
.json(&json!({
"slug": "stale-agent",
"display_name": "Stale Agent",
"description": "Stale revision test",
"instructions": {},
"tool_selection_policy": {}
}))
.send()
.await
.unwrap(),
)
.await;
let agent_id = agent["agent_id"].as_str().unwrap().to_owned();
assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!([{
"operation_id": operation_id,
"operation_version": 1,
"tool_name": "stale_agent_tool",
"tool_title": "Stale Agent Tool",
"enabled": true
}]))
.send()
.await
.unwrap(),
)
.await;
assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap(),
)
.await;
let response = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.json(&json!([]))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
reqwest::StatusCode::PRECONDITION_REQUIRED,
"Agent mutations after read must require current revision precondition"
);
let body = response.json::<Value>().await.unwrap();
assert_eq!(body["error"]["code"], "agent_precondition_required");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn unpublishes_and_archives_agent() {
let registry = test_registry().await;
let storage_root = test_storage_root("agent_statuses");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let operation = assert_success_json(
client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream_base_url,
"crm_create_lead_agent_status",
))
.send()
.await
.unwrap(),
)
.await;
let operation_id = operation["operation_id"].as_str().unwrap().to_owned();
client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap();
let created = assert_success_json(
client
.post(format!("{base_url}/agents"))
.json(&json!({
"slug": "sales-routing",
"display_name": "Sales Routing",
"description": "Routing agent",
"instructions": {},
"tool_selection_policy": {}
}))
.send()
.await
.unwrap(),
)
.await;
let agent_id = created["agent_id"].as_str().unwrap().to_owned();
client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!([
{
"operation_id": operation_id,
"operation_version": 1,
"tool_name": "crm_create_lead_agent_status",
"tool_title": "Create Lead",
"tool_description_override": null,
"enabled": true
}
]))
.send()
.await
.unwrap();
client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap();
let unpublished = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/unpublish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(unpublished["agent_id"], agent_id);
let draft_detail = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(draft_detail["status"], "draft");
assert_eq!(draft_detail["latest_published_version"], 1);
let archived = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/archive"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(archived["agent_id"], agent_id);
let archived_detail = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(archived_detail["status"], "archived");
}
@@ -11,17 +11,22 @@ use std::{
use async_trait::async_trait;
use axum::{Json, Router, routing::post};
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
ExecutionConfig, HttpMethod, MembershipRole, OperationSecurityLevel, Protocol,
ResponseCachePolicy, RestTarget, SecretKind, Target, ToolDescription, WorkspaceId,
};
use crank_core::{IdentityError, IdentityProvider, IdentityProviderKind, LoginOutcome};
use crank_mapping::{MappingRule, MappingSet};
use crank_registry::PostgresRegistry;
use crank_registry::{
CreateAdminBootstrapContractRequest, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
MigrationAuthority, PostgresRegistry,
};
use crank_runtime::SecretCrypto;
use crank_schema::{Schema, SchemaKind};
use serde_json::{Value, json};
use serial_test::serial;
use sha2::{Digest, Sha256};
use tokio::net::TcpListener;
use admin_api::{
@@ -36,7 +41,7 @@ const TEST_AUTH_EMAIL: &str = "owner@crank.local";
const TEST_AUTH_PASSWORD: &str = "test-password";
const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
struct TestServer {
base_url: String,
@@ -92,6 +97,48 @@ impl IdentityProvider for RejectingIdentityProvider {
}
}
async fn empty_registry(name: &str) -> PostgresRegistry {
let database_url = crank_test_support::postgres_schema_url(name).await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let secret_crypto = test_secret_crypto();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: secret_crypto.master_key_epoch(),
fingerprint: secret_crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
registry
}
fn bootstrap_token_hash(token: &str) -> String {
let digest = Sha256::digest(format!("bootstrap:{token}").as_bytes());
URL_SAFE_NO_PAD.encode(digest)
}
async fn login_client_without_csrf(root_url: &str) -> reqwest::Client {
let client = reqwest::Client::builder()
.cookie_store(true)
.build()
.unwrap();
client
.post(format!("{root_url}/api/auth/login"))
.json(&json!({
"email": TEST_AUTH_EMAIL,
"password": TEST_AUTH_PASSWORD,
}))
.send()
.await
.unwrap()
.error_for_status()
.unwrap();
client
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn rejects_rapid_login_requests_with_429() {
@@ -109,7 +156,7 @@ async fn rejects_rapid_login_requests_with_429() {
api_rate_limiter: crank_runtime::RequestRateLimiter::new(
crank_runtime::RequestRateLimitConfig::new(1, 1).unwrap(),
),
trust_forwarded_headers: false,
trusted_proxy_ips: Vec::new(),
});
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
let handle = tokio::spawn(async move {
@@ -195,3 +242,119 @@ async fn login_uses_identity_provider_when_configured() {
};
assert_eq!(error.to_string(), "invalid email or password");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn bootstrap_token_creates_first_admin_once_and_never_replays() {
let registry = empty_registry("test_admin_api_bootstrap_token").await;
let token = "bootstrap-token-story-1-12-local-operator-only";
registry
.create_admin_bootstrap_contract(CreateAdminBootstrapContractRequest {
id: "boot_test_story_1_12",
token_hash: &bootstrap_token_hash(token),
email: "first-owner@crank.local",
display_name: "First Owner",
expires_at: &(time::OffsetDateTime::now_utc() + time::Duration::minutes(15)),
})
.await
.unwrap();
let storage_root = test_storage_root("bootstrap_token");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let root_url = base_url
.as_ref()
.split("/api/admin/workspaces/")
.next()
.unwrap()
.to_owned();
let client = reqwest::Client::builder()
.cookie_store(true)
.build()
.unwrap();
let status = assert_success_json(
client
.get(format!("{root_url}/api/auth/bootstrap/status"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(status["bootstrap_required"], true);
let completed = assert_success_json(
client
.post(format!("{root_url}/api/auth/bootstrap/complete"))
.json(&json!({
"token": token,
"password": "first-owner-password-123"
}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(completed["user"]["email"], "first-owner@crank.local");
assert!(completed["csrf_token"].as_str().unwrap().len() >= 32);
let replay = client
.post(format!("{root_url}/api/auth/bootstrap/complete"))
.json(&json!({
"token": token,
"password": "second-password-should-not-work"
}))
.send()
.await
.unwrap();
assert_eq!(replay.status(), reqwest::StatusCode::UNAUTHORIZED);
let status_after = assert_success_json(
client
.get(format!("{root_url}/api/auth/bootstrap/status"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(status_after["bootstrap_required"], false);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn browser_mutations_require_csrf_and_reject_cross_origin_requests() {
let registry = test_registry().await;
let storage_root = test_storage_root("csrf_and_origin");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let root_url = base_url
.as_ref()
.split("/api/admin/workspaces/")
.next()
.unwrap()
.to_owned();
let client = login_client_without_csrf(&root_url).await;
let missing_csrf = client
.patch(format!("{root_url}/api/auth/profile"))
.json(&json!({
"display_name": "Blocked Without CSRF",
"email": TEST_AUTH_EMAIL
}))
.send()
.await
.unwrap();
assert_eq!(missing_csrf.status(), reqwest::StatusCode::FORBIDDEN);
let missing_body = missing_csrf.text().await.unwrap();
assert!(!missing_body.contains(TEST_SESSION_SECRET));
assert!(!missing_body.contains(TEST_AUTH_PASSWORD));
let cross_origin = client
.get(format!("{root_url}/api/auth/profile"))
.header("origin", "https://attacker.example")
.send()
.await
.unwrap();
assert_eq!(cross_origin.status(), reqwest::StatusCode::FORBIDDEN);
let cross_origin_body = cross_origin.text().await.unwrap();
assert!(!cross_origin_body.contains(TEST_SESSION_SECRET));
assert!(!cross_origin_body.contains(TEST_AUTH_PASSWORD));
}
+102 -8
View File
@@ -10,12 +10,12 @@ use std::{
use async_trait::async_trait;
use axum::{Json, Router, routing::post};
use crank_core::{
ExecutionConfig, HttpMethod, MembershipRole, OperationSecurityLevel, Protocol,
AuditSink, ExecutionConfig, HttpMethod, MembershipRole, OperationSecurityLevel, Protocol,
ResponseCachePolicy, RestTarget, SecretKind, Target, ToolDescription, WorkspaceId,
};
use crank_core::{IdentityError, IdentityProvider, IdentityProviderKind, LoginOutcome};
use crank_mapping::{MappingRule, MappingSet};
use crank_registry::PostgresRegistry;
use crank_registry::{MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresRegistry};
use crank_runtime::SecretCrypto;
use crank_schema::{Schema, SchemaKind};
use serde_json::{Value, json};
@@ -34,7 +34,7 @@ const TEST_AUTH_EMAIL: &str = "owner@crank.local";
const TEST_AUTH_PASSWORD: &str = "test-password";
const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
pub(super) struct TestServer {
base_url: String,
@@ -104,7 +104,32 @@ pub(super) fn build_test_app(
api_rate_limiter: crank_runtime::RequestRateLimiter::new(
crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
),
trust_forwarded_headers: false,
trusted_proxy_ips: Vec::new(),
})
}
pub(super) fn build_test_app_with_audit_sink(
registry: PostgresRegistry,
storage_root: std::path::PathBuf,
audit_sink: Arc<dyn AuditSink>,
) -> Router {
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
build_app(AppState {
service: AdminServiceBuilder::new(
registry,
storage_root,
test_auth_settings(),
test_secret_crypto(),
runtime,
)
.with_outbound_http_policy(outbound_policy)
.with_audit_sink(audit_sink)
.build(),
api_rate_limiter: crank_runtime::RequestRateLimiter::new(
crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
),
trusted_proxy_ips: Vec::new(),
})
}
@@ -169,12 +194,13 @@ pub(super) async fn authorized_client(workspace_base_url: impl AsRef<str>) -> re
.split("/api/admin/workspaces/")
.next()
.unwrap();
let client = reqwest::Client::builder()
.cookie_store(true)
let cookie_jar = Arc::new(reqwest::cookie::Jar::default());
let login_client = reqwest::Client::builder()
.cookie_provider(cookie_jar.clone())
.build()
.unwrap();
client
let login = login_client
.post(format!("{root_url}/api/auth/login"))
.json(&json!({
"email": TEST_AUTH_EMAIL,
@@ -184,9 +210,67 @@ pub(super) async fn authorized_client(workspace_base_url: impl AsRef<str>) -> re
.await
.unwrap()
.error_for_status()
.unwrap()
.json::<Value>()
.await
.unwrap();
let csrf_token = login["csrf_token"]
.as_str()
.expect("login response must include csrf_token");
let mut default_headers = reqwest::header::HeaderMap::new();
default_headers.insert(
"x-csrf-token",
reqwest::header::HeaderValue::from_str(csrf_token).unwrap(),
);
client
reqwest::Client::builder()
.cookie_provider(cookie_jar)
.default_headers(default_headers)
.build()
.unwrap()
}
pub(super) async fn operation_etag(
client: &reqwest::Client,
workspace_base_url: impl AsRef<str>,
operation_id: &str,
) -> String {
let response = client
.get(format!(
"{}/operations/{operation_id}",
workspace_base_url.as_ref()
))
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
response
.headers()
.get(reqwest::header::ETAG)
.expect("operation detail must expose an ETag")
.to_str()
.unwrap()
.to_owned()
}
pub(super) async fn agent_etag(
client: &reqwest::Client,
workspace_base_url: impl AsRef<str>,
agent_id: &str,
) -> String {
let response = client
.get(format!("{}/agents/{agent_id}", workspace_base_url.as_ref()))
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
response
.headers()
.get(reqwest::header::ETAG)
.expect("agent detail must expose an ETag")
.to_str()
.unwrap()
.to_owned()
}
pub(super) async fn assert_success_json(response: reqwest::Response) -> Value {
@@ -216,6 +300,16 @@ pub(super) async fn test_registry() -> PostgresRegistry {
.await
.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let secret_crypto = test_secret_crypto();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: secret_crypto.master_key_epoch(),
fingerprint: secret_crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &time::OffsetDateTime::now_utc(),
})
.await
.unwrap();
let password_hash = hash_password(TEST_AUTH_PASSWORD, TEST_PASSWORD_PEPPER).unwrap();
let user_id = registry
.upsert_bootstrap_user(TEST_AUTH_EMAIL, "Test Owner", &password_hash)
@@ -39,7 +39,7 @@ const TEST_AUTH_EMAIL: &str = "owner@crank.local";
const TEST_AUTH_PASSWORD: &str = "test-password";
const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
struct TestServer {
base_url: String,
@@ -478,6 +478,11 @@ async fn seeds_demo_assets_for_live_ui() {
&WorkspaceId::new(DEFAULT_WORKSPACE_ID),
&smoke_operation_id,
smoke_operation.version,
&crank_registry::OperationStateExpectation {
current_draft_version: smoke_operation.version,
status: crank_core::OperationStatus::Draft,
latest_published_version: None,
},
)
.await
.unwrap();
@@ -508,6 +513,7 @@ async fn seeds_demo_assets_for_live_ui() {
enabled: true,
}]
.into(),
None,
)
.await
.unwrap();
@@ -516,6 +522,7 @@ async fn seeds_demo_assets_for_live_ui() {
&WorkspaceId::new(DEFAULT_WORKSPACE_ID),
&smoke_agent_id,
smoke_agent.version,
None,
)
.await
.unwrap();
@@ -573,17 +580,22 @@ async fn seeds_demo_assets_for_live_ui() {
&default_workspace_id,
admin_api::service::LogsQuery {
level: None,
status: None,
search: None,
source: None,
operation_id: None,
agent_id: None,
outcome_group: None,
created_after: None,
created_before: None,
period: None,
cursor: None,
limit: Some(20),
},
)
.await
.unwrap();
assert!(!logs.is_empty());
assert!(!logs.items.is_empty());
}
#[tokio::test(flavor = "multi_thread")]
@@ -646,9 +658,11 @@ async fn updates_profile_and_changes_password() {
updated_profile["user"]["email"],
"updated-owner@crank.local"
);
let updated_csrf_token = updated_profile["csrf_token"].as_str().unwrap();
let password_status = client
.post(format!("{root_url}/api/auth/password"))
.header("x-csrf-token", updated_csrf_token)
.json(&json!({
"current_password": TEST_AUTH_PASSWORD,
"new_password": "updated-password-123"
@@ -671,7 +685,7 @@ async fn updates_profile_and_changes_password() {
.await
.unwrap()
.status();
assert_eq!(current_session_status, reqwest::StatusCode::OK);
assert_eq!(current_session_status, reqwest::StatusCode::UNAUTHORIZED);
assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
let relogin_client = reqwest::Client::builder()
@@ -741,87 +755,6 @@ async fn rejects_multi_workspace_session_switching_in_community() {
assert_eq!(session["current_workspace_id"], DEFAULT_WORKSPACE_ID);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn exposes_logs_and_usage_from_real_test_runs() {
let registry = test_registry().await;
let storage_root = test_storage_root("observability");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let created = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream_base_url,
"crm_observability",
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_id = created["operation_id"].as_str().unwrap().to_owned();
client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.json(&json!({
"version": 1,
"input": { "email": "user@example.com" }
}))
.send()
.await
.unwrap();
let logs = client
.get(format!("{base_url}/logs?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let log_id = logs["items"][0]["log"]["id"].as_str().unwrap().to_owned();
let log_detail = client
.get(format!("{base_url}/logs/{log_id}"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let usage = client
.get(format!("{base_url}/usage?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_usage = client
.get(format!(
"{base_url}/usage/operations/{operation_id}?period=7d"
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(logs["items"][0]["log"]["source"], "admin_test_run");
assert_eq!(logs["items"][0]["operation_name"], "crm_observability");
assert_eq!(log_detail["log"]["status"], "ok");
assert_eq!(usage["summary"]["rollup"]["calls_total"], 1);
assert_eq!(usage["summary"]["rollup"]["calls_ok"], 1);
assert_eq!(
usage["operations"][0]["operation_name"],
"crm_observability"
);
assert_eq!(operation_usage["rollup"]["calls_total"], 1);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn preserves_request_id_for_test_run_invocations() {
@@ -0,0 +1,994 @@
#![allow(dead_code, unused_imports)]
use super::common::*;
use std::sync::{Arc, Mutex};
use axum::{Json, Router, http::HeaderMap, routing::post};
use crank_core::{AuditError, AuditEvent, AuditSink, PlatformApiKeyKind, SecretKind};
use serde_json::{Value, json};
use serial_test::serial;
use tokio::net::TcpListener;
#[derive(Clone, Default)]
struct RecordingAuditSink {
events: Arc<Mutex<Vec<AuditEvent>>>,
}
impl RecordingAuditSink {
fn events(&self) -> Vec<AuditEvent> {
self.events.lock().unwrap().clone()
}
}
#[async_trait::async_trait]
impl AuditSink for RecordingAuditSink {
async fn record(&self, event: AuditEvent) -> Result<(), AuditError> {
self.events.lock().unwrap().push(event);
Ok(())
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn machine_key_raw_value_is_create_only_and_duplicate_name_is_typed_conflict() {
let registry = test_registry().await;
let storage_root = test_storage_root("credential_lifecycle_keys");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let agent_id = create_agent(&client, &base_url, "credential-lifecycle-agent").await;
let created = post_json(
&client,
format!("{base_url}/agents/{agent_id}/platform-api-keys"),
json!({"name": "primary-mcp-key", "key_kind": PlatformApiKeyKind::McpClient, "scopes": ["read", "write"]}),
)
.await;
let raw_key = created["secret"]
.as_str()
.expect("create response must disclose the raw key once")
.to_owned();
let random_part = raw_key
.strip_prefix("crk_")
.expect("MCP key must use the MCP bearer marker");
assert_eq!(random_part.len(), 43, "32 raw bytes in unpadded base64url");
assert!(
random_part
.bytes()
.all(|byte| { byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_') })
);
assert_eq!(
created["connection"]["endpoint"],
"http://localhost:3000/mcp/v1/default/credential-lifecycle-agent"
);
assert!(
created["connection"]["clients"]
.as_array()
.is_some_and(|clients| !clients.is_empty())
);
let approval = post_json(
&client,
format!("{base_url}/agents/{agent_id}/platform-api-keys"),
json!({"name": "approval-without-mcp-config", "key_kind": PlatformApiKeyKind::Approval, "scopes": ["read_pending"]}),
)
.await;
assert!(approval.get("connection").is_none());
let listed_text = client
.get(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
assert!(!listed_text.contains(&raw_key));
assert!(!listed_text.contains("secret_hash"));
assert!(!listed_text.contains("\"secret\""));
let duplicate = client
.post(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.json(&json!({
"name": "primary-mcp-key",
"key_kind": PlatformApiKeyKind::McpClient,
"scopes": ["read"]
}))
.send()
.await
.unwrap();
assert_eq!(duplicate.status(), reqwest::StatusCode::CONFLICT);
let duplicate_body = duplicate.text().await.unwrap();
assert!(!duplicate_body.contains(&raw_key));
assert!(!duplicate_body.contains("secret_hash"));
assert!(!duplicate_body.contains("duplicate key value"));
let duplicate_json: Value = serde_json::from_str(&duplicate_body).unwrap();
assert_eq!(
duplicate_json["error"]["code"],
"platform_api_key_name_conflict"
);
for payload in [
json!({"name": "n".repeat(129), "key_kind": PlatformApiKeyKind::McpClient, "scopes": ["read"]}),
json!({"name": "duplicate-scope", "key_kind": PlatformApiKeyKind::McpClient, "scopes": ["read", "read"]}),
json!({"name": "past-expiry", "key_kind": PlatformApiKeyKind::McpClient, "scopes": ["read"], "expires_at": "2020-01-01T00:00:00Z"}),
] {
assert_post_error(
&client,
format!("{base_url}/agents/{agent_id}/platform-api-keys"),
payload,
reqwest::StatusCode::BAD_REQUEST,
"validation_error",
)
.await;
}
let canonical = post_json(
&client,
format!("{base_url}/agents/{agent_id}/platform-api-keys"),
json!({"name": "canonical-approval-origin", "key_kind": PlatformApiKeyKind::Approval, "scopes": ["read_pending"], "allowed_origins": ["HTTPS://Example.COM:443/", "http://EXAMPLE.com:80/"]}),
)
.await;
assert_eq!(
canonical["api_key"]["api_key"]["allowed_origins"],
json!(["https://example.com", "http://example.com"])
);
for (index, origin) in [
"https://[::1",
"https://example.test:bad/",
"https://user:pass@example.test/",
"https://example.test/path",
"https://example.test/?token=origin-canary",
"https://example.test/#fragment",
]
.into_iter()
.enumerate()
{
assert_post_error(
&client,
format!("{base_url}/agents/{agent_id}/platform-api-keys"),
json!({"name": format!("malformed-origin-{index}"), "key_kind": PlatformApiKeyKind::Approval, "scopes": ["read_pending"], "allowed_origins": [origin]}),
reqwest::StatusCode::BAD_REQUEST,
"validation_error",
)
.await;
}
assert_post_error(
&client,
format!("{base_url}/agents/{agent_id}/platform-api-keys"),
json!({"name": "duplicate-canonical-origin", "key_kind": PlatformApiKeyKind::Approval, "scopes": ["read_pending"], "allowed_origins": ["https://EXAMPLE.test:443/", "https://example.test/"]}),
reqwest::StatusCode::BAD_REQUEST,
"validation_error",
)
.await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn deleted_machine_key_name_can_be_reused_and_terminal_state_cannot_regress() {
let registry = test_registry().await;
let storage_root = test_storage_root("credential_lifecycle_key_reuse");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let agent_id = create_agent(&client, &base_url, "credential-key-reuse-agent").await;
let first = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.json(&json!({
"name": "replaceable-key",
"key_kind": PlatformApiKeyKind::McpClient,
"scopes": ["read"]
}))
.send()
.await
.unwrap(),
)
.await;
let first_key_id = first["api_key"]["api_key"]["id"].as_str().unwrap();
let delete_response = client
.delete(format!(
"{base_url}/agents/{agent_id}/platform-api-keys/{first_key_id}"
))
.send()
.await
.unwrap();
assert_eq!(delete_response.status(), reqwest::StatusCode::NO_CONTENT);
let revoke_deleted = client
.post(format!(
"{base_url}/agents/{agent_id}/platform-api-keys/{first_key_id}/revoke"
))
.send()
.await
.unwrap();
assert_eq!(revoke_deleted.status(), reqwest::StatusCode::CONFLICT);
let revoke_body: Value = serde_json::from_str(&revoke_deleted.text().await.unwrap()).unwrap();
assert_eq!(revoke_body["error"]["code"], "platform_api_key_not_active");
let second = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.json(&json!({
"name": "replaceable-key",
"key_kind": PlatformApiKeyKind::McpClient,
"scopes": ["read"]
}))
.send()
.await
.unwrap(),
)
.await;
let second_key_id = second["api_key"]["api_key"]["id"].as_str().unwrap();
assert_ne!(first_key_id, second_key_id);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn secret_plaintext_is_write_only_and_auth_profile_stores_reference() {
let registry = test_registry().await;
let storage_root = test_storage_root("credential_lifecycle_secrets");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let canary = "secret-canary-1-10-do-not-leak";
let created_response = client
.post(format!("{base_url}/secrets"))
.json(&json!({
"name": "crm-bearer-token",
"kind": SecretKind::Token,
"value": { "token": canary }
}))
.send()
.await
.unwrap();
let created_text = created_response.text().await.unwrap();
assert!(!created_text.contains(canary));
assert!(!created_text.contains("ciphertext"));
let created: Value = serde_json::from_str(&created_text).unwrap();
let secret_id = created["id"].as_str().unwrap();
let listed_text = client
.get(format!("{base_url}/secrets"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
assert!(!listed_text.contains(canary));
assert!(!listed_text.contains("ciphertext"));
let fetched_text = client
.get(format!("{base_url}/secrets/{secret_id}"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
assert!(!fetched_text.contains(canary));
assert!(!fetched_text.contains("ciphertext"));
let auth_profile_text = client
.post(format!("{base_url}/auth-profiles"))
.json(&json!({
"name": "crm-bearer",
"kind": "bearer",
"config": {
"bearer": {
"header_name": "Authorization",
"secret_id": secret_id
}
}
}))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
assert!(!auth_profile_text.contains(canary));
assert!(!auth_profile_text.contains("ciphertext"));
let auth_profile: Value = serde_json::from_str(&auth_profile_text).unwrap();
assert_eq!(auth_profile["config"]["bearer"]["secret_id"], secret_id);
for (name, kind, value) in [
("null-token", SecretKind::Token, Value::Null),
(
"wrong-token-shape",
SecretKind::Token,
json!({"token": "token", "extra": "field"}),
),
(
"wrong-header-shape",
SecretKind::Header,
json!({"header_name": "X-Token"}),
),
(
"wrong-username-password-shape",
SecretKind::UsernamePassword,
json!({"username": "user"}),
),
] {
assert_post_error(
&client,
format!("{base_url}/secrets"),
json!({"name": name, "kind": kind, "value": value}),
reqwest::StatusCode::BAD_REQUEST,
"validation_error",
)
.await;
}
assert_post_error(
&client,
format!("{base_url}/secrets"),
json!({"name": "oversized-secret", "kind": SecretKind::Generic, "value": "x".repeat(65_537)}),
reqwest::StatusCode::BAD_REQUEST,
"validation_error",
)
.await;
for value in [
Value::Null,
json!({"token": "token", "extra": "field"}),
json!("x".repeat(65_537)),
] {
assert_post_error(
&client,
format!("{base_url}/secrets/{secret_id}/rotate"),
json!({"value": value}),
reqwest::StatusCode::BAD_REQUEST,
"validation_error",
)
.await;
}
let current = assert_success_json(
client
.get(format!("{base_url}/secrets/{secret_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(current["current_version"], 1);
for payload in [
json!({"name": "", "kind": "bearer", "config": {"bearer": {"header_name": "Authorization", "secret_id": secret_id}}}),
json!({"name": "n".repeat(129), "kind": "bearer", "config": {"bearer": {"header_name": "Authorization", "secret_id": secret_id}}}),
json!({"name": "invalid-header-name", "kind": "bearer", "config": {"bearer": {"header_name": "Bad Header", "secret_id": secret_id}}}),
json!({"name": "invalid-query-name", "kind": "api_key_query", "config": {"api_key_query": {"param_name": "bad name", "secret_id": secret_id}}}),
] {
assert_post_error(
&client,
format!("{base_url}/auth-profiles"),
payload,
reqwest::StatusCode::BAD_REQUEST,
"validation_error",
)
.await;
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn credential_mutations_emit_bounded_non_secret_audit_events() {
let registry = test_registry().await;
let storage_root = test_storage_root("credential_lifecycle_audit");
let audit_sink = RecordingAuditSink::default();
let base_url = spawn_admin_api(build_test_app_with_audit_sink(
registry,
storage_root,
Arc::new(audit_sink.clone()),
))
.await;
let client = authorized_client(&base_url).await;
let request_id = "req_credential_audit_1";
let traceparent = "00-11111111111111111111111111111111-2222222222222222-01";
let secret = assert_success_json(
client
.post(format!("{base_url}/secrets"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"name": "audit-token",
"kind": SecretKind::Token,
"value": { "token": "audit-secret-canary" }
}))
.send()
.await
.unwrap(),
)
.await;
let secret_id = secret["id"].as_str().unwrap();
assert_success_json(
client
.post(format!("{base_url}/secrets/{secret_id}/rotate"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"value": { "token": "audit-rotated-canary" }
}))
.send()
.await
.unwrap(),
)
.await;
let auth_profile = assert_success_json(
client
.post(format!("{base_url}/auth-profiles"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"name": "audit-profile",
"kind": "bearer",
"config": {
"bearer": {
"header_name": "Authorization",
"secret_id": secret_id
}
}
}))
.send()
.await
.unwrap(),
)
.await;
let auth_profile_id = auth_profile["id"].as_str().unwrap();
let delete_denied = client
.delete(format!("{base_url}/secrets/{secret_id}"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.send()
.await
.unwrap();
assert_eq!(delete_denied.status(), reqwest::StatusCode::CONFLICT);
let created_agent = assert_success_json(
client
.post(format!("{base_url}/agents"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"slug": "credential-audit-agent",
"display_name": "Credential Audit Agent",
"description": "Agent for credential audit tests.",
"instructions": {},
"tool_selection_policy": {}
}))
.send()
.await
.unwrap(),
)
.await;
let agent_id = created_agent["agent_id"].as_str().unwrap();
let created_key = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"name": "audit-mcp-key",
"key_kind": PlatformApiKeyKind::McpClient,
"scopes": ["read"]
}))
.send()
.await
.unwrap(),
)
.await;
let raw_key = created_key["secret"].as_str().unwrap();
let key_id = created_key["api_key"]["api_key"]["id"].as_str().unwrap();
let duplicate_key = client
.post(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"name": "audit-mcp-key",
"key_kind": PlatformApiKeyKind::McpClient,
"scopes": ["read"]
}))
.send()
.await
.unwrap();
assert_eq!(duplicate_key.status(), reqwest::StatusCode::CONFLICT);
let validation_key = client
.post(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"name": "n".repeat(129),
"key_kind": PlatformApiKeyKind::McpClient,
"scopes": ["read"]
}))
.send()
.await
.unwrap();
assert_eq!(validation_key.status(), reqwest::StatusCode::BAD_REQUEST);
let invalid_profile = client
.post(format!("{base_url}/auth-profiles"))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.json(&json!({
"name": "audit-invalid-profile",
"kind": "bearer",
"config": {
"bearer": {
"header_name": "Authorization",
"secret_id": "secret_missing_for_audit"
}
}
}))
.send()
.await
.unwrap();
assert_eq!(invalid_profile.status(), reqwest::StatusCode::NOT_FOUND);
client
.post(format!(
"{base_url}/agents/{agent_id}/platform-api-keys/{key_id}/revoke"
))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.send()
.await
.unwrap();
let delete_response = client
.delete(format!(
"{base_url}/agents/{agent_id}/platform-api-keys/{key_id}"
))
.header("x-request-id", request_id)
.header("traceparent", traceparent)
.send()
.await
.unwrap();
assert_eq!(delete_response.status(), reqwest::StatusCode::NO_CONTENT);
let listed_after_delete = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
.send()
.await
.unwrap(),
)
.await;
let deleted_metadata = listed_after_delete["items"]
.as_array()
.unwrap()
.iter()
.find(|item| item["api_key"]["id"] == key_id)
.expect("delete must preserve key metadata for provenance");
assert_eq!(deleted_metadata["api_key"]["status"], "deleted");
let listed_after_delete_text = serde_json::to_string(&listed_after_delete).unwrap();
assert!(!listed_after_delete_text.contains(raw_key));
let events = audit_sink.events();
let actions: Vec<&str> = events.iter().map(|event| event.action.as_str()).collect();
for expected in [
"credential.secret.created",
"credential.secret.rotated",
"credential.auth_profile.created",
"credential.secret.delete_denied",
"credential.platform_api_key.created",
"credential.platform_api_key.create_failed",
"credential.auth_profile.create_failed",
"credential.platform_api_key.revoked",
"credential.platform_api_key.deleted",
] {
assert!(
actions.contains(&expected),
"missing audit action {expected}"
);
}
let audit_json = serde_json::to_string(&events).unwrap();
assert!(audit_json.contains(request_id));
assert!(audit_json.contains("11111111111111111111111111111111"));
assert!(audit_json.contains(auth_profile_id));
assert!(!audit_json.contains("audit-secret-canary"));
assert!(!audit_json.contains("audit-rotated-canary"));
assert!(!audit_json.contains(raw_key));
assert!(!audit_json.contains("secret_hash"));
assert!(!audit_json.contains("ciphertext"));
assert!(!audit_json.contains("Bearer "));
for event in &events {
let reason = event.payload["reason"]
.as_str()
.expect("credential audit events must contain a reason code");
assert!(!reason.is_empty());
assert!(reason.len() <= 128, "audit reason must remain bounded");
}
assert!(events.iter().any(|event| {
event.action == "credential.platform_api_key.create_failed"
&& event.payload["reason"] == "validation_error"
}));
assert!(events.iter().any(|event| {
event.action == "credential.platform_api_key.create_failed"
&& event.payload["reason"] == "platform_api_key_name_conflict"
}));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn disabled_secret_fails_closed_for_reference_rotation_and_execution() {
let registry = test_registry().await;
let registry_control = registry.clone();
let storage_root = test_storage_root("credential_lifecycle_disabled_secret");
let (upstream_base_url, observed_authorizations) = spawn_auth_capture_upstream().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let secret = assert_success_json(
client
.post(format!("{base_url}/secrets"))
.json(&json!({
"name": "disabled-token",
"kind": SecretKind::Token,
"value": { "token": "disabled-secret-canary" }
}))
.send()
.await
.unwrap(),
)
.await;
let secret_id = secret["id"].as_str().unwrap();
let auth_profile = assert_success_json(
client
.post(format!("{base_url}/auth-profiles"))
.json(&json!({
"name": "disabled-profile",
"kind": "bearer",
"config": {
"bearer": {
"header_name": "Authorization",
"secret_id": secret_id
}
}
}))
.send()
.await
.unwrap(),
)
.await;
let auth_profile_id = auth_profile["id"].as_str().unwrap();
let mut operation_payload = serde_json::to_value(test_operation_payload(
&upstream_base_url,
"disabled_secret_execution",
))
.unwrap();
operation_payload["execution_config"]["auth_profile_ref"] = json!(auth_profile_id);
let created_operation = assert_success_json(
client
.post(format!("{base_url}/operations"))
.json(&operation_payload)
.send()
.await
.unwrap(),
)
.await;
let operation_id = created_operation["operation_id"].as_str().unwrap();
sqlx::query("update secrets set status = 'disabled' where id = $1")
.bind(secret_id)
.execute(registry_control.pool())
.await
.unwrap();
let ref_disabled = client
.post(format!("{base_url}/auth-profiles"))
.json(&json!({
"name": "disabled-profile-new-ref",
"kind": "bearer",
"config": {
"bearer": {
"header_name": "Authorization",
"secret_id": secret_id
}
}
}))
.send()
.await
.unwrap();
assert_eq!(ref_disabled.status(), reqwest::StatusCode::CONFLICT);
let ref_disabled_body: Value =
serde_json::from_str(&ref_disabled.text().await.unwrap()).unwrap();
assert_eq!(ref_disabled_body["error"]["code"], "secret_not_active");
let rotate_disabled = client
.post(format!("{base_url}/secrets/{secret_id}/rotate"))
.json(&json!({
"value": { "token": "should-not-rotate" }
}))
.send()
.await
.unwrap();
assert_eq!(rotate_disabled.status(), reqwest::StatusCode::CONFLICT);
let rotate_body: Value = serde_json::from_str(&rotate_disabled.text().await.unwrap()).unwrap();
assert_eq!(rotate_body["error"]["code"], "secret_not_active");
let test_run = assert_success_json(
client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.json(&json!({
"version": 1,
"input": { "email": "disabled@example.com" }
}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(test_run["ok"], false);
let body = serde_json::to_string(&test_run).unwrap();
assert!(!body.contains("disabled-secret-canary"));
assert!(observed_authorizations.lock().unwrap().is_empty());
let operation_get = client
.get(format!("{base_url}/operations/{operation_id}"))
.send()
.await
.unwrap();
let operation_etag = operation_get
.headers()
.get(reqwest::header::ETAG)
.unwrap()
.to_str()
.unwrap()
.to_owned();
let publish_disabled = client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(reqwest::header::IF_MATCH, operation_etag)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap();
assert_eq!(publish_disabled.status(), reqwest::StatusCode::CONFLICT);
let publish_body: Value =
serde_json::from_str(&publish_disabled.text().await.unwrap()).unwrap();
assert_eq!(publish_body["error"]["code"], "secret_not_active");
assert!(observed_authorizations.lock().unwrap().is_empty());
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn concurrent_secret_rotation_serializes_versions_without_storage_conflict() {
let registry = test_registry().await;
let storage_root = test_storage_root("credential_lifecycle_concurrent_rotation");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let secret = assert_success_json(
client
.post(format!("{base_url}/secrets"))
.json(&json!({
"name": "concurrent-token",
"kind": SecretKind::Token,
"value": { "token": "initial-concurrent-token" }
}))
.send()
.await
.unwrap(),
)
.await;
let secret_id = secret["id"].as_str().unwrap().to_owned();
let first = client
.post(format!("{base_url}/secrets/{secret_id}/rotate"))
.json(&json!({ "value": { "token": "concurrent-token-a" } }))
.send();
let second = client
.post(format!("{base_url}/secrets/{secret_id}/rotate"))
.json(&json!({ "value": { "token": "concurrent-token-b" } }))
.send();
let (first, second) = tokio::join!(first, second);
let first = assert_success_json(first.unwrap()).await;
let second = assert_success_json(second.unwrap()).await;
assert!(first["current_version"].as_u64().unwrap() >= 2);
assert!(second["current_version"].as_u64().unwrap() >= 2);
let current = assert_success_json(
client
.get(format!("{base_url}/secrets/{secret_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(current["current_version"], 3);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn auth_profile_secret_rotation_changes_next_admin_test_execution() {
let registry = test_registry().await;
let storage_root = test_storage_root("credential_lifecycle_rotation");
let (upstream_base_url, observed_authorizations) = spawn_auth_capture_upstream().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let secret = assert_success_json(
client
.post(format!("{base_url}/secrets"))
.json(&json!({
"name": "crm-rotating-token",
"kind": SecretKind::Token,
"value": { "token": "initial-rotation-token" }
}))
.send()
.await
.unwrap(),
)
.await;
let secret_id = secret["id"].as_str().unwrap();
let auth_profile = assert_success_json(
client
.post(format!("{base_url}/auth-profiles"))
.json(&json!({
"name": "crm-rotating-bearer",
"kind": "bearer",
"config": {
"bearer": {
"header_name": "Authorization",
"secret_id": secret_id
}
}
}))
.send()
.await
.unwrap(),
)
.await;
let auth_profile_id = auth_profile["id"].as_str().unwrap();
let mut operation_payload = serde_json::to_value(test_operation_payload(
&upstream_base_url,
"crm_auth_rotation",
))
.unwrap();
operation_payload["execution_config"]["auth_profile_ref"] = json!(auth_profile_id);
let created_operation = assert_success_json(
client
.post(format!("{base_url}/operations"))
.json(&operation_payload)
.send()
.await
.unwrap(),
)
.await;
let operation_id = created_operation["operation_id"].as_str().unwrap();
let first_run = assert_success_json(
client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.json(&json!({
"version": 1,
"input": { "email": "first@example.com" }
}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(first_run["ok"], true);
let rotated = assert_success_json(
client
.post(format!("{base_url}/secrets/{secret_id}/rotate"))
.json(&json!({
"value": { "token": "rotated-rotation-token" }
}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(rotated["current_version"], 2);
let second_run = assert_success_json(
client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.json(&json!({
"version": 1,
"input": { "email": "second@example.com" }
}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(second_run["ok"], true);
let captured = observed_authorizations.lock().unwrap().clone();
assert_eq!(
captured,
vec![
Some("Bearer initial-rotation-token".to_owned()),
Some("Bearer rotated-rotation-token".to_owned())
]
);
let auth_profile_after = client
.get(format!("{base_url}/auth-profiles/{auth_profile_id}"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
assert!(auth_profile_after.contains(secret_id));
assert!(!auth_profile_after.contains("initial-rotation-token"));
assert!(!auth_profile_after.contains("rotated-rotation-token"));
}
async fn spawn_auth_capture_upstream() -> (String, Arc<Mutex<Vec<Option<String>>>>) {
let observed_authorizations = Arc::new(Mutex::new(Vec::new()));
let captured = Arc::clone(&observed_authorizations);
let app = Router::new().route(
"/crm/leads",
post(move |headers: HeaderMap| {
let captured = Arc::clone(&captured);
async move {
let authorization = headers
.get("authorization")
.and_then(|value| value.to_str().ok())
.map(str::to_owned);
captured.lock().unwrap().push(authorization);
Json(json!({ "id": "lead_123" }))
}
}),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(format!("http://{address}"), observed_authorizations)
}
async fn assert_error_code(response: reqwest::Response, status: reqwest::StatusCode, code: &str) {
assert_eq!(response.status(), status);
let body = response.text().await.unwrap();
let json: Value = serde_json::from_str(&body).unwrap();
assert_eq!(json["error"]["code"], code, "unexpected error body: {body}");
}
async fn post_json(client: &reqwest::Client, url: String, payload: Value) -> Value {
assert_success_json(client.post(url).json(&payload).send().await.unwrap()).await
}
async fn create_agent(client: &reqwest::Client, base_url: impl AsRef<str>, slug: &str) -> String {
let base_url = base_url.as_ref();
post_json(
client,
format!("{base_url}/agents"),
json!({"slug": slug, "display_name": slug, "description": "Credential lifecycle test agent.", "instructions": {}, "tool_selection_policy": {}}),
)
.await["agent_id"]
.as_str()
.unwrap()
.to_owned()
}
async fn assert_post_error(
client: &reqwest::Client,
url: String,
payload: Value,
status: reqwest::StatusCode,
code: &str,
) {
assert_error_code(
client.post(url).json(&payload).send().await.unwrap(),
status,
code,
)
.await;
}
@@ -0,0 +1,152 @@
#![allow(dead_code, unused_imports)]
use super::common::*;
use serde_json::{Value, json};
use serial_test::serial;
use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn exposes_logs_and_usage_from_real_test_runs() {
let registry = test_registry().await;
let storage_root = test_storage_root("observability");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let created = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream_base_url,
"crm_observability",
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_id = created["operation_id"].as_str().unwrap().to_owned();
client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.json(&json!({
"version": 1,
"input": { "email": "user@example.com" }
}))
.send()
.await
.unwrap();
let logs = client
.get(format!("{base_url}/logs?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert!(logs["items"].is_array());
assert!(logs.get("next_cursor").is_some());
let log_id = logs["items"][0]["log"]["id"].as_str().unwrap().to_owned();
let created_at = OffsetDateTime::parse(
logs["items"][0]["log"]["created_at"].as_str().unwrap(),
&Rfc3339,
)
.unwrap();
let log_detail = client
.get(format!("{base_url}/logs/{log_id}"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let csv = client
.get(format!("{base_url}/logs/export.csv?period=7d&status=ok"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
let outcome_logs = client
.get(format!("{base_url}/logs?period=7d&outcome_group=success"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let created_after = (created_at - Duration::minutes(1))
.format(&Rfc3339)
.unwrap();
let created_before = (created_at + Duration::minutes(1))
.format(&Rfc3339)
.unwrap();
let usage = client
.get(format!("{base_url}/usage?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let windowed_usage = client
.get(format!(
"{base_url}/usage?created_after={created_after}&created_before={created_before}"
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let usage_csv = client
.get(format!("{base_url}/usage/export.csv?period=7d"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
let operation_usage = client
.get(format!(
"{base_url}/usage/operations/{operation_id}?period=7d"
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(logs["items"][0]["log"]["source"], "admin_test_run");
assert_eq!(logs["items"][0]["operation_name"], "crm_observability");
assert_eq!(log_detail["log"]["status"], "ok");
assert_eq!(log_detail["log"]["operation_version"], 1);
assert!(log_detail["log"]["request_id"].as_str().is_some());
assert!(
log_detail["log"]["trace_id"]
.as_str()
.is_some_and(|value| value.len() == 32)
);
assert!(csv.starts_with("created_at,level,status,source"));
assert!(csv.contains(log_detail["log"]["request_id"].as_str().unwrap()));
assert!(csv.contains(log_detail["log"]["trace_id"].as_str().unwrap()));
assert_eq!(outcome_logs["items"][0]["log"]["status"], "ok");
assert_eq!(usage["summary"]["rollup"]["calls_total"], 1);
assert_eq!(usage["summary"]["rollup"]["calls_ok"], 1);
assert_eq!(usage["outcomes"][0]["group"], "success");
assert_eq!(usage["outcomes"][0]["calls_total"], 1);
assert_eq!(windowed_usage["summary"]["rollup"]["calls_total"], 1);
assert!(usage_csv.starts_with("kind,name,group,error_code,calls_total"));
assert!(usage_csv.contains("operation"));
assert!(usage_csv.contains("outcome"));
assert_eq!(
usage["operations"][0]["operation_name"],
"crm_observability"
);
assert_eq!(operation_usage["rollup"]["calls_total"], 1);
}
@@ -0,0 +1,322 @@
#![allow(dead_code, unused_imports)]
use super::common::*;
use serde_json::{Value, json};
use serial_test::serial;
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn exposes_server_authoritative_onboarding_projection() {
let registry = test_registry().await;
let storage_root = test_storage_root("onboarding_projection");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let response = client
.get(format!("{base_url}/onboarding"))
.send()
.await
.unwrap();
let status = response.status();
let body = response.json::<Value>().await.unwrap();
assert_eq!(status, reqwest::StatusCode::OK, "body={body}");
assert_eq!(body["schema_version"], 1);
assert_eq!(body["status"], "in_progress");
assert!(body["revision"].as_i64().is_some());
assert!(body["eligible_since"].as_str().is_some());
assert_eq!(body["workspace_id"], "ws_default");
assert_eq!(body["completed"], false);
assert_eq!(
body["steps"]
.as_array()
.unwrap()
.iter()
.map(|step| step["id"].as_str().unwrap())
.collect::<Vec<_>>(),
vec![
"operation",
"test",
"publish_operation",
"agent",
"key",
"mcp_connection",
"first_call",
]
);
assert!(
body["steps"]
.as_array()
.unwrap()
.iter()
.all(|step| step["completed"] == false)
);
assert_eq!(body["steps"][0]["status"], "current");
assert!(
body["steps"]
.as_array()
.unwrap()
.iter()
.skip(1)
.all(|step| step["status"] == "pending")
);
// The server-owned eligibility denominator is recorded on the first GET;
// a repeat is idempotent and cannot advance browser-owned completion.
let repeated = client
.get(format!("{base_url}/onboarding"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(repeated["revision"], body["revision"]);
assert_eq!(repeated["steps"], body["steps"]);
assert_eq!(repeated["eligible_since"], body["eligible_since"]);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn ux_milestones_are_idempotent_and_revision_guarded() {
let registry = test_registry().await;
let storage_root = test_storage_root("onboarding_events");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let initial = client
.get(format!("{base_url}/onboarding"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let revision = initial["revision"].as_i64().unwrap();
let payload = json!({
"event": "started",
"idempotency_key": "onboarding-start-tab-a",
"expected_revision": revision,
});
let first = client
.post(format!("{base_url}/onboarding/events"))
.json(&payload)
.send()
.await
.unwrap();
let first_status = first.status();
let first = first.json::<Value>().await.unwrap();
assert_eq!(first_status, reqwest::StatusCode::OK, "body={first}");
assert_eq!(first["accepted"], true);
let accepted_revision = first["revision"].as_i64().unwrap();
assert_ne!(accepted_revision, revision);
let replay = client
.post(format!("{base_url}/onboarding/events"))
.json(&payload)
.send()
.await
.unwrap();
let replay_status = replay.status();
let replay = replay.json::<Value>().await.unwrap();
assert_eq!(replay_status, reqwest::StatusCode::OK, "body={replay}");
assert_eq!(replay["accepted"], false);
assert_eq!(replay["revision"], accepted_revision);
let semantic_conflict = client
.post(format!("{base_url}/onboarding/events"))
.json(&json!({
"event": "dismissed",
"idempotency_key": "onboarding-start-tab-a",
"expected_revision": revision,
}))
.send()
.await
.unwrap();
let semantic_conflict_status = semantic_conflict.status();
let semantic_conflict = semantic_conflict.json::<Value>().await.unwrap();
assert_eq!(
semantic_conflict_status,
reqwest::StatusCode::CONFLICT,
"body={semantic_conflict}"
);
assert_eq!(
semantic_conflict["error"]["code"],
"onboarding_idempotency_conflict"
);
let current_revision_conflict = client
.post(format!("{base_url}/onboarding/events"))
.json(&json!({
"event": "dismissed",
"idempotency_key": "onboarding-start-tab-a",
"expected_revision": accepted_revision,
}))
.send()
.await
.unwrap();
let current_revision_conflict_status = current_revision_conflict.status();
let current_revision_conflict = current_revision_conflict.json::<Value>().await.unwrap();
assert_eq!(
current_revision_conflict_status,
reqwest::StatusCode::CONFLICT,
"body={current_revision_conflict}"
);
assert_eq!(
current_revision_conflict["error"]["code"],
"onboarding_idempotency_conflict"
);
let stale = client
.post(format!("{base_url}/onboarding/events"))
.json(&json!({
"event": "dismissed",
"idempotency_key": "onboarding-dismiss-tab-b",
"expected_revision": revision,
}))
.send()
.await
.unwrap();
let stale_status = stale.status();
let stale = stale.json::<Value>().await.unwrap();
assert_eq!(stale_status, reqwest::StatusCode::CONFLICT, "body={stale}");
assert_eq!(stale["error"]["code"], "onboarding_stale_revision");
for payload in [
json!({
"event": "eligible",
"idempotency_key": "forged-eligibility",
"expected_revision": accepted_revision,
}),
json!({
"event": "completed",
"idempotency_key": "forged-completion",
"expected_revision": accepted_revision,
}),
json!({
"event": "started",
"idempotency_key": "forged-step",
"expected_revision": accepted_revision,
"completed_steps": ["operation", "first_call"],
}),
json!({
"event": "started",
"idempotency_key": "onboarding:completed:v1",
"expected_revision": accepted_revision,
}),
] {
let response = client
.post(format!("{base_url}/onboarding/events"))
.json(&payload)
.send()
.await
.unwrap();
let status = response.status();
let body = response.json::<Value>().await.unwrap();
assert_eq!(
status,
reqwest::StatusCode::UNPROCESSABLE_ENTITY,
"body={body}"
);
assert_eq!(body["error"]["code"], "onboarding_event_not_allowed");
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn direct_presentation_mutation_cannot_precede_server_eligibility() {
let registry = test_registry().await;
let storage_root = test_storage_root("onboarding_direct_event");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let direct = client
.post(format!("{base_url}/onboarding/events"))
.json(&json!({
"event": "started",
"idempotency_key": "direct-before-snapshot",
"expected_revision": 0,
}))
.send()
.await
.unwrap();
let direct_status = direct.status();
let direct = direct.json::<Value>().await.unwrap();
assert_eq!(
direct_status,
reqwest::StatusCode::CONFLICT,
"body={direct}"
);
assert_eq!(direct["error"]["code"], "onboarding_stale_revision");
let snapshot = client
.get(format!("{base_url}/onboarding"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert!(snapshot["eligible_since"].as_str().is_some());
assert_eq!(snapshot["status"], "in_progress");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn reset_selection_is_explicit_and_revision_guarded() {
let registry = test_registry().await;
let storage_root = test_storage_root("onboarding_reset_selection");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let initial = client
.get(format!("{base_url}/onboarding"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let revision = initial["revision"].as_i64().unwrap();
let reset = client
.post(format!("{base_url}/onboarding/reset-selection"))
.json(&json!({"expected_revision": revision}))
.send()
.await
.unwrap();
let reset_status = reset.status();
let reset = reset.json::<Value>().await.unwrap();
assert_eq!(reset_status, reqwest::StatusCode::OK, "body={reset}");
assert_eq!(reset["selection_reset"], true);
assert_ne!(reset["revision"], initial["revision"]);
assert_eq!(reset["workspace_id"], initial["workspace_id"]);
let stale = client
.post(format!("{base_url}/onboarding/reset-selection"))
.json(&json!({"expected_revision": revision}))
.send()
.await
.unwrap();
let stale_status = stale.status();
let stale = stale.json::<Value>().await.unwrap();
assert_eq!(stale_status, reqwest::StatusCode::CONFLICT, "body={stale}");
assert_eq!(stale["error"]["code"], "onboarding_stale_revision");
let forged_selection = client
.post(format!("{base_url}/onboarding/reset-selection"))
.json(&json!({
"expected_revision": reset["revision"],
"operation_id": "op_client_selected",
}))
.send()
.await
.unwrap();
assert_eq!(
forged_selection.status(),
reqwest::StatusCode::UNPROCESSABLE_ENTITY
);
}
@@ -0,0 +1,412 @@
#![allow(dead_code, unused_imports)]
use super::common::*;
use serde_json::Value;
use serial_test::serial;
const WORKSPACE_ID: &str = "ws_default";
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn portable_yaml_v2_is_redacted_and_semantic_replay_is_a_noop() {
let registry = test_registry().await;
let storage_root = test_storage_root("operation-yaml-v2");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let created = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload("http://127.0.0.1:9", "portable_v2"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_id = created["operation_id"].as_str().unwrap();
let yaml = client
.get(format!("{base_url}/operations/{operation_id}/export"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
assert!(yaml.contains("format_version: '2'") || yaml.contains("format_version: 2"));
let parsed: serde_yaml::Value = serde_yaml::from_str(&yaml).unwrap();
let operation = parsed["operation"].as_mapping().unwrap();
for forbidden in ["id", "status", "created_at", "wizard_state", "samples"] {
assert!(!operation.contains_key(serde_yaml::Value::String(forbidden.to_owned())));
}
let replay = client
.post(format!("{base_url}/operations/import?mode=upsert"))
.header("content-type", "application/yaml")
.body(yaml.clone())
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(replay["operation_id"], operation_id);
assert_eq!(replay["version"], 1);
let changed_yaml = yaml.replace(
"display_name: Create Lead",
"display_name: Portable Changed",
);
assert_ne!(changed_yaml, yaml);
let missing = client
.post(format!("{base_url}/operations/import?mode=upsert"))
.header("content-type", "application/yaml")
.body(changed_yaml.clone())
.send()
.await
.unwrap();
assert_eq!(missing.status(), reqwest::StatusCode::PRECONDITION_REQUIRED);
let missing = missing.json::<Value>().await.unwrap();
assert_eq!(missing["error"]["code"], "operation_precondition_required");
let changed = client
.post(format!("{base_url}/operations/import?mode=upsert"))
.header("content-type", "application/yaml")
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, operation_id).await,
)
.body(changed_yaml)
.send()
.await
.unwrap();
assert_eq!(changed.status(), reqwest::StatusCode::OK);
let changed = changed.json::<Value>().await.unwrap();
assert_eq!(changed["version"], 2);
let current_yaml = client
.get(format!("{base_url}/operations/{operation_id}/export"))
.send()
.await
.unwrap()
.text()
.await
.unwrap();
let archived = client
.post(format!("{base_url}/operations/{operation_id}/archive"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, operation_id).await,
)
.send()
.await
.unwrap();
assert!(archived.status().is_success());
let replay = client
.post(format!("{base_url}/operations/import?mode=upsert"))
.header("content-type", "application/yaml")
.body(current_yaml)
.send()
.await
.unwrap();
assert_eq!(replay.status(), reqwest::StatusCode::CONFLICT);
assert!(replay.text().await.unwrap().contains("operation_archived"));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn hostile_and_oversized_yaml_fail_before_mutation_without_echoing_canary() {
let registry = test_registry().await;
let storage_root = test_storage_root("operation-yaml-hostile");
let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await;
let client = authorized_client(&base_url).await;
let canary = "yaml-secret-canary-never-reflect";
let hostile = format!(
"format_version: '2'\nkind: operation\noperation:\n name: bad\n unknown_secret: {canary}\n"
);
let response = client
.post(format!("{base_url}/operations/import"))
.header("content-type", "application/yaml")
.body(hostile)
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
let body = response.text().await.unwrap();
assert!(!body.contains(canary));
assert!(body.contains("operation_yaml_invalid"));
let body: Value = serde_json::from_str(&body).unwrap();
assert_eq!(body["error"]["code"], "operation_yaml_invalid");
let oversized = "x".repeat(256 * 1024 + 1);
let response = client
.post(format!("{base_url}/operations/import"))
.header("content-type", "application/yaml")
.body(oversized)
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::PAYLOAD_TOO_LARGE);
let count = registry
.list_operations(&crank_core::WorkspaceId::new(WORKSPACE_ID))
.await
.unwrap()
.len();
assert_eq!(count, 0);
let alias = "format_version: '2'\nkind: operation\noperation: &shared\n name: alias\n";
let response = client
.post(format!("{base_url}/operations/import"))
.header("content-type", "application/yaml")
.body(alias)
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
assert!(
response
.text()
.await
.unwrap()
.contains("operation_yaml_unsupported")
);
let response = client
.post(format!("{base_url}/operations/import"))
.header("content-type", "application/yaml")
.body(vec![0xff, 0xfe, 0xfd])
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
assert!(
response
.text()
.await
.unwrap()
.contains("operation_yaml_invalid")
);
let mut credential_payload = serde_json::to_value(test_operation_payload(
"http://127.0.0.1:9",
"portable_credential_header",
))
.unwrap();
credential_payload["target"]["static_headers"]["X-Auth-Token"] =
Value::String("credential-canary".to_owned());
let created = client
.post(format!("{base_url}/operations"))
.json(&credential_payload)
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let credential_operation_id = created["operation_id"].as_str().unwrap();
let response = client
.get(format!(
"{base_url}/operations/{credential_operation_id}/export"
))
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
assert!(!response.text().await.unwrap().contains("credential-canary"));
let mut portable = serde_json::to_value(test_operation_payload(
"http://127.0.0.1:9",
"unknown_nested_yaml",
))
.unwrap();
portable.as_object_mut().unwrap().remove("wizard_state");
portable["target"]["unknown_nested"] = Value::String(canary.to_owned());
let document = serde_yaml::to_string(&serde_json::json!({
"format_version": "2",
"kind": "operation",
"operation": portable
}))
.unwrap();
let response = client
.post(format!("{base_url}/operations/import"))
.header("content-type", "application/yaml")
.body(document)
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
assert!(!response.text().await.unwrap().contains(canary));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn operation_mutations_require_identity_bound_etags_and_versions_are_stable() {
let registry = test_registry().await;
let storage_root = test_storage_root("operation-etag");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let first = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload("http://127.0.0.1:9", "etag_first"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let second = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload("http://127.0.0.1:9", "etag_second"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let first_id = first["operation_id"].as_str().unwrap();
let second_id = second["operation_id"].as_str().unwrap();
let missing = client
.post(format!("{base_url}/operations/{first_id}/archive"))
.send()
.await
.unwrap();
assert_eq!(missing.status(), reqwest::StatusCode::PRECONDITION_REQUIRED);
let missing = missing.json::<Value>().await.unwrap();
assert_eq!(missing["error"]["code"], "operation_precondition_required");
let first_etag = operation_etag(&client, &base_url, first_id).await;
let replay = client
.post(format!("{base_url}/operations/{second_id}/archive"))
.header(reqwest::header::IF_MATCH, first_etag)
.send()
.await
.unwrap();
assert_eq!(replay.status(), reqwest::StatusCode::CONFLICT);
let replay = replay.json::<Value>().await.unwrap();
assert_eq!(replay["error"]["code"], "operation_stale_version");
let v1 = client
.get(format!("{base_url}/operations/{first_id}/versions/1"))
.send()
.await
.unwrap();
assert_eq!(v1.status(), reqwest::StatusCode::OK);
let v1_etag = v1.headers()[reqwest::header::ETAG]
.to_str()
.unwrap()
.to_owned();
let mut update =
serde_json::to_value(test_operation_payload("http://127.0.0.1:9", "etag_first")).unwrap();
update["display_name"] = Value::String("Changed Draft".to_owned());
let updated = client
.patch(format!("{base_url}/operations/{first_id}"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, first_id).await,
)
.json(&update)
.send()
.await
.unwrap();
assert_eq!(updated.status(), reqwest::StatusCode::OK);
let v1_after = client
.get(format!("{base_url}/operations/{first_id}/versions/1"))
.send()
.await
.unwrap();
assert_eq!(v1_after.status(), reqwest::StatusCode::OK);
assert_eq!(
v1_after.headers()[reqwest::header::ETAG].to_str().unwrap(),
v1_etag
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn same_etag_concurrent_patches_have_exactly_one_winner() {
let registry = test_registry().await;
let storage_root = test_storage_root("operation-etag-race");
let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await;
let client = authorized_client(&base_url).await;
let created = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload("http://127.0.0.1:9", "etag_race"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_id = created["operation_id"].as_str().unwrap();
let etag = operation_etag(&client, &base_url, operation_id).await;
let mut first =
serde_json::to_value(test_operation_payload("http://127.0.0.1:9", "etag_race")).unwrap();
first["display_name"] = Value::String("first winner candidate".to_owned());
let mut second = first.clone();
second["display_name"] = Value::String("second winner candidate".to_owned());
let url = format!("{base_url}/operations/{operation_id}");
let first_request = client
.patch(&url)
.header(reqwest::header::IF_MATCH, &etag)
.json(&first)
.send();
let second_request = client
.patch(&url)
.header(reqwest::header::IF_MATCH, &etag)
.json(&second)
.send();
let (first_response, second_response) = tokio::join!(first_request, second_request);
let statuses = [
first_response.unwrap().status(),
second_response.unwrap().status(),
];
assert_eq!(
statuses.iter().filter(|status| status.is_success()).count(),
1
);
assert_eq!(
statuses
.iter()
.filter(|status| **status == reqwest::StatusCode::CONFLICT)
.count(),
1
);
let summary = registry
.get_operation_summary(
&crank_core::WorkspaceId::new(WORKSPACE_ID),
&crank_core::OperationId::new(operation_id),
)
.await
.unwrap()
.unwrap();
assert_eq!(summary.current_draft_version, 2);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn operation_rejects_unscoped_auth_profile_reference_before_save() {
let registry = test_registry().await;
let storage_root = test_storage_root("operation-auth-reference");
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let mut payload = serde_json::to_value(test_operation_payload(
"http://127.0.0.1:9",
"missing_auth_profile",
))
.unwrap();
payload["execution_config"]["auth_profile_ref"] =
Value::String("auth_foreign_or_missing".to_owned());
let response = client
.post(format!("{base_url}/operations"))
.json(&payload)
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
let body = response.text().await.unwrap();
assert!(body.contains("operation_auth_profile_invalid"));
assert!(!body.contains("auth_foreign_or_missing"));
}
@@ -36,7 +36,7 @@ const TEST_AUTH_EMAIL: &str = "owner@crank.local";
const TEST_AUTH_PASSWORD: &str = "test-password";
const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
struct TestServer {
base_url: String,
@@ -123,15 +123,6 @@ async fn creates_publishes_and_tests_rest_operation() {
.json::<Value>()
.await
.unwrap();
let published = client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let test_run_response = client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.header("x-request-id", "req_admin_test_run")
@@ -151,6 +142,19 @@ async fn creates_publishes_and_tests_rest_operation() {
"0af7651916cd43dd8448eb211c80319c"
);
let test_run = test_run_response.json::<Value>().await.unwrap();
let published = client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(listed["items"][0]["name"], "crm_create_lead");
assert_eq!(
@@ -160,20 +164,24 @@ async fn creates_publishes_and_tests_rest_operation() {
assert_eq!(listed["items"][0]["target_action"], "POST");
assert_eq!(published["published_version"], 1);
assert_eq!(test_run["ok"], true);
assert_eq!(
test_run["request_preview"]["body"]["email"],
"user@example.com"
);
assert_eq!(test_run["request_preview"]["body_configured"], true);
let safe_preview = test_run["request_preview"].to_string();
assert!(!safe_preview.contains("user@example.com"));
assert_eq!(test_run["response_preview"]["id"], "lead_123");
let logs = registry
.list_invocation_logs(crank_registry::ListInvocationLogsQuery {
workspace_id: &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(crank_core::InvocationSource::AdminTestRun),
operation_id: Some(&crank_core::OperationId::new(&operation_id)),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 10,
})
.await
@@ -267,6 +275,10 @@ async fn updates_archives_and_deletes_operation() {
let updated = client
.patch(format!("{base_url}/operations/{operation_id}"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({
"display_name": "Create Lead Updated",
"category": "marketing",
@@ -350,6 +362,10 @@ async fn updates_archives_and_deletes_operation() {
let archived = client
.post(format!("{base_url}/operations/{operation_id}/archive"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.send()
.await
.unwrap()
@@ -360,29 +376,29 @@ async fn updates_archives_and_deletes_operation() {
let deleted = client
.delete(format!("{base_url}/operations/{operation_id}"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.send()
.await
.unwrap();
assert_eq!(deleted.status(), reqwest::StatusCode::CONFLICT);
let deleted = deleted.json::<Value>().await.unwrap();
assert_eq!(
deleted["error"]["context"]["error_code"],
"operation_delete_forbidden"
);
let retained = client
.get(format!("{base_url}/operations/{operation_id}"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(deleted["operation_id"], operation_id);
let missing = client
.get(format!("{base_url}/operations/{operation_id}"))
.send()
.await
.unwrap();
let missing_status = missing.status();
let missing = missing.json::<Value>().await.unwrap();
assert_eq!(missing_status, reqwest::StatusCode::NOT_FOUND);
assert_eq!(missing["error"]["code"], "not_found");
assert_eq!(
missing["error"]["context"],
json!({
"operation_id": operation_id
})
);
assert_eq!(retained["status"], "archived");
}
#[tokio::test(flavor = "multi_thread")]
@@ -408,6 +424,10 @@ async fn creates_binds_and_publishes_agent() {
client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
@@ -432,6 +452,10 @@ async fn creates_binds_and_publishes_agent() {
let bindings = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!([
{
"operation_id": operation_id,
@@ -449,6 +473,10 @@ async fn creates_binds_and_publishes_agent() {
.unwrap();
let published = client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
@@ -489,6 +517,10 @@ async fn saves_and_previews_versioned_agent_tool_search_policy() {
assert_success_json(
client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({"version": 1}))
.send()
.await
@@ -535,6 +567,10 @@ async fn saves_and_previews_versioned_agent_tool_search_policy() {
let saved = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&catalog)
.send()
.await
@@ -565,6 +601,10 @@ async fn saves_and_previews_versioned_agent_tool_search_policy() {
let published = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({"version": 1}))
.send()
.await
@@ -576,7 +616,7 @@ async fn saves_and_previews_versioned_agent_tool_search_policy() {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {
async fn agent_binding_rejects_draft_operation_before_publish() {
let registry = test_registry().await;
let storage_root = test_storage_root("agent_publish_filters_drafts");
let upstream_base_url = spawn_upstream_server().await;
@@ -604,6 +644,10 @@ async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {
.post(format!(
"{base_url}/operations/{published_operation_id}/publish"
))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &published_operation_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
@@ -642,9 +686,47 @@ async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {
.await;
let agent_id = agent["agent_id"].as_str().unwrap().to_owned();
let rejected = client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!([
{
"operation_id": published_operation_id,
"operation_version": 1,
"tool_name": "crm_published_tool",
"tool_title": "Published Tool",
"tool_description_override": null,
"enabled": true
},
{
"operation_id": draft_operation_id,
"operation_version": 1,
"tool_name": "crm_draft_tool",
"tool_title": "Draft Tool",
"tool_description_override": null,
"enabled": true
}
]))
.send()
.await
.unwrap();
assert_eq!(rejected.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
let rejected_body = rejected.json::<Value>().await.unwrap();
assert_eq!(
rejected_body["error"]["code"],
"agent_binding_not_published"
);
assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!([
{
"operation_id": published_operation_id,
@@ -653,14 +735,6 @@ async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {
"tool_title": "Published Tool",
"tool_description_override": null,
"enabled": true
},
{
"operation_id": draft_operation_id,
"operation_version": 1,
"tool_name": "crm_draft_tool",
"tool_title": "Draft Tool",
"tool_description_override": null,
"enabled": true
}
]))
.send()
@@ -672,6 +746,10 @@ async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {
let published = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
@@ -694,24 +772,15 @@ async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {
.unwrap(),
)
.await;
let draft_version = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}/versions/2"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(published["published_version"], 1);
assert_eq!(agent_detail["current_draft_version"], 2);
assert_eq!(agent_detail["current_draft_version"], 1);
assert_eq!(agent_detail["latest_published_version"], 1);
assert_eq!(published_version["bindings"].as_array().unwrap().len(), 1);
assert_eq!(
published_version["bindings"][0]["operation_id"],
published_operation_id
);
assert_eq!(draft_version["bindings"].as_array().unwrap().len(), 2);
}
#[tokio::test(flavor = "multi_thread")]
@@ -737,6 +806,10 @@ async fn updates_lists_and_deletes_agent() {
client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.header(
reqwest::header::IF_MATCH,
operation_etag(&client, &base_url, &operation_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
@@ -761,6 +834,10 @@ async fn updates_lists_and_deletes_agent() {
client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!([
{
"operation_id": operation_id,
@@ -777,6 +854,10 @@ async fn updates_lists_and_deletes_agent() {
client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({ "version": 1 }))
.send()
.await
@@ -800,6 +881,10 @@ async fn updates_lists_and_deletes_agent() {
let updated = client
.patch(format!("{base_url}/agents/{agent_id}"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.json(&json!({
"slug": "support-escalation",
"display_name": "Support Escalation",
@@ -807,11 +892,17 @@ async fn updates_lists_and_deletes_agent() {
}))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(updated["agent_id"], agent_id);
assert_eq!(
updated.status(),
reqwest::StatusCode::CONFLICT,
"published Agent summary/slug must not mutate published MCP endpoint identity"
);
let updated = updated.json::<Value>().await.unwrap();
assert_eq!(
updated["error"]["context"]["error_code"],
"agent_published_summary_immutable"
);
let detail = client
.get(format!("{base_url}/agents/{agent_id}"))
@@ -821,148 +912,35 @@ async fn updates_lists_and_deletes_agent() {
.json::<Value>()
.await
.unwrap();
assert_eq!(detail["slug"], "support-escalation");
assert_eq!(detail["display_name"], "Support Escalation");
assert_eq!(detail["slug"], "support-team");
assert_eq!(detail["display_name"], "Support Team");
assert_eq!(detail["operation_count"], 1);
assert_eq!(detail["mcp_endpoint"], "/mcp/v1/default/support-escalation");
assert_eq!(detail["mcp_endpoint"], "/mcp/v1/default/support-team");
let deleted = client
let delete_response = client
.delete(format!("{base_url}/agents/{agent_id}"))
.header(
reqwest::header::IF_MATCH,
agent_etag(&client, &base_url, &agent_id).await,
)
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(deleted["agent_id"], agent_id);
assert_eq!(
delete_response.status(),
reqwest::StatusCode::CONFLICT,
"published Agent must not be hard-deleted because immutable versions/history must remain"
);
let delete_body = delete_response.json::<Value>().await.unwrap();
assert_eq!(
delete_body["error"]["context"]["error_code"],
"agent_delete_forbidden"
);
let missing = client
let still_present = client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap();
let missing_status = missing.status();
let missing = missing.json::<Value>().await.unwrap();
assert_eq!(missing_status, reqwest::StatusCode::NOT_FOUND);
assert_eq!(missing["error"]["code"], "not_found");
assert_eq!(
missing["error"]["context"],
json!({
"agent_id": agent_id
})
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn unpublishes_and_archives_agent() {
let registry = test_registry().await;
let storage_root = test_storage_root("agent_statuses");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let operation = assert_success_json(
client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream_base_url,
"crm_create_lead_agent_status",
))
.send()
.await
.unwrap(),
)
.await;
let operation_id = operation["operation_id"].as_str().unwrap().to_owned();
client
.post(format!("{base_url}/operations/{operation_id}/publish"))
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap();
let created = assert_success_json(
client
.post(format!("{base_url}/agents"))
.json(&json!({
"slug": "sales-routing",
"display_name": "Sales Routing",
"description": "Routing agent",
"instructions": {},
"tool_selection_policy": {}
}))
.send()
.await
.unwrap(),
)
.await;
let agent_id = created["agent_id"].as_str().unwrap().to_owned();
client
.post(format!("{base_url}/agents/{agent_id}/bindings"))
.json(&json!([
{
"operation_id": operation_id,
"operation_version": 1,
"tool_name": "crm_create_lead_agent_status",
"tool_title": "Create Lead",
"tool_description_override": null,
"enabled": true
}
]))
.send()
.await
.unwrap();
client
.post(format!("{base_url}/agents/{agent_id}/publish"))
.json(&json!({ "version": 1 }))
.send()
.await
.unwrap();
let unpublished = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/unpublish"))
.json(&json!({}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(unpublished["agent_id"], agent_id);
let draft_detail = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(draft_detail["status"], "draft");
assert_eq!(draft_detail["latest_published_version"], 1);
let archived = assert_success_json(
client
.post(format!("{base_url}/agents/{agent_id}/archive"))
.json(&json!({}))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(archived["agent_id"], agent_id);
let archived_detail = assert_success_json(
client
.get(format!("{base_url}/agents/{agent_id}"))
.send()
.await
.unwrap(),
)
.await;
assert_eq!(archived_detail["status"], "archived");
assert_eq!(still_present.status(), reqwest::StatusCode::OK);
}
@@ -36,7 +36,7 @@ const TEST_AUTH_EMAIL: &str = "owner@crank.local";
const TEST_AUTH_PASSWORD: &str = "test-password";
const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
struct TestServer {
base_url: String,
@@ -168,7 +168,7 @@ async fn manages_auth_profiles_and_yaml_upsert() {
secret_id
);
assert_eq!(imported["operation_id"], operation_id);
assert_eq!(imported["version"], 2);
assert_eq!(imported["version"], 1);
assert_eq!(imported["import_mode"], "upsert");
}
@@ -379,7 +379,7 @@ async fn rejects_deleting_secret_referenced_by_auth_profile() {
let body = response.json::<Value>().await.unwrap();
assert_eq!(status, reqwest::StatusCode::CONFLICT);
assert_eq!(body["error"]["code"], "conflict");
assert_eq!(body["error"]["code"], "secret_referenced_by_auth_profile");
assert_eq!(
body["error"]["message"],
format!("secret {secret_id} is referenced by auth profile {auth_profile_id}")
+349 -2
View File
@@ -1,5 +1,14 @@
use std::process::{Command, Output};
use crank_community_auth::{hash_password, verify_password};
use crank_core::{Secret, SecretId, SecretKind, SecretStatus, UserSessionId, WorkspaceId};
use crank_registry::{
CreateSecretRequest, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresRegistry,
};
use crank_runtime::SecretCrypto;
use serde_json::json;
use time::{Duration as TimeDuration, OffsetDateTime, format_description::well_known::Rfc3339};
fn command(arguments: &[&str], database_url: Option<&str>) -> Output {
let mut command = Command::new(env!("CARGO_BIN_EXE_crank-migrate"));
command.args(arguments);
@@ -16,6 +25,10 @@ fn command(arguments: &[&str], database_url: Option<&str>) -> Output {
command.output().expect("migration command must run")
}
fn timestamp(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).unwrap()
}
#[test]
fn plan_is_deterministic_and_committed_contract_is_current() {
let first = command(&["plan"], None);
@@ -27,7 +40,7 @@ fn plan_is_deterministic_and_committed_contract_is_current() {
);
assert_eq!(first.stdout, second.stdout);
let plan: serde_json::Value = serde_json::from_slice(&first.stdout).unwrap();
assert_eq!(plan["sequence"].as_array().unwrap().len(), 3);
assert_eq!(plan["sequence"].as_array().unwrap().len(), 11);
let checked = command(&["plan", "--check"], None);
assert!(
@@ -69,7 +82,147 @@ async fn database_only_config_can_apply_and_preflight_a_fresh_schema() {
);
let result: serde_json::Value = serde_json::from_slice(&preflight.stdout).unwrap();
assert_eq!(result["status"], "current");
assert_eq!(result["version"], 3);
assert_eq!(result["version"], 11);
}
#[tokio::test]
async fn admin_auth_bootstrap_create_outputs_bootstrap_secret_without_service_secrets() {
let database_url =
crank_test_support::postgres_schema_url("test_migration_admin_auth_bootstrap").await;
assert!(command(&["apply"], Some(&database_url)).status.success());
let created = command(
&[
"admin-auth",
"bootstrap-create",
"--email",
"operator@example.local",
"--display-name",
"Local Operator",
"--ttl-seconds",
"900",
],
Some(&database_url),
);
assert!(
created.status.success(),
"{}",
String::from_utf8_lossy(&created.stderr)
);
let payload: serde_json::Value = serde_json::from_slice(&created.stdout).unwrap();
assert_eq!(payload["status"], "bootstrap_created");
assert!(
payload["contract_id"]
.as_str()
.unwrap()
.starts_with("boot_")
);
assert!(payload["bootstrap_token"].as_str().unwrap().len() >= 32);
let stdout = String::from_utf8(created.stdout).unwrap();
assert!(!stdout.contains("CRANK_SESSION_SECRET"));
assert!(!stdout.contains("CRANK_PASSWORD_PEPPER"));
let duplicate = command(
&[
"admin-auth",
"bootstrap-create",
"--email",
"operator@example.local",
],
Some(&database_url),
);
assert!(!duplicate.status.success());
let diagnostic: serde_json::Value = serde_json::from_slice(&duplicate.stderr).unwrap();
assert_eq!(diagnostic["code"], "admin_bootstrap_unavailable");
}
#[tokio::test]
async fn admin_auth_recovery_requires_master_key_and_revokes_sessions_without_secret_output() {
let database_url =
crank_test_support::postgres_schema_url("test_migration_admin_auth_recovery").await;
assert!(command(&["apply"], Some(&database_url)).status.success());
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let now = timestamp("2026-08-21T00:00:00Z");
let master_key = "recovery-master-key-CANARY_SECRET_VALUE-000000000";
let crypto = SecretCrypto::new(master_key).unwrap();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
let pepper = "recovery-pepper-CANARY_SECRET_VALUE";
let old_password = "old-recovery-password-CANARY_SECRET_VALUE";
let old_hash = hash_password(old_password, pepper).unwrap();
let user_id = registry
.upsert_bootstrap_user("recover@example.local", "Recover Operator", &old_hash)
.await
.unwrap();
let session_id = UserSessionId::new("session_recovery_cli");
registry
.create_user_session(
&session_id,
&user_id,
Some(&WorkspaceId::new("ws_default")),
"session-secret-hash",
None,
&(OffsetDateTime::now_utc() + TimeDuration::hours(1)),
)
.await
.unwrap();
let key_dir =
std::env::temp_dir().join(format!("crank-admin-auth-recovery-{}", std::process::id()));
std::fs::create_dir_all(&key_dir).unwrap();
let password_path = key_dir.join("new-password.txt");
let pepper_path = key_dir.join("pepper.txt");
let master_key_path = key_dir.join("master.key");
let new_password = "new-recovery-password-CANARY_SECRET_VALUE";
std::fs::write(&password_path, format!("{new_password}\n")).unwrap();
std::fs::write(&pepper_path, format!("{pepper}\n")).unwrap();
std::fs::write(&master_key_path, format!("{master_key}\n")).unwrap();
let recovered = command(
&[
"admin-auth",
"recover",
"--email",
"recover@example.local",
"--password-file",
password_path.to_str().unwrap(),
"--password-pepper-file",
pepper_path.to_str().unwrap(),
"--master-key-file",
master_key_path.to_str().unwrap(),
],
Some(&database_url),
);
assert!(
recovered.status.success(),
"{}",
String::from_utf8_lossy(&recovered.stderr)
);
let stdout = String::from_utf8(recovered.stdout).unwrap();
assert!(stdout.contains("\"status\":\"admin_recovered\""));
assert!(!stdout.contains("CANARY_SECRET_VALUE"));
assert!(
registry
.get_user_session(&session_id, "session-secret-hash")
.await
.unwrap()
.is_none()
);
let user = registry
.get_auth_user_by_email("recover@example.local")
.await
.unwrap()
.unwrap();
assert!(verify_password(new_password, pepper, &user.password_hash));
assert!(!verify_password(old_password, pepper, &user.password_hash));
}
#[tokio::test]
@@ -88,3 +241,197 @@ async fn migration_error_json_preserves_affected_version() {
assert_eq!(diagnostic["code"], "checksum_mismatch");
assert_eq!(diagnostic["version"], 2);
}
#[tokio::test]
async fn master_key_rotation_command_is_resumable_and_redacted() {
let database_url =
crank_test_support::postgres_schema_url("test_master_key_rotation_cli").await;
assert!(command(&["apply"], Some(&database_url)).status.success());
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let now = timestamp("2026-08-21T00:00:00Z");
let current_key = "current-master-key-cli-canary-0000000000000000";
let target_key = "target-master-key-cli-canary-00000000000000000";
let current_crypto = SecretCrypto::new(current_key).unwrap();
registry
.verify_or_register_master_key_identity(MasterKeyIdentityCandidate {
epoch: 1,
fingerprint: current_crypto.master_key_fingerprint(),
cipher_contract: MASTER_KEY_CIPHER_CONTRACT,
observed_at: &now,
})
.await
.unwrap();
insert_secret(&registry, "cli_secret_a", &current_crypto, &now).await;
insert_secret(&registry, "cli_secret_b", &current_crypto, &now).await;
let key_dir =
std::env::temp_dir().join(format!("crank-master-key-rotation-{}", std::process::id()));
std::fs::create_dir_all(&key_dir).unwrap();
let current_path = key_dir.join("current.key");
let target_path = key_dir.join("target.key");
std::fs::write(&current_path, current_key).unwrap();
std::fs::write(&target_path, target_key).unwrap();
let current_path = current_path.to_string_lossy().to_string();
let target_path = target_path.to_string_lossy().to_string();
let preflight = command(
&[
"master-key",
"preflight",
"--current-key-file",
&current_path,
"--target-key-file",
&target_path,
"--backup-ref",
"offline-backup-ref",
],
Some(&database_url),
);
assert!(
preflight.status.success(),
"{}",
String::from_utf8_lossy(&preflight.stderr)
);
assert_redacted(&preflight, current_key, target_key);
let preflight_json: serde_json::Value = serde_json::from_slice(&preflight.stdout).unwrap();
assert_eq!(preflight_json["status"], "preflight_ok");
assert_eq!(preflight_json["affected_secret_versions"], 2);
assert!(
registry
.master_key_rotation_status()
.await
.unwrap()
.rotations
.is_empty()
);
let partial = command(
&[
"master-key",
"rotate",
"--current-key-file",
&current_path,
"--target-key-file",
&target_path,
"--backup-ref",
"offline-backup-ref",
"--max-versions",
"1",
],
Some(&database_url),
);
assert!(
partial.status.success(),
"{}",
String::from_utf8_lossy(&partial.stderr)
);
assert_redacted(&partial, current_key, target_key);
let partial_json: serde_json::Value = serde_json::from_slice(&partial.stdout).unwrap();
assert_eq!(partial_json["status"], "running");
assert_eq!(partial_json["processed_secret_versions"], 1);
let resumed = command(
&[
"master-key",
"rotate",
"--current-key-file",
&current_path,
"--target-key-file",
&target_path,
"--backup-ref",
"offline-backup-ref",
],
Some(&database_url),
);
assert!(
resumed.status.success(),
"{}",
String::from_utf8_lossy(&resumed.stderr)
);
let resumed_json: serde_json::Value = serde_json::from_slice(&resumed.stdout).unwrap();
assert_eq!(resumed_json["status"], "verifying");
assert_eq!(resumed_json["processed_secret_versions"], 2);
let verified = command(
&["master-key", "verify", "--target-key-file", &target_path],
Some(&database_url),
);
assert!(
verified.status.success(),
"{}",
String::from_utf8_lossy(&verified.stderr)
);
let verified_json: serde_json::Value = serde_json::from_slice(&verified.stdout).unwrap();
assert_eq!(verified_json["status"], "verified");
let promoted = command(
&["master-key", "promote", "--target-key-file", &target_path],
Some(&database_url),
);
assert!(
promoted.status.success(),
"{}",
String::from_utf8_lossy(&promoted.stderr)
);
assert_redacted(&promoted, current_key, target_key);
let promoted_json: serde_json::Value = serde_json::from_slice(&promoted.stdout).unwrap();
assert_eq!(promoted_json["status"], "promoted");
assert_eq!(promoted_json["active_epoch"], 2);
let old_key_rejected = command(
&[
"master-key",
"preflight",
"--current-key-file",
&current_path,
"--target-key-file",
&target_path,
],
Some(&database_url),
);
assert!(!old_key_rejected.status.success());
assert_redacted(&old_key_rejected, current_key, target_key);
let diagnostic: serde_json::Value = serde_json::from_slice(&old_key_rejected.stderr).unwrap();
assert_eq!(diagnostic["code"], "master_key_identity_mismatch");
}
async fn insert_secret(
registry: &PostgresRegistry,
id: &str,
crypto: &SecretCrypto,
now: &OffsetDateTime,
) {
let ciphertext = crypto
.encrypt(&json!({ "token": format!("{id}_value") }))
.unwrap();
let secret = Secret {
id: SecretId::new(id),
workspace_id: WorkspaceId::new("ws_default"),
name: id.to_owned(),
kind: SecretKind::Token,
status: SecretStatus::Active,
current_version: 1,
created_at: *now,
updated_at: *now,
last_used_at: None,
};
registry
.create_secret(CreateSecretRequest {
secret: &secret,
ciphertext: &ciphertext,
key_version: crypto.key_version(),
master_key_epoch: crypto.master_key_epoch(),
created_by: None,
})
.await
.unwrap();
}
fn assert_redacted(output: &Output, current_key: &str, target_key: &str) {
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(!stdout.contains(current_key));
assert!(!stdout.contains(target_key));
assert!(!stderr.contains(current_key));
assert!(!stderr.contains(target_key));
}
+2
View File
@@ -1,2 +1,4 @@
#[path = "unit/error.rs"]
mod error;
#[path = "unit/execution_parity.rs"]
mod execution_parity;
+55 -20
View File
@@ -1,6 +1,8 @@
use admin_api::error::{runtime_error_context, runtime_test_failure};
use admin_api::error::{
execution_test_failure, execution_test_failure_localized, runtime_test_failure,
};
use crank_core::{CorrelationContext, ExecutionErrorCode, ExecutionFailure, ExecutionLocale};
use crank_runtime::RuntimeError;
use serde_json::json;
#[test]
fn runtime_test_failure_includes_structured_context() {
@@ -9,23 +11,23 @@ fn runtime_test_failure_includes_structured_context() {
reason: "must be an object".to_owned(),
});
assert_eq!(payload["code"], "runtime_request_error");
assert_eq!(
payload["context"],
json!({
"field": "request.headers"
})
);
assert_eq!(payload["code"], "prepared_request_invalid");
assert_eq!(payload["stage"], "request_preparation");
assert_eq!(payload["retryability"], "never");
assert_eq!(payload["outcome_certainty"], "certain");
assert!(payload.get("context").is_none());
}
#[test]
fn runtime_error_context_does_not_expose_secret_crypto_details() {
let context = runtime_error_context(&RuntimeError::SecretCrypto {
let canary = "story18-secret-crypto-canary";
let payload = runtime_test_failure(&RuntimeError::SecretCrypto {
operation: "decode secret envelope",
details: "bad base64".to_owned(),
details: canary.to_owned(),
});
assert_eq!(context, None);
assert_eq!(payload["code"], "secret_invalid");
assert!(!payload.to_string().contains(canary));
}
#[test]
@@ -35,12 +37,45 @@ fn runtime_test_failure_includes_runtime_overload_context() {
limit: 16,
});
assert_eq!(payload["code"], "runtime_overloaded");
assert_eq!(
payload["context"],
json!({
"kind": "window",
"limit": 16
})
);
assert_eq!(payload["code"], "execution_overloaded");
assert_eq!(payload["stage"], "admission");
assert_eq!(payload["retryability"], "after_delay");
assert!(payload.get("context").is_none());
}
#[test]
fn every_execution_code_projects_the_canonical_semantics_without_raw_context() {
let canary = "story18-admin-projection-canary";
for code in ExecutionErrorCode::ALL {
let failure = ExecutionFailure::new(code, CorrelationContext::generate());
let payload = execution_test_failure(&failure);
assert_eq!(payload["code"], code.as_str());
assert_eq!(payload["stage"], code.stage().as_str());
assert_eq!(payload["retryability"], code.retryability().as_str());
assert_eq!(
payload["outcome_certainty"],
code.outcome_certainty().as_str()
);
assert!(!payload["message"].as_str().unwrap_or_default().is_empty());
assert!(!payload.to_string().contains(canary));
}
}
#[test]
fn confirmation_challenge_is_bounded_and_localized_at_the_admin_boundary() {
let failure = ExecutionFailure::new(
ExecutionErrorCode::ConfirmationRequired,
CorrelationContext::generate(),
)
.try_with_confirmation("confirmation-capability", 30_000)
.unwrap();
let payload = execution_test_failure_localized(&failure, ExecutionLocale::Ru);
assert_eq!(payload["message"], "Операция требует подтверждения.");
assert_eq!(
payload["context"]["confirmation_token"],
"confirmation-capability"
);
assert_eq!(payload["context"]["expires_in_ms"], 30_000);
assert!(!format!("{failure:?}").contains("confirmation-capability"));
}
@@ -0,0 +1,50 @@
use admin_api::error::execution_test_failure_localized;
use crank_community_mcp::tool_error::tool_error_contract_from_failure;
use crank_core::{
CorrelationContext, ExecutionErrorCode, ExecutionFailure, ExecutionLocale, Retryability,
};
#[test]
fn admin_and_mcp_project_every_execution_failure_with_identical_semantics() {
for locale in [ExecutionLocale::En, ExecutionLocale::Ru] {
for code in ExecutionErrorCode::ALL {
let failure = ExecutionFailure::new(code, CorrelationContext::generate());
let admin = execution_test_failure_localized(&failure, locale);
let mcp = tool_error_contract_from_failure(&failure, locale);
assert_eq!(admin["code"], mcp.error_code);
assert_eq!(admin["stage"], mcp.stage);
assert_eq!(admin["retryability"], mcp.retryability);
assert_eq!(admin["outcome_certainty"], mcp.outcome_certainty);
assert_eq!(admin["message"], mcp.message);
assert_eq!(
mcp.recoverable,
matches!(
code.retryability(),
Retryability::Safe
| Retryability::AfterDelay
| Retryability::RequiresConfirmation
)
);
}
}
}
#[test]
fn admin_and_mcp_preserve_dispatch_uncertainty_without_changing_the_code() {
for locale in [ExecutionLocale::En, ExecutionLocale::Ru] {
let failure = ExecutionFailure::new(
ExecutionErrorCode::UpstreamTransportError,
CorrelationContext::generate(),
)
.with_dispatch_uncertainty();
let admin = execution_test_failure_localized(&failure, locale);
let mcp = tool_error_contract_from_failure(&failure, locale);
assert_eq!(admin["retryability"], "manual_reconcile");
assert_eq!(admin["outcome_certainty"], "outcome_unknown");
assert_eq!(admin["retryability"], mcp.retryability);
assert_eq!(admin["outcome_certainty"], mcp.outcome_certainty);
assert!(!mcp.recoverable);
}
}