crank/apps/admin-api/src/app.rs

use axum::{
    Router, middleware,
    routing::{delete, get, post},
};

use crate::{
    auth::{require_session, require_workspace_session},
    rate_limit::apply_api_rate_limit,
    request_context::apply_request_context,
    routes::{
        access::export_workspace,
        agents::{
            archive_agent, create_agent, create_agent_platform_api_key, delete_agent,
            delete_agent_platform_api_key, get_agent, get_agent_version,
            list_agent_platform_api_keys, list_agents, publish_agent,
            revoke_agent_platform_api_key, save_agent_bindings, unpublish_agent, update_agent,
        },
        auth::{change_password, get_profile, get_session, login, logout, update_profile},
        auth_profiles::{create_auth_profile, get_auth_profile, list_auth_profiles},
        capabilities::get_capabilities,
        observability::{get_agent_usage, get_log, get_operation_usage, get_usage, list_logs},
        operations::{
            analyze_operation_quality, archive_operation, create_operation, create_version,
            delete_operation, export_operation, generate_draft, get_operation,
            get_operation_version, list_operations, publish_operation, run_test, update_operation,
            upload_input_json, upload_output_json,
        },
        secrets::{create_secret, delete_secret, get_secret, list_secrets, rotate_secret},
        upstreams::{create_upstream, list_upstreams, update_upstream},
        workspaces::{get_workspace, list_workspaces, update_workspace},
    },
    state::AppState,
};

pub fn build_app(state: AppState) -> Router {
    let workspace_router = Router::new()
        .route("/operations", get(list_operations).post(create_operation))
        .route(
            "/operations/analyze-quality",
            post(analyze_operation_quality),
        )
        .route("/operations/import", post(import_operation))
        .route(
            "/operations/{operation_id}",
            get(get_operation)
                .patch(update_operation)
                .delete(delete_operation),
        )
        .route("/operations/{operation_id}/versions", post(create_version))
        .route(
            "/operations/{operation_id}/versions/{version}",
            get(get_operation_version),
        )
        .route(
            "/operations/{operation_id}/publish",
            post(publish_operation),
        )
        .route(
            "/operations/{operation_id}/archive",
            post(archive_operation),
        )
        .route("/operations/{operation_id}/test-runs", post(run_test))
        .route(
            "/operations/{operation_id}/samples/input-json",
            post(upload_input_json),
        )
        .route(
            "/operations/{operation_id}/samples/output-json",
            post(upload_output_json),
        )
        .route(
            "/operations/{operation_id}/drafts/generate",
            post(generate_draft),
        )
        .route("/operations/{operation_id}/export", get(export_operation))
        .route("/agents", get(list_agents).post(create_agent))
        .route(
            "/agents/{agent_id}",
            get(get_agent).patch(update_agent).delete(delete_agent),
        )
        .route(
            "/agents/{agent_id}/versions/{version}",
            get(get_agent_version),
        )
        .route("/agents/{agent_id}/bindings", post(save_agent_bindings))
        .route("/agents/{agent_id}/publish", post(publish_agent))
        .route("/agents/{agent_id}/unpublish", post(unpublish_agent))
        .route("/agents/{agent_id}/archive", post(archive_agent))
        .route(
            "/agents/{agent_id}/platform-api-keys",
            get(list_agent_platform_api_keys).post(create_agent_platform_api_key),
        )
        .route(
            "/agents/{agent_id}/platform-api-keys/{key_id}/revoke",
            post(revoke_agent_platform_api_key),
        )
        .route(
            "/agents/{agent_id}/platform-api-keys/{key_id}",
            delete(delete_agent_platform_api_key),
        )
        .route(
            "/auth-profiles",
            get(list_auth_profiles).post(create_auth_profile),
        )
        .route("/auth-profiles/{auth_profile_id}", get(get_auth_profile))
        .route("/upstreams", get(list_upstreams).post(create_upstream))
        .route(
            "/upstreams/{upstream_id}",
            axum::routing::patch(update_upstream),
        )
        .route("/secrets", get(list_secrets).post(create_secret))
        .route(
            "/secrets/{secret_id}",
            get(get_secret).delete(delete_secret),
        )
        .route("/secrets/{secret_id}/rotate", post(rotate_secret))
        .route("/export", get(export_workspace))
        .route("/logs", get(list_logs))
        .route("/logs/{log_id}", get(get_log))
        .route("/usage", get(get_usage))
        .route("/usage/operations/{operation_id}", get(get_operation_usage))
        .route("/usage/agents/{agent_id}", get(get_agent_usage));

    let workspace_root_router = Router::new()
        .route("/capabilities", get(get_capabilities))
        .route("/workspaces", get(list_workspaces))
        .layer(middleware::from_fn_with_state(
            state.clone(),
            require_session,
        ));

    let workspace_scoped_router = Router::new()
        .route(
            "/workspaces/{workspace_id}",
            get(get_workspace).patch(update_workspace),
        )
        .nest("/workspaces/{workspace_id}", workspace_router)
        .layer(middleware::from_fn_with_state(
            state.clone(),
            require_workspace_session,
        ));

    let admin_router = workspace_root_router.merge(workspace_scoped_router);

    let protected_auth_router = Router::new()
        .route("/logout", post(logout))
        .route("/session", get(get_session))
        .route("/profile", get(get_profile).patch(update_profile))
        .route("/password", post(change_password))
        .layer(middleware::from_fn_with_state(
            state.clone(),
            require_session,
        ));

    Router::new()
        .route("/health", get(crate::routes::health))
        .nest(
            "/api/auth",
            Router::new()
                .route("/login", post(login))
                .merge(protected_auth_router),
        )
        .nest("/api/admin", admin_router)
        .layer(middleware::from_fn_with_state(
            state.clone(),
            apply_api_rate_limit,
        ))
        .layer(middleware::from_fn(apply_request_context))
        .with_state(state)
}

use crate::routes::operations::import_operation;

#[cfg(test)]
mod tests {
    use std::{
        collections::BTreeMap,
        env, fmt,
        sync::Arc,
        time::{SystemTime, UNIX_EPOCH},
    };

    use async_trait::async_trait;
    use axum::{Json, Router, routing::post};
    use crank_core::{
        ExecutionConfig, HttpMethod, MembershipRole, OperationSecurityLevel, Protocol,
        ResponseCachePolicy, RestTarget, SecretKind, Target, ToolDescription, WorkspaceId,
    };
    use crank_core::{IdentityError, IdentityProvider, IdentityProviderKind, LoginOutcome};
    use crank_mapping::{MappingRule, MappingSet};
    use crank_registry::PostgresRegistry;
    use crank_runtime::SecretCrypto;
    use crank_schema::{Schema, SchemaKind};
    use serde_json::{Value, json};
    use serial_test::serial;
    use sqlx::{Connection, Executor, PgConnection};
    use tokio::net::TcpListener;

    use crate::{
        app::build_app,
        auth::{AuthSettings, BootstrapAdminConfig, hash_password},
        service::{AdminService, AdminServiceBuilder, OperationPayload},
        state::AppState,
    };

    const DEFAULT_WORKSPACE_ID: &str = "ws_default";
    const TEST_AUTH_EMAIL: &str = "owner@crank.local";
    const TEST_AUTH_PASSWORD: &str = "test-password";
    const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
    const TEST_SESSION_SECRET: &str = "test-session-secret";
    const TEST_MASTER_KEY: &str = "test-master-key";

    struct TestServer {
        base_url: String,
        shutdown: Option<tokio::sync::oneshot::Sender<()>>,
        handle: Option<tokio::task::JoinHandle<()>>,
    }

    impl Drop for TestServer {
        fn drop(&mut self) {
            let shutdown = self.shutdown.take();
            let handle = self.handle.take();

            tokio::task::block_in_place(|| {
                if let Some(shutdown) = shutdown {
                    let _ = shutdown.send(());
                }
                if let Some(handle) = handle {
                    let _ = tokio::runtime::Handle::current().block_on(handle);
                }
            });
        }
    }

    impl fmt::Display for TestServer {
        fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
            formatter.write_str(&self.base_url)
        }
    }

    impl AsRef<str> for TestServer {
        fn as_ref(&self) -> &str {
            &self.base_url
        }
    }

    struct RejectingIdentityProvider;

    #[async_trait]
    impl IdentityProvider for RejectingIdentityProvider {
        fn id(&self) -> &str {
            "rejecting-test-provider"
        }

        fn kind(&self) -> IdentityProviderKind {
            IdentityProviderKind::Password
        }

        async fn login_password(
            &self,
            _payload: crank_core::LoginPayload,
        ) -> Result<LoginOutcome, IdentityError> {
            Err(IdentityError::BadCredentials)
        }
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn creates_publishes_and_tests_rest_operation() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("lifecycle");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_create_lead",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap().to_owned();

        let listed = client
            .get(format!("{base_url}/operations"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let published = client
            .post(format!("{base_url}/operations/{operation_id}/publish"))
            .json(&json!({ "version": 1 }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let test_run = client
            .post(format!("{base_url}/operations/{operation_id}/test-runs"))
            .json(&json!({
                "version": 1,
                "input": { "email": "user@example.com" }
            }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();

        assert_eq!(listed["items"][0]["name"], "crm_create_lead");
        assert_eq!(
            listed["items"][0]["target_url"],
            format!("{upstream_base_url}/crm/leads")
        );
        assert_eq!(listed["items"][0]["target_action"], "POST");
        assert_eq!(published["published_version"], 1);
        assert_eq!(test_run["ok"], true);
        assert_eq!(
            test_run["request_preview"]["body"]["email"],
            "user@example.com"
        );
        assert_eq!(test_run["response_preview"]["id"], "lead_123");
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn updates_archives_and_deletes_operation() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("operation_mutations");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_mutable_operation",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap().to_owned();

        let listed = client
            .get(format!("{base_url}/operations"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(listed["total"], 1);
        assert_eq!(listed["items"][0]["category"], "sales");
        assert_eq!(
            listed["items"][0]["target_url"],
            format!("{upstream_base_url}/crm/leads")
        );
        assert_eq!(listed["items"][0]["target_action"], "POST");

        let updated = client
            .patch(format!("{base_url}/operations/{operation_id}"))
            .json(&json!({
                "display_name": "Create Lead Updated",
                "category": "marketing",
                "target": {
                    "kind": "rest",
                    "base_url": upstream_base_url,
                    "method": "POST",
                    "path_template": "/crm/leads",
                    "static_headers": {}
                },
                "input_schema": {
                    "type": "object",
                    "required": true,
                    "nullable": false,
                    "fields": {
                        "email": {
                            "type": "string",
                            "required": true,
                            "nullable": false
                        }
                    }
                },
                "output_schema": {
                    "type": "object",
                    "required": true,
                    "nullable": false,
                    "fields": {
                        "id": {
                            "type": "string",
                            "required": true,
                            "nullable": false
                        }
                    }
                },
                "input_mapping": {
                    "rules": [
                        {
                            "source": "$.mcp.email",
                            "target": "$.request.body.email",
                            "required": true
                        }
                    ]
                },
                "output_mapping": {
                    "rules": [
                        {
                            "source": "$.response.body.id",
                            "target": "$.output.id",
                            "required": true
                        }
                    ]
                },
                "execution_config": {
                    "timeout_ms": 1000,
                    "headers": {}
                },
                "tool_description": {
                    "title": "Create Lead Updated",
                    "description": "Creates a CRM lead",
                    "tags": ["crm"]
                }
            }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(updated["status"], "draft");

        let detail = client
            .get(format!("{base_url}/operations/{operation_id}"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(detail["display_name"], "Create Lead Updated");
        assert_eq!(detail["category"], "marketing");

        let archived = client
            .post(format!("{base_url}/operations/{operation_id}/archive"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(archived["status"], "archived");

        let deleted = client
            .delete(format!("{base_url}/operations/{operation_id}"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(deleted["operation_id"], operation_id);

        let missing = client
            .get(format!("{base_url}/operations/{operation_id}"))
            .send()
            .await
            .unwrap();
        let missing_status = missing.status();
        let missing = missing.json::<Value>().await.unwrap();
        assert_eq!(missing_status, reqwest::StatusCode::NOT_FOUND);
        assert_eq!(missing["error"]["code"], "not_found");
        assert_eq!(
            missing["error"]["context"],
            json!({
                "operation_id": operation_id
            })
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn creates_binds_and_publishes_agent() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("agent_lifecycle");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let operation = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_create_lead_agent",
            ))
            .send()
            .await
            .unwrap();
        let operation = assert_success_json(operation).await;
        let operation_id = operation["operation_id"].as_str().unwrap().to_owned();

        client
            .post(format!("{base_url}/operations/{operation_id}/publish"))
            .json(&json!({ "version": 1 }))
            .send()
            .await
            .unwrap();

        let agent = client
            .post(format!("{base_url}/agents"))
            .json(&json!({
                "slug": "sales-assistant",
                "display_name": "Sales Assistant",
                "description": "Curated sales toolset",
                "instructions": {},
                "tool_selection_policy": {}
            }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let agent_id = agent["agent_id"].as_str().unwrap().to_owned();

        let bindings = client
            .post(format!("{base_url}/agents/{agent_id}/bindings"))
            .json(&json!([
                {
                    "operation_id": operation_id,
                    "operation_version": 1,
                    "tool_name": "crm_create_lead_agent",
                    "tool_title": "Create Lead",
                    "enabled": true
                }
            ]))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let published = client
            .post(format!("{base_url}/agents/{agent_id}/publish"))
            .json(&json!({ "version": 1 }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();

        assert_eq!(
            bindings["bindings"][0]["tool_name"],
            "crm_create_lead_agent"
        );
        assert_eq!(published["published_version"], 1);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("agent_publish_filters_drafts");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let published_operation = assert_success_json(
            client
                .post(format!("{base_url}/operations"))
                .json(&test_operation_payload(
                    &upstream_base_url,
                    "crm_published_tool",
                ))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let published_operation_id = published_operation["operation_id"]
            .as_str()
            .unwrap()
            .to_owned();
        assert_success_json(
            client
                .post(format!(
                    "{base_url}/operations/{published_operation_id}/publish"
                ))
                .json(&json!({ "version": 1 }))
                .send()
                .await
                .unwrap(),
        )
        .await;

        let draft_operation = assert_success_json(
            client
                .post(format!("{base_url}/operations"))
                .json(&test_operation_payload(
                    &upstream_base_url,
                    "crm_draft_tool",
                ))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let draft_operation_id = draft_operation["operation_id"].as_str().unwrap().to_owned();

        let agent = assert_success_json(
            client
                .post(format!("{base_url}/agents"))
                .json(&json!({
                    "slug": "collaborative-agent",
                    "display_name": "Collaborative Agent",
                    "description": "Agent with published and draft tools",
                    "instructions": {},
                    "tool_selection_policy": {}
                }))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let agent_id = agent["agent_id"].as_str().unwrap().to_owned();

        assert_success_json(
            client
                .post(format!("{base_url}/agents/{agent_id}/bindings"))
                .json(&json!([
                    {
                        "operation_id": published_operation_id,
                        "operation_version": 1,
                        "tool_name": "crm_published_tool",
                        "tool_title": "Published Tool",
                        "tool_description_override": null,
                        "enabled": true
                    },
                    {
                        "operation_id": draft_operation_id,
                        "operation_version": 1,
                        "tool_name": "crm_draft_tool",
                        "tool_title": "Draft Tool",
                        "tool_description_override": null,
                        "enabled": true
                    }
                ]))
                .send()
                .await
                .unwrap(),
        )
        .await;

        let published = assert_success_json(
            client
                .post(format!("{base_url}/agents/{agent_id}/publish"))
                .json(&json!({ "version": 1 }))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let agent_detail = assert_success_json(
            client
                .get(format!("{base_url}/agents/{agent_id}"))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let published_version = assert_success_json(
            client
                .get(format!("{base_url}/agents/{agent_id}/versions/1"))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let draft_version = assert_success_json(
            client
                .get(format!("{base_url}/agents/{agent_id}/versions/2"))
                .send()
                .await
                .unwrap(),
        )
        .await;

        assert_eq!(published["published_version"], 1);
        assert_eq!(agent_detail["current_draft_version"], 2);
        assert_eq!(agent_detail["latest_published_version"], 1);
        assert_eq!(published_version["bindings"].as_array().unwrap().len(), 1);
        assert_eq!(
            published_version["bindings"][0]["operation_id"],
            published_operation_id
        );
        assert_eq!(draft_version["bindings"].as_array().unwrap().len(), 2);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn updates_lists_and_deletes_agent() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("agent_mutations");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let operation = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_create_lead_agents_page",
            ))
            .send()
            .await
            .unwrap();
        let operation = assert_success_json(operation).await;
        let operation_id = operation["operation_id"].as_str().unwrap().to_owned();

        client
            .post(format!("{base_url}/operations/{operation_id}/publish"))
            .json(&json!({ "version": 1 }))
            .send()
            .await
            .unwrap();

        let created = client
            .post(format!("{base_url}/agents"))
            .json(&json!({
                "slug": "support-team",
                "display_name": "Support Team",
                "description": "Support workflows",
                "instructions": {},
                "tool_selection_policy": {}
            }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let agent_id = created["agent_id"].as_str().unwrap().to_owned();

        client
            .post(format!("{base_url}/agents/{agent_id}/bindings"))
            .json(&json!([
                {
                    "operation_id": operation_id,
                    "operation_version": 1,
                    "tool_name": "crm_create_lead_agents_page",
                    "tool_title": "Create Lead",
                    "tool_description_override": null,
                    "enabled": true
                }
            ]))
            .send()
            .await
            .unwrap();

        client
            .post(format!("{base_url}/agents/{agent_id}/publish"))
            .json(&json!({ "version": 1 }))
            .send()
            .await
            .unwrap();

        let listed = client
            .get(format!("{base_url}/agents"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(listed["items"][0]["operation_count"], 1);
        assert_eq!(listed["items"][0]["operation_ids"][0], operation_id);
        assert_eq!(
            listed["items"][0]["mcp_endpoint"],
            "/mcp/v1/default/support-team"
        );
        assert_eq!(listed["items"][0]["status"], "published");

        let updated = client
            .patch(format!("{base_url}/agents/{agent_id}"))
            .json(&json!({
                "slug": "support-escalation",
                "display_name": "Support Escalation",
                "description": "Escalation workflows"
            }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(updated["agent_id"], agent_id);

        let detail = client
            .get(format!("{base_url}/agents/{agent_id}"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(detail["slug"], "support-escalation");
        assert_eq!(detail["display_name"], "Support Escalation");
        assert_eq!(detail["operation_count"], 1);
        assert_eq!(detail["mcp_endpoint"], "/mcp/v1/default/support-escalation");

        let deleted = client
            .delete(format!("{base_url}/agents/{agent_id}"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        assert_eq!(deleted["agent_id"], agent_id);

        let missing = client
            .get(format!("{base_url}/agents/{agent_id}"))
            .send()
            .await
            .unwrap();
        let missing_status = missing.status();
        let missing = missing.json::<Value>().await.unwrap();
        assert_eq!(missing_status, reqwest::StatusCode::NOT_FOUND);
        assert_eq!(missing["error"]["code"], "not_found");
        assert_eq!(
            missing["error"]["context"],
            json!({
                "agent_id": agent_id
            })
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn unpublishes_and_archives_agent() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("agent_statuses");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let operation = assert_success_json(
            client
                .post(format!("{base_url}/operations"))
                .json(&test_operation_payload(
                    &upstream_base_url,
                    "crm_create_lead_agent_status",
                ))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let operation_id = operation["operation_id"].as_str().unwrap().to_owned();

        client
            .post(format!("{base_url}/operations/{operation_id}/publish"))
            .json(&json!({ "version": 1 }))
            .send()
            .await
            .unwrap();

        let created = assert_success_json(
            client
                .post(format!("{base_url}/agents"))
                .json(&json!({
                    "slug": "sales-routing",
                    "display_name": "Sales Routing",
                    "description": "Routing agent",
                    "instructions": {},
                    "tool_selection_policy": {}
                }))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let agent_id = created["agent_id"].as_str().unwrap().to_owned();

        client
            .post(format!("{base_url}/agents/{agent_id}/bindings"))
            .json(&json!([
                {
                    "operation_id": operation_id,
                    "operation_version": 1,
                    "tool_name": "crm_create_lead_agent_status",
                    "tool_title": "Create Lead",
                    "tool_description_override": null,
                    "enabled": true
                }
            ]))
            .send()
            .await
            .unwrap();

        client
            .post(format!("{base_url}/agents/{agent_id}/publish"))
            .json(&json!({ "version": 1 }))
            .send()
            .await
            .unwrap();

        let unpublished = assert_success_json(
            client
                .post(format!("{base_url}/agents/{agent_id}/unpublish"))
                .json(&json!({}))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(unpublished["agent_id"], agent_id);

        let draft_detail = assert_success_json(
            client
                .get(format!("{base_url}/agents/{agent_id}"))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(draft_detail["status"], "draft");
        assert_eq!(draft_detail["latest_published_version"], 1);

        let archived = assert_success_json(
            client
                .post(format!("{base_url}/agents/{agent_id}/archive"))
                .json(&json!({}))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(archived["agent_id"], agent_id);

        let archived_detail = assert_success_json(
            client
                .get(format!("{base_url}/agents/{agent_id}"))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(archived_detail["status"], "archived");
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn rejects_workspace_access_management_in_community() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("platform_access");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let members_status = client
            .get(format!("{base_url}/members"))
            .send()
            .await
            .unwrap()
            .status();
        let create_invitation_status = client
            .post(format!("{base_url}/invitations"))
            .json(&json!({
                "email": "operator@example.com",
                "role": "operator"
            }))
            .send()
            .await
            .unwrap()
            .status();

        assert_eq!(members_status, reqwest::StatusCode::NOT_FOUND);
        assert_eq!(create_invitation_status, reqwest::StatusCode::NOT_FOUND);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn manages_agent_platform_api_keys() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("agent_platform_keys");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created_agent = assert_success_json(
            client
                .post(format!("{base_url}/agents"))
                .json(&json!({
                    "slug": "sales-routing",
                    "display_name": "Sales Routing",
                    "description": "Routing agent",
                    "instructions": {},
                    "tool_selection_policy": {}
                }))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let agent_id = created_agent["agent_id"].as_str().unwrap().to_owned();

        let created_key = assert_success_json(
            client
                .post(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
                .json(&json!({
                    "name": "sales-routing-primary",
                    "scopes": ["read", "write"]
                }))
                .send()
                .await
                .unwrap(),
        )
        .await;
        let key_id = created_key["api_key"]["api_key"]["id"]
            .as_str()
            .unwrap()
            .to_owned();

        let listed_keys = assert_success_json(
            client
                .get(format!("{base_url}/agents/{agent_id}/platform-api-keys"))
                .send()
                .await
                .unwrap(),
        )
        .await;

        let revoke_status = client
            .post(format!(
                "{base_url}/agents/{agent_id}/platform-api-keys/{key_id}/revoke"
            ))
            .send()
            .await
            .unwrap()
            .status();
        let delete_status = client
            .delete(format!(
                "{base_url}/agents/{agent_id}/platform-api-keys/{key_id}"
            ))
            .send()
            .await
            .unwrap()
            .status();

        assert_eq!(created_key["api_key"]["api_key"]["agent_id"], agent_id);
        assert_eq!(
            listed_keys["items"][0]["api_key"]["agent_id"],
            json!(agent_id)
        );
        assert_eq!(
            listed_keys["items"][0]["api_key"]["name"],
            "sales-routing-primary"
        );
        assert!(created_key["secret"].as_str().unwrap().starts_with("crk_"));
        assert_eq!(revoke_status, reqwest::StatusCode::NO_CONTENT);
        assert_eq!(delete_status, reqwest::StatusCode::NO_CONTENT);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn returns_community_capabilities() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("community_capabilities");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;
        let root_url = base_url
            .as_ref()
            .split("/api/admin/workspaces/")
            .next()
            .unwrap();

        let response = assert_success_json(
            client
                .get(format!("{root_url}/api/admin/capabilities"))
                .send()
                .await
                .unwrap(),
        )
        .await;

        assert_eq!(response["edition"], "community");
        assert_eq!(response["supported_protocols"], json!(["rest"]));
        assert_eq!(response["supported_security_levels"], json!(["standard"]));
        assert_eq!(
            response["machine_access_modes"],
            json!(["static_agent_key"])
        );
        assert_eq!(response["limits"]["max_workspaces"], 1);
        assert_eq!(response["limits"]["max_users_per_workspace"], 1);
        assert_eq!(response["limits"]["max_agents_per_workspace"], Value::Null);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn rejects_response_cache_for_non_get_rest_operation_create() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("community_response_cache_post_reject");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let mut payload = test_operation_payload(&upstream_base_url, "crm_cache_post");
        payload.execution_config.response_cache = Some(ResponseCachePolicy { ttl_ms: 5_000 });

        let response = client
            .post(format!("{base_url}/operations"))
            .json(&payload)
            .send()
            .await
            .unwrap();
        let status = response.status();
        let body = response.json::<Value>().await.unwrap();

        assert_eq!(status, reqwest::StatusCode::BAD_REQUEST);
        assert_eq!(body["error"]["code"], "validation_error");
        assert_eq!(
            body["error"]["context"]["field"],
            "execution_config.response_cache"
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn rejects_response_cache_for_operation_with_auth_profile() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("community_response_cache_auth_reject");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let mut payload = test_operation_payload(&upstream_base_url, "crm_cache_auth");
        payload.target = Target::Rest(RestTarget {
            base_url: upstream_base_url,
            method: HttpMethod::Get,
            path_template: "/crm/leads".to_owned(),
            static_headers: BTreeMap::new(),
        });
        payload.execution_config.response_cache = Some(ResponseCachePolicy { ttl_ms: 5_000 });
        payload.execution_config.auth_profile_ref = Some("auth_profile_01".into());

        let response = client
            .post(format!("{base_url}/operations"))
            .json(&payload)
            .send()
            .await
            .unwrap();
        let status = response.status();
        let body = response.json::<Value>().await.unwrap();

        assert_eq!(status, reqwest::StatusCode::BAD_REQUEST);
        assert_eq!(body["error"]["code"], "validation_error");
        assert_eq!(
            body["error"]["context"]["field"],
            "execution_config.auth_profile_ref"
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn exports_single_workspace_but_rejects_access_lifecycle() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("workspace_access");
        let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await;
        let client = authorized_client(&base_url).await;

        let second_user_id = registry
            .upsert_bootstrap_user("operator-2@crank.local", "Operator Two", "external-hash")
            .await
            .unwrap();
        registry
            .ensure_membership(
                &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
                &second_user_id,
                MembershipRole::Viewer,
            )
            .await
            .unwrap();

        let update_member_status = client
            .patch(format!("{base_url}/members/{}", second_user_id.as_str()))
            .json(&json!({ "role": "admin" }))
            .send()
            .await
            .unwrap()
            .status();
        assert_eq!(update_member_status, reqwest::StatusCode::NOT_FOUND);

        let exported = assert_success_json(
            client
                .get(format!("{base_url}/export"))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(
            exported["workspace"]["workspace"]["id"],
            DEFAULT_WORKSPACE_ID
        );
        assert!(exported.get("memberships").is_none());
        assert!(exported.get("invitations").is_none());

        let delete_member_status = client
            .delete(format!("{base_url}/members/{}", second_user_id.as_str()))
            .send()
            .await
            .unwrap()
            .status();
        assert_eq!(delete_member_status, reqwest::StatusCode::NOT_FOUND);

        let delete_workspace_status = client
            .delete(base_url.as_ref())
            .send()
            .await
            .unwrap()
            .status();
        assert_eq!(
            delete_workspace_status,
            reqwest::StatusCode::METHOD_NOT_ALLOWED
        );

        let root_url = base_url
            .as_ref()
            .split("/api/admin/workspaces/")
            .next()
            .unwrap();
        let still_available = assert_success_json(
            client
                .get(format!(
                    "{root_url}/api/admin/workspaces/{DEFAULT_WORKSPACE_ID}"
                ))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(still_available["workspace"]["id"], DEFAULT_WORKSPACE_ID);
    }

    #[tokio::test]
    #[serial]
    async fn seeds_demo_assets_for_live_ui() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("demo_seed");
        let service = AdminService::new(
            registry.clone(),
            storage_root,
            test_auth_settings(),
            test_secret_crypto(),
        );

        service.bootstrap_admin_user().await.unwrap();
        service.seed_demo_assets().await.unwrap();
        service.seed_demo_assets().await.unwrap();

        let owner = registry
            .get_auth_user_by_email(TEST_AUTH_EMAIL)
            .await
            .unwrap()
            .unwrap();
        let workspaces = service
            .list_workspaces_for_user(&owner.user.id)
            .await
            .unwrap();
        assert!(
            workspaces
                .iter()
                .any(|item| item.workspace.slug == "default")
        );
        assert_eq!(workspaces.len(), 1);

        let default_workspace_id = WorkspaceId::new(DEFAULT_WORKSPACE_ID);
        let operations = service
            .list_operations(&default_workspace_id)
            .await
            .unwrap();
        assert!(operations.len() >= 2);

        let agents = service.list_agents(&default_workspace_id).await.unwrap();
        assert!(!agents.is_empty());

        assert!(agents.iter().any(|agent| agent.key_count > 0));

        let logs = service
            .list_logs(
                &default_workspace_id,
                crate::service::LogsQuery {
                    level: None,
                    search: None,
                    source: None,
                    operation_id: None,
                    agent_id: None,
                    period: None,
                    limit: Some(20),
                },
            )
            .await
            .unwrap();
        assert!(!logs.is_empty());
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn updates_profile_and_changes_password() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("settings_profile");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let root_url = base_url
            .as_ref()
            .split("/api/admin/workspaces/")
            .next()
            .unwrap()
            .to_owned();
        let client = authorized_client(&base_url).await;

        let profile = assert_success_json(
            client
                .get(format!("{root_url}/api/auth/profile"))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(profile["user"]["email"], TEST_AUTH_EMAIL);

        let updated_profile = assert_success_json(
            client
                .patch(format!("{root_url}/api/auth/profile"))
                .json(&json!({
                    "display_name": "Updated Owner",
                    "email": "updated-owner@crank.local"
                }))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(updated_profile["user"]["display_name"], "Updated Owner");
        assert_eq!(
            updated_profile["user"]["email"],
            "updated-owner@crank.local"
        );

        let password_status = client
            .post(format!("{root_url}/api/auth/password"))
            .json(&json!({
                "current_password": TEST_AUTH_PASSWORD,
                "new_password": "updated-password-123"
            }))
            .send()
            .await
            .unwrap()
            .status();
        assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);

        let relogin_client = reqwest::Client::builder()
            .cookie_store(true)
            .build()
            .unwrap();
        let relogin_status = relogin_client
            .post(format!("{root_url}/api/auth/login"))
            .json(&json!({
                "email": "updated-owner@crank.local",
                "password": "updated-password-123"
            }))
            .send()
            .await
            .unwrap()
            .status();
        assert_eq!(relogin_status, reqwest::StatusCode::OK);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn rejects_multi_workspace_session_switching_in_community() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("session_workspace");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let root_url = base_url
            .as_ref()
            .split("/api/admin/workspaces/")
            .next()
            .unwrap()
            .to_owned();
        let client = authorized_client(&base_url).await;

        let create_workspace_status = client
            .post(format!("{root_url}/api/admin/workspaces"))
            .json(&json!({
                "slug": "growth-lab",
                "display_name": "Growth Lab",
                "settings": {}
            }))
            .send()
            .await
            .unwrap()
            .status();
        assert_eq!(
            create_workspace_status,
            reqwest::StatusCode::METHOD_NOT_ALLOWED
        );

        let switch_status = client
            .post(format!("{root_url}/api/auth/current-workspace"))
            .json(&json!({ "workspace_id": DEFAULT_WORKSPACE_ID }))
            .send()
            .await
            .unwrap()
            .status();
        assert_eq!(switch_status, reqwest::StatusCode::NOT_FOUND);

        let session = assert_success_json(
            client
                .get(format!("{root_url}/api/auth/session"))
                .send()
                .await
                .unwrap(),
        )
        .await;
        assert_eq!(session["current_workspace_id"], DEFAULT_WORKSPACE_ID);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn exposes_logs_and_usage_from_real_test_runs() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("observability");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_observability",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap().to_owned();

        client
            .post(format!("{base_url}/operations/{operation_id}/test-runs"))
            .json(&json!({
                "version": 1,
                "input": { "email": "user@example.com" }
            }))
            .send()
            .await
            .unwrap();

        let logs = client
            .get(format!("{base_url}/logs?period=7d"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let log_id = logs["items"][0]["log"]["id"].as_str().unwrap().to_owned();
        let log_detail = client
            .get(format!("{base_url}/logs/{log_id}"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let usage = client
            .get(format!("{base_url}/usage?period=7d"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_usage = client
            .get(format!(
                "{base_url}/usage/operations/{operation_id}?period=7d"
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();

        assert_eq!(logs["items"][0]["log"]["source"], "admin_test_run");
        assert_eq!(logs["items"][0]["operation_name"], "crm_observability");
        assert_eq!(log_detail["log"]["status"], "ok");
        assert_eq!(usage["summary"]["rollup"]["calls_total"], 1);
        assert_eq!(usage["summary"]["rollup"]["calls_ok"], 1);
        assert_eq!(
            usage["operations"][0]["operation_name"],
            "crm_observability"
        );
        assert_eq!(operation_usage["rollup"]["calls_total"], 1);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn preserves_request_id_for_test_run_invocations() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("observability_request_id");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_observability_request_id",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap().to_owned();

        let response = client
            .post(format!("{base_url}/operations/{operation_id}/test-runs"))
            .header("x-request-id", "req_test_123")
            .json(&json!({
                "version": 1,
                "input": { "email": "user@example.com" }
            }))
            .send()
            .await
            .unwrap();

        assert_eq!(
            response.headers()["x-request-id"].to_str().unwrap(),
            "req_test_123"
        );
        response.error_for_status().unwrap();

        let logs = client
            .get(format!("{base_url}/logs?period=7d"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();

        assert_eq!(logs["items"][0]["log"]["request_id"], "req_test_123");
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn generates_request_id_for_test_run_invocations() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("observability_generated_request_id");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_observability_generated_request_id",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap().to_owned();

        let response = client
            .post(format!("{base_url}/operations/{operation_id}/test-runs"))
            .json(&json!({
                "version": 1,
                "input": { "email": "user@example.com" }
            }))
            .send()
            .await
            .unwrap();
        let request_id = response
            .headers()
            .get("x-request-id")
            .unwrap()
            .to_str()
            .unwrap()
            .to_owned();

        assert!(!request_id.is_empty());
        response.error_for_status().unwrap();

        let logs = client
            .get(format!("{base_url}/logs?period=7d"))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();

        assert_eq!(logs["items"][0]["log"]["request_id"], request_id);
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn returns_structured_context_for_missing_operation_usage() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("missing_operation_usage");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_missing_usage",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap().to_owned();

        let response = client
            .get(format!(
                "{base_url}/usage/operations/{operation_id}?period=7d"
            ))
            .send()
            .await
            .unwrap();
        let status = response.status();
        let body = response.json::<Value>().await.unwrap();

        assert_eq!(status, reqwest::StatusCode::NOT_FOUND);
        assert_eq!(body["error"]["code"], "not_found");
        assert_eq!(
            body["error"]["message"],
            format!("usage for operation {operation_id} was not found")
        );
        assert_eq!(
            body["error"]["context"],
            json!({
                "operation_id": operation_id
            })
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn manages_auth_profiles_and_yaml_upsert() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("yaml");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;
        let secret = client
            .post(format!("{base_url}/secrets"))
            .json(&json!({
                "name": "crm-api-token",
                "kind": SecretKind::Token,
                "value": { "token": "super-secret-token" }
            }))
            .send()
            .await
            .unwrap();
        let secret = assert_success_json(secret).await;
        let secret_id = secret["id"].as_str().unwrap();

        let auth_profile = client
            .post(format!("{base_url}/auth-profiles"))
            .json(&json!({
                "name": "crm-header",
                "kind": "api_key_header",
                "config": {
                    "api_key_header": {
                        "header_name": "X-Api-Key",
                        "secret_id": secret_id
                    }
                }
            }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_export_target",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap();
        let yaml = client
            .get(format!("{base_url}/operations/{operation_id}/export"))
            .send()
            .await
            .unwrap()
            .text()
            .await
            .unwrap();
        let imported = client
            .post(format!("{base_url}/operations/import?mode=upsert"))
            .body(yaml)
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();

        assert_eq!(auth_profile["kind"], "api_key_header");
        assert_eq!(
            auth_profile["config"]["api_key_header"]["secret_id"],
            secret_id
        );
        assert_eq!(imported["operation_id"], operation_id);
        assert_eq!(imported["version"], 2);
        assert_eq!(imported["import_mode"], "upsert");
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn manages_workspace_secrets_without_exposing_plaintext() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("secrets");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/secrets"))
            .json(&json!({
                "name": "crm-api-token",
                "kind": SecretKind::Token,
                "value": { "token": "super-secret-token" }
            }))
            .send()
            .await
            .unwrap();
        let created = assert_success_json(created).await;
        let secret_id = created["id"].as_str().unwrap().to_owned();

        let listed = client
            .get(format!("{base_url}/secrets"))
            .send()
            .await
            .unwrap();
        let listed = assert_success_json(listed).await;

        let fetched = client
            .get(format!("{base_url}/secrets/{secret_id}"))
            .send()
            .await
            .unwrap();
        let fetched = assert_success_json(fetched).await;

        let rotated = client
            .post(format!("{base_url}/secrets/{secret_id}/rotate"))
            .json(&json!({
                "value": { "token": "rotated-token" }
            }))
            .send()
            .await
            .unwrap();
        let rotated = assert_success_json(rotated).await;

        let deleted = client
            .delete(format!("{base_url}/secrets/{secret_id}"))
            .send()
            .await
            .unwrap();
        let deleted = assert_success_json(deleted).await;

        let missing = client
            .get(format!("{base_url}/secrets/{secret_id}"))
            .send()
            .await
            .unwrap();
        let missing_status = missing.status();
        let missing = missing.json::<Value>().await.unwrap();

        assert_eq!(created["name"], "crm-api-token");
        assert_eq!(created["kind"], "token");
        assert_eq!(created["current_version"], 1);
        assert!(created.get("value").is_none());
        assert_eq!(listed["items"].as_array().unwrap().len(), 1);
        assert_eq!(fetched["id"], secret_id);
        assert!(fetched.get("value").is_none());
        assert_eq!(rotated["current_version"], 2);
        assert_eq!(deleted["ok"], true);
        assert_eq!(missing_status, reqwest::StatusCode::NOT_FOUND);
        assert_eq!(missing["error"]["code"], "not_found");
        assert_eq!(
            missing["error"]["context"],
            json!({
                "secret_id": secret_id
            })
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn returns_structured_context_for_missing_operation_version() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("missing_operation_version");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_missing_operation_version",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap().to_owned();

        let response = client
            .get(format!("{base_url}/operations/{operation_id}/versions/99"))
            .send()
            .await
            .unwrap();
        let status = response.status();
        let body = response.json::<Value>().await.unwrap();

        assert_eq!(status, reqwest::StatusCode::NOT_FOUND);
        assert_eq!(body["error"]["code"], "not_found");
        assert_eq!(
            body["error"]["message"],
            format!("operation version 99 for {operation_id} was not found")
        );
        assert_eq!(
            body["error"]["context"],
            json!({
                "operation_id": operation_id,
                "version": 99
            })
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn rejects_auth_profile_with_missing_secret() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("missing_secret_auth");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let response = client
            .post(format!("{base_url}/auth-profiles"))
            .json(&json!({
                "name": "crm-header",
                "kind": "api_key_header",
                "config": {
                    "api_key_header": {
                        "header_name": "X-Api-Key",
                        "secret_id": "secret_missing"
                    }
                }
            }))
            .send()
            .await
            .unwrap();
        let status = response.status();
        let body = response.json::<Value>().await.unwrap();

        assert_eq!(status, reqwest::StatusCode::NOT_FOUND);
        assert_eq!(body["error"]["code"], "not_found");
        assert_eq!(
            body["error"]["message"],
            "secret secret_missing was not found"
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn rejects_deleting_secret_referenced_by_auth_profile() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("secret_references");
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let secret = client
            .post(format!("{base_url}/secrets"))
            .json(&json!({
                "name": "crm-api-token",
                "kind": SecretKind::Token,
                "value": { "token": "super-secret-token" }
            }))
            .send()
            .await
            .unwrap();
        let secret = assert_success_json(secret).await;
        let secret_id = secret["id"].as_str().unwrap();

        let auth_profile = client
            .post(format!("{base_url}/auth-profiles"))
            .json(&json!({
                "name": "crm-header",
                "kind": "api_key_header",
                "config": {
                    "api_key_header": {
                        "header_name": "X-Api-Key",
                        "secret_id": secret_id
                    }
                }
            }))
            .send()
            .await
            .unwrap();
        let auth_profile = assert_success_json(auth_profile).await;
        let auth_profile_id = auth_profile["id"].as_str().unwrap();

        let response = client
            .delete(format!("{base_url}/secrets/{secret_id}"))
            .send()
            .await
            .unwrap();
        let status = response.status();
        let body = response.json::<Value>().await.unwrap();

        assert_eq!(status, reqwest::StatusCode::CONFLICT);
        assert_eq!(body["error"]["code"], "conflict");
        assert_eq!(
            body["error"]["message"],
            format!("secret {secret_id} is referenced by auth profile {auth_profile_id}")
        );
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn uploads_samples_and_generates_draft() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("draft");
        let upstream_base_url = spawn_upstream_server().await;
        let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
        let client = authorized_client(&base_url).await;

        let created = client
            .post(format!("{base_url}/operations"))
            .json(&test_operation_payload(
                &upstream_base_url,
                "crm_draft_target",
            ))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();
        let operation_id = created["operation_id"].as_str().unwrap();

        client
            .post(format!(
                "{base_url}/operations/{operation_id}/samples/input-json"
            ))
            .json(&json!({ "email": "user@example.com", "name": "Ada" }))
            .send()
            .await
            .unwrap();
        client
            .post(format!(
                "{base_url}/operations/{operation_id}/samples/output-json"
            ))
            .json(&json!({ "id": "lead_123", "status": "created" }))
            .send()
            .await
            .unwrap();

        let generated = client
            .post(format!(
                "{base_url}/operations/{operation_id}/drafts/generate"
            ))
            .json(&json!({
                "sources": ["input_json_sample", "output_json_sample"]
            }))
            .send()
            .await
            .unwrap()
            .json::<Value>()
            .await
            .unwrap();

        assert_eq!(generated["generated_draft"]["status"], "available");
        assert_eq!(
            generated["input_schema"]["fields"]["email"]["type"],
            "string"
        );
        assert_eq!(
            generated["output_mapping"]["rules"][0]["target"],
            "$.output.id"
        );
    }

    fn build_test_app(registry: PostgresRegistry, storage_root: std::path::PathBuf) -> Router {
        build_app(AppState {
            service: AdminService::new(
                registry,
                storage_root,
                test_auth_settings(),
                test_secret_crypto(),
            ),
            api_rate_limiter: crank_runtime::RequestRateLimiter::new(
                crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
            ),
        })
    }

    async fn spawn_upstream_server() -> String {
        let app = Router::new().route("/crm/leads", post(create_lead));
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();

        tokio::spawn(async move {
            axum::serve(listener, app).await.unwrap();
        });

        format!("http://{}", address)
    }

    async fn spawn_admin_api(app: Router) -> TestServer {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();

        let handle = tokio::spawn(async move {
            axum::serve(listener, app)
                .with_graceful_shutdown(async move {
                    let _ = shutdown_rx.await;
                })
                .await
                .unwrap();
        });

        TestServer {
            base_url: format!(
                "http://{}/api/admin/workspaces/{}",
                address, DEFAULT_WORKSPACE_ID
            ),
            shutdown: Some(shutdown_tx),
            handle: Some(handle),
        }
    }

    async fn authorized_client(workspace_base_url: impl AsRef<str>) -> reqwest::Client {
        let root_url = workspace_base_url
            .as_ref()
            .split("/api/admin/workspaces/")
            .next()
            .unwrap();
        let client = reqwest::Client::builder()
            .cookie_store(true)
            .build()
            .unwrap();

        client
            .post(format!("{root_url}/api/auth/login"))
            .json(&json!({
                "email": TEST_AUTH_EMAIL,
                "password": TEST_AUTH_PASSWORD,
            }))
            .send()
            .await
            .unwrap()
            .error_for_status()
            .unwrap();

        client
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn rejects_rapid_login_requests_with_429() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("login_rate_limit");
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let address = listener.local_addr().unwrap();
        let app = build_app(AppState {
            service: AdminService::new(
                registry,
                storage_root,
                test_auth_settings(),
                test_secret_crypto(),
            ),
            api_rate_limiter: crank_runtime::RequestRateLimiter::new(
                crank_runtime::RequestRateLimitConfig::new(1, 1).unwrap(),
            ),
        });
        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
        let handle = tokio::spawn(async move {
            axum::serve(listener, app)
                .with_graceful_shutdown(async move {
                    let _ = shutdown_rx.await;
                })
                .await
                .unwrap();
        });

        let client = reqwest::Client::new();
        let root_url = format!("http://{address}");
        let first_response = client
            .post(format!("{root_url}/api/auth/login"))
            .header("x-request-id", "req_admin_rate_01")
            .json(&json!({
                "email": TEST_AUTH_EMAIL,
                "password": TEST_AUTH_PASSWORD,
            }))
            .send()
            .await
            .unwrap();
        assert_eq!(first_response.status(), reqwest::StatusCode::OK);

        let second_response = client
            .post(format!("{root_url}/api/auth/login"))
            .header("x-request-id", "req_admin_rate_02")
            .json(&json!({
                "email": TEST_AUTH_EMAIL,
                "password": TEST_AUTH_PASSWORD,
            }))
            .send()
            .await
            .unwrap();

        assert_eq!(
            second_response.status(),
            reqwest::StatusCode::TOO_MANY_REQUESTS
        );
        assert_eq!(
            second_response.headers()["x-request-id"].to_str().unwrap(),
            "req_admin_rate_02"
        );
        let payload = second_response.json::<Value>().await.unwrap();
        assert_eq!(payload["error"]["code"], "rate_limited");
        let retry_after_ms = payload["error"]["context"]["retry_after_ms"]
            .as_u64()
            .unwrap();
        assert!((1..=1000).contains(&retry_after_ms));

        let _ = shutdown_tx.send(());
        let _ = handle.await;
    }

    #[tokio::test(flavor = "multi_thread")]
    #[serial]
    async fn login_uses_identity_provider_when_configured() {
        let registry = test_registry().await;
        let storage_root = test_storage_root("identity_provider_login");
        let service = AdminServiceBuilder::new(
            registry,
            storage_root,
            test_auth_settings(),
            test_secret_crypto(),
            crank_runtime::community_default().build(),
        )
        .with_identity_provider(Arc::new(RejectingIdentityProvider))
        .build();

        let error = match service
            .login(crate::service::LoginPayload {
                email: TEST_AUTH_EMAIL.to_owned(),
                password: TEST_AUTH_PASSWORD.to_owned(),
            })
            .await
        {
            Ok(_) => panic!("login should delegate to identity provider"),
            Err(error) => error,
        };
        assert_eq!(error.to_string(), "invalid email or password");
    }

    async fn assert_success_json(response: reqwest::Response) -> Value {
        let status = response.status();
        let body = response.text().await.unwrap();

        assert!(
            status.is_success(),
            "request failed with status {status}: {body}"
        );

        serde_json::from_str(&body).unwrap()
    }

    async fn create_lead(Json(payload): Json<Value>) -> Json<Value> {
        Json(json!({
            "id": "lead_123",
            "status": "created",
            "email": payload["email"]
        }))
    }

    async fn test_registry() -> PostgresRegistry {
        let database_url = env::var("TEST_DATABASE_URL")
            .unwrap_or_else(|_| "postgres://crank:crank@127.0.0.1:15432/crank".to_owned());
        let mut admin_connection = PgConnection::connect(&database_url).await.unwrap();
        let schema = format!(
            "test_admin_api_{}_{}",
            std::process::id(),
            SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        );

        admin_connection
            .execute(sqlx::query(&format!("create schema {schema}")))
            .await
            .unwrap();
        let registry =
            PostgresRegistry::connect(&format!("{database_url}?options=-csearch_path%3D{schema}"))
                .await
                .unwrap();
        let password_hash = hash_password(TEST_AUTH_PASSWORD, TEST_PASSWORD_PEPPER).unwrap();
        let user_id = registry
            .upsert_bootstrap_user(TEST_AUTH_EMAIL, "Test Owner", &password_hash)
            .await
            .unwrap();
        registry
            .ensure_membership(
                &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
                &user_id,
                MembershipRole::Owner,
            )
            .await
            .unwrap();

        registry
    }

    fn test_storage_root(name: &str) -> std::path::PathBuf {
        env::temp_dir().join(format!(
            "crank_admin_api_{name}_{}_{}",
            std::process::id(),
            SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ))
    }

    fn test_auth_settings() -> AuthSettings {
        AuthSettings {
            session_secret: TEST_SESSION_SECRET.to_owned(),
            password_pepper: TEST_PASSWORD_PEPPER.to_owned(),
            session_ttl_hours: 24,
            cookie_secure: false,
            bootstrap_admin: BootstrapAdminConfig {
                email: TEST_AUTH_EMAIL.to_owned(),
                password: TEST_AUTH_PASSWORD.to_owned(),
                display_name: "Test Owner".to_owned(),
            },
        }
    }

    fn test_secret_crypto() -> SecretCrypto {
        SecretCrypto::new(TEST_MASTER_KEY).unwrap()
    }

    fn test_operation_payload(base_url: &str, name: &str) -> OperationPayload {
        OperationPayload {
            name: name.to_owned(),
            display_name: "Create Lead".to_owned(),
            category: "sales".to_owned(),
            protocol: Protocol::Rest,
            security_level: OperationSecurityLevel::Standard,
            target: Target::Rest(RestTarget {
                base_url: base_url.to_owned(),
                method: HttpMethod::Post,
                path_template: "/crm/leads".to_owned(),
                static_headers: BTreeMap::new(),
            }),
            input_schema: object_schema("email"),
            output_schema: object_schema("id"),
            input_mapping: MappingSet {
                rules: vec![MappingRule {
                    source: "$.mcp.email".to_owned(),
                    target: "$.request.body.email".to_owned(),
                    required: true,
                    default_value: None,
                    transform: None,
                    condition: None,
                    notes: None,
                }],
            },
            output_mapping: MappingSet {
                rules: vec![MappingRule {
                    source: "$.response.body.id".to_owned(),
                    target: "$.output.id".to_owned(),
                    required: true,
                    default_value: None,
                    transform: None,
                    condition: None,
                    notes: None,
                }],
            },
            execution_config: ExecutionConfig {
                timeout_ms: 1_000,
                retry_policy: None,
                response_cache: None,
                idempotency: None,
                auth_profile_ref: None,
                headers: BTreeMap::new(),
            },
            tool_description: ToolDescription {
                title: "Create Lead".to_owned(),
                description: "Creates a CRM lead".to_owned(),
                tags: vec!["crm".to_owned()],
                examples: Vec::new(),
            },
            wizard_state: None,
        }
    }

    fn object_schema(field_name: &str) -> Schema {
        Schema {
            kind: SchemaKind::Object,
            description: None,
            required: true,
            nullable: false,
            default_value: None,
            fields: BTreeMap::from([(
                field_name.to_owned(),
                Schema {
                    kind: SchemaKind::String,
                    description: None,
                    required: true,
                    nullable: false,
                    default_value: None,
                    fields: BTreeMap::new(),
                    items: None,
                    enum_values: Vec::new(),
                    variants: Vec::new(),
                },
            )]),
            items: None,
            enum_values: Vec::new(),
            variants: Vec::new(),
        }
    }
}