исправить: упорядочить запуск Community UI

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+jobs = 2

.env.example

@@ -29,7 +29,30 @@ CRANK_RUNTIME_MAX_CONCURRENT_JOBS=16
 CRANK_OUTBOUND_ALLOWED_HOSTS=
 CRANK_OUTBOUND_DENIED_HOSTS=
 CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
+CRANK_ENVIRONMENT=development
 CRANK_LOG_LEVEL=info
+# Пустое значение отключает канал критических ошибок.
+CRANK_SENTRY_DSN=
+# Prometheus endpoints use separate listeners and stay on loopback by default.
+CRANK_METRICS_ENABLED=true
+CRANK_ADMIN_METRICS_BIND=127.0.0.1:9464
+CRANK_MCP_METRICS_BIND=127.0.0.1:9465
+# Required when either metrics listener uses a non-loopback address.
+CRANK_METRICS_BEARER_TOKEN=
+CRANK_INVOCATION_LOG_RETENTION_DAYS=30
+# Пустой endpoint полностью отключает экспорт трасс.
+OTEL_EXPORTER_OTLP_ENDPOINT=
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=
+OTEL_EXPORTER_OTLP_TIMEOUT=10000
+OTEL_EXPORTER_OTLP_TRACES_TIMEOUT=
+OTEL_EXPORTER_OTLP_HEADERS=
+OTEL_EXPORTER_OTLP_TRACES_HEADERS=
+OTEL_BSP_MAX_QUEUE_SIZE=2048
+OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512
+OTEL_BSP_SCHEDULE_DELAY=5000
+OTEL_BSP_EXPORT_TIMEOUT=30000
 CRANK_MASTER_KEY=change-me-master-key
 CRANK_SESSION_SECRET=change-me-session-secret
 CRANK_PASSWORD_PEPPER=change-me-password-pepper

.gitea/workflows/ci.yml

@@ -55,6 +55,9 @@ jobs:
           docker --version
           docker info
 
+      - name: Install dependency policy tool
+        run: cargo install cargo-deny --version 0.20.2 --locked
+
       - name: Run tooling unit tests
         run: python3 -m unittest discover -s tests/unit
 
@@ -67,6 +70,9 @@ jobs:
       - name: Check Rust code health
         run: scripts/check-rust-code-health.sh
 
+      - name: Check dependency licenses and advisories
+        run: cargo deny --locked check advisories bans licenses sources
+
       - name: Check Rust boundaries
         run: scripts/check-rust-boundaries.sh
 
@@ -95,6 +101,10 @@ jobs:
         working-directory: apps/ui
         run: npm ci
 
+      - name: Audit UI dependencies
+        working-directory: apps/ui
+        run: npm audit --audit-level=high
+
       - name: Build UI bundle
         working-directory: apps/ui
         run: npm run build
@@ -179,9 +189,11 @@ jobs:
           find .tmp/ui-e2e/logs -maxdepth 1 -type f -print -exec sed -n '1,220p' {} \; || true
 
   deployment:
-    name: Deployment Manifests
+    name: Community Image Smoke
     runs-on: ubuntu-latest
-    needs: ui
+    needs:
+      - rust
+      - ui
 
     steps:
       - name: Checkout
@@ -190,6 +202,65 @@ jobs:
       - name: Validate Community deployment manifest
         run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
 
+      - name: Build Community images
+        run: |
+          docker build -f apps/admin-api/Dockerfile -t crank/admin-api:ci .
+          docker build -f apps/mcp-server/Dockerfile -t crank/mcp-server:ci .
+          docker build -f apps/ui/Dockerfile -t crank/ui:ci .
+
+      - name: Start Community image stack
+        run: |
+          mkdir -p .tmp
+          cat > .tmp/community-smoke.env <<'EOF'
+          COMPOSE_PROJECT_NAME=crank-ci-smoke-${{ github.run_id }}-${{ github.run_attempt }}
+          POSTGRES_HOST=postgres
+          POSTGRES_PORT=5432
+          POSTGRES_PUBLISH_PORT=0
+          POSTGRES_DB=crank
+          POSTGRES_USER=crank
+          POSTGRES_PASSWORD=crank-ci-password
+          CRANK_ADMIN_API_IMAGE=crank/admin-api:ci
+          CRANK_MCP_SERVER_IMAGE=crank/mcp-server:ci
+          CRANK_UI_IMAGE=crank/ui:ci
+          CRANK_MASTER_KEY=0000000000000000000000000000000000000000000000000000000000000000
+          CRANK_SESSION_SECRET=ci-session-secret
+          CRANK_PASSWORD_PEPPER=ci-password-pepper
+          CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
+          CRANK_BOOTSTRAP_ADMIN_PASSWORD=ci-admin-password
+          CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner
+          CRANK_BASE_URL=http://127.0.0.1
+          CRANK_PUBLISH_BIND=127.0.0.1
+          CRANK_ADMIN_PUBLISH_PORT=0
+          CRANK_MCP_PUBLISH_PORT=0
+          CRANK_UI_PUBLISH_PORT=0
+          CRANK_DEMO_SEED=true
+          EOF
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db up -d --wait
+          ui_address="$(docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env port ui 3000)"
+          printf 'http://%s\n' "$ui_address" > .tmp/community-smoke.url
+
+      - name: Run authenticated Community image smoke
+        env:
+          CRANK_STAGING_ADMIN_EMAIL: owner@crank.test
+          CRANK_STAGING_ADMIN_PASSWORD: ci-admin-password
+        run: scripts/authenticated-product-smoke.sh "$(cat .tmp/community-smoke.url)"
+
+      - name: Show Community image logs
+        if: failure()
+        run: |
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db ps || true
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db logs --no-color || true
+
+      - name: Stop Community image stack
+        if: always()
+        run: |
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db down -v --remove-orphans || true
+
   deploy:
     name: Deploy
     runs-on: ubuntu-latest
@@ -245,6 +316,10 @@ jobs:
             -t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \
             -t '${{ env.UI_IMAGE }}:main' \
             .
+          scripts/scan-images.sh \
+            '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:main'
           docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}'
@@ -274,9 +349,14 @@ jobs:
         run: |
           . "$OPENBAO_ENV_FILE"
           ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
-            "mkdir -p '$DEPLOY_PATH'"
+            "mkdir -p '$DEPLOY_PATH' && \
+             if [ -f '$DEPLOY_PATH/docker-compose.yml' ]; then \
+               cp '$DEPLOY_PATH/docker-compose.yml' '$DEPLOY_PATH/docker-compose.previous.yml'; \
+             fi"
           rsync -az -e "ssh -p $DEPLOY_PORT" deploy/community/docker-compose.yml \
             "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/docker-compose.yml"
+          rsync -az -e "ssh -p $DEPLOY_PORT" scripts/deploy-community.sh \
+            "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/deploy-community.sh"
 
       - name: Write environment file
         run: |
@@ -292,6 +372,11 @@ jobs:
           append_if_set POSTGRES_USER "$POSTGRES_USER"
           append_if_set POSTGRES_PASSWORD "$POSTGRES_PASSWORD"
           append_if_set POSTGRES_HOST "$POSTGRES_HOST"
+          append_if_set POSTGRES_MAX_CONNECTIONS "${POSTGRES_MAX_CONNECTIONS:-}"
+          append_if_set POSTGRES_MIN_CONNECTIONS "${POSTGRES_MIN_CONNECTIONS:-}"
+          append_if_set POSTGRES_ACQUIRE_TIMEOUT_MS "${POSTGRES_ACQUIRE_TIMEOUT_MS:-}"
+          append_if_set POSTGRES_IDLE_TIMEOUT_MS "${POSTGRES_IDLE_TIMEOUT_MS:-}"
+          append_if_set POSTGRES_MAX_LIFETIME_MS "${POSTGRES_MAX_LIFETIME_MS:-}"
           if [ -n "${POSTGRES_PORT:-}" ]; then
             append_if_set POSTGRES_PORT "$POSTGRES_PORT"
           elif [ -n "${PGBOUNCER_PORT:-}" ]; then
@@ -302,10 +387,35 @@ jobs:
           append_if_set CRANK_ADMIN_BIND "$CRANK_ADMIN_BIND"
           append_if_set CRANK_MCP_BIND "$CRANK_MCP_BIND"
           append_if_set CRANK_MCP_REFRESH_MS "$CRANK_MCP_REFRESH_MS"
+          append_if_set CRANK_ADMIN_RATE_LIMIT_RPS "${CRANK_ADMIN_RATE_LIMIT_RPS:-}"
+          append_if_set CRANK_ADMIN_RATE_LIMIT_BURST "${CRANK_ADMIN_RATE_LIMIT_BURST:-}"
+          append_if_set CRANK_MCP_RATE_LIMIT_RPS "${CRANK_MCP_RATE_LIMIT_RPS:-}"
+          append_if_set CRANK_MCP_RATE_LIMIT_BURST "${CRANK_MCP_RATE_LIMIT_BURST:-}"
+          append_if_set CRANK_RUNTIME_MAX_CONCURRENT_UNARY "${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-}"
+          append_if_set CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS "${CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS:-}"
           append_if_set CRANK_OUTBOUND_ALLOWED_HOSTS "${CRANK_OUTBOUND_ALLOWED_HOSTS:-}"
           append_if_set CRANK_OUTBOUND_DENIED_HOSTS "${CRANK_OUTBOUND_DENIED_HOSTS:-}"
           append_if_set CRANK_OUTBOUND_MAX_RESPONSE_BYTES "${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-}"
+          append_if_set CRANK_ENVIRONMENT "${CRANK_ENVIRONMENT:-production}"
           append_if_set CRANK_LOG_LEVEL "$CRANK_LOG_LEVEL"
+          append_if_set CRANK_SENTRY_DSN "${CRANK_SENTRY_DSN:-}"
+          append_if_set CRANK_METRICS_ENABLED "${CRANK_METRICS_ENABLED:-}"
+          append_if_set CRANK_ADMIN_METRICS_BIND "${CRANK_ADMIN_METRICS_BIND:-}"
+          append_if_set CRANK_MCP_METRICS_BIND "${CRANK_MCP_METRICS_BIND:-}"
+          append_if_set CRANK_METRICS_BEARER_TOKEN "${CRANK_METRICS_BEARER_TOKEN:-}"
+          append_if_set CRANK_INVOCATION_LOG_RETENTION_DAYS "${CRANK_INVOCATION_LOG_RETENTION_DAYS:-}"
+          append_if_set OTEL_EXPORTER_OTLP_ENDPOINT "${OTEL_EXPORTER_OTLP_ENDPOINT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_PROTOCOL "${OTEL_EXPORTER_OTLP_PROTOCOL:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_PROTOCOL "${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TIMEOUT "${OTEL_EXPORTER_OTLP_TIMEOUT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_TIMEOUT "${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_HEADERS "${OTEL_EXPORTER_OTLP_HEADERS:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_HEADERS "${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-}"
+          append_if_set OTEL_BSP_MAX_QUEUE_SIZE "${OTEL_BSP_MAX_QUEUE_SIZE:-}"
+          append_if_set OTEL_BSP_MAX_EXPORT_BATCH_SIZE "${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-}"
+          append_if_set OTEL_BSP_SCHEDULE_DELAY "${OTEL_BSP_SCHEDULE_DELAY:-}"
+          append_if_set OTEL_BSP_EXPORT_TIMEOUT "${OTEL_BSP_EXPORT_TIMEOUT:-}"
           append_if_set CRANK_MASTER_KEY "$CRANK_MASTER_KEY"
           append_if_set CRANK_BASE_URL "$CRANK_BASE_URL"
           append_if_set CRANK_CACHE_BACKEND "$CRANK_CACHE_BACKEND"
@@ -325,7 +435,10 @@ jobs:
             printf 'CRANK_UI_IMAGE=%s:%s\n' '${{ env.UI_IMAGE }}' '${{ env.IMAGE_TAG }}'
           } >> "$tmp_env"
           cat "$tmp_env" | ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
-            "mkdir -p '$DEPLOY_PATH' && cat > '$DEPLOY_PATH/.env'"
+            "mkdir -p '$DEPLOY_PATH' && \
+             if [ -f '$DEPLOY_PATH/.env' ]; then \
+               cp '$DEPLOY_PATH/.env' '$DEPLOY_PATH/.env.previous'; \
+             fi && cat > '$DEPLOY_PATH/.env'"
           rm -f "$tmp_env"
 
       - name: Validate required environment variables
@@ -359,48 +472,12 @@ jobs:
       - name: Deploy with Docker Compose
         run: |
           . "$OPENBAO_ENV_FILE"
-          ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" "
-            set -e
-            cd '$DEPLOY_PATH'
-            compose_profiles=''
-            cache_backend=\$(grep -E '^CRANK_CACHE_BACKEND=' .env | tail -n1 | cut -d= -f2- || true)
-            if [ \"\$cache_backend\" = 'valkey' ] || [ \"\$cache_backend\" = 'redis' ]; then
-              compose_profiles='--profile cache'
-            fi
-            echo '$DEPLOY_REGISTRY_TOKEN' | docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin
-            docker compose \$compose_profiles config -q
-            docker compose \$compose_profiles pull
-            docker compose \$compose_profiles down --remove-orphans
-            for container in \
-              crank-ui-1 \
-              crank-admin-api-1 \
-              crank-mcp-server-1 \
-              crank-postgres-1 \
-              crank-valkey-1 \
-              crank-community-ui-1 \
-              crank-community-admin-api-1 \
-              crank-community-mcp-server-1 \
-              crank-community-postgres-1 \
-              crank-community-valkey-1; do
-              if docker ps -a --format '{{.Names}}' | grep -Fx \"\$container\" >/dev/null; then
-                docker rm -f \"\$container\"
-              fi
-            done
-            echo 'Docker containers before freeing required ports:'
-            docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Ports}}'
-            for port in 3000 3001 3002; do
-              container_ids=\$(docker ps -aq --filter \"publish=\$port\")
-              if [ -n \"\$container_ids\" ]; then
-                echo \"Removing containers publishing port \$port\"
-                docker inspect --format '{{.Name}} {{json .NetworkSettings.Ports}}' \$container_ids || true
-                docker rm -f \$container_ids
-              fi
-            done
-            if command -v ss >/dev/null 2>&1; then
-              ss -ltnp '( sport = :3000 or sport = :3001 or sport = :3002 )' || true
-            fi
-            docker compose \$compose_profiles up -d --remove-orphans
-          "
+          printf '%s' "$DEPLOY_REGISTRY_TOKEN" | ssh -p "$DEPLOY_PORT" \
+            "$DEPLOY_USER@$DEPLOY_HOST" \
+            "docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin"
+          ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
+            "chmod 700 '$DEPLOY_PATH/deploy-community.sh' && \
+             '$DEPLOY_PATH/deploy-community.sh' '$DEPLOY_PATH'"
 
       - name: Verify health endpoints
         run: |
@@ -410,8 +487,8 @@ jobs:
             cd '$DEPLOY_PATH'
             for attempt in \$(seq 1 30); do
               if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
-                && curl --fail --silent http://127.0.0.1:3001/health >/dev/null \
-                && curl --fail --silent http://127.0.0.1:3002/health >/dev/null; then
+                && curl --fail --silent http://127.0.0.1:3001/ready >/dev/null \
+                && curl --fail --silent http://127.0.0.1:3002/ready >/dev/null; then
                 exit 0
               fi
               sleep 2

.gitea/workflows/release.yml

@@ -49,6 +49,16 @@ jobs:
           command -v bao
           bao version
 
+      - name: Install dependency policy tool
+        run: cargo install cargo-deny --version 0.20.2 --locked
+
+      - name: Run release quality gates
+        run: |
+          cargo fmt --all --check
+          cargo clippy --workspace --all-targets --all-features -- -D warnings
+          cargo test --workspace --all-targets
+          cargo deny --locked check advisories bans licenses sources
+
       - name: Build release binaries
         run: cargo build --release -p admin-api -p mcp-server
 
@@ -56,10 +66,25 @@ jobs:
         working-directory: apps/ui
         run: npm ci
 
+      - name: Audit UI dependencies
+        working-directory: apps/ui
+        run: npm audit --audit-level=high
+
       - name: Build UI dist
         working-directory: apps/ui
         run: npm run build
 
+      - name: Install Playwright browser
+        working-directory: apps/ui
+        run: npx playwright install --with-deps chromium
+
+      - name: Run release end-to-end tests
+        working-directory: apps/ui
+        run: npm run e2e
+
+      - name: Validate deployment manifest
+        run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
+
       - name: Package release artifacts
         run: |
           mkdir -p dist/release
@@ -105,6 +130,10 @@ jobs:
           docker build -f apps/ui/Dockerfile \
             -t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \
             -t '${{ env.UI_IMAGE }}:latest' .
+          scripts/scan-images.sh \
+            '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:latest'
           docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}'

Cargo.toml

@@ -8,9 +8,11 @@ members = [
     "crates/crank-import",
     "crates/crank-schema",
     "crates/crank-mapping",
+    "crates/crank-observability",
     "crates/crank-registry",
     "crates/crank-runtime",
     "crates/crank-test-support",
+    "crates/crank-trace",
     "crates/crank-adapter-rest",
 ]
 resolver = "3"
@@ -20,6 +22,7 @@ edition = "2024"
 license = "AGPL-3.0-only"
 rust-version = "1.96"
 version = "0.3.1"
+publish = false
 
 [workspace.dependencies]
 aes-gcm = "0.10"
@@ -28,18 +31,43 @@ axum = "0.8"
 axum-extra = { version = "0.12", features = ["cookie"] }
 base64 = "0.22"
 hkdf = "0.12"
+metrics = "0.24.6"
+metrics-exporter-prometheus = { version = "0.18.3", default-features = false }
+opentelemetry = { version = "0.32.0", default-features = false, features = ["trace"] }
+opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["http-proto", "reqwest-blocking-client", "reqwest-rustls", "trace"] }
+opentelemetry-proto = { version = "0.32.0", default-features = false, features = ["gen-tonic-messages", "trace"] }
+opentelemetry_sdk = { version = "0.32.1", default-features = false, features = ["trace"] }
+percent-encoding = "2"
+prost = "0.14"
 rand = "0.10"
 reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "rustls-tls"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
+sentry = { version = "0.49.0", default-features = false, features = ["backtrace", "panic", "rustls", "ureq"] }
 sha2 = "0.10"
 sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "macros", "json", "time", "uuid"] }
+subtle = "2.6"
 thiserror = "2"
 time = { version = "0.3.53", features = ["formatting", "parsing", "serde"] }
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+tower = "0.5"
 tracing = "0.1"
+tracing-opentelemetry = { version = "0.33.0", default-features = false }
 tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+url = "2"
 uuid = { version = "1", features = ["serde", "v7"] }
 testcontainers = { version = "0.27", features = ["blocking"] }
 testcontainers-modules = { version = "0.15", features = ["postgres", "blocking"] }
+
+[profile.dev]
+debug = "line-tables-only"
+
+[profile.dev.package."*"]
+debug = false
+
+[profile.test]
+debug = "line-tables-only"
+
+[profile.test.package."*"]
+debug = false

apps/admin-api/Cargo.toml

@@ -3,6 +3,7 @@ name = "admin-api"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [[bin]]
@@ -18,9 +19,12 @@ crank-community-auth = { path = "../../crates/crank-community-auth" }
 crank-core = { path = "../../crates/crank-core" }
 crank-import = { path = "../../crates/crank-import" }
 crank-mapping = { path = "../../crates/crank-mapping" }
+crank-observability = { path = "../../crates/crank-observability" }
 crank-registry = { path = "../../crates/crank-registry" }
 crank-runtime = { path = "../../crates/crank-runtime" }
 crank-schema = { path = "../../crates/crank-schema" }
+crank-trace = { path = "../../crates/crank-trace" }
+metrics.workspace = true
 rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -37,5 +41,9 @@ uuid.workspace = true
 [dev-dependencies]
 async-trait = "0.1"
 crank-test-support = { path = "../../crates/crank-test-support" }
+opentelemetry.workspace = true
+opentelemetry_sdk.workspace = true
 reqwest.workspace = true
 serial_test = "3"
+tower.workspace = true
+tracing-opentelemetry.workspace = true

apps/admin-api/src/app.rs

@@ -12,7 +12,7 @@ use crate::{
         agents::{
             archive_agent, create_agent, create_agent_platform_api_key, delete_agent,
             delete_agent_platform_api_key, get_agent, get_agent_version,
-            list_agent_platform_api_keys, list_agents, publish_agent,
+            list_agent_platform_api_keys, list_agents, preview_tool_search, publish_agent,
             revoke_agent_platform_api_key, save_agent_bindings, unpublish_agent, update_agent,
         },
         auth::{change_password, get_profile, get_session, login, logout, update_profile},
@@ -83,6 +83,7 @@ pub fn build_app(state: AppState) -> Router {
         )
         .route("/operations/{operation_id}/export", get(export_operation))
         .route("/agents", get(list_agents).post(create_agent))
+        .route("/agents/tool-search/preview", post(preview_tool_search))
         .route(
             "/agents/{agent_id}",
             get(get_agent).patch(update_agent).delete(delete_agent),
@@ -165,6 +166,7 @@ pub fn build_app(state: AppState) -> Router {
 
     Router::new()
         .route("/health", get(crate::routes::health))
+        .route("/ready", get(crate::routes::readiness))
         .nest(
             "/api/auth",
             Router::new()
@@ -177,6 +179,9 @@ pub fn build_app(state: AppState) -> Router {
             apply_api_rate_limit,
         ))
         .layer(middleware::from_fn(apply_request_context))
+        .layer(middleware::from_fn(
+            crank_observability::record_http_request,
+        ))
         .with_state(state)
 }
 

apps/admin-api/src/dto.rs

@@ -2,7 +2,7 @@ use crank_core::{
     AgentId, AgentStatus, ApprovalRequestStatus, AuthConfig, AuthKind, ExecutionMode, ExportMode,
     GeneratedDraft, InvocationLevel, InvocationSource, InvocationStatus, OperationSecurityLevel,
     OperationStatus, PlatformApiKeyKind, PlatformApiKeyScope, Protocol, SecretKind, Target,
-    UsagePeriod, WizardState, WorkspaceId, WorkspaceStatus,
+    ToolSelectionPolicy, UsagePeriod, WizardState, WorkspaceId, WorkspaceStatus,
 };
 use crank_mapping::MappingSet;
 use crank_registry::{
@@ -144,7 +144,7 @@ pub struct AgentPayload {
     #[serde(default)]
     pub instructions: Value,
     #[serde(default)]
-    pub tool_selection_policy: Value,
+    pub tool_selection_policy: ToolSelectionPolicy,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -165,6 +165,43 @@ pub struct AgentBindingPayload {
     pub enabled: bool,
 }
 
+#[derive(Clone, Debug, Deserialize)]
+pub struct ToolSearchPreviewPayload {
+    pub query: String,
+    #[serde(default)]
+    pub group_ids: Vec<String>,
+    pub bindings: Vec<AgentBindingPayload>,
+    pub tool_selection_policy: ToolSelectionPolicy,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(untagged)]
+pub enum AgentCatalogPayload {
+    Bindings(Vec<AgentBindingPayload>),
+    Config {
+        bindings: Vec<AgentBindingPayload>,
+        tool_selection_policy: ToolSelectionPolicy,
+    },
+}
+
+impl AgentCatalogPayload {
+    pub fn into_parts(self) -> (Vec<AgentBindingPayload>, Option<ToolSelectionPolicy>) {
+        match self {
+            Self::Bindings(bindings) => (bindings, None),
+            Self::Config {
+                bindings,
+                tool_selection_policy,
+            } => (bindings, Some(tool_selection_policy)),
+        }
+    }
+}
+
+impl From<Vec<AgentBindingPayload>> for AgentCatalogPayload {
+    fn from(bindings: Vec<AgentBindingPayload>) -> Self {
+        Self::Bindings(bindings)
+    }
+}
+
 #[derive(Clone, Debug, Serialize)]
 pub struct CreatedAgentResponse {
     pub agent_id: String,
@@ -196,6 +233,7 @@ pub struct AgentSummaryView {
     pub published_at: Option<String>,
     pub operation_count: usize,
     pub operation_ids: Vec<String>,
+    pub tool_selection_policy: ToolSelectionPolicy,
     pub key_count: usize,
     pub calls_today: u64,
     pub mcp_endpoint: String,
@@ -231,7 +269,12 @@ pub struct CreatedPlatformApiKeyResponse {
 }
 
 #[derive(Clone, Debug, Serialize)]
-pub struct WorkspaceExportResponse {
+pub struct WorkspaceCatalogSnapshotResponse {
+    pub kind: String,
+    pub format_version: String,
+    pub restorable: bool,
+    pub included: Vec<String>,
+    pub excluded: Vec<String>,
     pub workspace: WorkspaceRecord,
     pub operations: Vec<OperationSummaryView>,
     pub agents: Vec<AgentSummaryView>,

apps/admin-api/src/error.rs

@@ -137,16 +137,24 @@ impl ApiError {
 impl IntoResponse for ApiError {
     fn into_response(self) -> Response {
         match &self {
-            Self::Internal { message, .. } => {
-                error!(error_code = self.code(), error_message = %message)
+            Self::Internal { .. } => {
+                error!(
+                    name: "admin.response.internal_error",
+                    error_code = self.code(),
+                    "internal API error response"
+                )
             }
-            Self::Unauthorized { message, .. }
-            | Self::Forbidden { message, .. }
-            | Self::Validation { message, .. }
-            | Self::NotFound { message, .. }
-            | Self::Conflict { message, .. }
-            | Self::RateLimited { message, .. } => {
-                warn!(error_code = self.code(), error_message = %message)
+            Self::Unauthorized { .. }
+            | Self::Forbidden { .. }
+            | Self::Validation { .. }
+            | Self::NotFound { .. }
+            | Self::Conflict { .. }
+            | Self::RateLimited { .. } => {
+                warn!(
+                    name: "admin.response.rejected",
+                    error_code = self.code(),
+                    "API request rejected"
+                )
             }
         }
 
@@ -351,6 +359,10 @@ impl From<RegistryError> for ApiError {
                 format!("import job {job_id} was not found"),
                 json!({ "job_id": job_id }),
             ),
+            RegistryError::ImportJobAlreadyApplied { job_id } => Self::conflict_with_context(
+                format!("import job {job_id} was already applied with different parameters"),
+                json!({ "job_id": job_id }),
+            ),
             RegistryError::Storage(_) | RegistryError::Serialization(_) => {
                 Self::internal(value.to_string())
             }
@@ -407,6 +419,10 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "runtime_confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "runtime_confirmation_error",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "runtime_confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "runtime_idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "runtime_idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "runtime_idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "runtime_idempotency_outcome_unknown",
         RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error",
         RuntimeError::MissingAuthProfile { .. } => "runtime_auth_profile_error",
         RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
@@ -434,7 +450,11 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
             "safety_class": safety_class,
         })),
         RuntimeError::InvalidConfirmationToken { operation_id }
-        | RuntimeError::ConfirmationStoreUnavailable { operation_id } => Some(json!({
+        | RuntimeError::ConfirmationStoreUnavailable { operation_id }
+        | RuntimeError::IdempotencyStoreUnavailable { operation_id }
+        | RuntimeError::IdempotencyInProgress { operation_id }
+        | RuntimeError::IdempotencyConflict { operation_id }
+        | RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({
             "operation_id": operation_id,
         })),
         RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({

apps/admin-api/src/main.rs

@@ -1,4 +1,4 @@
-use std::{env, net::SocketAddr, path::PathBuf};
+use std::{env, net::SocketAddr, path::PathBuf, time::Duration};
 
 use admin_api::{
     app::build_app,
@@ -7,22 +7,50 @@ use admin_api::{
     state::AppState,
 };
 use crank_community_auth::PasswordIdentityProvider;
+use crank_observability::{
+    CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
+    capture_critical_error,
+};
 use crank_registry::{PostgresPoolConfig, PostgresRegistry};
 use crank_runtime::{
     RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
     RuntimeLimits, SecretCrypto,
 };
-use sqlx::postgres::PgConnectOptions;
+use sqlx::{PgPool, postgres::PgConnectOptions};
 use tokio::net::TcpListener;
-use tracing::info;
+use tracing::{info, warn};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    tracing_subscriber::fmt()
-        .with_env_filter(
-            env::var("CRANK_LOG_LEVEL").unwrap_or_else(|_| "admin_api=info,tower_http=info".into()),
-        )
-        .init();
+    let observability = crank_observability::init(ObservabilityConfig::from_env(
+        "admin-api",
+        env!("CARGO_PKG_VERSION"),
+        "admin_api=info,tower_http=info",
+    )?)?;
+    let mut startup_completed = false;
+    let result = run(&observability, &mut startup_completed).await;
+    if result.is_err() {
+        capture_critical_error(if startup_completed {
+            CriticalErrorCategory::Internal
+        } else {
+            CriticalErrorCategory::Startup
+        });
+    }
+    result
+}
+
+async fn run(
+    observability: &ObservabilityLifecycle,
+    startup_completed: &mut bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let metrics_config =
+        MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?;
+    let metrics_enabled = metrics_config.enabled();
+    let metrics_server = if metrics_config.enabled() {
+        Some(observability.metrics_surface(metrics_config).bind().await?)
+    } else {
+        None
+    };
 
     let storage_root = PathBuf::from(
         env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()),
@@ -36,6 +64,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         pool_config,
     )
     .await?;
+    if metrics_enabled {
+        spawn_postgres_pool_metrics(registry.pool().clone());
+    }
     let auth_settings = AuthSettings {
         session_secret: env::var("CRANK_SESSION_SECRET")?,
         password_pepper: env::var("CRANK_PASSWORD_PEPPER")?,
@@ -74,10 +105,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     .with_outbound_http_policy(outbound_http_policy)
     .with_identity_provider(std::sync::Arc::new(identity_provider))
     .build();
+    let invocation_log_retention_days =
+        positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
     service.bootstrap_admin_user().await?;
     if env_flag("CRANK_DEMO_SEED") {
         service.seed_demo_assets().await?;
     }
+    spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days);
     let state = AppState {
         service,
         api_rate_limiter: if cache_config.backend.is_external() {
@@ -92,6 +126,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
 
     info!(
+        name: "admin.postgres_pool.configured",
         runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
         admin_rate_limit_rps = api_rate_limit.requests_per_second,
         admin_rate_limit_burst = api_rate_limit.burst,
@@ -101,15 +136,76 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         acquire_timeout_ms = pool_config.acquire_timeout_ms,
         idle_timeout_ms = pool_config.idle_timeout_ms,
         max_lifetime_ms = pool_config.max_lifetime_ms,
+        invocation_log_retention_days,
         "postgres pool configured"
     );
-    info!("admin-api listening on {}", socket_addr);
+    info!(
+        name: "admin.server.listening",
+        bind_address = %socket_addr,
+        "admin-api listening"
+    );
+    *startup_completed = true;
 
+    if let Some(metrics_server) = metrics_server {
+        tokio::select! {
+            result = axum::serve(listener, make_service) => result?,
+            result = metrics_server.serve() => result?,
+        }
+    } else {
         axum::serve(listener, make_service).await?;
+    }
 
     Ok(())
 }
 
+fn positive_i64_from_env(
+    name: &'static str,
+    default: i64,
+) -> Result<i64, Box<dyn std::error::Error>> {
+    let value = match env::var(name) {
+        Ok(raw) => raw.parse::<i64>()?,
+        Err(env::VarError::NotPresent) => default,
+        Err(error) => return Err(error.into()),
+    };
+    if value <= 0 {
+        return Err(format!("{name} must be greater than zero").into());
+    }
+    Ok(value)
+}
+
+fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(60 * 60));
+        loop {
+            interval.tick().await;
+            let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days);
+            match service.cleanup_invocation_logs_before(cutoff).await {
+                Ok(removed) if removed > 0 => info!(
+                    name: "admin.invocation_log_cleanup.completed",
+                    removed,
+                    "expired invocation logs removed"
+                ),
+                Ok(_) => {}
+                Err(_) => warn!(
+                    name: "admin.invocation_log_cleanup.failed",
+                    error_category = "registry_cleanup",
+                    "failed to remove expired invocation logs"
+                ),
+            }
+        }
+    });
+}
+
+fn spawn_postgres_pool_metrics(pool: PgPool) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(5));
+        loop {
+            interval.tick().await;
+            crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
+        }
+    });
+}
+
 fn env_flag(name: &str) -> bool {
     matches!(
         env::var(name)

apps/admin-api/src/rate_limit.rs

@@ -6,7 +6,7 @@ use axum::{
     middleware::Next,
     response::Response,
 };
-use crank_runtime::RateLimitRejection;
+use crank_runtime::{RateLimitCheckError, RateLimitRejection};
 
 use crate::{error::ApiError, state::AppState};
 
@@ -25,11 +25,16 @@ pub async fn apply_api_rate_limit(
         peer_ip,
         state.trust_forwarded_headers,
     );
-    if let Err(rejection) = state.api_rate_limiter.check(&key).await {
-        return Err(ApiError::rate_limited_with_context(
+    if let Err(error) = state.api_rate_limiter.check(&key).await {
+        return match error {
+            RateLimitCheckError::Rejected(rejection) => Err(ApiError::rate_limited_with_context(
                 "request rate limit exceeded",
                 rejection_context(rejection),
-        ));
+            )),
+            RateLimitCheckError::StoreUnavailable => {
+                Err(ApiError::internal("rate limit service unavailable"))
+            }
+        };
     }
 
     Ok(next.run(request).await)

apps/admin-api/src/request_context.rs

@@ -1,14 +1,13 @@
 use axum::{
     extract::Request,
-    http::{HeaderMap, HeaderName, HeaderValue},
+    http::{HeaderName, HeaderValue},
     middleware::Next,
     response::Response,
 };
-use tracing::info;
-use uuid::Uuid;
+use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
+use tracing::{Instrument, info, info_span};
 
 pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
-const MAX_REQUEST_ID_LEN: usize = 128;
 
 #[derive(Clone, Debug)]
 pub struct RequestContext {
@@ -17,14 +16,22 @@ pub struct RequestContext {
 
 pub async fn apply_request_context(mut request: Request, next: Next) -> Response {
     let context = RequestContext {
-        request_id: resolve_request_id(request.headers()),
+        request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
     };
     let method = request.method().clone();
     let path = request.uri().path().to_owned();
+    let span = info_span!(
+        target: "crank::trace",
+        "http.request",
+        request_id = %context.request_id,
+    );
+    set_remote_trace_parent(&span, request.headers());
     request.extensions_mut().insert(context.clone());
 
-    let mut response = next.run(request).await;
+    with_request_correlation(context.request_id.clone(), async move {
+        let mut response = next.run(request).instrument(span).await;
         info!(
+            name: "admin.request.completed",
             request_id = %context.request_id,
             method = %method,
             path,
@@ -35,131 +42,22 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
             response.headers_mut().insert(REQUEST_ID_HEADER, value);
         }
         response
-}
-
-fn resolve_request_id(headers: &HeaderMap) -> String {
-    headers
-        .get(&REQUEST_ID_HEADER)
-        .and_then(|value| value.to_str().ok())
-        .map(str::trim)
-        .filter(|value| is_valid_request_id(value))
-        .map(ToOwned::to_owned)
-        .unwrap_or_else(|| Uuid::now_v7().to_string())
-}
-
-fn is_valid_request_id(value: &str) -> bool {
-    !value.is_empty()
-        && value.len() <= MAX_REQUEST_ID_LEN
-        && value
-            .bytes()
-            .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
+    })
+    .await
 }
 
 #[cfg(test)]
 mod tests {
-    use std::io;
-    use std::sync::{Arc, Mutex};
-
-    use axum::{Router, routing::get};
-    use reqwest::Client;
-    use tokio::net::TcpListener;
-    use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
-
-    use super::{REQUEST_ID_HEADER, apply_request_context, is_valid_request_id};
-
     #[test]
     fn accepts_visible_ascii_request_ids() {
-        assert!(is_valid_request_id("req_test_123"));
-        assert!(is_valid_request_id("trace-123/abc"));
+        assert!(crank_observability::RequestId::is_valid("req_test_123"));
+        assert!(crank_observability::RequestId::is_valid("trace-123/abc"));
     }
 
     #[test]
     fn rejects_empty_or_control_request_ids() {
-        assert!(!is_valid_request_id(""));
-        assert!(!is_valid_request_id("bad value"));
-        assert!(!is_valid_request_id("bad\nvalue"));
-    }
-
-    #[derive(Clone, Default)]
-    struct SharedLogWriter {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    impl SharedLogWriter {
-        fn output(&self) -> String {
-            String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
-        }
-    }
-
-    impl<'a> MakeWriter<'a> for SharedLogWriter {
-        type Writer = SharedLogGuard;
-
-        fn make_writer(&'a self) -> Self::Writer {
-            SharedLogGuard {
-                buffer: Arc::clone(&self.buffer),
-            }
-        }
-    }
-
-    struct SharedLogGuard {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    impl io::Write for SharedLogGuard {
-        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
-            self.buffer.lock().unwrap().extend_from_slice(bytes);
-            Ok(bytes.len())
-        }
-
-        fn flush(&mut self) -> io::Result<()> {
-            Ok(())
-        }
-    }
-
-    #[tokio::test]
-    async fn logs_request_completion_with_request_id() {
-        let writer = SharedLogWriter::default();
-        let subscriber = tracing_subscriber::registry().with(
-            tracing_subscriber::fmt::layer()
-                .with_writer(writer.clone())
-                .without_time()
-                .with_ansi(false)
-                .with_target(false)
-                .compact()
-                .with_filter(LevelFilter::INFO),
-        );
-        let dispatch = tracing::Dispatch::new(subscriber);
-        let app = Router::new()
-            .route("/probe", get(|| async { "ok" }))
-            .layer(axum::middleware::from_fn(apply_request_context));
-        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let address = listener.local_addr().unwrap();
-
-        let _guard = tracing::dispatcher::set_default(&dispatch);
-        tokio::spawn(async move {
-            axum::serve(listener, app).await.unwrap();
-        });
-
-        let response = Client::new()
-            .get(format!("http://{address}/probe"))
-            .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
-            .send()
-            .await
-            .unwrap();
-
-        assert_eq!(response.status(), reqwest::StatusCode::OK);
-        assert_eq!(
-            response.headers()[REQUEST_ID_HEADER.as_str()]
-                .to_str()
-                .unwrap(),
-            "req_admin_trace_123"
-        );
-
-        let logs = writer.output();
-        assert!(logs.contains("admin request completed"));
-        assert!(logs.contains("req_admin_trace_123"));
-        assert!(logs.contains("GET"));
-        assert!(logs.contains("/probe"));
-        assert!(logs.contains("status=200"));
+        assert!(!crank_observability::RequestId::is_valid(""));
+        assert!(!crank_observability::RequestId::is_valid("bad value"));
+        assert!(!crank_observability::RequestId::is_valid("bad\nvalue"));
     }
 }

apps/admin-api/src/routes.rs

@@ -10,12 +10,36 @@ pub mod secrets;
 pub mod upstreams;
 pub mod workspaces;
 
-use axum::Json;
+use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
 use serde_json::json;
 
+use crate::state::AppState;
+
 pub async fn health() -> Json<serde_json::Value> {
     Json(json!({
         "service": "admin-api",
         "status": "ok"
     }))
 }
+
+pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
+    match state.service.readiness().await {
+        Ok(()) => (
+            StatusCode::OK,
+            Json(json!({
+                "service": "admin-api",
+                "status": "ready",
+                "checks": { "postgres": "ready" }
+            })),
+        ),
+        Err(error) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "service": "admin-api",
+                "status": "not_ready",
+                "checks": { "postgres": "not_ready" },
+                "error": error.to_string()
+            })),
+        ),
+    }
+}

apps/admin-api/src/routes/access.rs

@@ -18,7 +18,7 @@ pub async fn export_workspace(
 ) -> Result<Json<Value>, ApiError> {
     let exported = state
         .service
-        .export_workspace(&path.workspace_id.as_str().into())
+        .export_workspace_catalog_snapshot(&path.workspace_id.as_str().into())
         .await?;
     Ok(Json(json!(exported)))
 }

apps/admin-api/src/routes/agents.rs

@@ -8,8 +8,8 @@ use serde_json::{Value, json};
 use crate::{
     error::ApiError,
     service::{
-        AgentBindingPayload, AgentPayload, PlatformApiKeyPayload, PublishPayload,
-        UpdateAgentPayload,
+        AgentCatalogPayload, AgentPayload, PlatformApiKeyPayload, PublishPayload,
+        ToolSearchPreviewPayload, UpdateAgentPayload,
     },
     state::AppState,
 };
@@ -50,6 +50,18 @@ pub async fn list_agents(
     Ok(Json(json!({ "items": items })))
 }
 
+pub async fn preview_tool_search(
+    Path(path): Path<WorkspacePath>,
+    State(state): State<AppState>,
+    Json(payload): Json<ToolSearchPreviewPayload>,
+) -> Result<Json<Value>, ApiError> {
+    let items = state
+        .service
+        .preview_tool_search(&path.workspace_id.as_str().into(), payload)
+        .await?;
+    Ok(Json(json!({"items": items})))
+}
+
 pub async fn create_agent(
     Path(path): Path<WorkspacePath>,
     State(state): State<AppState>,
@@ -124,7 +136,7 @@ pub async fn get_agent_version(
 pub async fn save_agent_bindings(
     Path(path): Path<WorkspaceAgentPath>,
     State(state): State<AppState>,
-    Json(payload): Json<Vec<AgentBindingPayload>>,
+    Json(payload): Json<AgentCatalogPayload>,
 ) -> Result<Json<Value>, ApiError> {
     let record = state
         .service

apps/admin-api/src/routes/auth.rs

@@ -103,7 +103,7 @@ pub async fn change_password(
 ) -> Result<StatusCode, ApiError> {
     state
         .service
-        .change_password(&session.user.id, payload)
+        .change_password(&session.user.id, &session.session_id, payload)
         .await?;
     Ok(StatusCode::NO_CONTENT)
 }

apps/admin-api/src/service.rs

@@ -5,23 +5,25 @@ use std::sync::Arc;
 use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
 use crank_core::{
     AuditSink, AuthProfile, CapabilityProfile, CommunityCapabilityProfile, EditionCapabilities,
-    ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId, NoopAuditSink,
-    OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine, ProductEdition, Protocol,
-    ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind, ToolQualitySchemaNode,
-    UsagePeriod, WorkspaceId,
+    ExecutionMode, IdentityError, IdentityProvider, InvocationLog, InvocationLogId,
+    InvocationSource, NoopAuditSink, OperationSecurityLevel, OwnerOnlyPolicyEngine, PolicyEngine,
+    ProductEdition, Protocol, ToolQualityMappingRule, ToolQualityMappingSet, ToolQualitySchemaKind,
+    ToolQualitySchemaNode, UsagePeriod, WorkspaceId,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_registry::{
-    AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary,
-    OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
+    AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef,
+    OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
 };
 use crank_runtime::{
     OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
 };
 use crank_schema::{Schema, SchemaKind};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use serde_json::{Value, json};
 use sha2::{Digest, Sha256};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tracing::Instrument;
 use uuid::Uuid;
 
 mod agents;
@@ -74,6 +76,11 @@ pub struct AdminServiceBuilder {
 pub use crate::dto::*;
 
 impl AdminService {
+    pub async fn readiness(&self) -> Result<(), ApiError> {
+        self.registry.ping().await?;
+        Ok(())
+    }
+
     #[cfg(test)]
     pub fn new(
         registry: PostgresRegistry,
@@ -199,16 +206,33 @@ impl AdminServiceBuilder {
 }
 
 impl AdminService {
-    pub async fn export_workspace(
+    pub async fn export_workspace_catalog_snapshot(
         &self,
         workspace_id: &WorkspaceId,
-    ) -> Result<WorkspaceExportResponse, ApiError> {
+    ) -> Result<WorkspaceCatalogSnapshotResponse, ApiError> {
         let workspace = self.get_workspace(workspace_id).await?;
         let operations = self.list_operations(workspace_id).await?;
         let agents = self.list_agents(workspace_id).await?;
         let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?;
 
-        Ok(WorkspaceExportResponse {
+        Ok(WorkspaceCatalogSnapshotResponse {
+            kind: "workspace_catalog_snapshot".to_owned(),
+            format_version: "1".to_owned(),
+            restorable: false,
+            included: vec![
+                "workspace_settings".to_owned(),
+                "operation_summaries".to_owned(),
+                "agent_summaries".to_owned(),
+                "platform_api_key_metadata".to_owned(),
+            ],
+            excluded: vec![
+                "operation_versions_and_samples".to_owned(),
+                "agent_versions_and_bindings".to_owned(),
+                "secret_metadata_and_values".to_owned(),
+                "secret_values".to_owned(),
+                "invocation_logs_and_usage".to_owned(),
+                "authentication_sessions".to_owned(),
+            ],
             workspace,
             operations,
             agents,
@@ -239,9 +263,13 @@ impl AdminService {
             return Ok(None);
         };
 
-        let auth_profile = self
-            .registry
-            .get_auth_profile(workspace_id, auth_profile_id)
+        let span = Stage::AuthResolve.span();
+        let result = async {
+            let auth_profile = observe_db_query(
+                DbOperation::AuthProfileRead,
+                self.registry
+                    .get_auth_profile(workspace_id, auth_profile_id),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "load auth profile",
@@ -255,6 +283,17 @@ impl AdminService {
                 .await
                 .map(Some)
         }
+        .instrument(span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&span),
+            Err(_) => {
+                StageOutcome::Error.record(&span);
+                ErrorCategory::Configuration.record(&span);
+            }
+        }
+        result
+    }
 
     async fn resolve_auth_profile(
         &self,
@@ -265,9 +304,10 @@ impl AdminService {
         let used_at = OffsetDateTime::now_utc();
 
         for secret_id in auth_profile.config.secret_ids() {
-            let secret = self
-                .registry
-                .get_secret(workspace_id, secret_id)
+            let secret = observe_db_query(
+                DbOperation::SecretRead,
+                self.registry.get_secret(workspace_id, secret_id),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "load secret",
@@ -276,9 +316,11 @@ impl AdminService {
             .ok_or_else(|| RuntimeError::MissingSecret {
                 secret_id: secret_id.as_str().to_owned(),
             })?;
-            let version = self
-                .registry
-                .get_current_secret_version(workspace_id, secret_id)
+            let version = observe_db_query(
+                DbOperation::SecretRead,
+                self.registry
+                    .get_current_secret_version(workspace_id, secret_id),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "load current secret version",
@@ -292,8 +334,11 @@ impl AdminService {
                 &version.secret_version.key_version,
                 &version.secret_version.ciphertext,
             )?;
+            observe_db_query(
+                DbOperation::SecretTouch,
                 self.registry
-                .touch_secret(workspace_id, secret_id, &used_at)
+                    .touch_secret(workspace_id, secret_id, &used_at),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "touch secret",
@@ -412,7 +457,7 @@ impl AdminService {
     async fn record_invocation(
         &self,
         request: InvocationRecordRequest<'_>,
-    ) -> Result<(), ApiError> {
+    ) -> InvocationHistoryWriteOutcome {
         let log = InvocationLog {
             id: InvocationLogId::new(new_prefixed_id("log")),
             workspace_id: request.workspace_id.clone(),
@@ -432,11 +477,75 @@ impl AdminService {
             created_at: OffsetDateTime::now_utc(),
         };
 
-        self.registry
+        let history_span = crank_trace::Stage::HistoryWrite.span();
+        let (outcome, db_span) = async {
+            let db_span = crank_trace::DbOperation::InvocationHistoryWrite.span();
+            let outcome = self
+                .registry
                 .create_invocation_log(CreateInvocationLogRequest { log: &log })
-            .await?;
+                .instrument(db_span.clone())
+                .await;
+            (outcome, db_span)
+        }
+        .instrument(history_span.clone())
+        .await;
+        match outcome {
+            InvocationHistoryWriteOutcome::Recorded => {
+                crank_trace::StageOutcome::Success.record(&db_span);
+                crank_trace::StageOutcome::Success.record(&history_span);
+            }
+            InvocationHistoryWriteOutcome::Lost(_) => {
+                crank_trace::StageOutcome::Error.record(&db_span);
+                crank_trace::ErrorCategory::Database.record(&db_span);
+                crank_trace::StageOutcome::Error.record(&history_span);
+                crank_trace::ErrorCategory::History.record(&history_span);
+            }
+        }
+        drop(db_span);
+        drop(history_span);
+        observe_invocation_history_outcome(
+            outcome,
+            request.request_id,
+            request.status,
+            request.source,
+        );
+        outcome
+    }
+}
 
-        Ok(())
+fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: crank_core::InvocationStatus,
+    source: InvocationSource,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    tracing::warn!(
+        name: "admin.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source = invocation_source_label(source),
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_source_label(source: InvocationSource) -> &'static str {
+    match source {
+        InvocationSource::AdminTestRun => "admin_test_run",
+        InvocationSource::AgentToolCall => "agent_tool_call",
+    }
+}
+
+fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
+    match status {
+        crank_core::InvocationStatus::Ok => "ok",
+        crank_core::InvocationStatus::Error => "error",
     }
 }
 
@@ -502,9 +611,9 @@ fn new_prefixed_id(prefix: &str) -> String {
     format!("{prefix}_{}", Uuid::now_v7().simple())
 }
 
-fn generate_access_secret(prefix: &str) -> String {
+fn generate_access_secret(marker: &str) -> String {
     let random = URL_SAFE_NO_PAD.encode(Uuid::now_v7().as_bytes());
-    format!("{prefix}_{random}")
+    format!("{marker}{random}")
 }
 
 fn hash_access_secret(secret: &str) -> String {
@@ -543,6 +652,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
         RuntimeError::RestAdapter(_) => "rest_error",
         RuntimeError::ProtocolAdapter(_) => "adapter_error",
         RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
@@ -698,6 +811,7 @@ fn map_agent_summary_view(summary: AgentSummary) -> AgentSummaryView {
         published_at: summary.published_at.map(format_timestamp),
         operation_count: 0,
         operation_ids: Vec::new(),
+        tool_selection_policy: Default::default(),
         key_count: 0,
         calls_today: 0,
         mcp_endpoint: String::new(),
@@ -760,7 +874,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule {
 #[cfg(test)]
 #[allow(clippy::items_after_test_module)]
 mod tests {
-    use super::{validate_profile_display_name, validate_profile_email};
+    use std::{
+        io,
+        sync::{Arc, Mutex},
+    };
+
+    use crank_core::{InvocationSource, InvocationStatus};
+    use crank_observability::{
+        ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+        operational_incident_total,
+    };
+    use crank_registry::{
+        InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
+    };
+    use serde_json::Value;
+    use tracing_subscriber::fmt::MakeWriter;
+
+    use super::{
+        observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email,
+    };
 
     #[test]
     fn validates_profile_identity_fields() {
@@ -781,6 +913,81 @@ mod tests {
         assert!(validate_profile_display_name(&"x".repeat(81)).is_err());
         assert!(validate_profile_email("owner <html>@crank.local").is_err());
     }
+
+    #[test]
+    fn emits_bounded_history_loss_incident() {
+        let writer = SharedLogWriter::default();
+        let subscriber = crank_observability::build_subscriber(
+            ObservabilityConfig::new(
+                ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+                "info",
+                RedactionLimits::default(),
+            ),
+            writer.clone(),
+        )
+        .unwrap();
+        let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let _guard = tracing::dispatcher::set_default(&dispatch);
+
+        observe_invocation_history_outcome(
+            InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
+                category: InvocationHistoryLossCategory::InvalidRecord,
+            }),
+            Some("req_admin_dc08"),
+            InvocationStatus::Error,
+            InvocationSource::AgentToolCall,
+        );
+
+        let output = writer.output();
+        assert!(!output.contains("dc08-canary-secret"));
+        let event: Value = output
+            .lines()
+            .map(|line| serde_json::from_str(line).unwrap())
+            .find(|event: &Value| event["event"] == "admin.invocation_history.lost")
+            .unwrap();
+        assert_eq!(event["request_id"], "req_admin_dc08");
+        assert_eq!(event["fields"]["source"], "agent_tool_call");
+        assert_eq!(event["fields"]["invocation_status"], "error");
+        assert_eq!(event["fields"]["error_category"], "invalid_record");
+        assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+    }
+
+    #[derive(Clone, Default)]
+    struct SharedLogWriter {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl SharedLogWriter {
+        fn output(&self) -> String {
+            String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+        }
+    }
+
+    impl<'a> MakeWriter<'a> for SharedLogWriter {
+        type Writer = SharedLogGuard;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            SharedLogGuard {
+                buffer: Arc::clone(&self.buffer),
+            }
+        }
+    }
+
+    struct SharedLogGuard {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl io::Write for SharedLogGuard {
+        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+            self.buffer.lock().unwrap().extend_from_slice(bytes);
+            Ok(bytes.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
 }
 
 fn enrich_operation_summary(

apps/admin-api/src/service/agents.rs

@@ -1,12 +1,12 @@
 use std::collections::BTreeMap;
 
 use crank_core::{
-    Agent, AgentId, AgentOperationBinding, AgentStatus, AgentVersion, OperationId, UsagePeriod,
-    WorkspaceId,
+    Agent, AgentId, AgentOperationBinding, AgentStatus, AgentVersion, OperationId, SearchableTool,
+    ToolAccessMode, ToolSelectionPolicy, UsagePeriod, WorkspaceId, search_tool_catalog,
 };
 use crank_registry::{
     AgentVersionRecord, CreateAgentDraftVersionRequest, CreateAgentRequest, PublishAgentRequest,
-    SaveAgentBindingsRequest, UsageBucket, UsageQuery,
+    SaveAgentBindingsRequest, SaveAgentCatalogConfigRequest, UsageBucket, UsageQuery,
 };
 use serde_json::json;
 use time::OffsetDateTime;
@@ -15,13 +15,93 @@ use tracing::{info, instrument};
 use crate::{
     error::ApiError,
     service::{
-        AdminService, AgentBindingPayload, AgentMutationResult, AgentPayload, AgentSummaryView,
-        CreatedAgentResponse, PublishAgentResponse, UpdateAgentPayload, agent_mcp_endpoint,
-        format_timestamp, map_agent_summary_view, new_prefixed_id, today_start_utc,
+        AdminService, AgentCatalogPayload, AgentMutationResult, AgentPayload, AgentSummaryView,
+        CreatedAgentResponse, PublishAgentResponse, ToolSearchPreviewPayload, UpdateAgentPayload,
+        agent_mcp_endpoint, format_timestamp, map_agent_summary_view, new_prefixed_id,
+        today_start_utc,
     },
 };
 
 impl AdminService {
+    #[instrument(skip(self, payload), fields(workspace_id = %workspace_id.as_str()))]
+    pub async fn preview_tool_search(
+        &self,
+        workspace_id: &WorkspaceId,
+        payload: ToolSearchPreviewPayload,
+    ) -> Result<Vec<crank_core::ToolSearchMatch>, ApiError> {
+        self.ensure_workspace_exists(workspace_id).await?;
+        if payload.tool_selection_policy.mode != ToolAccessMode::Search {
+            return Err(ApiError::validation_with_context(
+                "tool search preview requires search mode",
+                json!({"field": "tool_selection_policy.mode"}),
+            ));
+        }
+        let bindings = payload
+            .bindings
+            .iter()
+            .map(|binding| AgentOperationBinding {
+                operation_id: OperationId::new(binding.operation_id.clone()),
+                operation_version: binding.operation_version,
+                tool_name: binding.tool_name.clone(),
+                tool_title: binding.tool_title.clone(),
+                tool_description_override: binding.tool_description_override.clone(),
+                enabled: binding.enabled,
+            })
+            .collect::<Vec<_>>();
+        validate_tool_selection_policy(&payload.tool_selection_policy, &bindings)?;
+
+        let mut tools = Vec::new();
+        for binding in bindings.iter().filter(|binding| binding.enabled) {
+            let version = self
+                .registry
+                .get_operation_version(
+                    workspace_id,
+                    &binding.operation_id,
+                    binding.operation_version,
+                )
+                .await?
+                .ok_or_else(|| {
+                    ApiError::not_found_with_context(
+                        format!("operation {} was not found", binding.operation_id.as_str()),
+                        json!({"operation_id": binding.operation_id.as_str()}),
+                    )
+                })?;
+            let groups = payload
+                .tool_selection_policy
+                .groups
+                .iter()
+                .filter(|group| {
+                    group
+                        .tool_names
+                        .iter()
+                        .any(|name| name == &binding.tool_name)
+                })
+                .collect::<Vec<_>>();
+            tools.push(SearchableTool {
+                name: binding.tool_name.clone(),
+                title: binding.tool_title.clone(),
+                description: binding
+                    .tool_description_override
+                    .clone()
+                    .unwrap_or(version.snapshot.tool_description.description),
+                input_schema: serde_json::Value::Null,
+                group_ids: groups.iter().map(|group| group.id.clone()).collect(),
+                group_context: groups
+                    .iter()
+                    .map(|group| format!("{} {}", group.name, group.description))
+                    .collect::<Vec<_>>()
+                    .join(" "),
+            });
+        }
+        let max_results = payload.tool_selection_policy.search.max_results;
+        Ok(search_tool_catalog(
+            &tools,
+            &payload.query,
+            &payload.group_ids,
+            max_results,
+        ))
+    }
+
     #[instrument(skip(self))]
     pub async fn list_agents(
         &self,
@@ -69,6 +149,7 @@ impl AdminService {
             items.push(AgentSummaryView {
                 operation_count: operation_ids.len(),
                 operation_ids,
+                tool_selection_policy: version.snapshot.tool_selection_policy,
                 key_count: key_counts.get(summary.id.as_str()).copied().unwrap_or(0),
                 calls_today: calls_today.get(summary.id.as_str()).copied().unwrap_or(0),
                 mcp_endpoint: agent_mcp_endpoint(
@@ -130,6 +211,7 @@ impl AdminService {
         Ok(AgentSummaryView {
             operation_count: operation_ids.len(),
             operation_ids,
+            tool_selection_policy: version.snapshot.tool_selection_policy,
             key_count,
             calls_today: usage.map(|item| item.rollup.calls_total).unwrap_or(0),
             mcp_endpoint: agent_mcp_endpoint(
@@ -214,7 +296,12 @@ impl AdminService {
                 bindings: &[],
             })
             .await?;
-        info!(agent_id = %agent_id.as_str(), version = 1, "agent created");
+        info!(
+            name: "admin.agent.created",
+            agent_id = %agent_id.as_str(),
+            version = 1,
+            "agent created"
+        );
 
         Ok(CreatedAgentResponse {
             agent_id: agent_id.as_str().to_owned(),
@@ -304,9 +391,12 @@ impl AdminService {
         &self,
         workspace_id: &WorkspaceId,
         agent_id: &AgentId,
-        payload: Vec<AgentBindingPayload>,
+        payload: AgentCatalogPayload,
     ) -> Result<AgentVersionRecord, ApiError> {
-        let agent = self.get_agent(workspace_id, agent_id).await?;
+        let current_version = self
+            .ensure_editable_agent_version(workspace_id, agent_id)
+            .await?;
+        let (payload, requested_policy) = payload.into_parts();
         let bindings = payload
             .into_iter()
             .map(|binding| AgentOperationBinding {
@@ -318,23 +408,63 @@ impl AdminService {
                 enabled: binding.enabled,
             })
             .collect::<Vec<_>>();
+        let tool_selection_policy = requested_policy
+            .unwrap_or_else(|| current_version.snapshot.tool_selection_policy.clone());
+        validate_tool_selection_policy(&tool_selection_policy, &bindings)?;
 
         self.registry
-            .save_agent_bindings(SaveAgentBindingsRequest {
+            .save_agent_catalog_config(SaveAgentCatalogConfigRequest {
                 workspace_id,
                 agent_id,
-                agent_version: agent.current_draft_version,
+                agent_version: current_version.version,
                 bindings: &bindings,
+                tool_selection_policy: &tool_selection_policy,
             })
             .await?;
         info!(
+            name: "admin.agent.bindings_saved",
             agent_id = %agent_id.as_str(),
-            version = agent.current_draft_version,
+            version = current_version.version,
             binding_count = bindings.len(),
             "agent bindings saved"
         );
 
-        self.get_agent_version(workspace_id, agent_id, agent.current_draft_version)
+        self.get_agent_version(workspace_id, agent_id, current_version.version)
+            .await
+    }
+
+    async fn ensure_editable_agent_version(
+        &self,
+        workspace_id: &WorkspaceId,
+        agent_id: &AgentId,
+    ) -> Result<AgentVersionRecord, ApiError> {
+        let agent = self.get_agent(workspace_id, agent_id).await?;
+        let current = self
+            .get_agent_version(workspace_id, agent_id, agent.current_draft_version)
+            .await?;
+        if agent.latest_published_version != Some(agent.current_draft_version) {
+            return Ok(current);
+        }
+
+        let now = OffsetDateTime::now_utc();
+        let draft = AgentVersion {
+            agent_id: agent_id.clone(),
+            version: current.version + 1,
+            status: AgentStatus::Draft,
+            instructions: current.snapshot.instructions.clone(),
+            tool_selection_policy: current.snapshot.tool_selection_policy.clone(),
+            created_at: now,
+        };
+        self.registry
+            .create_agent_draft_version(CreateAgentDraftVersionRequest {
+                workspace_id,
+                agent_id,
+                version: &draft,
+                bindings: &current.bindings,
+                updated_at: &now,
+            })
+            .await?;
+        self.get_agent_version(workspace_id, agent_id, draft.version)
             .await
     }
 
@@ -351,6 +481,10 @@ impl AdminService {
         let published_bindings = self
             .published_agent_bindings(workspace_id, &agent_version.bindings)
             .await?;
+        validate_tool_selection_policy(
+            &agent_version.snapshot.tool_selection_policy,
+            &published_bindings,
+        )?;
 
         if published_bindings.is_empty() {
             return Err(ApiError::conflict_with_context(
@@ -403,7 +537,12 @@ impl AdminService {
                 published_by: None,
             })
             .await?;
-        info!(agent_id = %agent_id.as_str(), version, "agent published");
+        info!(
+            name: "admin.agent.published",
+            agent_id = %agent_id.as_str(),
+            version,
+            "agent published"
+        );
 
         Ok(PublishAgentResponse {
             agent_id: agent_id.as_str().to_owned(),
@@ -461,7 +600,11 @@ impl AdminService {
         self.registry
             .unpublish_agent(workspace_id, agent_id, &updated_at)
             .await?;
-        info!(agent_id = %agent_id.as_str(), "agent moved to draft");
+        info!(
+            name: "admin.agent.unpublished",
+            agent_id = %agent_id.as_str(),
+            "agent moved to draft"
+        );
 
         Ok(AgentMutationResult {
             agent_id: agent_id.as_str().to_owned(),
@@ -481,7 +624,11 @@ impl AdminService {
         self.registry
             .archive_agent(workspace_id, agent_id, &updated_at)
             .await?;
-        info!(agent_id = %agent_id.as_str(), "agent archived");
+        info!(
+            name: "admin.agent.archived",
+            agent_id = %agent_id.as_str(),
+            "agent archived"
+        );
 
         Ok(AgentMutationResult {
             agent_id: agent_id.as_str().to_owned(),
@@ -490,3 +637,22 @@ impl AdminService {
         })
     }
 }
+
+fn validate_tool_selection_policy(
+    policy: &ToolSelectionPolicy,
+    bindings: &[AgentOperationBinding],
+) -> Result<(), ApiError> {
+    policy
+        .validate_for_tools(
+            bindings
+                .iter()
+                .filter(|binding| binding.enabled)
+                .map(|binding| binding.tool_name.as_str()),
+        )
+        .map_err(|error| {
+            ApiError::validation_with_context(
+                error.to_string(),
+                json!({"field": "tool_selection_policy"}),
+            )
+        })
+}

apps/admin-api/src/service/api_keys.rs

@@ -65,10 +65,7 @@ impl AdminService {
             ),
             None => None,
         };
-        let secret = generate_access_secret(match payload.key_kind {
-            PlatformApiKeyKind::McpClient => "crk",
-            PlatformApiKeyKind::Approval => "crk_appr",
-        });
+        let secret = generate_access_secret(payload.key_kind.secret_marker());
         let api_key = PlatformApiKeyRecord {
             api_key: PlatformApiKey {
                 id: PlatformApiKeyId::new(new_prefixed_id("pk")),

apps/admin-api/src/service/auth.rs

@@ -22,7 +22,7 @@ impl AdminService {
         )?;
         let user_id = self
             .registry
-            .upsert_bootstrap_user(
+            .ensure_bootstrap_user(
                 &self.auth_settings.bootstrap_admin.email,
                 &self.auth_settings.bootstrap_admin.display_name,
                 &password_hash,
@@ -227,6 +227,7 @@ impl AdminService {
     pub async fn change_password(
         &self,
         user_id: &crank_core::UserId,
+        current_session_id: &UserSessionId,
         payload: ChangePasswordPayload,
     ) -> Result<(), ApiError> {
         if payload.new_password.len() < 12 {
@@ -257,7 +258,11 @@ impl AdminService {
         let password_hash =
             hash_password(&payload.new_password, &self.auth_settings.password_pepper)?;
         self.registry
-            .update_user_password(user_id, &password_hash)
+            .update_user_password_and_revoke_other_sessions(
+                user_id,
+                current_session_id,
+                &password_hash,
+            )
             .await?;
 
         Ok(())

apps/admin-api/src/service/demo.rs

@@ -131,6 +131,7 @@ impl AdminService {
             Ok(()) => Ok(()),
             Err(RegistryError::OperationHasPublishedAgentBindings { .. }) => {
                 tracing::warn!(
+                    name: "admin.demo_operation.cleanup_skipped",
                     operation_id = %operation_id.as_str(),
                     "legacy demo operation is still bound to a published agent; leaving it in place"
                 );
@@ -272,7 +273,7 @@ impl AdminService {
         publish: bool,
     ) -> Result<(), ApiError> {
         let summary = self.get_agent(workspace_id, agent_id).await?;
-        self.save_agent_bindings(workspace_id, agent_id, bindings)
+        self.save_agent_bindings(workspace_id, agent_id, bindings.into())
             .await?;
         if publish && summary.latest_published_version.is_none() {
             self.publish_agent(workspace_id, agent_id, summary.current_draft_version)
@@ -335,7 +336,7 @@ impl AdminService {
             }),
             response_preview: demo_rest_response_sample(),
         })
-        .await?;
+        .await;
         Ok(())
     }
 }
@@ -348,10 +349,7 @@ fn demo_currency_agent_payload() -> AgentPayload {
         instructions: json!({
             "system": "Используй инструменты Frankfurter только для запросов о курсах валют."
         }),
-        tool_selection_policy: json!({
-            "max_tools": 4,
-            "prefer_tag": ["currency", "exchange-rate"]
-        }),
+        tool_selection_policy: Default::default(),
     }
 }
 

apps/admin-api/src/service/import_export.rs

@@ -110,6 +110,7 @@ impl AdminService {
                         warnings,
                     };
                     info!(
+                        name: "admin.operation.imported",
                         operation_id = %response.operation_id,
                         version = response.version,
                         "operation imported by upsert"
@@ -125,6 +126,7 @@ impl AdminService {
                         warnings,
                     };
                     info!(
+                        name: "admin.operation.imported",
                         operation_id = %response.operation_id,
                         version = response.version,
                         "operation imported by upsert"

apps/admin-api/src/service/imports.rs

@@ -8,9 +8,11 @@ use crank_import::rest::{
     ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
 };
 use crank_registry::{
-    CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus,
+    ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind,
+    ImportJobStatus, ImportOperationDraft,
 };
 use serde_json::json;
+use sha2::{Digest, Sha256};
 use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
 use tracing::{info, instrument};
 
@@ -50,7 +52,7 @@ impl AdminService {
                 kind: ImportJobKind::OpenApi,
                 source_format: &preview.source.format,
                 source_version: preview.source.version.as_deref(),
-                status: ImportJobStatus::Completed,
+                status: ImportJobStatus::Pending,
                 preview_payload: &preview_payload,
                 created_at: &now,
                 expires_at: &expires_at,
@@ -99,8 +101,12 @@ impl AdminService {
             return Err(ApiError::validation("import job kind is not openapi"));
         }
 
-        let preview: crank_import::rest::ImportPreview =
-            serde_json::from_value(job.preview_payload.clone())
+        let stored_preview = job
+            .preview_payload
+            .get("preview")
+            .cloned()
+            .unwrap_or_else(|| job.preview_payload.clone());
+        let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview)
             .map_err(|error| ApiError::internal(error.to_string()))?;
         let selected = payload
             .selected_operation_keys
@@ -120,10 +126,8 @@ impl AdminService {
             }
         }
 
-        let mut created = Vec::new();
         let mut skipped = Vec::new();
-        let mut findings = Vec::new();
-        let mut created_ids = Vec::new();
+        let mut operations = Vec::new();
 
         for operation_key in selected {
             let Some(candidate) = candidates.get(&operation_key) else {
@@ -137,44 +141,7 @@ impl AdminService {
             let mut draft =
                 operation_draft_from_candidate(candidate, payload.server_url.as_deref());
             attach_import_findings(&mut draft, candidate);
-            if let Some(existing_name) = self
-                .find_operation_by_name(workspace_id, &draft.name)
-                .await?
-                .map(|operation| operation.name)
-            {
-                if payload.conflict_mode == "skip" {
-                    skipped.push(OpenApiImportSkippedOperation {
-                        operation_key: candidate.key.clone(),
-                        name: draft.name.clone(),
-                        reason: "operation with this name already exists".to_owned(),
-                    });
-                    findings.push(ImportFinding {
-                        code: "operation_name_conflict".to_owned(),
-                        severity: ImportFindingSeverity::Warning,
-                        message: format!(
-                            "Операция {} уже существует и была пропущена.",
-                            draft.name
-                        ),
-                        operation_key: Some(candidate.key.clone()),
-                    });
-                    continue;
-                }
-
-                let renamed = self
-                    .next_available_operation_name(workspace_id, &draft.name)
-                    .await?;
-                findings.push(ImportFinding {
-                    code: "operation_name_renamed".to_owned(),
-                    severity: ImportFindingSeverity::Info,
-                    message: format!(
-                        "Операция {existing_name} уже существует, новый черновик создан как {renamed}."
-                    ),
-                    operation_key: Some(candidate.key.clone()),
-                });
-                draft.name = renamed;
-            }
-
-            let payload = OperationPayload {
+            let operation = self.new_operation_snapshot(OperationPayload {
                 name: draft.name.clone(),
                 display_name: draft.display_name.clone(),
                 category: draft.category,
@@ -197,27 +164,74 @@ impl AdminService {
                 },
                 tool_description: draft.tool_description,
                 wizard_state: draft.wizard_state,
-            };
-            let result = self.create_operation(workspace_id, payload).await?;
-            created_ids.push(result.operation_id.clone());
-            created.push(OpenApiImportCreatedOperation {
-                operation_id: result.operation_id,
-                name: draft.name,
-                version: result.version,
+            })?;
+            operations.push(ImportOperationDraft {
+                operation_key: candidate.key.clone(),
+                operation,
             });
         }
 
         let finished_at = OffsetDateTime::now_utc();
-        self.registry
-            .finish_import_job(FinishImportJobRequest {
+        let application_key = openapi_application_key(&payload)?;
+        let conflict_mode = if payload.conflict_mode == "skip" {
+            ImportConflictMode::Skip
+        } else {
+            ImportConflictMode::Rename
+        };
+        let applied = self
+            .registry
+            .apply_import_job(ApplyImportJobRequest {
                 id: job_id,
-                status: ImportJobStatus::Completed,
-                created_operation_ids: &json!(created_ids),
-                error_text: None,
+                workspace_id,
+                application_key: &application_key,
+                conflict_mode,
+                operations: &operations,
                 finished_at: &finished_at,
             })
             .await?;
+
+        let created = applied
+            .created
+            .iter()
+            .map(|operation| OpenApiImportCreatedOperation {
+                operation_id: operation.operation_id.as_str().to_owned(),
+                name: operation.name.clone(),
+                version: operation.version,
+            })
+            .collect::<Vec<_>>();
+        let mut findings = applied
+            .created
+            .iter()
+            .filter_map(|operation| {
+                operation.renamed_from.as_ref().map(|previous_name| ImportFinding {
+                    code: "operation_name_renamed".to_owned(),
+                    severity: ImportFindingSeverity::Info,
+                    message: format!(
+                        "Операция {previous_name} уже существует, новый черновик создан как {}.",
+                        operation.name
+                    ),
+                    operation_key: Some(operation.operation_key.clone()),
+                })
+            })
+            .collect::<Vec<_>>();
+        for operation in applied.skipped {
+            skipped.push(OpenApiImportSkippedOperation {
+                operation_key: operation.operation_key.clone(),
+                name: operation.name.clone(),
+                reason: "operation with this name already exists".to_owned(),
+            });
+            findings.push(ImportFinding {
+                code: operation.reason,
+                severity: ImportFindingSeverity::Warning,
+                message: format!(
+                    "Операция {} уже существует и была пропущена.",
+                    operation.name
+                ),
+                operation_key: Some(operation.operation_key),
+            });
+        }
         info!(
+            name: "admin.openapi_import.completed",
             created = created.len(),
             skipped = skipped.len(),
             "openapi import created drafts"
@@ -229,25 +243,21 @@ impl AdminService {
             findings,
         })
     }
+}
 
-    async fn next_available_operation_name(
-        &self,
-        workspace_id: &WorkspaceId,
-        base_name: &str,
-    ) -> Result<String, ApiError> {
-        for index in 2.. {
-            let candidate = format!("{base_name}_{index}");
-            if self
-                .find_operation_by_name(workspace_id, &candidate)
-                .await?
-                .is_none()
-            {
-                return Ok(candidate);
-            }
-        }
-
-        unreachable!()
-    }
+fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
+    let selected_operation_keys = payload
+        .selected_operation_keys
+        .iter()
+        .cloned()
+        .collect::<BTreeSet<_>>();
+    let canonical = serde_json::to_vec(&json!({
+        "selected_operation_keys": selected_operation_keys,
+        "server_url": payload.server_url.as_deref(),
+        "conflict_mode": payload.conflict_mode.as_str(),
+    }))
+    .map_err(|error| ApiError::internal(error.to_string()))?;
+    Ok(format!("{:x}", Sha256::digest(canonical)))
 }
 
 fn attach_import_findings(

apps/admin-api/src/service/observability.rs

@@ -19,6 +19,16 @@ use crate::{
 };
 
 impl AdminService {
+    pub async fn cleanup_invocation_logs_before(
+        &self,
+        cutoff: OffsetDateTime,
+    ) -> Result<u64, ApiError> {
+        self.registry
+            .delete_invocation_logs_before(cutoff)
+            .await
+            .map_err(ApiError::from)
+    }
+
     #[instrument(skip(self))]
     pub async fn list_logs(
         &self,

apps/admin-api/src/service/operations.rs

@@ -141,7 +141,6 @@ impl AdminService {
         workspace_id: &WorkspaceId,
         payload: OperationPayload,
     ) -> Result<CreatedOperationResponse, ApiError> {
-        self.validate_operation_payload(&payload)?;
         self.ensure_workspace_exists(workspace_id).await?;
 
         if self
@@ -155,10 +154,36 @@ impl AdminService {
             ));
         }
 
+        let snapshot = self.new_operation_snapshot(payload)?;
+        let operation_id = snapshot.id.clone();
+
+        self.registry
+            .create_operation(workspace_id, &snapshot, None)
+            .await?;
+        info!(
+            name: "admin.operation.created",
+            operation_id = %operation_id.as_str(),
+            version = 1,
+            "operation created"
+        );
+
+        Ok(CreatedOperationResponse {
+            operation_id: operation_id.as_str().to_owned(),
+            workspace_id: workspace_id.as_str().to_owned(),
+            version: 1,
+            status: OperationStatus::Draft,
+            updated_at: format_timestamp(snapshot.updated_at),
+        })
+    }
+
+    pub(super) fn new_operation_snapshot(
+        &self,
+        payload: OperationPayload,
+    ) -> Result<RegistryOperation, ApiError> {
+        self.validate_operation_payload(&payload)?;
         let now = OffsetDateTime::now_utc();
-        let operation_id = OperationId::new(new_prefixed_id("op"));
-        let snapshot = RegistryOperation {
-            id: operation_id.clone(),
+        Ok(RegistryOperation {
+            id: OperationId::new(new_prefixed_id("op")),
             name: payload.name,
             display_name: payload.display_name,
             category: payload.category,
@@ -183,19 +208,6 @@ impl AdminService {
             created_at: now,
             updated_at: now,
             published_at: None,
-        };
-
-        self.registry
-            .create_operation(workspace_id, &snapshot, None)
-            .await?;
-        info!(operation_id = %operation_id.as_str(), version = 1, "operation created");
-
-        Ok(CreatedOperationResponse {
-            operation_id: operation_id.as_str().to_owned(),
-            workspace_id: workspace_id.as_str().to_owned(),
-            version: 1,
-            status: OperationStatus::Draft,
-            updated_at: format_timestamp(snapshot.updated_at),
         })
     }
 
@@ -279,7 +291,12 @@ impl AdminService {
                 created_by: None,
             })
             .await?;
-        info!(operation_id = %operation_id.as_str(), version, "operation version created");
+        info!(
+            name: "admin.operation.version_created",
+            operation_id = %operation_id.as_str(),
+            version,
+            "operation version created"
+        );
 
         Ok(CreatedOperationResponse {
             operation_id: operation_id.as_str().to_owned(),
@@ -364,7 +381,12 @@ impl AdminService {
                 published_by: None,
             })
             .await?;
-        info!(operation_id = %operation_id.as_str(), version, "operation published");
+        info!(
+            name: "admin.operation.published",
+            operation_id = %operation_id.as_str(),
+            version,
+            "operation published"
+        );
 
         Ok(PublishResponse {
             operation_id: operation_id.as_str().to_owned(),
@@ -431,10 +453,15 @@ impl AdminService {
             .await?;
         let runtime = RuntimeOperation::from(record.snapshot.clone());
         let mode = ExecutionMode::Unary;
-        let request_preview =
-            match build_request_preview(&record.snapshot.input_mapping, &payload.input) {
+        let preview_span = crank_trace::Stage::RuntimeArgumentsMap.span();
+        let preview_result = preview_span
+            .in_scope(|| build_request_preview(&record.snapshot.input_mapping, &payload.input));
+        let request_preview = match preview_result {
             Ok(preview) => preview,
             Err(error) => {
+                crank_trace::StageOutcome::Error.record(&preview_span);
+                crank_trace::ErrorCategory::Mapping.record(&preview_span);
+                drop(preview_span);
                 self.record_invocation(InvocationRecordRequest {
                     workspace_id,
                     agent_id: None,
@@ -450,7 +477,7 @@ impl AdminService {
                     request_preview: Value::Null,
                     response_preview: Value::Null,
                 })
-                    .await?;
+                .await;
                 return Ok(TestRunResult {
                     ok: false,
                     mode,
@@ -462,6 +489,8 @@ impl AdminService {
                 });
             }
         };
+        crank_trace::StageOutcome::Success.record(&preview_span);
+        drop(preview_span);
 
         let resolved_auth = self
             .resolve_operation_auth(workspace_id, &runtime.execution_config)
@@ -497,7 +526,7 @@ impl AdminService {
                     request_preview: request_preview.clone(),
                     response_preview: response_preview.clone(),
                 })
-                .await?;
+                .await;
                 Ok(TestRunResult {
                     ok: true,
                     mode,
@@ -524,7 +553,7 @@ impl AdminService {
                     request_preview: request_preview.clone(),
                     response_preview: Value::Null,
                 })
-                .await?;
+                .await;
                 Ok(TestRunResult {
                     ok: false,
                     mode,

apps/admin-api/src/service/samples.rs

@@ -50,6 +50,7 @@ impl AdminService {
             .save_sample_metadata(SaveSampleMetadataRequest { sample: &metadata })
             .await?;
         info!(
+            name: "admin.sample.saved",
             operation_id = %operation_id.as_str(),
             sample_id = %metadata.id.as_str(),
             version,
@@ -119,7 +120,11 @@ impl AdminService {
             input_mapping,
             output_mapping,
         };
-        info!(operation_id = %operation_id.as_str(), "draft generated from samples");
+        info!(
+            name: "admin.operation_draft.generated",
+            operation_id = %operation_id.as_str(),
+            "draft generated from samples"
+        );
 
         Ok(result)
     }

apps/admin-api/src/service/secrets.rs

@@ -92,7 +92,11 @@ impl AdminService {
                 created_by,
             })
             .await?;
-        info!(secret_id = %secret.id.as_str(), "secret created");
+        info!(
+            name: "admin.secret.created",
+            secret_id = %secret.id.as_str(),
+            "secret created"
+        );
 
         Ok(secret)
     }
@@ -126,7 +130,11 @@ impl AdminService {
                 created_by,
             })
             .await?;
-        info!(secret_id = %secret_id.as_str(), "secret rotated");
+        info!(
+            name: "admin.secret.rotated",
+            secret_id = %secret_id.as_str(),
+            "secret rotated"
+        );
 
         self.get_secret(workspace_id, secret_id).await
     }
@@ -152,7 +160,11 @@ impl AdminService {
             .into());
         }
         self.registry.delete_secret(workspace_id, secret_id).await?;
-        info!(secret_id = %secret_id.as_str(), "secret deleted");
+        info!(
+            name: "admin.secret.deleted",
+            secret_id = %secret_id.as_str(),
+            "secret deleted"
+        );
         Ok(())
     }
 
@@ -201,7 +213,11 @@ impl AdminService {
                 profile: &profile,
             })
             .await?;
-        info!(auth_profile_id = %profile.id.as_str(), "auth profile created");
+        info!(
+            name: "admin.auth_profile.created",
+            auth_profile_id = %profile.id.as_str(),
+            "auth profile created"
+        );
 
         Ok(profile)
     }

apps/admin-api/src/service/upstreams.rs

@@ -75,7 +75,11 @@ impl AdminService {
                 upstream: &upstream,
             })
             .await?;
-        info!(upstream_id = %upstream.id.as_str(), "workspace upstream saved");
+        info!(
+            name: "admin.upstream.saved",
+            upstream_id = %upstream.id.as_str(),
+            "workspace upstream saved"
+        );
 
         Ok(upstream)
     }

apps/admin-api/tests/dc08.rs

@@ -0,0 +1,208 @@
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+use std::{io, sync::Mutex};
+
+use axum::{Json, Router, extract::State, routing::post};
+use crank_core::{OperationId, WorkspaceId};
+use serde_json::{Value, json};
+use serial_test::serial;
+use tokio::{net::TcpListener, sync::Notify};
+use tracing_subscriber::fmt::MakeWriter;
+
+#[path = "integration/common.rs"]
+mod common;
+
+use common::*;
+
+const DEFAULT_WORKSPACE_ID: &str = "ws_default";
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn preserves_external_success_when_invocation_history_is_lost() {
+    let registry = test_registry().await;
+    let registry_for_failure = registry.clone();
+    let storage_root = test_storage_root("observability_history_loss");
+    let upstream = spawn_blocking_upstream_server().await;
+    let log_writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        crank_observability::ObservabilityConfig::new(
+            crank_observability::ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            "info",
+            crank_observability::RedactionLimits::default(),
+        ),
+        log_writer.clone(),
+    )
+    .unwrap();
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+    let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
+    let client = authorized_client(&base_url).await;
+
+    let created = client
+        .post(format!("{base_url}/operations"))
+        .json(&test_operation_payload(
+            &upstream.base_url,
+            "crm_history_loss",
+        ))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    let operation_id = created["operation_id"].as_str().unwrap().to_owned();
+    let request_client = client.clone();
+    let request_url = format!("{base_url}/operations/{operation_id}/test-runs");
+    let before = crank_observability::operational_incident_total(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+
+    let request = tokio::spawn(async move {
+        request_client
+            .post(request_url)
+            .header("x-request-id", "req_dc08_admin")
+            .json(&json!({
+                "version": 1,
+                "input": { "email": "dc08-canary-secret@example.com" }
+            }))
+            .send()
+            .await
+            .unwrap()
+    });
+
+    upstream.started.notified().await;
+    registry_for_failure
+        .delete_operation(
+            &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
+            &OperationId::new(operation_id.clone()),
+        )
+        .await
+        .unwrap();
+    upstream.release.notify_one();
+
+    let response = request.await.unwrap();
+    assert_eq!(response.status(), reqwest::StatusCode::OK);
+    assert_eq!(
+        response.headers()["x-request-id"].to_str().unwrap(),
+        "req_dc08_admin"
+    );
+    let body = response.json::<Value>().await.unwrap();
+    assert_eq!(body["ok"], true);
+    assert_eq!(body["response_preview"]["id"], "lead_123");
+    assert_eq!(upstream.calls.load(Ordering::SeqCst), 1);
+    assert!(
+        crank_observability::operational_incident_total(
+            crank_observability::OperationalIncident::InvocationHistoryLost
+        ) > before
+    );
+
+    let logs = client
+        .get(format!("{base_url}/logs?period=7d"))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    assert!(logs["items"].as_array().unwrap().is_empty());
+
+    let output = log_writer.output();
+    assert!(!output.contains("dc08-canary-secret"));
+    let incident = output
+        .lines()
+        .filter_map(|line| serde_json::from_str::<Value>(line).ok())
+        .find(|event| event["event"] == "admin.invocation_history.lost")
+        .expect("DC-08 incident");
+    assert_eq!(incident["request_id"], "req_dc08_admin");
+    assert_eq!(incident["fields"]["source"], "admin_test_run");
+}
+
+struct BlockingUpstream {
+    base_url: String,
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+#[derive(Clone)]
+struct BlockingUpstreamState {
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+async fn spawn_blocking_upstream_server() -> BlockingUpstream {
+    let state = BlockingUpstreamState {
+        started: Arc::new(Notify::new()),
+        release: Arc::new(Notify::new()),
+        calls: Arc::new(AtomicUsize::new(0)),
+    };
+    let app = Router::new()
+        .route("/crm/leads", post(blocking_create_lead))
+        .with_state(state.clone());
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    BlockingUpstream {
+        base_url: format!("http://{address}"),
+        started: state.started,
+        release: state.release,
+        calls: state.calls,
+    }
+}
+
+async fn blocking_create_lead(
+    State(state): State<BlockingUpstreamState>,
+    Json(payload): Json<Value>,
+) -> Json<Value> {
+    state.calls.fetch_add(1, Ordering::SeqCst);
+    state.started.notify_one();
+    state.release.notified().await;
+
+    Json(json!({
+        "id": "lead_123",
+        "status": "created",
+        "email": payload["email"]
+    }))
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

apps/admin-api/tests/integration/community_access_usage.rs

@@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind};
 use serde_json::{Value, json};
 use serial_test::serial;
 use tokio::net::TcpListener;
+use uuid::Version;
 
 use admin_api::{
     app::build_app,
@@ -385,6 +386,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() {
         exported["workspace"]["workspace"]["id"],
         DEFAULT_WORKSPACE_ID
     );
+    assert_eq!(exported["kind"], "workspace_catalog_snapshot");
+    assert_eq!(exported["format_version"], "1");
+    assert_eq!(exported["restorable"], false);
+    assert_eq!(
+        exported["included"],
+        json!([
+            "workspace_settings",
+            "operation_summaries",
+            "agent_summaries",
+            "platform_api_key_metadata"
+        ])
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("secret_values"))
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("invocation_logs_and_usage"))
+    );
     assert!(exported.get("memberships").is_none());
     assert!(exported.get("invitations").is_none());
 
@@ -464,7 +489,7 @@ async fn seeds_demo_assets_for_live_ui() {
                 display_name: "Legacy Smoke Agent".to_owned(),
                 description: "Keeps a legacy smoke operation published".to_owned(),
                 instructions: json!({}),
-                tool_selection_policy: json!({}),
+                tool_selection_policy: Default::default(),
             },
         )
         .await
@@ -481,7 +506,8 @@ async fn seeds_demo_assets_for_live_ui() {
                 tool_title: "Legacy health smoke".to_owned(),
                 tool_description_override: None,
                 enabled: true,
-            }],
+            }]
+            .into(),
         )
         .await
         .unwrap();
@@ -573,6 +599,25 @@ async fn updates_profile_and_changes_password() {
         .unwrap()
         .to_owned();
     let client = authorized_client(&base_url).await;
+    let second_client = reqwest::Client::builder()
+        .cookie_store(true)
+        .build()
+        .unwrap();
+    let second_login = second_client
+        .post(format!("{root_url}/api/auth/login"))
+        .json(&json!({
+            "email": TEST_AUTH_EMAIL,
+            "password": TEST_AUTH_PASSWORD,
+        }))
+        .send()
+        .await
+        .unwrap();
+    let second_login_status = second_login.status();
+    let second_login_body = second_login.text().await.unwrap();
+    assert!(
+        second_login_status.is_success(),
+        "second login failed with {second_login_status}: {second_login_body}"
+    );
 
     let profile = assert_success_json(
         client
@@ -614,6 +659,21 @@ async fn updates_profile_and_changes_password() {
         .status();
     assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);
 
+    let current_session_status = client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    let other_session_status = second_client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    assert_eq!(current_session_status, reqwest::StatusCode::OK);
+    assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
+
     let relogin_client = reqwest::Client::builder()
         .cookie_store(true)
         .build()
@@ -854,7 +914,10 @@ async fn generates_request_id_for_test_run_invocations() {
         .unwrap()
         .to_owned();
 
-    assert!(!request_id.is_empty());
+    assert_eq!(
+        uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
     response.error_for_status().unwrap();
 
     let logs = client

apps/admin-api/tests/integration/openapi_import.rs

@@ -1,5 +1,6 @@
 use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
 use crank_core::WorkspaceId;
+use crank_registry::ImportJobStatus;
 use serial_test::serial;
 
 use super::common::{
@@ -42,7 +43,7 @@ paths:
 async fn previews_openapi_and_creates_draft_operations() {
     let registry = test_registry().await;
     let service = test_service(
-        registry,
+        registry.clone(),
         test_storage_root("openapi_import"),
         test_auth_settings(),
         test_secret_crypto(),
@@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() {
         preview.preview.groups[0].operations[0].suggested_name,
         "latest_rates"
     );
+    let preview_job = registry
+        .get_import_job(&workspace_id, &preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(preview_job.status, ImportJobStatus::Pending);
 
     let created = service
         .create_openapi_import(
@@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() {
             .any(|finding| finding.code == "openapi_import.weak_tool_description")
     );
 
+    let skip_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let skipped = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &skip_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(skipped.skipped[0].name, "latest_rates");
     assert_eq!(skipped.findings[0].code, "operation_name_conflict");
 
+    let rename_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let renamed = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &rename_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(renamed.created[0].name, "latest_rates_2");
     assert_eq!(renamed.findings[0].code, "operation_name_renamed");
 }
+
+#[tokio::test]
+#[serial]
+async fn concurrent_openapi_import_replays_the_same_atomic_result() {
+    let registry = test_registry().await;
+    let service = test_service(
+        registry,
+        test_storage_root("openapi_import_replay"),
+        test_auth_settings(),
+        test_secret_crypto(),
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    let job_id = preview.job_id.as_str().into();
+    let payload = OpenApiImportCreatePayload {
+        selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+        server_url: Some("https://api.frankfurter.dev".to_owned()),
+        conflict_mode: "rename".to_owned(),
+    };
+
+    let (first, second) = tokio::join!(
+        service.create_openapi_import(&workspace_id, &job_id, payload.clone()),
+        service.create_openapi_import(&workspace_id, &job_id, payload),
+    );
+    let first = first.unwrap();
+    let second = second.unwrap();
+
+    assert_eq!(first.created.len(), 1);
+    assert_eq!(second.created.len(), 1);
+    assert_eq!(
+        first.created[0].operation_id,
+        second.created[0].operation_id
+    );
+    assert_eq!(first.created[0].name, second.created[0].name);
+    assert_eq!(
+        service.list_operations(&workspace_id).await.unwrap().len(),
+        1
+    );
+
+    let conflicting_replay = service
+        .create_openapi_import(
+            &workspace_id,
+            &job_id,
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+                server_url: Some("https://api.frankfurter.dev".to_owned()),
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await;
+    assert!(conflicting_replay.is_err());
+}

apps/admin-api/tests/integration/operations_agents.rs

@@ -398,6 +398,116 @@ async fn creates_binds_and_publishes_agent() {
     assert_eq!(published["published_version"], 1);
 }
 
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn saves_and_previews_versioned_agent_tool_search_policy() {
+    let registry = test_registry().await;
+    let storage_root = test_storage_root("agent_tool_search");
+    let upstream_base_url = spawn_upstream_server().await;
+    let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
+    let client = authorized_client(&base_url).await;
+
+    let operation = assert_success_json(
+        client
+            .post(format!("{base_url}/operations"))
+            .json(&test_operation_payload(
+                &upstream_base_url,
+                "finance_create_invoice",
+            ))
+            .send()
+            .await
+            .unwrap(),
+    )
+    .await;
+    let operation_id = operation["operation_id"].as_str().unwrap().to_owned();
+    assert_success_json(
+        client
+            .post(format!("{base_url}/operations/{operation_id}/publish"))
+            .json(&json!({"version": 1}))
+            .send()
+            .await
+            .unwrap(),
+    )
+    .await;
+
+    let agent = assert_success_json(
+        client
+            .post(format!("{base_url}/agents"))
+            .json(&json!({
+                "slug": "finance-agent",
+                "display_name": "Finance Agent",
+                "description": "Finance workflows",
+                "instructions": {},
+                "tool_selection_policy": {}
+            }))
+            .send()
+            .await
+            .unwrap(),
+    )
+    .await;
+    let agent_id = agent["agent_id"].as_str().unwrap().to_owned();
+    let catalog = json!({
+        "bindings": [{
+            "operation_id": operation_id,
+            "operation_version": 1,
+            "tool_name": "finance_create_invoice",
+            "tool_title": "Create Lead",
+            "tool_description_override": "Creates an invoice for a customer",
+            "enabled": true
+        }],
+        "tool_selection_policy": {
+            "mode": "search",
+            "groups": [{
+                "id": "finance",
+                "name": "Finance",
+                "description": "Invoices and payments",
+                "tool_names": ["finance_create_invoice"]
+            }],
+            "search": {"max_results": 5}
+        }
+    });
+    let saved = assert_success_json(
+        client
+            .post(format!("{base_url}/agents/{agent_id}/bindings"))
+            .json(&catalog)
+            .send()
+            .await
+            .unwrap(),
+    )
+    .await;
+    assert_eq!(saved["snapshot"]["tool_selection_policy"]["mode"], "search");
+
+    let preview = assert_success_json(
+        client
+            .post(format!("{base_url}/agents/tool-search/preview"))
+            .json(&json!({
+                "query": "create invoice",
+                "group_ids": ["finance"],
+                "bindings": catalog["bindings"],
+                "tool_selection_policy": catalog["tool_selection_policy"]
+            }))
+            .send()
+            .await
+            .unwrap(),
+    )
+    .await;
+    assert_eq!(
+        preview["items"][0]["tool"]["name"],
+        "finance_create_invoice"
+    );
+
+    let published = assert_success_json(
+        client
+            .post(format!("{base_url}/agents/{agent_id}/publish"))
+            .json(&json!({"version": 1}))
+            .send()
+            .await
+            .unwrap(),
+    )
+    .await;
+    assert_eq!(published["published_version"], 1);
+}
+
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
 async fn agent_publish_skips_draft_operation_bindings_and_preserves_draft() {

apps/admin-api/tests/integration/request_context.rs

@@ -0,0 +1,240 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
+use axum::{
+    Router,
+    body::Body,
+    http::{HeaderMap, HeaderValue, Request, StatusCode},
+    routing::get,
+};
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
+};
+use opentelemetry::{global, trace::TracerProvider as _};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
+use tower::ServiceExt;
+use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
+use uuid::Version;
+
+static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+#[tokio::test(flavor = "current_thread")]
+async fn logs_request_completion_and_rejects_untrusted_values() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let app = probe_app();
+
+    let response = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(
+        response.headers()[REQUEST_ID_HEADER.as_str()]
+            .to_str()
+            .unwrap(),
+        "req_admin_trace_123"
+    );
+    let event: serde_json::Value = writer
+        .output()
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
+        .unwrap();
+    assert_eq!(event["request_id"], "req_admin_trace_123");
+    assert_eq!(event["fields"]["status"], 200);
+
+    let invalid_response = app
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "bad,value")
+                .header("traceparent", "canary-invalid-traceparent")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+    let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
+        .to_str()
+        .unwrap();
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
+    assert!(!writer.output().contains("canary-invalid-traceparent"));
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn covers_valid_invalid_and_absent_traceparent() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("admin-request-context-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let app = trace_probe_app();
+
+    let valid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header(
+                        "traceparent",
+                        "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+                    )
+                    .header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap(),
+    );
+    let invalid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header("traceparent", "canary-invalid-traceparent")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap(),
+    );
+    let absent = observed_trace_id(
+        app.oneshot(
+            Request::builder()
+                .uri("/trace")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap(),
+    );
+
+    assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c");
+    assert_ne!(invalid, valid);
+    assert_ne!(absent, valid);
+    assert_ne!(invalid, absent);
+    provider.shutdown().unwrap();
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    let dispatch = tracing::Dispatch::new(tracing_subscriber::registry());
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let mut request = Request::builder()
+        .uri("/probe")
+        .body(Body::empty())
+        .unwrap();
+    request.headers_mut().append(
+        REQUEST_ID_HEADER,
+        HeaderValue::from_static("first-request-id"),
+    );
+    request.headers_mut().append(
+        REQUEST_ID_HEADER,
+        HeaderValue::from_static("second-request-id"),
+    );
+
+    let response = probe_app().oneshot(request).await.unwrap();
+    let generated = response.headers()[REQUEST_ID_HEADER].to_str().unwrap();
+
+    assert_ne!(generated, "first-request-id");
+    assert_ne!(generated, "second-request-id");
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
+}
+
+fn probe_app() -> Router {
+    Router::new()
+        .route("/probe", get(|| async { "ok" }))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+fn trace_probe_app() -> Router {
+    Router::new()
+        .route("/trace", get(observed_traceparent))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+async fn observed_traceparent() -> HeaderMap {
+    let mut trace_headers = HeaderMap::new();
+    inject_current_trace_context(&mut trace_headers);
+    let mut response_headers = HeaderMap::new();
+    if let Some(traceparent) = trace_headers.remove("traceparent") {
+        response_headers.insert("x-observed-traceparent", traceparent);
+    }
+    response_headers
+}
+
+fn observed_trace_id(response: axum::response::Response) -> String {
+    let traceparent = response.headers()["x-observed-traceparent"]
+        .to_str()
+        .unwrap();
+    traceparent[3..35].to_owned()
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

apps/admin-api/tests/request_context.rs

@@ -0,0 +1,2 @@
+#[path = "integration/request_context.rs"]
+mod request_context;

apps/mcp-server/Cargo.toml

@@ -3,6 +3,7 @@ name = "mcp-server"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [[bin]]
@@ -15,10 +16,12 @@ axum.workspace = true
 base64.workspace = true
 crank-community-mcp = { path = "../../crates/crank-community-mcp" }
 crank-core = { path = "../../crates/crank-core" }
+crank-observability = { path = "../../crates/crank-observability" }
 crank-registry = { path = "../../crates/crank-registry" }
 crank-runtime = { path = "../../crates/crank-runtime" }
 crank-schema = { path = "../../crates/crank-schema" }
 futures-util = "0.3"
+metrics.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
@@ -34,4 +37,10 @@ uuid.workspace = true
 crank-mapping = { path = "../../crates/crank-mapping" }
 crank-schema = { path = "../../crates/crank-schema" }
 crank-test-support = { path = "../../crates/crank-test-support" }
+opentelemetry.workspace = true
+opentelemetry-proto.workspace = true
+opentelemetry_sdk.workspace = true
+prost.workspace = true
 reqwest.workspace = true
+tower.workspace = true
+tracing-opentelemetry.workspace = true

apps/mcp-server/src/main.rs

@@ -1,26 +1,53 @@
 use std::{env, net::SocketAddr, time::Duration};
 
 use crank_community_mcp::{
-    auth::CommunityMachineCredentialVerifier, build_app_with_background_workers,
+    auth::CommunityMachineCredentialVerifier, build_app_with_background_workers_and_limits,
     session::PostgresTransportSessionStore,
 };
+use crank_observability::{
+    CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
+    capture_critical_error,
+};
 use crank_registry::{PostgresPoolConfig, PostgresRegistry};
 use crank_runtime::{
     RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
     RuntimeLimits, SecretCrypto,
 };
-use sqlx::postgres::PgConnectOptions;
+use sqlx::{PgPool, postgres::PgConnectOptions};
 use tokio::net::TcpListener;
 use tracing::info;
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    tracing_subscriber::fmt()
-        .with_env_filter(
-            env::var("CRANK_LOG_LEVEL")
-                .unwrap_or_else(|_| "mcp_server=info,tower_http=info".into()),
-        )
-        .init();
+    let observability = crank_observability::init(ObservabilityConfig::from_env(
+        "mcp-server",
+        env!("CARGO_PKG_VERSION"),
+        "mcp_server=info,tower_http=info",
+    )?)?;
+    let mut startup_completed = false;
+    let result = run(&observability, &mut startup_completed).await;
+    if result.is_err() {
+        capture_critical_error(if startup_completed {
+            CriticalErrorCategory::Internal
+        } else {
+            CriticalErrorCategory::Startup
+        });
+    }
+    result
+}
+
+async fn run(
+    observability: &ObservabilityLifecycle,
+    startup_completed: &mut bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let metrics_config =
+        MetricsConfig::from_env("CRANK_MCP_METRICS_BIND", "127.0.0.1:9465".parse()?)?;
+    let metrics_enabled = metrics_config.enabled();
+    let metrics_server = if metrics_config.enabled() {
+        Some(observability.metrics_surface(metrics_config).bind().await?)
+    } else {
+        None
+    };
 
     let bind_addr = env::var("CRANK_MCP_BIND").unwrap_or_else(|_| "0.0.0.0:3002".into());
     let base_url = env::var("CRANK_BASE_URL").ok();
@@ -36,23 +63,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
     let api_rate_limit = mcp_api_rate_limit_config_from_env()?;
     let database_options = database_options_from_env()?;
-    let registry = PostgresRegistry::connect_with_options_and_pool_config(
-        database_options.clone(),
-        pool_config,
-    )
-    .await?;
-    let session_store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
-        database_options,
-        pool_config,
-    )
+    let registry =
+        PostgresRegistry::connect_with_options_and_pool_config(database_options, pool_config)
             .await?;
+    if metrics_enabled {
+        spawn_postgres_pool_metrics(registry.pool().clone());
+    }
+    let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
     let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?;
     let runtime = crank_runtime::community_from_env()?
         .with_limits(runtime_limits)
         .with_response_cache(cache_stores.response.clone())
         .with_coordination_store(cache_stores.coordination.clone())
         .build();
-    let app = build_app_with_background_workers(
+    let app = build_app_with_background_workers_and_limits(
         registry,
         refresh_interval,
         base_url,
@@ -66,11 +90,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         cache_stores.coordination.clone(),
         std::sync::Arc::new(session_store),
         std::sync::Arc::new(CommunityMachineCredentialVerifier),
+        runtime_limits.max_concurrent_sessions,
     );
     let listener = TcpListener::bind(socket_addr).await?;
 
     info!(
+        name: "mcp.postgres_pool.configured",
         runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
+        runtime_max_concurrent_sessions = runtime_limits.max_concurrent_sessions,
         mcp_rate_limit_rps = api_rate_limit.requests_per_second,
         mcp_rate_limit_burst = api_rate_limit.burst,
         cache_backend = %cache_config.backend,
@@ -81,9 +108,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         max_lifetime_ms = pool_config.max_lifetime_ms,
         "postgres pool configured"
     );
-    info!("mcp-server listening on {}", socket_addr);
+    info!(
+        name: "mcp.server.listening",
+        bind_address = %socket_addr,
+        "mcp-server listening"
+    );
+    *startup_completed = true;
 
+    if let Some(metrics_server) = metrics_server {
+        tokio::select! {
+            result = axum::serve(listener, app) => result?,
+            result = metrics_server.serve() => result?,
+        }
+    } else {
         axum::serve(listener, app).await?;
+    }
 
     Ok(())
 }
@@ -123,3 +162,13 @@ fn mcp_api_rate_limit_config_from_env() -> Result<RequestRateLimitConfig, Box<dy
 
     Ok(RequestRateLimitConfig::new(requests_per_second, burst)?)
 }
+
+fn spawn_postgres_pool_metrics(pool: PgPool) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(5));
+        loop {
+            interval.tick().await;
+            crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
+        }
+    });
+}

apps/mcp-server/tests/dc08.rs

@@ -0,0 +1,227 @@
+use std::{
+    io,
+    sync::{
+        Arc, Mutex,
+        atomic::{AtomicUsize, Ordering},
+    },
+    time::Duration,
+};
+
+use axum::{Json, Router, routing::post};
+use crank_core::{PlatformApiKeyScope, WorkspaceId};
+use crank_observability::{
+    ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+    operational_incident_total,
+};
+use crank_registry::{ListInvocationLogsQuery, PublishRequest};
+use serde_json::{Value, json};
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tokio::net::TcpListener;
+use tracing_subscriber::fmt::MakeWriter;
+
+#[path = "integration/common.rs"]
+mod common;
+
+use common::*;
+
+const CANARY_SECRET: &str = "dc08-canary-secret";
+
+#[tokio::test]
+async fn preserves_mcp_result_when_postgres_rejects_invocation_history() {
+    let registry = test_registry().await;
+    let upstream_calls = Arc::new(AtomicUsize::new(0));
+    let upstream_base_url = spawn_counted_upstream(Arc::clone(&upstream_calls)).await;
+    let operation = test_operation(&upstream_base_url, "crm_dc08");
+
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    registry
+        .publish_operation(PublishRequest {
+            workspace_id: &test_workspace_id(),
+            operation_id: &operation.id,
+            version: 1,
+            published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
+            published_by: Some("alice"),
+        })
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-dc08").await;
+    let api_key = create_platform_api_key(
+        &registry,
+        "sales-dc08",
+        "mcp-dc08",
+        &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
+    )
+    .await;
+    install_history_failure_trigger(&registry).await;
+
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+    let base_url = spawn_mcp_server(build_test_app(
+        registry.clone(),
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+    ))
+    .await;
+    let client = reqwest::Client::new();
+    let mcp_url = agent_mcp_url(&base_url, "sales-dc08");
+    let session_id = initialize_session(&client, &mcp_url, &api_key).await;
+    let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+
+    let result = post_jsonrpc_response(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&session_id),
+        Some("req_mcp_dc08"),
+        json!({
+            "jsonrpc": "2.0",
+            "id": 3,
+            "method": "tools/call",
+            "params": {
+                "name": "crm_dc08",
+                "arguments": {
+                    "email": format!("{CANARY_SECRET}@example.com")
+                }
+            }
+        }),
+    )
+    .await
+    .json::<Value>()
+    .await
+    .unwrap();
+
+    assert_eq!(
+        result["result"]["structuredContent"],
+        json!({"id": "lead_123"})
+    );
+    assert_eq!(result["result"]["isError"], false);
+    assert_eq!(upstream_calls.load(Ordering::SeqCst), 1);
+    assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+
+    let logs = registry
+        .list_invocation_logs(ListInvocationLogsQuery {
+            workspace_id: &WorkspaceId::new("ws_default"),
+            level: None,
+            search_text: None,
+            source: None,
+            operation_id: Some(&operation.id),
+            agent_id: None,
+            created_after: None,
+            limit: 10,
+        })
+        .await
+        .unwrap();
+    assert!(logs.is_empty());
+
+    let output = writer.output();
+    assert!(!output.contains(CANARY_SECRET));
+    let incidents = output
+        .lines()
+        .filter_map(|line| serde_json::from_str::<Value>(line).ok())
+        .filter(|event| event["event"] == "mcp.invocation_history.lost")
+        .collect::<Vec<_>>();
+    assert_eq!(incidents.len(), 1);
+    assert_eq!(incidents[0]["request_id"], "req_mcp_dc08");
+    assert_eq!(incidents[0]["fields"]["source"], "agent_tool_call");
+    assert_eq!(incidents[0]["fields"]["invocation_status"], "ok");
+}
+
+async fn install_history_failure_trigger(registry: &crank_registry::PostgresRegistry) {
+    sqlx::query(
+        r#"
+        create function fail_dc08_invocation_history() returns trigger
+        language plpgsql
+        as $$
+        begin
+            if new.request_id = 'req_mcp_dc08' then
+                raise exception 'forced DC-08 invocation history failure';
+            end if;
+            return new;
+        end;
+        $$
+        "#,
+    )
+    .execute(registry.pool())
+    .await
+    .unwrap();
+    sqlx::query(
+        r#"
+        create trigger fail_dc08_invocation_history
+        before insert on invocation_logs
+        for each row execute function fail_dc08_invocation_history()
+        "#,
+    )
+    .execute(registry.pool())
+    .await
+    .unwrap();
+}
+
+async fn spawn_counted_upstream(calls: Arc<AtomicUsize>) -> String {
+    let app = Router::new().route(
+        "/crm/leads",
+        post(move |Json(payload): Json<Value>| {
+            let calls = Arc::clone(&calls);
+            async move {
+                calls.fetch_add(1, Ordering::SeqCst);
+                Json(json!({
+                    "id": "lead_123",
+                    "email": payload["email"]
+                }))
+            }
+        }),
+    );
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+    format!("http://{address}")
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

apps/mcp-server/tests/execution_stages.rs

@@ -0,0 +1,4 @@
+#[path = "integration/common.rs"]
+mod common;
+#[path = "integration/execution_stages.rs"]
+mod execution_stages;

apps/mcp-server/tests/integration.rs

@@ -1,5 +1,6 @@
 mod integration {
     mod catalog_access;
     mod common;
+    mod tool_search;
     mod transport_protocol;
 }

apps/mcp-server/tests/integration/catalog_access/approval_access.rs

@@ -89,11 +89,16 @@ async fn approval_key_lists_and_decides_pending_requests() {
     let approved = client
         .post(&approve_url)
         .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .header("x-request-id", "req_approval_execute_123")
         .json(&json!({ "approve": "yes", "note": "confirmed by test" }))
         .send()
         .await
         .unwrap();
     assert_eq!(approved.status(), reqwest::StatusCode::OK);
+    assert_eq!(
+        approved.headers()["x-request-id"].to_str().unwrap(),
+        "req_approval_execute_123"
+    );
     let approved_body = approved.json::<Value>().await.unwrap();
     assert_eq!(
         approved_body["approval"]["status"],
@@ -169,6 +174,10 @@ async fn approval_key_lists_and_decides_pending_requests() {
         .await
         .unwrap();
     assert_eq!(logs.len(), 1);
+    assert_eq!(
+        logs[0].log.request_id.as_deref(),
+        Some("req_approval_execute_123")
+    );
 }
 
 #[tokio::test]
@@ -401,12 +410,7 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
     let mcp_url = agent_mcp_url(&base_url, "sales-gated");
     let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
 
-    let tool_result = post_jsonrpc(
-        &client,
-        &mcp_url,
-        &api_key,
-        Some(&initialized_session),
-        json!({
+    let tool_call = json!({
         "jsonrpc": "2.0",
         "id": 9,
         "method": "tools/call",
@@ -416,7 +420,13 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
                 "email": "ada@example.com"
             }
         }
-        }),
+    });
+    let tool_result = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&initialized_session),
+        tool_call.clone(),
     )
     .await;
 
@@ -430,6 +440,19 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
         .unwrap();
     assert!(approval_id.starts_with("approval_"));
 
+    let repeated_tool_result = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&initialized_session),
+        tool_call,
+    )
+    .await;
+    assert_eq!(
+        repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
+        "deduplicated tools/call must return the persisted approval id",
+    );
+
     let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
     let pending = client
         .get(&approvals_url)
@@ -448,6 +471,238 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
     );
 }
 
+#[tokio::test]
+async fn approval_http_endpoints_enforce_request_rate_limit() {
+    let registry = test_registry().await;
+    let upstream_base_url = spawn_upstream_server().await;
+    let operation = test_operation(&upstream_base_url, "crm_approval_rate_limit");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-approval-rate-limit").await;
+    let approval_key = create_approval_platform_api_key(
+        &registry,
+        "sales-approval-rate-limit",
+        "approval-rate-limit",
+    )
+    .await;
+    let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+        RequestRateLimitConfig::new(1, 1).unwrap(),
+    ))
+    .await;
+    let approvals_url = format!(
+        "{}/approvals",
+        agent_mcp_url(&base_url, "sales-approval-rate-limit")
+    );
+    let client = reqwest::Client::new();
+
+    let allowed = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(allowed.status(), reqwest::StatusCode::OK);
+
+    let limited = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
+    assert!(limited.headers().contains_key(header::RETRY_AFTER));
+}
+
+#[tokio::test]
+async fn unverified_session_ids_do_not_create_approval_rate_limit_buckets() {
+    let registry = test_registry().await;
+    let upstream_base_url = spawn_upstream_server().await;
+    let operation = test_operation(&upstream_base_url, "crm_approval_session_rate_limit");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-approval-session-rate-limit").await;
+    let approval_key = create_approval_platform_api_key(
+        &registry,
+        "sales-approval-session-rate-limit",
+        "approval-session-rate-limit",
+    )
+    .await;
+    let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+        RequestRateLimitConfig::new(1, 1).unwrap(),
+    ))
+    .await;
+    let approvals_url = format!(
+        "{}/approvals",
+        agent_mcp_url(&base_url, "sales-approval-session-rate-limit")
+    );
+    let client = reqwest::Client::new();
+
+    let allowed = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .header("MCP-Session-Id", "unverified-session-a")
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(allowed.status(), reqwest::StatusCode::OK);
+
+    let limited = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .header("MCP-Session-Id", "unverified-session-b")
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
+    assert!(limited.headers().contains_key(header::RETRY_AFTER));
+}
+
+#[tokio::test]
+async fn recovery_does_not_repeat_interrupted_mutating_approval() {
+    let registry = test_registry().await;
+    let (upstream_base_url, upstream_calls) = spawn_counted_approval_upstream().await;
+    let operation = test_operation(&upstream_base_url, "crm_interrupted_approval");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-interrupted-approval").await;
+    let approval_key_name = "approval-interrupted";
+    create_approval_platform_api_key(&registry, "sales-interrupted-approval", approval_key_name)
+        .await;
+    let now = OffsetDateTime::now_utc();
+    let approval = ApprovalRequest {
+        id: ApprovalRequestId::new("approval_interrupted_mutation"),
+        workspace_id: test_workspace_id(),
+        agent_id: test_agent_id("sales-interrupted-approval"),
+        operation_id: operation.id.clone(),
+        operation_version: operation.version,
+        status: ApprovalRequestStatus::Pending,
+        risk_level: OperationApprovalRiskLevel::Dangerous,
+        request_payload: json!({"email": "interrupted@example.com"}),
+        response_payload: None,
+        created_at: now - time::Duration::minutes(10),
+        expires_at: now + time::Duration::minutes(5),
+        decided_at: None,
+        decided_by_key_id: None,
+        decision_note: None,
+    };
+    registry
+        .create_approval_request(CreateApprovalRequest {
+            approval: &approval,
+        })
+        .await
+        .unwrap();
+    let approval_key_id = PlatformApiKeyId::new(format!("pk_{approval_key_name}"));
+    registry
+        .decide_approval_request(crank_registry::DecideApprovalRequest {
+            workspace_id: &approval.workspace_id,
+            agent_id: &approval.agent_id,
+            approval_id: &approval.id,
+            status: ApprovalRequestStatus::Approved,
+            decided_at: now - time::Duration::minutes(10),
+            decided_by_key_id: &approval_key_id,
+            response_payload: Some(json!({"approve": "yes"})),
+            decision_note: None,
+        })
+        .await
+        .unwrap()
+        .unwrap();
+    registry
+        .claim_approval_request(
+            &approval.workspace_id,
+            &approval.agent_id,
+            &approval.id,
+            now - time::Duration::minutes(7),
+        )
+        .await
+        .unwrap()
+        .unwrap();
+
+    let _app = build_test_app_with_approval_recovery(registry.clone());
+    let failed = tokio::time::timeout(Duration::from_secs(2), async {
+        loop {
+            let current = registry
+                .get_approval_request_for_agent(
+                    &approval.workspace_id,
+                    &approval.agent_id,
+                    &approval.id,
+                )
+                .await
+                .unwrap()
+                .unwrap();
+            if current.approval.status == ApprovalRequestStatus::Failed {
+                break current;
+            }
+            tokio::time::sleep(Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("recovery must quarantine interrupted execution");
+
+    assert_eq!(
+        failed.approval.response_payload.unwrap()["error"]["code"],
+        "approval_execution_outcome_unknown"
+    );
+    assert_eq!(
+        upstream_calls.load(std::sync::atomic::Ordering::SeqCst),
+        0,
+        "recovery must not repeat a mutating upstream request"
+    );
+}
+
+fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router {
+    crank_community_mcp::build_app_with_background_workers(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+        SecretCrypto::new("test-master-key").unwrap(),
+        crank_runtime::community_with_outbound_policy(
+            crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
+        )
+        .build(),
+        RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
+        Arc::new(InMemoryCoordinationStateStore::default()),
+        Arc::new(InMemorySessionStore::default()),
+        Arc::new(CommunityMachineCredentialVerifier),
+    )
+}
+
+async fn spawn_counted_approval_upstream() -> (String, Arc<std::sync::atomic::AtomicUsize>) {
+    let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let handler_calls = Arc::clone(&calls);
+    let app = Router::new().route(
+        "/crm/leads",
+        post(move |Json(payload): Json<Value>| {
+            let calls = Arc::clone(&handler_calls);
+            async move {
+                calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                Json(json!({
+                    "id": "lead_123",
+                    "email": payload["email"]
+                }))
+            }
+        }),
+    );
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    (format!("http://{address}"), calls)
+}
+
 #[tokio::test]
 async fn elicitation_approval_requires_client_capability() {
     let registry = test_registry().await;

apps/mcp-server/tests/integration/common.rs

@@ -18,7 +18,7 @@ use crank_core::{
     Agent, AgentId, AgentOperationBinding, AgentStatus, AgentVersion, ExecutionConfig, HttpMethod,
     Operation, OperationId, OperationStatus, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind,
     PlatformApiKeyScope, PlatformApiKeyStatus, Protocol, RestTarget, Target, ToolDescription,
-    WorkspaceId,
+    ToolSelectionPolicy, WorkspaceId,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_registry::{
@@ -45,7 +45,7 @@ use crank_community_mcp::{
     session::{InMemorySessionStore, SharedSessionStore, TransportSessionStore},
 };
 
-fn test_workspace_id() -> WorkspaceId {
+pub(super) fn test_workspace_id() -> WorkspaceId {
     WorkspaceId::new("ws_default")
 }
 
@@ -93,7 +93,7 @@ fn test_agent_id(agent_slug: &str) -> AgentId {
     AgentId::new(format!("agent_{agent_slug}"))
 }
 
-fn build_test_app(
+pub(super) fn build_test_app(
     registry: PostgresRegistry,
     refresh_interval: Duration,
     public_base_url: Option<String>,
@@ -363,6 +363,21 @@ pub(super) async fn publish_agent_with_bindings(
     registry: &PostgresRegistry,
     agent_slug: &str,
     bindings: Vec<AgentOperationBinding>,
+) {
+    publish_agent_with_policy(
+        registry,
+        agent_slug,
+        bindings,
+        ToolSelectionPolicy::default(),
+    )
+    .await;
+}
+
+pub(super) async fn publish_agent_with_policy(
+    registry: &PostgresRegistry,
+    agent_slug: &str,
+    bindings: Vec<AgentOperationBinding>,
+    tool_selection_policy: ToolSelectionPolicy,
 ) {
     let agent_id = AgentId::new(format!("agent_{agent_slug}"));
     let agent = Agent {
@@ -383,7 +398,7 @@ pub(super) async fn publish_agent_with_bindings(
         version: 1,
         status: AgentStatus::Draft,
         instructions: json!({}),
-        tool_selection_policy: json!({}),
+        tool_selection_policy,
         created_at: OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
     };
 

apps/mcp-server/tests/integration/execution_stages.rs

@@ -0,0 +1,376 @@
+use std::{
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use axum::{
+    Json, Router,
+    body::{Body, Bytes, to_bytes},
+    extract::State,
+    http::{HeaderMap, Request, StatusCode, header},
+    routing::post,
+};
+use crank_core::{InvocationSource, PlatformApiKeyScope};
+use crank_observability::{
+    OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider,
+};
+use crank_registry::{ListInvocationLogsQuery, PublishRequest};
+use opentelemetry::global;
+use opentelemetry_proto::tonic::{
+    collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value, trace::v1::Span,
+};
+use opentelemetry_sdk::propagation::TraceContextPropagator;
+use prost::Message;
+use serde_json::{Value, json};
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tokio::net::TcpListener;
+use tower::ServiceExt;
+use tracing::instrument::WithSubscriber;
+use tracing_subscriber::{Layer, filter::filter_fn, layer::SubscriberExt};
+
+use super::common::{
+    build_test_app, create_platform_api_key, publish_agent_for_operation, test_operation,
+    test_registry, test_workspace_id,
+};
+
+const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
+const REQUEST_ID: &str = "req_stage_end_to_end";
+const CANARY: &str = "dc-stage-canary-secret";
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn exports_real_tool_stages_without_sensitive_data() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let (otlp_endpoint, request_rx, collector) = spawn_otlp_collector().await;
+    let trace_config = OtlpTraceConfig::try_new(
+        Some(otlp_endpoint),
+        Some("http/protobuf".to_owned()),
+        Duration::from_secs(2),
+        OtlpBatchConfig::try_new(128, 128, Duration::from_secs(300), Duration::from_secs(2))
+            .unwrap(),
+    )
+    .unwrap();
+    let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "integration-test").unwrap();
+    let (provider, tracer) = build_tracer_provider(&identity, &trace_config)
+        .unwrap()
+        .expect("enabled OTLP provider");
+    let subscriber = tracing_subscriber::registry().with(
+        tracing_opentelemetry::layer()
+            .with_tracer(tracer)
+            .with_filter(filter_fn(|metadata| {
+                metadata.is_span() && metadata.target() == "crank::trace"
+            })),
+    );
+    let dispatch = tracing::Dispatch::new(subscriber);
+
+    let registry = test_registry().await;
+    let observed_traceparent = Arc::new(Mutex::new(None));
+    let upstream_base_url = spawn_upstream(Arc::clone(&observed_traceparent)).await;
+    let operation = test_operation(&upstream_base_url, "stage_end_to_end");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("test"))
+        .await
+        .unwrap();
+    registry
+        .publish_operation(PublishRequest {
+            workspace_id: &test_workspace_id(),
+            operation_id: &operation.id,
+            version: 1,
+            published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
+            published_by: Some("test"),
+        })
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "stage-agent").await;
+    let api_key = create_platform_api_key(
+        &registry,
+        "stage-agent",
+        "stage-key",
+        &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
+    )
+    .await;
+    let app = build_test_app(registry.clone(), Duration::ZERO, None);
+
+    let call_result = async {
+        let initialized = send_jsonrpc(
+            app.clone(),
+            &api_key,
+            None,
+            None,
+            None,
+            json!({
+                "jsonrpc": "2.0",
+                "id": 1,
+                "method": "initialize",
+                "params": {
+                    "protocolVersion": "2025-11-25",
+                    "capabilities": {}
+                }
+            }),
+        )
+        .await;
+        assert_eq!(initialized.status(), StatusCode::OK);
+        let session_id = initialized
+            .headers()
+            .get("MCP-Session-Id")
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .to_owned();
+
+        let notification = send_jsonrpc(
+            app.clone(),
+            &api_key,
+            Some(&session_id),
+            None,
+            None,
+            json!({
+                "jsonrpc": "2.0",
+                "method": "notifications/initialized",
+                "params": {}
+            }),
+        )
+        .await;
+        assert_eq!(notification.status(), StatusCode::ACCEPTED);
+
+        send_jsonrpc(
+            app,
+            &api_key,
+            Some(&session_id),
+            Some(REQUEST_ID),
+            Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+            json!({
+                "jsonrpc": "2.0",
+                "id": 2,
+                "method": "tools/call",
+                "params": {
+                    "name": "stage_end_to_end",
+                    "arguments": { "email": CANARY }
+                }
+            }),
+        )
+        .await
+    }
+    .with_subscriber(dispatch)
+    .await;
+
+    assert_eq!(call_result.status(), StatusCode::OK);
+    let body = to_bytes(call_result.into_body(), 1024 * 1024)
+        .await
+        .unwrap();
+    let body: Value = serde_json::from_slice(&body).unwrap();
+    assert_eq!(body["result"]["isError"], false);
+    provider.force_flush().unwrap();
+    let request = tokio::time::timeout(Duration::from_secs(2), request_rx)
+        .await
+        .unwrap()
+        .unwrap();
+    collector.abort();
+    assert_eq!(
+        request
+            .headers
+            .get(header::CONTENT_TYPE)
+            .and_then(|value| value.to_str().ok()),
+        Some("application/x-protobuf")
+    );
+    assert!(
+        !request
+            .body
+            .windows(CANARY.len())
+            .any(|window| window == CANARY.as_bytes())
+    );
+    let export = ExportTraceServiceRequest::decode(request.body).unwrap();
+    let spans = export
+        .resource_spans
+        .iter()
+        .flat_map(|resource| &resource.scope_spans)
+        .flat_map(|scope| &scope.spans)
+        .collect::<Vec<_>>();
+
+    let traceparent = observed_traceparent
+        .lock()
+        .unwrap()
+        .clone()
+        .expect("upstream traceparent");
+    assert_eq!(&traceparent[3..35], REMOTE_TRACE_ID);
+
+    let expected_trace_id = decode_trace_id(REMOTE_TRACE_ID);
+    let trace_spans = spans
+        .iter()
+        .copied()
+        .filter(|span| span.trace_id.as_slice() == expected_trace_id)
+        .collect::<Vec<_>>();
+    for expected in [
+        "mcp.request",
+        "mcp.rate_limit",
+        "mcp.access.check",
+        "mcp.catalog.load",
+        "mcp.tools.resolve",
+        "runtime.execute",
+        "runtime.arguments.map",
+        "upstream.http",
+        "runtime.response.transform",
+        "history.write",
+        "db.query",
+    ] {
+        assert!(
+            trace_spans.iter().any(|span| span.name == expected),
+            "missing span {expected}"
+        );
+    }
+    assert!(!trace_spans.iter().any(|span| span.name == "approval.check"));
+    assert!(
+        !trace_spans
+            .iter()
+            .any(|span| span.name == "runtime.idempotency")
+    );
+
+    let root = trace_spans
+        .iter()
+        .find(|span| span.name == "mcp.request")
+        .expect("mcp root");
+    let runtime = trace_spans
+        .iter()
+        .find(|span| span.name == "runtime.execute")
+        .expect("runtime");
+    assert!(!runtime.parent_span_id.is_empty());
+    assert_eq!(runtime.parent_span_id, root.span_id);
+
+    let history = trace_spans
+        .iter()
+        .find(|span| span.name == "history.write")
+        .expect("history write");
+    let history_db = trace_spans
+        .iter()
+        .find(|span| {
+            span.name == "db.query"
+                && string_attribute(span, "db.operation") == Some("invocation_history.write")
+        })
+        .expect("history PostgreSQL write");
+    assert_eq!(history_db.parent_span_id, history.span_id);
+
+    let logs = registry
+        .list_invocation_logs(ListInvocationLogsQuery {
+            workspace_id: &test_workspace_id(),
+            level: None,
+            search_text: None,
+            source: Some(InvocationSource::AgentToolCall),
+            operation_id: Some(&operation.id),
+            agent_id: None,
+            created_after: None,
+            limit: 10,
+        })
+        .await
+        .unwrap();
+    assert_eq!(logs.len(), 1);
+    assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID));
+
+    provider.shutdown().unwrap();
+}
+
+struct OtlpRequest {
+    headers: HeaderMap,
+    body: Bytes,
+}
+
+async fn spawn_otlp_collector() -> (
+    String,
+    tokio::sync::oneshot::Receiver<OtlpRequest>,
+    tokio::task::JoinHandle<()>,
+) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    let (request_tx, request_rx) = tokio::sync::oneshot::channel();
+    let sender = Arc::new(Mutex::new(Some(request_tx)));
+    let app = Router::new().route(
+        "/v1/traces",
+        post({
+            let sender = Arc::clone(&sender);
+            move |headers: HeaderMap, body: Bytes| {
+                let sender = Arc::clone(&sender);
+                async move {
+                    if let Some(sender) = sender.lock().unwrap().take() {
+                        let _ = sender.send(OtlpRequest { headers, body });
+                    }
+                    StatusCode::OK
+                }
+            }
+        }),
+    );
+    let collector = tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+    (format!("http://{address}/v1/traces"), request_rx, collector)
+}
+
+fn decode_trace_id(value: &str) -> [u8; 16] {
+    let mut bytes = [0_u8; 16];
+    for (index, byte) in bytes.iter_mut().enumerate() {
+        *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap();
+    }
+    bytes
+}
+
+fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> {
+    span.attributes.iter().find_map(|attribute| {
+        let value = attribute.value.as_ref()?.value.as_ref()?;
+        (attribute.key == key)
+            .then_some(value)
+            .and_then(|value| match value {
+                any_value::Value::StringValue(value) => Some(value.as_str()),
+                _ => None,
+            })
+    })
+}
+
+async fn send_jsonrpc(
+    app: Router,
+    api_key: &str,
+    session_id: Option<&str>,
+    request_id: Option<&str>,
+    traceparent: Option<&str>,
+    payload: Value,
+) -> axum::response::Response {
+    let mut request = Request::builder()
+        .method("POST")
+        .uri("/v1/default/stage-agent")
+        .header(header::CONTENT_TYPE, "application/json")
+        .header(header::ACCEPT, "application/json, text/event-stream")
+        .header(header::AUTHORIZATION, format!("Bearer {api_key}"))
+        .header("MCP-Protocol-Version", "2025-11-25");
+    if let Some(session_id) = session_id {
+        request = request.header("MCP-Session-Id", session_id);
+    }
+    if let Some(request_id) = request_id {
+        request = request.header("x-request-id", request_id);
+    }
+    if let Some(traceparent) = traceparent {
+        request = request.header("traceparent", traceparent);
+    }
+    app.oneshot(request.body(Body::from(payload.to_string())).unwrap())
+        .await
+        .unwrap()
+}
+
+async fn spawn_upstream(observed: Arc<Mutex<Option<String>>>) -> String {
+    async fn create_lead(
+        State(observed): State<Arc<Mutex<Option<String>>>>,
+        headers: HeaderMap,
+        Json(_payload): Json<Value>,
+    ) -> Json<Value> {
+        *observed.lock().unwrap() = headers
+            .get("traceparent")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned);
+        Json(json!({ "id": "lead_123" }))
+    }
+
+    let app = Router::new()
+        .route("/crm/leads", post(create_lead))
+        .with_state(observed);
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+    format!("http://{address}")
+}

apps/mcp-server/tests/integration/request_context.rs

@@ -0,0 +1,163 @@
+use std::{
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use axum::{
+    body::Body,
+    http::{HeaderValue, Request, StatusCode},
+};
+use opentelemetry::{
+    global,
+    trace::{TraceId, TracerProvider as _},
+};
+use opentelemetry_sdk::{
+    error::OTelSdkResult,
+    propagation::TraceContextPropagator,
+    trace::{SdkTracerProvider, SpanData, SpanExporter},
+};
+use tower::ServiceExt;
+use tracing::instrument::WithSubscriber;
+use tracing_subscriber::layer::SubscriberExt;
+
+use super::common::{build_test_app, test_registry};
+
+const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
+static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+#[tokio::test(flavor = "current_thread")]
+async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let exported = Arc::new(Mutex::new(Vec::new()));
+    let provider = SdkTracerProvider::builder()
+        .with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
+        .build();
+    let tracer = provider.tracer("mcp-request-context-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = build_test_app(test_registry().await, Duration::ZERO, None);
+
+    let (valid, invalid, absent) = async {
+        let valid = send_health(
+            app.clone(),
+            Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+            Some("request-id-is-separate"),
+        )
+        .await;
+        let invalid = send_health(
+            app.clone(),
+            Some("canary-invalid-traceparent"),
+            Some("bad,value"),
+        )
+        .await;
+        let absent = send_health(app, None, None).await;
+        (valid, invalid, absent)
+    }
+    .with_subscriber(dispatch)
+    .await;
+    provider.force_flush().unwrap();
+
+    assert_eq!(valid.status, StatusCode::OK);
+    assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate"));
+    assert_eq!(invalid.status, StatusCode::OK);
+    assert_eq!(absent.status, StatusCode::OK);
+    assert!(valid.traceparent_response.is_none());
+    assert!(invalid.traceparent_response.is_none());
+    assert!(absent.traceparent_response.is_none());
+
+    let trace_ids: Vec<_> = exported
+        .lock()
+        .unwrap()
+        .iter()
+        .filter(|span| span.name.as_ref() == "mcp.request")
+        .map(|span| span.span_context.trace_id())
+        .collect();
+    assert_eq!(trace_ids.len(), 3);
+    assert_eq!(trace_ids[0].to_string(), REMOTE_TRACE_ID);
+    assert_ne!(trace_ids[1], trace_ids[0]);
+    assert_ne!(trace_ids[2], trace_ids[0]);
+    assert_ne!(trace_ids[1], trace_ids[2]);
+    assert!(!trace_ids.contains(&TraceId::INVALID));
+    provider.shutdown().unwrap();
+}
+
+#[tokio::test]
+async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    let app = build_test_app(test_registry().await, Duration::ZERO, None);
+    let mut request = Request::builder()
+        .uri("/health")
+        .body(Body::empty())
+        .unwrap();
+    request
+        .headers_mut()
+        .append("x-request-id", HeaderValue::from_static("first-request-id"));
+    request.headers_mut().append(
+        "x-request-id",
+        HeaderValue::from_static("second-request-id"),
+    );
+
+    let response = app
+        .oneshot(request)
+        .with_subscriber(tracing_subscriber::registry())
+        .await
+        .unwrap();
+    let generated = response.headers()["x-request-id"].to_str().unwrap();
+
+    assert_ne!(generated, "first-request-id");
+    assert_ne!(generated, "second-request-id");
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(uuid::Version::SortRand)
+    );
+}
+
+async fn send_health(
+    app: axum::Router,
+    traceparent: Option<&str>,
+    request_id: Option<&str>,
+) -> ProbeResponse {
+    let mut request = Request::builder().uri("/health");
+    if let Some(traceparent) = traceparent {
+        request = request.header("traceparent", traceparent);
+    }
+    if let Some(request_id) = request_id {
+        request = request.header("x-request-id", request_id);
+    }
+    let response = app
+        .oneshot(request.body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+
+    ProbeResponse {
+        status: response.status(),
+        request_id: response
+            .headers()
+            .get("x-request-id")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned),
+        traceparent_response: response
+            .headers()
+            .get("traceparent")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned),
+    }
+}
+
+struct ProbeResponse {
+    status: StatusCode,
+    request_id: Option<String>,
+    traceparent_response: Option<String>,
+}
+
+#[derive(Clone, Debug)]
+struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
+
+impl SpanExporter for CapturingExporter {
+    async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
+        self.0.lock().unwrap().extend(batch);
+        Ok(())
+    }
+}

apps/mcp-server/tests/integration/tool_search.rs

@@ -0,0 +1,154 @@
+use super::common::*;
+
+use std::time::Duration;
+
+use crank_core::{
+    PlatformApiKeyScope, ToolAccessMode, ToolGroup, ToolSearchSettings, ToolSelectionPolicy,
+};
+use crank_registry::PublishRequest;
+use serde_json::json;
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+
+#[tokio::test]
+async fn search_mode_discovers_and_calls_tools_through_meta_tools() {
+    let registry = test_registry().await;
+    let upstream_base_url = spawn_upstream_server().await;
+    let invoice = test_operation(&upstream_base_url, "create_invoice");
+    let ticket = test_operation(&upstream_base_url, "create_support_ticket");
+    for operation in [&invoice, &ticket] {
+        registry
+            .create_operation(&test_workspace_id(), operation, Some("alice"))
+            .await
+            .unwrap();
+        registry
+            .publish_operation(PublishRequest {
+                workspace_id: &test_workspace_id(),
+                operation_id: &operation.id,
+                version: 1,
+                published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
+                published_by: Some("alice"),
+            })
+            .await
+            .unwrap();
+    }
+    publish_agent_with_policy(
+        &registry,
+        "business-search",
+        vec![
+            binding_for_operation(&invoice),
+            binding_for_operation(&ticket),
+        ],
+        ToolSelectionPolicy {
+            mode: ToolAccessMode::Search,
+            groups: vec![
+                ToolGroup {
+                    id: "finance".to_owned(),
+                    name: "Finance".to_owned(),
+                    description: "Invoices and payments".to_owned(),
+                    tool_names: vec![invoice.name.clone()],
+                },
+                ToolGroup {
+                    id: "support".to_owned(),
+                    name: "Support".to_owned(),
+                    description: "Customer support tickets".to_owned(),
+                    tool_names: vec![ticket.name.clone()],
+                },
+            ],
+            search: ToolSearchSettings { max_results: 5 },
+        },
+    )
+    .await;
+    let api_key = create_platform_api_key(
+        &registry,
+        "business-search",
+        "mcp-search",
+        &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
+    )
+    .await;
+    let base_url = spawn_mcp_server(build_test_app(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+    ))
+    .await;
+    let client = reqwest::Client::new();
+    let mcp_url = agent_mcp_url(&base_url, "business-search");
+    let session = initialize_session(&client, &mcp_url, &api_key).await;
+
+    let listed = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&session),
+        json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}),
+    )
+    .await;
+    assert_eq!(
+        listed["result"]["tools"]
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|tool| tool["name"].as_str().unwrap())
+            .collect::<Vec<_>>(),
+        vec!["search_tools", "call_tool"]
+    );
+
+    let search = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&session),
+        json!({
+            "jsonrpc":"2.0","id":3,"method":"tools/call",
+            "params":{"name":"search_tools","arguments":{"query":"invoice","group_ids":["finance"]}}
+        }),
+    )
+    .await;
+    assert_eq!(
+        search["result"]["structuredContent"]["tools"][0]["name"],
+        "create_invoice"
+    );
+    assert_eq!(
+        search["result"]["structuredContent"]["catalog_revision"],
+        "agent-version-1"
+    );
+
+    let stale_call = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&session),
+        json!({
+            "jsonrpc":"2.0","id":4,"method":"tools/call",
+            "params":{"name":"call_tool","arguments":{
+                "name":"create_invoice",
+                "arguments":{"email":"user@example.com"},
+                "catalog_revision":"agent-version-0"
+            }}
+        }),
+    )
+    .await;
+    assert_eq!(stale_call["result"]["isError"], true);
+    assert_eq!(
+        stale_call["result"]["structuredContent"]["error"]["code"],
+        "catalog_revision_changed"
+    );
+
+    let call = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&session),
+        json!({
+            "jsonrpc":"2.0","id":5,"method":"tools/call",
+            "params":{"name":"call_tool","arguments":{
+                "name":"create_invoice",
+                "arguments":{"email":"user@example.com"},
+                "catalog_revision":"agent-version-1"
+            }}
+        }),
+    )
+    .await;
+    assert_eq!(call["result"]["isError"], false);
+    assert_eq!(call["result"]["structuredContent"]["id"], "lead_123");
+}

apps/mcp-server/tests/integration/transport_protocol.rs

@@ -19,7 +19,8 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
 use crank_core::{
     Agent, AgentId, AgentOperationBinding, AgentStatus, AgentVersion, ExecutionConfig, HttpMethod,
     Operation, OperationId, OperationStatus, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyScope,
-    PlatformApiKeyStatus, Protocol, RestTarget, Target, ToolDescription, WorkspaceId,
+    PlatformApiKeyStatus, Protocol, RestTarget, Target, ToolAccessMode, ToolDescription, ToolGroup,
+    ToolSearchSettings, ToolSelectionPolicy, WorkspaceId,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_registry::{
@@ -37,7 +38,8 @@ use sha2::{Digest, Sha256};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::net::TcpListener;
 use tokio::time::sleep;
-use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
+use tracing_subscriber::fmt::MakeWriter;
+use uuid::Version;
 
 use crank_community_mcp::{
     auth::{CommunityMachineCredentialVerifier, SharedMachineCredentialVerifier},
@@ -45,6 +47,9 @@ use crank_community_mcp::{
     catalog::PublishedToolCatalog,
     session::{InMemorySessionStore, SharedSessionStore, TransportSessionStore},
 };
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber,
+};
 
 fn test_workspace_id() -> WorkspaceId {
     WorkspaceId::new("ws_default")
@@ -403,7 +408,10 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
         .to_str()
         .unwrap()
         .to_owned();
-    assert!(!request_id.is_empty());
+    assert_eq!(
+        uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
 
     let call_result = response.json::<Value>().await.unwrap();
     assert_eq!(call_result["result"]["isError"], false);
@@ -426,7 +434,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
     assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str()));
 }
 
-#[tokio::test]
+#[tokio::test(flavor = "current_thread")]
 async fn emits_request_id_in_mcp_ingress_logs() {
     let registry = test_registry().await;
     let upstream_base_url = spawn_upstream_server().await;
@@ -455,6 +463,18 @@ async fn emits_request_id_in_mcp_ingress_logs() {
     )
     .await;
 
+    let writer = SharedLogWriter::default();
+    let subscriber = build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _guard = tracing::dispatcher::set_default(&dispatch);
     let base_url = spawn_mcp_server(build_test_app(
         registry,
         Duration::from_millis(0),
@@ -463,18 +483,7 @@ async fn emits_request_id_in_mcp_ingress_logs() {
     .await;
     let client = reqwest::Client::new();
     let mcp_url = agent_mcp_url(&base_url, "sales-request-trace");
-    let writer = SharedLogWriter::default();
-    let subscriber = tracing_subscriber::registry().with(
-        tracing_subscriber::fmt::layer()
-            .with_writer(writer.clone())
-            .without_time()
-            .with_ansi(false)
-            .with_target(false)
-            .compact()
-            .with_filter(LevelFilter::INFO),
-    );
 
-    let _ = tracing::subscriber::set_global_default(subscriber);
     let response = post_jsonrpc_response(
         &client,
         &mcp_url,
@@ -499,14 +508,16 @@ async fn emits_request_id_in_mcp_ingress_logs() {
     assert_eq!(response.status(), reqwest::StatusCode::OK);
 
     let logs = writer.output();
-    assert!(
-        logs.contains("mcp request received"),
-        "captured logs did not include ingress marker: {logs}"
-    );
-    assert!(logs.contains("req_mcp_trace_123"));
-    assert!(logs.contains("sales-request-trace"));
-    assert!(logs.contains("default"));
-    assert!(logs.contains("initialize"));
+    let event = logs
+        .lines()
+        .filter_map(|line| serde_json::from_str::<Value>(line).ok())
+        .find(|event| event["event"] == "mcp.request.received")
+        .unwrap();
+    assert_eq!(event["service"], "mcp-server");
+    assert_eq!(event["request_id"], "req_mcp_trace_123");
+    assert_eq!(event["fields"]["agent_slug"], "sales-request-trace");
+    assert_eq!(event["fields"]["workspace_slug"], "default");
+    assert_eq!(event["fields"]["jsonrpc_method"], "initialize");
 }
 
 #[tokio::test]
@@ -802,11 +813,30 @@ async fn get_requires_session_header() {
         .get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
         .header(header::ACCEPT, "text/event-stream")
         .header(header::AUTHORIZATION, format!("Bearer {api_key}"))
+        .header("x-request-id", "req_early_mcp_error")
         .send()
         .await
         .unwrap();
 
     assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
+    assert_eq!(
+        response.headers()["x-request-id"].to_str().unwrap(),
+        "req_early_mcp_error"
+    );
+
+    let invalid_response = client
+        .get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
+        .header(header::ACCEPT, "text/event-stream")
+        .header(header::AUTHORIZATION, format!("Bearer {api_key}"))
+        .header("x-request-id", "bad,value")
+        .send()
+        .await
+        .unwrap();
+    let generated = invalid_response.headers()["x-request-id"].to_str().unwrap();
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
 }
 
 #[tokio::test]

apps/mcp-server/tests/request_context.rs

@@ -0,0 +1,4 @@
+#[path = "integration/common.rs"]
+mod common;
+#[path = "integration/request_context.rs"]
+mod request_context;

apps/ui/css/pages.css

@@ -1403,6 +1403,177 @@
 .agents-rec-callout svg { flex-shrink: 0; color: #d2991f; margin-top: 1px; }
 .agents-rec-callout strong { color: var(--text-primary); }
 
+.tool-access-options {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 8px;
+}
+.tool-access-option {
+  min-height: 112px;
+  padding: 14px;
+  border: 1px solid var(--border);
+  border-radius: 7px;
+  background: var(--bg-canvas);
+  color: var(--text-secondary);
+  text-align: left;
+  cursor: pointer;
+}
+.tool-access-option:hover { border-color: var(--border-muted, #444c56); }
+.tool-access-option.active {
+  border-color: var(--accent);
+  background: rgba(45, 212, 191, 0.07);
+}
+.tool-access-option-title {
+  display: block;
+  color: var(--text-primary);
+  font-size: 13px;
+  font-weight: 600;
+  line-height: 1.35;
+}
+.tool-access-option.active .tool-access-option-title { color: var(--accent); }
+.tool-access-option-body {
+  display: block;
+  margin-top: 7px;
+  font-size: 11px;
+  line-height: 1.5;
+}
+.tool-search-config {
+  display: flex;
+  flex-direction: column;
+  gap: 12px;
+  margin-top: 14px;
+  padding-top: 14px;
+  border-top: 1px solid var(--border-subtle);
+}
+.tool-search-config-header,
+.tool-group-card-header,
+.tool-search-preview-controls,
+.tool-search-result,
+.tool-group-assignment-row {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 10px;
+}
+.tool-search-config-title {
+  color: var(--text-primary);
+  font-size: 12px;
+  font-weight: 600;
+}
+.tool-search-config-header .drawer-section-sub { margin: 3px 0 0; }
+.tool-search-config-header .btn-ghost-sm { white-space: nowrap; }
+.tool-group-empty {
+  padding: 12px;
+  border: 1px dashed var(--border);
+  border-radius: 7px;
+  color: var(--text-muted);
+  font-size: 11px;
+  line-height: 1.5;
+}
+.tool-group-card,
+.tool-search-preview {
+  padding: 12px;
+  border: 1px solid var(--border);
+  border-radius: 7px;
+  background: var(--bg-canvas);
+}
+.tool-group-card-header { margin-bottom: 10px; }
+.tool-group-card-header strong { font-size: 12px; color: var(--text-primary); }
+.tool-group-card-header .agent-action-btn img { width: 13px; height: 13px; }
+.tool-group-fields {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
+  gap: 8px;
+}
+.tool-group-fields label,
+.tool-group-description,
+.tool-search-limit {
+  display: flex;
+  flex-direction: column;
+  gap: 5px;
+  color: var(--text-secondary);
+  font-size: 11px;
+}
+.tool-group-description { margin-top: 9px; }
+.tool-group-assignments {
+  border: 1px solid var(--border);
+  border-radius: 7px;
+  overflow: hidden;
+}
+.tool-group-assignments > .tool-search-config-title {
+  padding: 10px 12px;
+  border-bottom: 1px solid var(--border-subtle);
+  background: var(--bg-canvas);
+}
+.tool-group-assignment-row {
+  align-items: flex-start;
+  padding: 10px 12px;
+  border-bottom: 1px solid var(--border-subtle);
+}
+.tool-group-assignment-row:last-child { border-bottom: 0; }
+.tool-group-assignment-tool { min-width: 130px; }
+.tool-group-assignment-tool strong,
+.tool-group-assignment-tool code,
+.tool-search-result strong,
+.tool-search-result code {
+  display: block;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+.tool-group-assignment-tool strong,
+.tool-search-result strong { color: var(--text-primary); font-size: 11px; }
+.tool-group-assignment-tool code,
+.tool-search-result code { margin-top: 3px; color: var(--text-muted); font-size: 10px; }
+.tool-group-chips {
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: flex-end;
+  gap: 5px;
+}
+.tool-group-chip {
+  max-width: 150px;
+  padding: 4px 7px;
+  border: 1px solid var(--border);
+  border-radius: 5px;
+  background: transparent;
+  color: var(--text-muted);
+  font-size: 10px;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  cursor: pointer;
+}
+.tool-group-chip.active {
+  border-color: var(--accent);
+  background: rgba(45, 212, 191, 0.08);
+  color: var(--accent);
+}
+.tool-search-limit { max-width: 190px; }
+.tool-search-preview .drawer-section-sub { margin-top: 4px; }
+.tool-search-preview-controls {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) 132px auto;
+}
+.tool-search-results {
+  margin-top: 10px;
+  border-top: 1px solid var(--border-subtle);
+}
+.tool-search-result { padding: 9px 0; border-bottom: 1px solid var(--border-subtle); }
+.tool-search-result:last-child { border-bottom: 0; }
+.tool-search-result > div { min-width: 0; }
+.tool-search-result > span { color: var(--text-muted); font-size: 10px; }
+.tool-search-preview > .tool-group-empty { margin-top: 10px; }
+
+@media (max-width: 540px) {
+  .tool-access-options,
+  .tool-group-fields,
+  .tool-search-preview-controls { grid-template-columns: 1fr; }
+  .tool-group-assignment-row { flex-direction: column; }
+  .tool-group-chips { justify-content: flex-start; }
+  .tool-search-limit { max-width: none; }
+}
+
 
 /* ═══════════════════════════════════════════════════
    Settings — Members enhanced

apps/ui/html/agents.html

@@ -359,24 +359,122 @@
           <!-- Footer count -->
           <div class="ops-picker-footer" x-show="form.selectedOps.length > 0">
             <span x-text="tfKey('agents.drawer.ops_selected', { count: form.selectedOps.length })"></span>
-            <button @click="form.selectedOps = []" style="background:none;border:none;color:var(--text-muted);font-size:12px;cursor:pointer;margin-left:8px;" data-i18n="agents.drawer.clear_all">Clear all</button>
+            <button @click="clearSelectedOperations()" style="background:none;border:none;color:var(--text-muted);font-size:12px;cursor:pointer;margin-left:8px;" data-i18n="agents.drawer.clear_all">Clear all</button>
           </div>
         </div>
 
         <!-- Recommendation callout -->
-        <div class="agents-rec-callout" x-show="form.selectedOps.length > 15">
+        <div class="agents-rec-callout" x-show="form.selectedOps.length > 15 && form.accessMode === 'direct'">
           <svg width="13" height="13" viewBox="0 0 16 16" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round"><polygon points="8,1 15,14 1,14" fill="none"/><path d="M8 6v4M8 11.5v.5"/></svg>
           <span x-text="tfKey('agents.drawer.recommendation', { count: form.selectedOps.length })">You've selected tools.</span>
         </div>
       </div>
 
+      <div class="drawer-section">
+        <div class="drawer-section-title" data-i18n="agents.drawer.access.title">Tool access</div>
+        <div class="drawer-section-sub" data-i18n="agents.drawer.access.subtitle">Choose how the model receives this agent's tool catalog.</div>
+
+        <div class="tool-access-options">
+          <button class="tool-access-option" :class="{ active: form.accessMode === 'direct' }" @click="setAccessMode('direct')">
+            <span class="tool-access-option-title" data-i18n="agents.drawer.access.direct">Show tools immediately</span>
+            <span class="tool-access-option-body" data-i18n="agents.drawer.access.direct_hint">Best for a small curated catalog. MCP clients receive every tool in tools/list.</span>
+          </button>
+          <button class="tool-access-option" :class="{ active: form.accessMode === 'search' }" @click="setAccessMode('search')">
+            <span class="tool-access-option-title" data-i18n="agents.drawer.access.search">Select tools on demand</span>
+            <span class="tool-access-option-body" data-i18n="agents.drawer.access.search_hint">The model sees search_tools and call_tool, then discovers only relevant schemas.</span>
+          </button>
+        </div>
+
+        <div class="tool-search-config" x-show="form.accessMode === 'search'" style="display:none">
+          <div class="tool-search-config-header">
+            <div>
+              <div class="tool-search-config-title" data-i18n="agents.drawer.groups.title">Catalog sections</div>
+              <div class="drawer-section-sub" data-i18n="agents.drawer.groups.subtitle">Sections help the model narrow a search without blocking catalog-wide discovery.</div>
+            </div>
+            <button class="btn-ghost-sm" @click="addToolGroup()" data-i18n="agents.drawer.groups.add">Add section</button>
+          </div>
+
+          <div class="tool-group-empty" x-show="form.groups.length === 0" data-i18n="agents.drawer.groups.empty">No sections yet. Search will use the entire selected catalog.</div>
+          <template x-for="(group, groupIndex) in form.groups" :key="groupIndex">
+            <div class="tool-group-card">
+              <div class="tool-group-card-header">
+                <strong x-text="group.name || tKey('agents.drawer.groups.untitled')"></strong>
+                <button class="agent-action-btn danger" @click="removeToolGroup(groupIndex)" :title="tKey('agents.drawer.groups.remove')">
+                  <img src="/icons/general/trash.svg" alt="">
+                </button>
+              </div>
+              <div class="tool-group-fields">
+                <label>
+                  <span data-i18n="agents.drawer.groups.name">Name</span>
+                  <input class="form-input" type="text" :value="group.name" @input="onToolGroupName(groupIndex, $event.target.value)" data-i18n-ph="agents.drawer.groups.name_placeholder" placeholder="Finance">
+                </label>
+                <label>
+                  <span data-i18n="agents.drawer.groups.id">Identifier</span>
+                  <input class="form-input input-mono" type="text" :value="group.id" @input="onToolGroupId(groupIndex, $event.target.value)" placeholder="finance">
+                </label>
+              </div>
+              <label class="tool-group-description">
+                <span data-i18n="agents.drawer.groups.description">Description for the model</span>
+                <textarea class="form-textarea" rows="2" x-model="group.description" @input="resetSearchPreview()" data-i18n-ph="agents.drawer.groups.description_placeholder" placeholder="Invoices, payments and refunds"></textarea>
+              </label>
+            </div>
+          </template>
+
+          <div class="tool-group-assignments" x-show="form.groups.length > 0 && selectedOperations.length > 0">
+            <div class="tool-search-config-title" data-i18n="agents.drawer.groups.assign">Assign tools to sections</div>
+            <template x-for="operation in selectedOperations" :key="operation.id">
+              <div class="tool-group-assignment-row">
+                <div class="tool-group-assignment-tool">
+                  <strong x-text="operation.display_name || operation.name"></strong>
+                  <code x-text="operation.name"></code>
+                </div>
+                <div class="tool-group-chips">
+                  <template x-for="(group, groupIndex) in form.groups" :key="groupIndex">
+                    <button class="tool-group-chip" :class="{ active: toolInGroup(groupIndex, operation.name) }" @click="toggleToolGroup(groupIndex, operation.name)" x-text="group.name || group.id || '—'"></button>
+                  </template>
+                </div>
+              </div>
+            </template>
+          </div>
+
+          <label class="tool-search-limit">
+            <span data-i18n="agents.drawer.search.limit">Maximum results per search</span>
+            <input class="form-input" type="number" min="1" max="20" x-model.number="form.searchMaxResults" @input="resetSearchPreview()">
+          </label>
+
+          <div class="tool-search-preview">
+            <div class="tool-search-config-title" data-i18n="agents.drawer.search.preview_title">Test tool selection</div>
+            <div class="drawer-section-sub" data-i18n="agents.drawer.search.preview_subtitle">Enter a task and verify which tools the model will receive.</div>
+            <div class="tool-search-preview-controls">
+              <input class="form-input" type="text" x-model="searchPreviewQuery" @keydown.enter.prevent="previewToolSearch()" data-i18n-ph="agents.drawer.search.query_placeholder" placeholder="Create an invoice for a customer">
+              <select class="form-select" x-model="searchPreviewGroup">
+                <option value="" data-i18n="agents.drawer.search.all_groups">All sections</option>
+                <template x-for="group in form.groups" :key="group.id">
+                  <option :value="group.id" x-text="group.name || group.id"></option>
+                </template>
+              </select>
+              <button class="btn-primary-sm" :disabled="searchPreviewLoading || !searchPreviewQuery.trim() || !catalogConfigValid" @click="previewToolSearch()" x-text="searchPreviewLoading ? tKey('agents.drawer.search.testing') : tKey('agents.drawer.search.test')">Test</button>
+            </div>
+            <div class="tool-search-results" x-show="searchPreviewItems.length > 0">
+              <template x-for="item in searchPreviewItems" :key="item.tool.name">
+                <div class="tool-search-result">
+                  <div><strong x-text="item.tool.title"></strong><code x-text="item.tool.name"></code></div>
+                  <span x-text="item.tool.group_ids.join(', ')"></span>
+                </div>
+              </template>
+            </div>
+            <div class="tool-group-empty" x-show="searchPreviewRan && !searchPreviewLoading && searchPreviewItems.length === 0" data-i18n="agents.drawer.search.no_results">No tools matched this task.</div>
+          </div>
+        </div>
+      </div>
+
     </div><!-- /drawer-body -->
 
     <!-- Drawer footer -->
     <div class="drawer-footer">
       <button class="btn-ghost-sm" style="padding: 8px 16px; font-size: 13px;" @click="closeDrawer()" data-i18n="agents.drawer.cancel">Cancel</button>
       <button class="btn-primary-sm" style="padding: 8px 20px; font-size: 13px;"
-              :disabled="!form.display_name.trim() || !form.slug.trim()"
+              :disabled="!form.display_name.trim() || !form.slug.trim() || !catalogConfigValid"
               @click="saveAgent()"
               x-text="drawerMode === 'create' ? tKey('agents.drawer.create') : tKey('agents.drawer.save')">
         Create agent

apps/ui/html/workspace-setup.html

@@ -97,8 +97,8 @@
           </div>
           <div class="danger-zone-action">
             <div class="danger-zone-text">
-              <div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export all data</div>
-              <div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.</div>
+              <div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export workspace catalog</div>
+              <div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.</div>
             </div>
             <button class="btn-danger" id="export-workspace-btn" type="button" data-i18n="workspace_setup.export">Export</button>
           </div>

apps/ui/js/agents.js

@@ -14,6 +14,7 @@ function mapAgent(agent) {
     raw_status: agent.status,
     operation_count: agent.operation_count || 0,
     operation_ids: agent.operation_ids || [],
+    tool_selection_policy: agent.tool_selection_policy || { mode: 'direct', groups: [], search: { max_results: 8 } },
     key_count: agent.key_count || 0,
     calls_today: agent.calls_today || 0,
     created_at: agent.created_at,
@@ -66,10 +67,18 @@ document.addEventListener('alpine:init', function() {
         description: '',
         status: 'published',
         selectedOps: [],
+        accessMode: 'direct',
+        groups: [],
+        searchMaxResults: 8,
       },
 
       opSearch: '',
       slugManuallyEdited: false,
+      searchPreviewQuery: '',
+      searchPreviewGroup: '',
+      searchPreviewItems: [],
+      searchPreviewLoading: false,
+      searchPreviewRan: false,
 
       async init() {
         var self = this;
@@ -205,7 +214,7 @@ document.addEventListener('alpine:init', function() {
       get agentToolFindings() {
         var findings = [];
         var selected = this.selectedOperations;
-        if (selected.length > 8) {
+        if (selected.length > 8 && this.form.accessMode === 'direct') {
           findings.push(this.tKey('agents.drawer.finding.too_many_tools'));
         }
 
@@ -242,13 +251,18 @@ document.addEventListener('alpine:init', function() {
           description: '',
           status: 'published',
           selectedOps: [],
+          accessMode: 'direct',
+          groups: [],
+          searchMaxResults: 8,
         };
         this.opSearch = '';
         this.slugManuallyEdited = false;
+        this.resetSearchPreview();
         this.drawerOpen = true;
       },
 
       openEdit(agent) {
+        var policy = agent.tool_selection_policy || {};
         this.drawerMode = 'edit';
         this.editingId = agent.id;
         this.form = {
@@ -257,9 +271,22 @@ document.addEventListener('alpine:init', function() {
           description: agent.description,
           status: agent.raw_status || agent.status || 'draft',
           selectedOps: [].concat(agent.operation_ids || []),
+          accessMode: policy.mode === 'search' ? 'search' : 'direct',
+          groups: (policy.groups || []).map(function(group) {
+            return {
+              id: group.id || '',
+              name: group.name || '',
+              description: group.description || '',
+              tool_names: [].concat(group.tool_names || []),
+            };
+          }),
+          searchMaxResults: policy.search && policy.search.max_results
+            ? policy.search.max_results
+            : 8,
         };
         this.opSearch = '';
         this.slugManuallyEdited = true;
+        this.resetSearchPreview();
         this.drawerOpen = true;
       },
 
@@ -289,13 +316,170 @@ document.addEventListener('alpine:init', function() {
           this.form.selectedOps.push(operationId);
         } else {
           this.form.selectedOps.splice(index, 1);
+          var operation = this.operations.find(function(item) { return item.id === operationId; });
+          if (operation) {
+            this.form.groups.forEach(function(group) {
+              group.tool_names = group.tool_names.filter(function(name) {
+                return name !== operation.name;
+              });
+            });
           }
+        }
+        this.resetSearchPreview();
       },
 
       isOpSelected(operationId) {
         return this.form.selectedOps.includes(operationId);
       },
 
+      clearSelectedOperations() {
+        this.form.selectedOps = [];
+        this.form.groups.forEach(function(group) { group.tool_names = []; });
+        this.resetSearchPreview();
+      },
+
+      setAccessMode(mode) {
+        this.form.accessMode = mode === 'search' ? 'search' : 'direct';
+        this.resetSearchPreview();
+      },
+
+      addToolGroup() {
+        this.form.groups.push({ id: '', name: '', description: '', tool_names: [] });
+        this.resetSearchPreview();
+      },
+
+      removeToolGroup(index) {
+        this.form.groups.splice(index, 1);
+        this.resetSearchPreview();
+      },
+
+      onToolGroupName(index, value) {
+        var group = this.form.groups[index];
+        if (!group) return;
+        var previousSlug = this.slugifyGroupName(group.name);
+        group.name = value;
+        if (!group.id || group.id === previousSlug) {
+          group.id = this.slugifyGroupName(value);
+        }
+        this.resetSearchPreview();
+      },
+
+      onToolGroupId(index, value) {
+        var group = this.form.groups[index];
+        if (!group) return;
+        group.id = this.slugifyGroupName(value);
+        this.resetSearchPreview();
+      },
+
+      slugifyGroupName(value) {
+        return String(value || '')
+          .toLowerCase()
+          .replace(/\s+/g, '-')
+          .replace(/[^a-z0-9-]/g, '')
+          .replace(/-+/g, '-')
+          .replace(/^-|-$/g, '');
+      },
+
+      toggleToolGroup(groupIndex, toolName) {
+        var group = this.form.groups[groupIndex];
+        if (!group) return;
+        var index = group.tool_names.indexOf(toolName);
+        if (index === -1) group.tool_names.push(toolName);
+        else group.tool_names.splice(index, 1);
+        this.resetSearchPreview();
+      },
+
+      toolInGroup(groupIndex, toolName) {
+        var group = this.form.groups[groupIndex];
+        return Boolean(group && group.tool_names.includes(toolName));
+      },
+
+      toolSelectionPolicy() {
+        return {
+          mode: this.form.accessMode,
+          groups: this.form.accessMode === 'search'
+            ? this.form.groups.map(function(group) {
+              return {
+                id: group.id.trim(),
+                name: group.name.trim(),
+                description: group.description.trim(),
+                tool_names: [].concat(group.tool_names || []),
+              };
+            })
+            : [],
+          search: {
+            max_results: Math.max(1, Math.min(20, Number(this.form.searchMaxResults) || 8)),
+          },
+        };
+      },
+
+      get catalogConfigValid() {
+        if (this.form.accessMode !== 'search') return true;
+        var ids = [];
+        for (var index = 0; index < this.form.groups.length; index += 1) {
+          var group = this.form.groups[index];
+          if (!group.id.trim() || !group.name.trim() || !group.description.trim()) return false;
+          if (ids.includes(group.id.trim())) return false;
+          ids.push(group.id.trim());
+        }
+        return true;
+      },
+
+      agentBindings() {
+        var self = this;
+        return this.form.selectedOps.map(function(operationId) {
+          var operation = self.operations.find(function(item) { return item.id === operationId; });
+          return {
+            operation_id: operationId,
+            operation_version: operation && operation.latest_published_version
+              ? operation.latest_published_version
+              : operation && operation.current_draft_version
+                ? operation.current_draft_version
+                : 1,
+            tool_name: operation ? operation.name : operationId,
+            tool_title: operation ? (operation.display_name || operation.name) : operationId,
+            tool_description_override: null,
+            enabled: true,
+          };
+        });
+      },
+
+      resetSearchPreview() {
+        this.searchPreviewItems = [];
+        this.searchPreviewRan = false;
+      },
+
+      async previewToolSearch() {
+        if (
+          this.searchPreviewLoading
+          || !this.workspaceId
+          || !this.searchPreviewQuery.trim()
+          || this.form.accessMode !== 'search'
+        ) return;
+        this.searchPreviewLoading = true;
+        this.searchPreviewRan = false;
+        try {
+          var response = await window.CrankApi.previewAgentToolSearch(this.workspaceId, {
+            query: this.searchPreviewQuery.trim(),
+            group_ids: this.searchPreviewGroup ? [this.searchPreviewGroup] : [],
+            bindings: this.agentBindings(),
+            tool_selection_policy: this.toolSelectionPolicy(),
+          });
+          this.searchPreviewItems = response.items || [];
+          this.searchPreviewRan = true;
+        } catch (error) {
+          this.searchPreviewItems = [];
+          if (window.CrankUi) {
+            window.CrankUi.error(
+              error.message || this.tKey('agents.drawer.search.preview_error'),
+              this.tKey('agents.drawer.search.preview_error_title')
+            );
+          }
+        } finally {
+          this.searchPreviewLoading = false;
+        }
+      },
+
       operationsLookSimilar(left, right) {
         var leftTokens = this.operationTokens(left);
         var rightTokens = this.operationTokens(right);
@@ -345,7 +529,7 @@ document.addEventListener('alpine:init', function() {
               display_name: this.form.display_name,
               description: this.form.description,
               instructions: {},
-              tool_selection_policy: {},
+              tool_selection_policy: this.toolSelectionPolicy(),
             });
             agentId = created.agent_id;
             currentVersion = created.version || 1;
@@ -359,27 +543,15 @@ document.addEventListener('alpine:init', function() {
             currentVersion = agent.current_draft_version || 1;
           }
 
-          await window.CrankApi.saveAgentBindings(
+          var savedVersion = await window.CrankApi.saveAgentBindings(
             this.workspaceId,
             agentId,
-            this.form.selectedOps.map(function(operationId) {
-              var operation = self.operations.find(function(item) {
-                return item.id === operationId;
-              });
-              return {
-                operation_id: operationId,
-                operation_version: operation && operation.latest_published_version
-                  ? operation.latest_published_version
-                  : operation && operation.current_draft_version
-                    ? operation.current_draft_version
-                    : 1,
-                tool_name: operation ? operation.name : operationId,
-                tool_title: operation ? (operation.display_name || operation.name) : operationId,
-                tool_description_override: null,
-                enabled: true,
-              };
-            }),
+            {
+              bindings: this.agentBindings(),
+              tool_selection_policy: this.toolSelectionPolicy(),
+            },
           );
+          currentVersion = savedVersion.version || currentVersion;
 
           if (this.form.status === 'published') {
             await window.CrankApi.publishAgent(this.workspaceId, agentId, {

apps/ui/js/api.js

@@ -262,6 +262,9 @@
     saveAgentBindings: function(workspaceId, agentId, payload) {
       return post('/workspaces/' + encodeURIComponent(workspaceId) + '/agents/' + encodeURIComponent(agentId) + '/bindings', payload);
     },
+    previewAgentToolSearch: function(workspaceId, payload) {
+      return post('/workspaces/' + encodeURIComponent(workspaceId) + '/agents/tool-search/preview', payload);
+    },
     publishAgent: function(workspaceId, agentId, payload) {
       return post('/workspaces/' + encodeURIComponent(workspaceId) + '/agents/' + encodeURIComponent(agentId) + '/publish', payload);
     },

apps/ui/js/i18n.js

@@ -442,8 +442,8 @@ var TRANSLATIONS = {
     'workspace_setup.create.subtitle': 'This Community installation uses one workspace for MCP operations and agents.',
     'workspace_setup.create.footer': 'This Community installation uses one workspace.',
     'workspace_setup.danger.title': 'Danger zone',
-    'workspace_setup.danger.export_title': 'Export all data',
-    'workspace_setup.danger.export_body': 'Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.',
+    'workspace_setup.danger.export_title': 'Export workspace catalog',
+    'workspace_setup.danger.export_body': 'Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.',
     'workspace_setup.export': 'Export',
     'workspace_setup.role.owner': 'Owner',
     'workspace_setup.role.admin': 'Admin',
@@ -819,13 +819,41 @@ var TRANSLATIONS = {
     'agents.drawer.operations_sub': 'Select the MCP tools available to this agent.',
     'agents.drawer.operations_sub_community': 'Select the MCP tools available to this agent.',
     'agents.drawer.finding.title': 'Recommendation',
-    'agents.drawer.finding.too_many_tools': 'This agent has many tools. Keep only the tools needed for one concrete task.',
+    'agents.drawer.finding.too_many_tools': 'This agent has many tools. Use on-demand selection or keep only the tools needed for one concrete task.',
     'agents.drawer.finding.similar_tools': 'Tools “{left}” and “{right}” look similar. Rename them more precisely or keep one of them.',
     'agents.drawer.filter_ops': 'Filter operations…',
     'agents.drawer.ops_no_match': 'No operations match "{query}"',
     'agents.drawer.ops_selected': '{count} operations selected',
     'agents.drawer.clear_all': 'Clear all',
-    'agents.drawer.recommendation': "You've selected {count} tools. LLMs usually work best when an agent has fewer than 15 tools. Consider splitting this into separate agents by use case.",
+    'agents.drawer.recommendation': "You've selected {count} tools. Switch to on-demand selection so the model receives only relevant schemas.",
+    'agents.drawer.access.title': 'Tool access',
+    'agents.drawer.access.subtitle': "Choose how the model receives this agent's tool catalog.",
+    'agents.drawer.access.direct': 'Show tools immediately',
+    'agents.drawer.access.direct_hint': 'Best for a small curated catalog. MCP clients receive every tool in tools/list.',
+    'agents.drawer.access.search': 'Select tools on demand',
+    'agents.drawer.access.search_hint': 'The model sees search_tools and call_tool, then discovers only relevant schemas.',
+    'agents.drawer.groups.title': 'Catalog sections',
+    'agents.drawer.groups.subtitle': 'Sections help the model narrow a search without blocking catalog-wide discovery.',
+    'agents.drawer.groups.add': 'Add section',
+    'agents.drawer.groups.empty': 'No sections yet. Search will use the entire selected catalog.',
+    'agents.drawer.groups.untitled': 'Untitled section',
+    'agents.drawer.groups.remove': 'Remove section',
+    'agents.drawer.groups.name': 'Name',
+    'agents.drawer.groups.name_placeholder': 'Finance',
+    'agents.drawer.groups.id': 'Identifier',
+    'agents.drawer.groups.description': 'Description for the model',
+    'agents.drawer.groups.description_placeholder': 'Invoices, payments and refunds',
+    'agents.drawer.groups.assign': 'Assign tools to sections',
+    'agents.drawer.search.limit': 'Maximum results per search',
+    'agents.drawer.search.preview_title': 'Test tool selection',
+    'agents.drawer.search.preview_subtitle': 'Enter a task and verify which tools the model will receive.',
+    'agents.drawer.search.query_placeholder': 'Create an invoice for a customer',
+    'agents.drawer.search.all_groups': 'All sections',
+    'agents.drawer.search.test': 'Test',
+    'agents.drawer.search.testing': 'Testing…',
+    'agents.drawer.search.no_results': 'No preview results yet.',
+    'agents.drawer.search.preview_error': 'Failed to test tool selection',
+    'agents.drawer.search.preview_error_title': 'Tool selection test failed',
     'agents.drawer.cancel': 'Cancel',
     'agents.drawer.create': 'Create agent',
     'agents.drawer.save': 'Save changes',
@@ -1306,8 +1334,8 @@ var TRANSLATIONS = {
     'workspace_setup.create.subtitle': 'В Community используется один воркспейс для MCP-операций и агентов.',
     'workspace_setup.create.footer': 'В Community используется один воркспейс.',
     'workspace_setup.danger.title': 'Опасная зона',
-    'workspace_setup.danger.export_title': 'Экспортировать все данные',
-    'workspace_setup.danger.export_body': 'Скачать JSON-снимок настроек воркспейса, операций, агентов, секретов, данных использования и ключей доступа агентов.',
+    'workspace_setup.danger.export_title': 'Экспорт каталога рабочего пространства',
+    'workspace_setup.danger.export_body': 'Скачать невосстанавливаемый JSON-каталог настроек рабочего пространства, сводок операций и агентов, а также метаданных ключей API.',
     'workspace_setup.export': 'Экспорт',
     'workspace_setup.role.owner': 'Владелец',
     'workspace_setup.role.admin': 'Администратор',
@@ -1683,13 +1711,41 @@ var TRANSLATIONS = {
     'agents.drawer.operations_sub': 'Выберите MCP инструменты, которые будут доступны для этого агента.',
     'agents.drawer.operations_sub_community': 'Выберите MCP инструменты, которые будут доступны для этого агента.',
     'agents.drawer.finding.title': 'Рекомендация',
-    'agents.drawer.finding.too_many_tools': 'У агента выбрано много инструментов. Оставьте только те, которые нужны для одной конкретной задачи.',
+    'agents.drawer.finding.too_many_tools': 'У агента выбрано много инструментов. Включите подбор по запросу или оставьте только инструменты для одной конкретной задачи.',
     'agents.drawer.finding.similar_tools': 'Инструменты «{left}» и «{right}» похожи. Переименуйте их точнее или оставьте один вариант.',
     'agents.drawer.filter_ops': 'Фильтр операций…',
     'agents.drawer.ops_no_match': 'Нет операций по запросу "{query}"',
     'agents.drawer.ops_selected': 'Выбрано операций: {count}',
     'agents.drawer.clear_all': 'Очистить все',
-    'agents.drawer.recommendation': 'Сейчас выбрано {count} инструментов. LLM лучше работает, когда у агента меньше 15 инструментов. Подумайте о разбиении по сценариям.',
+    'agents.drawer.recommendation': 'Сейчас выбрано {count} инструментов. Включите подбор по запросу, чтобы модель получала только подходящие схемы.',
+    'agents.drawer.access.title': 'Доступ к инструментам',
+    'agents.drawer.access.subtitle': 'Выберите, как модель будет получать каталог инструментов этого агента.',
+    'agents.drawer.access.direct': 'Показывать сразу',
+    'agents.drawer.access.direct_hint': 'Для небольшого отобранного каталога. MCP-клиент получает все инструменты через tools/list.',
+    'agents.drawer.access.search': 'Подбирать по запросу',
+    'agents.drawer.access.search_hint': 'Модель видит search_tools и call_tool, а затем получает только подходящие схемы.',
+    'agents.drawer.groups.title': 'Разделы каталога',
+    'agents.drawer.groups.subtitle': 'Разделы сужают область поиска, но не мешают искать по всему каталогу.',
+    'agents.drawer.groups.add': 'Добавить раздел',
+    'agents.drawer.groups.empty': 'Разделов пока нет. Поиск будет выполняться по всему выбранному каталогу.',
+    'agents.drawer.groups.untitled': 'Раздел без названия',
+    'agents.drawer.groups.remove': 'Удалить раздел',
+    'agents.drawer.groups.name': 'Название',
+    'agents.drawer.groups.name_placeholder': 'Расчёты',
+    'agents.drawer.groups.id': 'Идентификатор',
+    'agents.drawer.groups.description': 'Описание для модели',
+    'agents.drawer.groups.description_placeholder': 'Счета, платежи и возвраты',
+    'agents.drawer.groups.assign': 'Распределение инструментов по разделам',
+    'agents.drawer.search.limit': 'Максимум результатов за один поиск',
+    'agents.drawer.search.preview_title': 'Проверка подбора',
+    'agents.drawer.search.preview_subtitle': 'Введите задачу и проверьте, какие инструменты получит модель.',
+    'agents.drawer.search.query_placeholder': 'Создать счёт для клиента',
+    'agents.drawer.search.all_groups': 'Все разделы',
+    'agents.drawer.search.test': 'Проверить',
+    'agents.drawer.search.testing': 'Проверяем…',
+    'agents.drawer.search.no_results': 'Результатов проверки пока нет.',
+    'agents.drawer.search.preview_error': 'Не удалось проверить подбор инструментов',
+    'agents.drawer.search.preview_error_title': 'Ошибка проверки подбора',
     'agents.drawer.cancel': 'Отмена',
     'agents.drawer.create': 'Создать агента',
     'agents.drawer.save': 'Сохранить изменения',

apps/ui/js/logs.js

@@ -8,6 +8,8 @@ document.addEventListener('DOMContentLoaded', function () {
     openId: null,
     liveMode: true,
     timer: null,
+    searchTimer: null,
+    refreshPromise: null,
     workspaceId: null,
     loading: false,
     loadError: '',
@@ -457,7 +459,13 @@ document.addEventListener('DOMContentLoaded', function () {
   }
 
   async function refreshOperationalData() {
-    await Promise.all([loadLogs(), loadApprovals()]);
+    if (state.refreshPromise) {
+      return state.refreshPromise;
+    }
+    state.refreshPromise = Promise.all([loadLogs(), loadApprovals()]).finally(function () {
+      state.refreshPromise = null;
+    });
+    return state.refreshPromise;
   }
 
   async function loadLogDetail(logId) {
@@ -494,10 +502,14 @@ document.addEventListener('DOMContentLoaded', function () {
 
   function startPolling() {
     stopPolling();
-    if (!state.liveMode) {
+    if (!state.liveMode || document.hidden) {
       return;
     }
-    state.timer = setInterval(refreshOperationalData, 4000);
+    state.timer = setTimeout(async function poll() {
+      state.timer = null;
+      await refreshOperationalData();
+      startPolling();
+    }, 4000);
   }
 
   function toggleLive() {
@@ -526,7 +538,13 @@ document.addEventListener('DOMContentLoaded', function () {
   if (logSearch) {
     logSearch.addEventListener('input', function () {
       state.search = this.value.trim();
+      if (state.searchTimer) {
+        clearTimeout(state.searchTimer);
+      }
+      state.searchTimer = setTimeout(function () {
+        state.searchTimer = null;
         loadLogs();
+      }, 250);
     });
   }
 
@@ -572,6 +590,24 @@ document.addEventListener('DOMContentLoaded', function () {
     refreshOperationalData();
   });
 
+  document.addEventListener('visibilitychange', function () {
+    if (document.hidden) {
+      stopPolling();
+      return;
+    }
+    if (state.liveMode) {
+      refreshOperationalData().finally(startPolling);
+    }
+  });
+
+  window.addEventListener('pagehide', function () {
+    stopPolling();
+    if (state.searchTimer) {
+      clearTimeout(state.searchTimer);
+      state.searchTimer = null;
+    }
+  });
+
   setLiveState();
   startPolling();
 

apps/ui/js/workspace-setup.js

@@ -196,7 +196,7 @@ async function exportWorkspaceSnapshot() {
     var slug = workspaceFormState.workspaceRecord && workspaceFormState.workspaceRecord.workspace
       ? workspaceFormState.workspaceRecord.workspace.slug
       : tKey('settings.nav.workspace');
-    downloadJsonFile(slug + '-snapshot.json', snapshot);
+    downloadJsonFile(slug + '-catalog-snapshot.json', snapshot);
   } catch (error) {
     if (window.CrankUi) {
       window.CrankUi.error(error.message || tKey('workspace_setup.export_error'), tKey('workspace_setup.export_error_title'));

apps/ui/package-lock.json

@@ -9,7 +9,7 @@
         "@fontsource/inter": "5.2.8",
         "@fontsource/jetbrains-mono": "5.2.8",
         "alpinejs": "3.15.12",
-        "js-yaml": "5.2.1"
+        "js-yaml": "5.2.2"
       },
       "devDependencies": {
         "@playwright/test": "1.61.1",
@@ -580,9 +580,9 @@
       }
     },
     "node_modules/js-yaml": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz",
-      "integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==",
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz",
+      "integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==",
       "funding": [
         {
           "type": "github",

apps/ui/package.json

@@ -11,7 +11,7 @@
     "@fontsource/inter": "5.2.8",
     "@fontsource/jetbrains-mono": "5.2.8",
     "alpinejs": "3.15.12",
-    "js-yaml": "5.2.1"
+    "js-yaml": "5.2.2"
   },
   "devDependencies": {
     "@playwright/test": "1.61.1",

apps/ui/tests/e2e/agents.spec.js

@@ -21,3 +21,26 @@ test('agents page shows demo cards and edit drawer opens', async ({ page }) => {
   );
   await expect(page.locator('.drawer')).toContainText(/mcp/i);
 });
+
+test('agent drawer configures on-demand tool discovery and catalog sections', async ({ page }) => {
+  await login(page);
+  await page.goto('/agents');
+  await page.getByRole('button', { name: localized('New agent', 'Новый агент') }).click();
+  await page.locator('.ops-picker-item').first().click();
+
+  await page.locator('.tool-access-option').nth(1).click();
+  await expect(page.locator('.tool-search-config')).toBeVisible();
+  await page.getByRole('button', { name: localized('Add section', 'Добавить раздел') }).click();
+
+  const group = page.locator('.tool-group-card').first();
+  await group.locator('input').nth(0).fill('Finance');
+  await group.locator('input').nth(1).fill('finance');
+  await group.locator('textarea').fill('Invoices, payments and refunds');
+
+  await expect(group.locator('input').nth(1)).toHaveValue('finance');
+  await expect(page.locator('.tool-search-preview')).toBeVisible();
+  await page.locator('.tool-group-chip').first().click();
+  await page.locator('.tool-search-preview input').fill('currency rate');
+  await page.locator('.tool-search-preview .btn-primary-sm').click();
+  await expect(page.locator('.tool-search-result').first()).toBeVisible();
+});

apps/ui/tests/e2e/wizard.spec.js

@@ -161,15 +161,15 @@ test('wizard builds visual request mappings from JSON sample and path params', a
 
   await page.goto('/wizard/');
   await page.locator('[data-testid="wizard-protocol-rest"]').click();
-  await page.evaluate(() => window.CrankWizardShell.doGoToStep(2));
+  await page.evaluate(() => window.CrankWizardShell.goToStep(2));
   await expect(page.locator('#step-panel-2')).toBeVisible();
   await page.locator('#endpoint-path').fill('/rates/{date}');
 
-  await page.evaluate(() => window.CrankWizardShell.doGoToStep(3));
+  await page.evaluate(() => window.CrankWizardShell.goToStep(3));
   await expect(page.locator('#step-panel-3-rest')).toBeVisible();
   await page.locator('.method-card[data-method="GET"]').click();
 
-  await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
+  await page.evaluate(() => window.CrankWizardShell.goToStep(5));
   await expect(page.locator('#step-panel-5')).toBeVisible();
 
   await page.locator('#wizard-input-sample').fill(JSON.stringify({

apps/ui/tests/e2e/workspace-settings.spec.js

@@ -11,6 +11,12 @@ test('workspace and settings pages show live session data', async ({ page }) =>
   await expect(page.locator('#section-members')).toHaveCount(0);
   await expect(page.locator('#section-invite')).toHaveCount(0);
   await expect(page.locator('#delete-workspace-btn')).toHaveCount(0);
+  await expect(page.locator('[data-i18n="workspace_setup.danger.export_title"]')).toHaveText(
+    localized('Export workspace catalog', 'Экспорт каталога рабочего пространства'),
+  );
+  await expect(page.locator('[data-i18n="workspace_setup.danger.export_body"]')).not.toContainText(
+    localized('all data', 'все данные'),
+  );
 
   await page.goto('/settings');
   await expect(page.locator('.page-title')).toHaveText(localized('Account settings', 'Настройки аккаунта'));

crates/crank-adapter-rest/Cargo.toml

@@ -3,18 +3,26 @@ name = "crank-adapter-rest"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]
 async-trait = "0.1"
 crank-core = { path = "../crank-core" }
+crank-trace = { path = "../crank-trace" }
 futures-util = "0.3"
+metrics.workspace = true
+opentelemetry.workspace = true
 reqwest = { workspace = true, features = ["stream"] }
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
 
 [dev-dependencies]
 axum.workspace = true
+opentelemetry_sdk.workspace = true
 tokio.workspace = true
+tracing-subscriber.workspace = true

crates/crank-adapter-rest/src/client.rs

@@ -7,7 +7,9 @@ use std::{
 };
 
 use crank_core::{HttpMethod, RestTarget};
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
 use futures_util::StreamExt;
+use opentelemetry::{global, propagation::Injector, trace::TraceContextExt};
 use reqwest::{
     Client,
     dns::{Addrs, Name, Resolve, Resolving},
@@ -15,6 +17,8 @@ use reqwest::{
     redirect,
 };
 use serde_json::Value;
+use tracing::{Instrument, Span};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 use crate::{RestAdapterError, RestRequest, RestResponse};
 
@@ -66,10 +70,37 @@ impl RestAdapter {
         &self,
         target: &RestTarget,
         request: &RestRequest,
+    ) -> Result<RestResponse, RestAdapterError> {
+        let started_at = std::time::Instant::now();
+        let result = self.execute_inner(target, request).await;
+        let outcome = match &result {
+            Ok(_) => "success",
+            Err(error) => upstream_outcome(error),
+        };
+        metrics::counter!(
+            "crank_upstream_requests_total",
+            "operation_kind" => "rest",
+            "outcome" => outcome
+        )
+        .increment(1);
+        metrics::histogram!(
+            "crank_upstream_request_duration_seconds",
+            "operation_kind" => "rest",
+            "outcome" => outcome
+        )
+        .record(started_at.elapsed().as_secs_f64());
+        result
+    }
+
+    async fn execute_inner(
+        &self,
+        target: &RestTarget,
+        request: &RestRequest,
     ) -> Result<RestResponse, RestAdapterError> {
         let url = build_url(target, request)?;
         self.policy.validate_url(&url)?;
-        let headers = build_headers(target, request)?;
+        let mut headers = build_headers(target, request)?;
+        apply_current_trace_context(&mut headers);
         let client =
             self.client
                 .as_ref()
@@ -85,6 +116,8 @@ impl RestAdapter {
             builder = builder.json(body);
         }
 
+        let upstream_span = Stage::UpstreamHttp.span();
+        let result = async {
             let response = builder.send().await?;
             let status = response.status();
             let headers = normalize_headers(response.headers());
@@ -103,6 +136,41 @@ impl RestAdapter {
                 body,
             })
         }
+        .instrument(upstream_span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&upstream_span),
+            Err(_) => {
+                StageOutcome::Error.record(&upstream_span);
+                ErrorCategory::Upstream.record(&upstream_span);
+            }
+        }
+        result
+    }
+}
+
+fn upstream_outcome(error: &RestAdapterError) -> &'static str {
+    match error {
+        RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
+            "client_error"
+        }
+        RestAdapterError::UnexpectedStatus { status, .. } if (500..600).contains(status) => {
+            "server_error"
+        }
+        RestAdapterError::UnexpectedStatus { .. } => "unexpected_status",
+        RestAdapterError::Transport(error) if error.is_timeout() => "timeout",
+        RestAdapterError::Transport(_) => "transport_error",
+        RestAdapterError::ResponseTooLarge { .. } => "response_too_large",
+        RestAdapterError::TargetNotAllowed { .. } => "rejected",
+        RestAdapterError::WindowExpired => "window_expired",
+        RestAdapterError::InvalidSseEvent => "invalid_response",
+        RestAdapterError::InvalidBaseUrl { .. }
+        | RestAdapterError::InvalidPathParameter { .. }
+        | RestAdapterError::InvalidQueryParameter { .. }
+        | RestAdapterError::InvalidHeaderName { .. }
+        | RestAdapterError::InvalidHeaderValue { .. } => "invalid_request",
+        RestAdapterError::InvalidConfiguration { .. } => "configuration",
+    }
 }
 
 impl Default for OutboundHttpPolicy {
@@ -407,6 +475,9 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
         HeaderName::try_from(name).map_err(|_| RestAdapterError::InvalidHeaderName {
             header: name.to_owned(),
         })?;
+    if is_trace_propagation_header(&header_name) {
+        return Ok(());
+    }
     let header_value =
         HeaderValue::try_from(value).map_err(|_| RestAdapterError::InvalidHeaderValue {
             header: name.to_owned(),
@@ -416,6 +487,38 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
     Ok(())
 }
 
+fn is_trace_propagation_header(name: &HeaderName) -> bool {
+    matches!(name.as_str(), "traceparent" | "tracestate" | "baggage")
+}
+
+fn apply_current_trace_context(headers: &mut HeaderMap) {
+    for header in ["traceparent", "tracestate", "baggage"] {
+        headers.remove(header);
+    }
+
+    let context = Span::current().context();
+    if !context.span().span_context().is_valid() {
+        return;
+    }
+    global::get_text_map_propagator(|propagator| {
+        propagator.inject_context(&context, &mut ReqwestHeaderInjector(headers));
+    });
+}
+
+struct ReqwestHeaderInjector<'a>(&'a mut HeaderMap);
+
+impl Injector for ReqwestHeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(name) = HeaderName::try_from(key) else {
+            return;
+        };
+        let Ok(value) = HeaderValue::try_from(value) else {
+            return;
+        };
+        self.0.insert(name, value);
+    }
+}
+
 async fn decode_body(
     response: reqwest::Response,
     max_response_bytes: usize,

crates/crank-adapter-rest/src/lib.rs

@@ -26,13 +26,15 @@ impl ProtocolAdapter for RestAdapter {
         &self,
         target: &Target,
         prepared: &PreparedRequest,
-        _context: &RuntimeRequestContext,
+        context: &RuntimeRequestContext,
     ) -> Result<AdapterResponse, ProtocolAdapterError> {
         let target = rest_target(target)?;
+        let mut headers = prepared.headers.clone();
+        headers.extend(context.outbound_headers());
         let request = RestRequest {
             path_params: prepared.path_params.clone(),
             query_params: prepared.query_params.clone(),
-            headers: prepared.headers.clone(),
+            headers,
             body: prepared.body.clone(),
             timeout_ms: prepared.timeout_ms,
         };

crates/crank-adapter-rest/tests/integration/client.rs

@@ -8,9 +8,19 @@ use axum::{
     routing::{get, post},
 };
 use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
-use crank_core::{HttpMethod, RestTarget};
+use crank_core::{
+    HttpMethod, PreparedRequest, ProtocolAdapter, RestTarget, RuntimeRequestContext, Target,
+};
+use opentelemetry::{
+    global,
+    trace::{TraceContextExt, TracerProvider as _},
+};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
 use serde_json::{Value, json};
 use tokio::net::TcpListener;
+use tracing::Instrument;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+use tracing_subscriber::layer::SubscriberExt;
 
 #[tokio::test]
 async fn executes_rest_request_and_normalizes_json_response() {
@@ -45,6 +55,120 @@ async fn executes_rest_request_and_normalizes_json_response() {
     );
 }
 
+#[tokio::test]
+async fn protocol_context_overrides_mapped_correlation_headers() {
+    let base_url = spawn_test_server().await;
+    let adapter = test_adapter();
+    let target = Target::Rest(RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::from([
+            ("x-request-id".to_owned(), "static-request".to_owned()),
+            (
+                "x-correlation-id".to_owned(),
+                "static-correlation".to_owned(),
+            ),
+        ]),
+    });
+    let prepared = PreparedRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        headers: BTreeMap::from([
+            ("x-request-id".to_owned(), "mapped-request".to_owned()),
+            (
+                "x-correlation-id".to_owned(),
+                "mapped-correlation".to_owned(),
+            ),
+        ]),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+        ..PreparedRequest::default()
+    };
+    let context = RuntimeRequestContext::new("req-runtime", "corr-runtime");
+
+    let response = adapter
+        .invoke_unary(&target, &prepared, &context)
+        .await
+        .unwrap();
+
+    assert_eq!(response.body["request_id"], "req-runtime");
+    assert_eq!(response.body["correlation_id"], "corr-runtime");
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn current_trace_context_overrides_mapped_traceparent() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("rest-propagation-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let span = tracing::info_span!("runtime.execute");
+    let context = span.context();
+    let expected_trace_id = context.span().span_context().trace_id().to_string();
+    let base_url = spawn_test_server().await;
+    let target = RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::new(),
+    };
+    let request = RestRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        query_params: BTreeMap::new(),
+        headers: BTreeMap::from([(
+            "traceparent".to_owned(),
+            "00-11111111111111111111111111111111-2222222222222222-01".to_owned(),
+        )]),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+    };
+
+    let response = test_adapter()
+        .execute(&target, &request)
+        .instrument(span)
+        .await
+        .unwrap();
+
+    assert_eq!(
+        &response.body["traceparent"].as_str().unwrap()[3..35],
+        expected_trace_id
+    );
+    provider.shutdown().unwrap();
+}
+
+#[tokio::test]
+async fn user_configured_propagation_headers_are_removed_without_trusted_context() {
+    let base_url = spawn_test_server().await;
+    let target = RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::from([
+            (
+                "traceparent".to_owned(),
+                "untrusted\ninvalid-value".to_owned(),
+            ),
+            ("tracestate".to_owned(), "vendor=value".to_owned()),
+            ("baggage".to_owned(), "secret=must-not-leave".to_owned()),
+        ]),
+    };
+    let request = RestRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        query_params: BTreeMap::new(),
+        headers: BTreeMap::new(),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+    };
+
+    let response = test_adapter().execute(&target, &request).await.unwrap();
+
+    assert!(response.body.get("traceparent").is_none());
+    assert!(response.body.get("tracestate").is_none());
+    assert!(response.body.get("baggage").is_none());
+}
+
 #[tokio::test]
 async fn returns_unexpected_status_with_normalized_body() {
     let base_url = spawn_test_server().await;
@@ -186,14 +310,57 @@ async fn create_user(
         .get("x-static")
         .and_then(|value| value.to_str().ok())
         .unwrap_or_default();
+    let request_id = headers
+        .get("x-request-id")
+        .and_then(|value| value.to_str().ok());
+    let correlation_id = headers
+        .get("x-correlation-id")
+        .and_then(|value| value.to_str().ok());
+    let traceparent = headers
+        .get("traceparent")
+        .and_then(|value| value.to_str().ok());
+    let tracestate = headers
+        .get("tracestate")
+        .and_then(|value| value.to_str().ok());
+    let baggage = headers.get("baggage").and_then(|value| value.to_str().ok());
 
-    Json(json!({
+    let mut response = json!({
         "id": user_id,
         "query": query.get("expand").cloned().unwrap_or_default(),
         "trace": trace,
         "static": static_header,
         "payload": payload
-    }))
+    });
+    let response = response.as_object_mut().unwrap();
+    if let Some(request_id) = request_id {
+        response.insert(
+            "request_id".to_owned(),
+            Value::String(request_id.to_owned()),
+        );
+    }
+    if let Some(correlation_id) = correlation_id {
+        response.insert(
+            "correlation_id".to_owned(),
+            Value::String(correlation_id.to_owned()),
+        );
+    }
+    if let Some(traceparent) = traceparent {
+        response.insert(
+            "traceparent".to_owned(),
+            Value::String(traceparent.to_owned()),
+        );
+    }
+    if let Some(tracestate) = tracestate {
+        response.insert(
+            "tracestate".to_owned(),
+            Value::String(tracestate.to_owned()),
+        );
+    }
+    if let Some(baggage) = baggage {
+        response.insert("baggage".to_owned(), Value::String(baggage.to_owned()));
+    }
+
+    Json(Value::Object(response.clone()))
 }
 
 async fn fail() -> (axum::http::StatusCode, Json<Value>) {

crates/crank-community-auth/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-community-auth"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-community-auth/src/password_provider.rs

@@ -52,7 +52,11 @@ impl IdentityProvider for PasswordIdentityProvider {
             &self.password_pepper,
             &user.password_hash,
         ) {
-            debug!(email = %payload.email, "password identity provider rejected credentials");
+            debug!(
+                name: "auth.password.rejected",
+                identity_provider = "password",
+                "password identity provider rejected credentials"
+            );
             return Err(IdentityError::BadCredentials);
         }
 

crates/crank-community-mcp/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-community-mcp"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]
@@ -11,10 +12,13 @@ axum.workspace = true
 base64.workspace = true
 crank-adapter-rest = { path = "../crank-adapter-rest" }
 crank-core = { path = "../crank-core" }
+crank-observability = { path = "../crank-observability" }
 crank-registry = { path = "../crank-registry" }
 crank-runtime = { path = "../crank-runtime" }
 crank-schema = { path = "../crank-schema" }
+crank-trace = { path = "../crank-trace" }
 futures-util = "0.3"
+metrics.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -29,3 +33,7 @@ uuid.workspace = true
 [dev-dependencies]
 crank-mapping = { path = "../crank-mapping" }
 crank-test-support = { path = "../crank-test-support" }
+opentelemetry.workspace = true
+opentelemetry_sdk.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true

crates/crank-community-mcp/src/access.rs

@@ -1,27 +1,54 @@
 use std::sync::Arc;
 
-use axum::http::{HeaderMap, StatusCode, header::AUTHORIZATION};
+use axum::{
+    http::{HeaderMap, StatusCode, header::AUTHORIZATION},
+    response::{IntoResponse, Response},
+};
 use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
 use crank_core::{OperationSecurityLevel, PlatformApiKeyScope};
+use crank_trace::{DbOperation, ErrorCategory, StageOutcome, observe_db_query};
 use sha2::{Digest, Sha256};
 use time::OffsetDateTime;
+use tracing::Instrument;
 
 use crate::{
     app::{AgentRoutePath, AppState},
     auth::VerifiedMachineCredential,
 };
 
+#[derive(Clone, Copy, Debug)]
+pub(super) enum MachineAccessError {
+    Denied(StatusCode),
+    Unavailable,
+}
+
+impl MachineAccessError {
+    pub(super) fn is_denied(self) -> bool {
+        matches!(self, Self::Denied(_))
+    }
+}
+
+impl IntoResponse for MachineAccessError {
+    fn into_response(self) -> Response {
+        match self {
+            Self::Denied(status) => status.into_response(),
+            Self::Unavailable => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
+        }
+    }
+}
+
 pub(super) async fn require_machine_access(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     headers: &HeaderMap,
     required_scope: PlatformApiKeyScope,
-) -> Result<VerifiedMachineCredential, StatusCode> {
-    let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?;
+) -> Result<VerifiedMachineCredential, MachineAccessError> {
+    let secret =
+        bearer_token(headers).ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))?;
     let credential = resolve_machine_credential(state, path, secret).await?;
 
     if !allows_scope(&credential.scopes, required_scope) {
-        return Err(StatusCode::FORBIDDEN);
+        return Err(MachineAccessError::Denied(StatusCode::FORBIDDEN));
     }
 
     Ok(credential)
@@ -35,12 +62,15 @@ pub(super) async fn require_approval_access(
 ) -> Result<crank_registry::PlatformApiKeyRecord, StatusCode> {
     let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?;
     let secret_hash = hash_access_secret(secret);
-    let Some(api_key) = state
+    let Some(api_key) = observe_db_query(
+        DbOperation::MachineAccessRead,
+        state
             .registry
             .get_approval_api_key_by_secret_for_agent_slug(
                 &path.workspace_slug,
                 &path.agent_slug,
                 &secret_hash,
+            ),
     )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
@@ -53,9 +83,14 @@ pub(super) async fn require_approval_access(
     }
 
     let used_at = OffsetDateTime::now_utc();
-    state
-        .registry
-        .touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
+    observe_db_query(
+        DbOperation::MachineAccessTouch,
+        state.registry.touch_platform_api_key(
+            &api_key.api_key.workspace_id,
+            &api_key.api_key.id,
+            &used_at,
+        ),
+    )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
 
@@ -100,7 +135,7 @@ async fn resolve_machine_credential(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     token: &str,
-) -> Result<VerifiedMachineCredential, StatusCode> {
+) -> Result<VerifiedMachineCredential, MachineAccessError> {
     if let Some(credential) = verify_static_agent_key(state, path, token).await? {
         return Ok(credential);
     }
@@ -109,35 +144,64 @@ async fn resolve_machine_credential(
         .credential_verifier
         .verify_bearer_token(&path.workspace_slug, &path.agent_slug, token)
         .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
-        .ok_or(StatusCode::UNAUTHORIZED)
+        .map_err(|_| MachineAccessError::Unavailable)?
+        .ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))
 }
 
 async fn verify_static_agent_key(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     secret: &str,
-) -> Result<Option<VerifiedMachineCredential>, StatusCode> {
+) -> Result<Option<VerifiedMachineCredential>, MachineAccessError> {
     let secret_hash = hash_access_secret(secret);
-    let Some(api_key) = state
+    let read_span = DbOperation::MachineAccessRead.span();
+    let api_key_result = state
         .registry
         .get_platform_api_key_by_secret_for_agent_slug(
             &path.workspace_slug,
             &path.agent_slug,
             &secret_hash,
         )
+        .instrument(read_span.clone())
         .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
-    else {
+        .map_err(|_| MachineAccessError::Unavailable);
+    let api_key = match api_key_result {
+        Ok(api_key) => {
+            StageOutcome::Success.record(&read_span);
+            drop(read_span);
+            api_key
+        }
+        Err(status) => {
+            StageOutcome::Error.record(&read_span);
+            ErrorCategory::Database.record(&read_span);
+            drop(read_span);
+            return Err(status);
+        }
+    };
+    let Some(api_key) = api_key else {
         return Ok(None);
     };
 
     let used_at = OffsetDateTime::now_utc();
-    state
+    let touch_span = DbOperation::MachineAccessTouch.span();
+    let touch_result = state
         .registry
         .touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
+        .instrument(touch_span.clone())
         .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+        .map_err(|_| MachineAccessError::Unavailable);
+    match touch_result {
+        Ok(()) => {
+            StageOutcome::Success.record(&touch_span);
+            drop(touch_span);
+        }
+        Err(status) => {
+            StageOutcome::Error.record(&touch_span);
+            ErrorCategory::Database.record(&touch_span);
+            drop(touch_span);
+            return Err(status);
+        }
+    }
 
     Ok(Some(VerifiedMachineCredential {
         machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,

crates/crank-community-mcp/src/app/invocation_history.rs

@@ -0,0 +1,122 @@
+use std::{sync::Arc, time::Duration};
+
+use crank_core::{
+    InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus,
+};
+use crank_registry::{
+    CreateInvocationLogRequest, InvocationHistoryWriteOutcome, PublishedAgentTool,
+};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
+use serde_json::Value;
+use time::OffsetDateTime;
+use tracing::{Instrument, warn};
+
+use super::AppState;
+
+pub(crate) struct InvocationRecord<'a> {
+    pub(crate) request_id: Option<&'a str>,
+    pub(crate) tool_name: &'a str,
+    pub(crate) status: InvocationStatus,
+    pub(crate) level: InvocationLevel,
+    pub(crate) message: &'a str,
+    pub(crate) status_code: Option<u16>,
+    pub(crate) error_kind: Option<&'a str>,
+    pub(crate) duration: Duration,
+    pub(crate) request_preview: Value,
+    pub(crate) response_preview: Value,
+}
+
+pub(crate) async fn persist_invocation(
+    state: &Arc<AppState>,
+    tool: &PublishedAgentTool,
+    record: InvocationRecord<'_>,
+) -> InvocationHistoryWriteOutcome {
+    let log = InvocationLog {
+        id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
+        workspace_id: tool.workspace_id.clone(),
+        agent_id: Some(tool.agent_id.clone()),
+        operation_id: tool.operation.id.clone(),
+        source: InvocationSource::AgentToolCall,
+        level: record.level,
+        status: record.status,
+        tool_name: record.tool_name.to_owned(),
+        message: record.message.to_owned(),
+        request_id: record.request_id.map(ToOwned::to_owned),
+        status_code: record.status_code,
+        duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX),
+        error_kind: record.error_kind.map(ToOwned::to_owned),
+        request_preview: record.request_preview,
+        response_preview: record.response_preview,
+        created_at: OffsetDateTime::now_utc(),
+    };
+
+    let history_span = Stage::HistoryWrite.span();
+    let (outcome, db_span) = async {
+        let db_span = DbOperation::InvocationHistoryWrite.span();
+        let outcome = state
+            .registry
+            .create_invocation_log(CreateInvocationLogRequest { log: &log })
+            .instrument(db_span.clone())
+            .await;
+        (outcome, db_span)
+    }
+    .instrument(history_span.clone())
+    .await;
+    match outcome {
+        InvocationHistoryWriteOutcome::Recorded => {
+            StageOutcome::Success.record(&db_span);
+            StageOutcome::Success.record(&history_span);
+        }
+        InvocationHistoryWriteOutcome::Lost(_) => {
+            StageOutcome::Error.record(&db_span);
+            ErrorCategory::Database.record(&db_span);
+            StageOutcome::Error.record(&history_span);
+            ErrorCategory::History.record(&history_span);
+        }
+    }
+    drop(db_span);
+    drop(history_span);
+    observe_invocation_history_outcome(
+        outcome,
+        record.request_id,
+        record.status,
+        InvocationSource::AgentToolCall,
+    );
+    outcome
+}
+
+pub(super) fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: InvocationStatus,
+    source: InvocationSource,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    warn!(
+        name: "mcp.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source = invocation_source_label(source),
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_source_label(source: InvocationSource) -> &'static str {
+    match source {
+        InvocationSource::AdminTestRun => "admin_test_run",
+        InvocationSource::AgentToolCall => "agent_tool_call",
+    }
+}
+
+fn invocation_status_label(status: InvocationStatus) -> &'static str {
+    match status {
+        InvocationStatus::Ok => "ok",
+        InvocationStatus::Error => "error",
+    }
+}

crates/crank-community-mcp/src/app/metrics.rs

@@ -0,0 +1,93 @@
+use std::sync::Arc;
+
+use axum::http::StatusCode;
+use serde_json::Value;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
+
+use crate::{
+    jsonrpc::{is_notification, is_response, method_name},
+    transport::ResponseMode,
+};
+
+pub(super) struct McpRequestMetrics {
+    method: &'static str,
+    response_mode: &'static str,
+    outcome: &'static str,
+}
+
+impl McpRequestMetrics {
+    pub(super) fn new(message: &Value) -> Self {
+        Self {
+            method: normalized_mcp_method(message),
+            response_mode: "unknown",
+            outcome: "rejected",
+        }
+    }
+
+    pub(super) fn set_response_mode(&mut self, mode: ResponseMode) -> ResponseMode {
+        self.response_mode = match mode {
+            ResponseMode::Json => "json",
+            ResponseMode::Sse => "sse",
+        };
+        mode
+    }
+
+    pub(super) fn complete(&mut self, status: StatusCode) {
+        self.outcome = match status.as_u16() {
+            200..=299 => "success",
+            400..=499 => "client_error",
+            500..=599 => "server_error",
+            _ => "other",
+        };
+    }
+}
+
+impl Drop for McpRequestMetrics {
+    fn drop(&mut self) {
+        ::metrics::counter!(
+            "crank_mcp_requests_total",
+            "method" => self.method,
+            "response_mode" => self.response_mode,
+            "outcome" => self.outcome
+        )
+        .increment(1);
+    }
+}
+
+pub(super) fn normalized_mcp_method(message: &Value) -> &'static str {
+    match method_name(message) {
+        Some("initialize") => "initialize",
+        Some("notifications/initialized") => "initialized",
+        Some("ping") => "ping",
+        Some("tools/list") => "tools_list",
+        Some("tools/call") => "tools_call",
+        Some(_) if is_notification(message) => "notification",
+        Some(_) => "unsupported",
+        None if is_response(message) => "response",
+        None => "invalid",
+    }
+}
+
+pub(super) struct ActiveSessionGuard {
+    _permit: OwnedSemaphorePermit,
+}
+
+impl ActiveSessionGuard {
+    pub(super) fn try_acquire(slots: &Arc<Semaphore>) -> Result<Self, ()> {
+        let permit = Arc::clone(slots).try_acquire_owned().map_err(|_| {
+            ::metrics::counter!(
+                "crank_runtime_limit_rejections_total",
+                "stage" => "mcp_session"
+            )
+            .increment(1);
+        })?;
+        ::metrics::gauge!("crank_mcp_active_sessions").increment(1.0);
+        Ok(Self { _permit: permit })
+    }
+}
+
+impl Drop for ActiveSessionGuard {
+    fn drop(&mut self) {
+        ::metrics::gauge!("crank_mcp_active_sessions").decrement(1.0);
+    }
+}

crates/crank-community-mcp/src/app/stages.rs

@@ -0,0 +1,86 @@
+use std::sync::Arc;
+
+use axum::http::{HeaderMap, StatusCode};
+use crank_core::PlatformApiKeyScope;
+use crank_registry::PlatformApiKeyRecord;
+use crank_runtime::RateLimitCheckError;
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
+use tracing::Instrument;
+
+use crate::{
+    access::{MachineAccessError, require_approval_access, require_machine_access},
+    app::{AgentRoutePath, AppState},
+    auth::VerifiedMachineCredential,
+    rate_limit::enforce_transport_rate_limit,
+};
+
+pub(super) async fn enforce_traced_rate_limit(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+) -> Result<(), RateLimitCheckError> {
+    let span = Stage::McpRateLimit.span();
+    let result = enforce_transport_rate_limit(state, path, headers)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(()) => StageOutcome::Allowed.record(&span),
+        Err(RateLimitCheckError::Rejected(_)) => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::RateLimit.record(&span);
+        }
+        Err(RateLimitCheckError::StoreUnavailable) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}
+
+pub(super) async fn require_traced_machine_access(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+    required_scope: PlatformApiKeyScope,
+) -> Result<VerifiedMachineCredential, MachineAccessError> {
+    let span = Stage::McpAccessCheck.span();
+    let result = require_machine_access(state, path, headers, required_scope)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(_) => StageOutcome::Allowed.record(&span),
+        Err(error) if error.is_denied() => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::Access.record(&span);
+        }
+        Err(_) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}
+
+pub(super) async fn require_traced_approval_access(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+    required_scope: PlatformApiKeyScope,
+) -> Result<PlatformApiKeyRecord, StatusCode> {
+    let span = Stage::McpAccessCheck.span();
+    let result = require_approval_access(state, path, headers, required_scope)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(_) => StageOutcome::Allowed.record(&span),
+        Err(status) if *status == StatusCode::UNAUTHORIZED || *status == StatusCode::FORBIDDEN => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::Access.record(&span);
+        }
+        Err(_) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}

crates/crank-community-mcp/src/app/tests.rs

@@ -0,0 +1,149 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use axum::body::to_bytes;
+use crank_core::InvocationStatus;
+use crank_observability::{
+    ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+    operational_incident_total,
+};
+use crank_registry::{
+    InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
+};
+use serde_json::{Value, json};
+use tracing_subscriber::fmt::MakeWriter;
+
+use super::{
+    ResponseMode, metrics::normalized_mcp_method, observe_invocation_history_outcome,
+    tool_error_response,
+};
+use crate::jsonrpc::CURRENT_PROTOCOL_VERSION;
+use crate::tool_error::generic_tool_error_contract;
+
+#[tokio::test]
+async fn tool_error_response_includes_structured_context() {
+    let response = tool_error_response(
+        &json!({"jsonrpc": "2.0", "id": "req-1"}),
+        ResponseMode::Json,
+        CURRENT_PROTOCOL_VERSION,
+        generic_tool_error_contract(
+            "streaming_payload_error",
+            "request root must be an object",
+            "req-1",
+            false,
+            Some("Проверьте параметры вызова инструмента."),
+        ),
+    );
+
+    let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let payload: Value = serde_json::from_slice(&body).unwrap();
+
+    assert_eq!(
+        payload["result"]["structuredContent"]["error"],
+        json!({
+            "code": "streaming_payload_error",
+            "error_code": "streaming_payload_error",
+            "message": "request root must be an object",
+            "recoverable": false,
+            "request_id": "req-1",
+            "suggested_action": "Проверьте параметры вызова инструмента."
+        })
+    );
+}
+
+#[test]
+fn emits_bounded_history_loss_incident() {
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _guard = tracing::dispatcher::set_default(&dispatch);
+
+    observe_invocation_history_outcome(
+        InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
+            category: InvocationHistoryLossCategory::Unavailable,
+        }),
+        Some("req_mcp_dc08"),
+        InvocationStatus::Ok,
+        crank_core::InvocationSource::AgentToolCall,
+    );
+
+    let output = writer.output();
+    assert!(!output.contains("dc08-canary-secret"));
+    let event: Value = output
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &Value| event["event"] == "mcp.invocation_history.lost")
+        .unwrap();
+    assert_eq!(event["request_id"], "req_mcp_dc08");
+    assert_eq!(event["fields"]["source"], "agent_tool_call");
+    assert_eq!(event["fields"]["invocation_status"], "ok");
+    assert_eq!(event["fields"]["error_category"], "unavailable");
+    assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+}
+
+#[test]
+fn mcp_metric_method_is_always_from_a_closed_set() {
+    assert_eq!(
+        normalized_mcp_method(&json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call"})),
+        "tools_call"
+    );
+    assert_eq!(
+        normalized_mcp_method(
+            &json!({"jsonrpc": "2.0", "id": 2, "method": "customer-controlled-method"})
+        ),
+        "unsupported"
+    );
+    assert_eq!(
+        normalized_mcp_method(
+            &json!({"jsonrpc": "2.0", "method": "customer-controlled-notification"})
+        ),
+        "notification"
+    );
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

crates/crank-community-mcp/src/approval_execution.rs

@@ -5,11 +5,13 @@ use axum::{
     response::{IntoResponse, Response},
 };
 use crank_core::{ApprovalRequestStatus, InvocationLevel, InvocationSource, InvocationStatus};
+use crank_observability::RequestId;
 use crank_registry::{ApprovalRequestRecord, FinishApprovalRequest};
 use crank_runtime::{RuntimeExecutionRequest, RuntimeRequestContext};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use serde_json::json;
 use time::OffsetDateTime;
-use tracing::warn;
+use tracing::{Instrument, warn};
 
 use crate::{
     app::{
@@ -35,32 +37,81 @@ pub(super) fn spawn_approval_recovery(state: Arc<AppState>) {
 }
 
 async fn recover_approved_requests(state: &Arc<AppState>) {
+    fail_interrupted_requests(state).await;
+
     for _ in 0..32 {
         let now = OffsetDateTime::now_utc();
-        let approval = match state
+        let approval = match observe_db_query(
+            DbOperation::ApprovalWrite,
+            state
                 .registry
-            .claim_next_recoverable_approval_request(
-                now,
-                now - RECOVERY_GRACE,
-                now - EXECUTION_LEASE,
+                .claim_next_recoverable_approval_request(now, now - RECOVERY_GRACE),
         )
         .await
         {
             Ok(Some(approval)) => approval,
             Ok(None) => break,
-            Err(error) => {
-                warn!(error = %error, "approval recovery query failed");
+            Err(_) => {
+                warn!(
+                    name: "mcp.approval_recovery.query_failed",
+                    error_category = "registry",
+                    "approval recovery query failed"
+                );
                 break;
             }
         };
         let Some(path) = approval_agent_path(state, &approval).await else {
             continue;
         };
-        if execute_approved_request(state, &path, approval)
+        let recovery_span = Stage::ApprovalRecovery.span();
+        let result = execute_approved_request(state, &path, approval, None)
+            .instrument(recovery_span.clone())
+            .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&recovery_span),
+            Err(_) => {
+                StageOutcome::Error.record(&recovery_span);
+                ErrorCategory::Approval.record(&recovery_span);
+            }
+        }
+        drop(recovery_span);
+        if result.is_err() {
+            warn!(
+                name: "mcp.approval_recovery.execution_failed",
+                error_category = "runtime",
+                "recovered approval execution did not finish"
+            );
+        }
+    }
+}
+
+async fn fail_interrupted_requests(state: &Arc<AppState>) {
+    for _ in 0..32 {
+        let stale_before = OffsetDateTime::now_utc() - EXECUTION_LEASE;
+        match observe_db_query(
+            DbOperation::ApprovalWrite,
+            state
+                .registry
+                .fail_next_interrupted_approval_request(stale_before),
+        )
         .await
-            .is_err()
         {
-            warn!("recovered approval execution did not finish");
+            Ok(Some(approval)) => {
+                warn!(
+                    name: "mcp.approval_recovery.interrupted",
+                    approval_id = approval.approval.id.as_str(),
+                    "interrupted approval execution was not retried because its outcome is unknown"
+                );
+            }
+            Ok(None) => break,
+            Err(_) => {
+                warn!(
+                    name: "mcp.approval_recovery.interrupted_query_failed",
+                    error_category = "registry",
+                    "interrupted approval recovery query failed"
+                );
+                break;
+            }
         }
     }
 }
@@ -76,8 +127,12 @@ async fn approval_agent_path(
     {
         Ok(Some(workspace)) => workspace,
         Ok(None) => return None,
-        Err(error) => {
-            warn!(error = %error, "approval workspace lookup failed");
+        Err(_) => {
+            warn!(
+                name: "mcp.approval_recovery.workspace_lookup_failed",
+                error_category = "registry",
+                "approval workspace lookup failed"
+            );
             return None;
         }
     };
@@ -88,8 +143,12 @@ async fn approval_agent_path(
     {
         Ok(Some(agent)) => agent,
         Ok(None) => return None,
-        Err(error) => {
-            warn!(error = %error, "approval agent lookup failed");
+        Err(_) => {
+            warn!(
+                name: "mcp.approval_recovery.agent_lookup_failed",
+                error_category = "registry",
+                "approval agent lookup failed"
+            );
             return None;
         }
     };
@@ -103,7 +162,9 @@ pub(super) async fn execute_approved_request(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     approval: ApprovalRequestRecord,
+    request_id: Option<&str>,
 ) -> Result<ApprovalRequestRecord, Response> {
+    let request_id = RequestId::resolve(request_id).into_string();
     let tools = state
         .catalog
         .list_tools(&path.workspace_slug, &path.agent_slug)
@@ -123,8 +184,7 @@ pub(super) async fn execute_approved_request(
         &approval.approval.request_payload,
     );
     let started_at = Instant::now();
-    let runtime_request_context =
-        RuntimeRequestContext::from_request_id(approval.approval.id.as_str().to_owned())
+    let runtime_request_context = RuntimeRequestContext::from_request_id(request_id.clone())
         .with_response_cache_scope(
             tool.workspace_id.as_str().to_owned(),
             tool.agent_id.as_str().to_owned(),
@@ -176,11 +236,11 @@ pub(super) async fn execute_approved_request(
             ),
         };
 
-    if let Err(error) = persist_invocation(
+    persist_invocation(
         state,
         &tool,
         InvocationRecord {
-            request_id: Some(approval.approval.id.as_str()),
+            request_id: Some(&request_id),
             tool_name: &tool.tool_name,
             status: invocation_status,
             level: invocation_level,
@@ -192,11 +252,10 @@ pub(super) async fn execute_approved_request(
             response_preview: response_payload.clone(),
         },
     )
-    .await
-    {
-        warn!(error = %error, "approved invocation log write failed");
-    }
+    .await;
 
+    observe_db_query(
+        DbOperation::ApprovalWrite,
         state
             .registry
             .finish_approval_request(FinishApprovalRequest {
@@ -206,7 +265,8 @@ pub(super) async fn execute_approved_request(
                 status,
                 response_payload: Some(response_payload),
                 decision_note: None,
-        })
+            }),
+    )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
     .ok_or_else(|| StatusCode::CONFLICT.into_response())
@@ -216,6 +276,8 @@ async fn finish_unavailable_approval(
     state: &Arc<AppState>,
     approval: &ApprovalRequestRecord,
 ) -> Result<ApprovalRequestRecord, Response> {
+    observe_db_query(
+        DbOperation::ApprovalWrite,
         state
             .registry
             .finish_approval_request(FinishApprovalRequest {
@@ -230,7 +292,8 @@ async fn finish_unavailable_approval(
                     }
                 })),
                 decision_note: None,
-        })
+            }),
+    )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
     .ok_or_else(|| StatusCode::CONFLICT.into_response())

crates/crank-community-mcp/src/approval_response.rs

@@ -0,0 +1,38 @@
+use crank_core::{ApprovalRequest, OperationApprovalPolicy};
+use crank_registry::PublishedAgentTool;
+use serde_json::{Value, json};
+
+pub(super) fn approval_required_response(
+    tool: &PublishedAgentTool,
+    approval: &ApprovalRequest,
+    policy: &OperationApprovalPolicy,
+) -> Value {
+    let approval_url = format!(
+        "/v1/{}/{}/approvals/{}",
+        tool.workspace_slug,
+        tool.agent_slug,
+        approval.id.as_str()
+    );
+    json!({
+        "status": "approval_required",
+        "approval_id": approval.id.as_str(),
+        "approval_url": approval_url,
+        "approve": {
+            "method": "POST",
+            "url": format!("{approval_url}/approve"),
+            "body": { "approve": "yes" }
+        },
+        "deny": {
+            "method": "POST",
+            "url": format!("{approval_url}/deny"),
+            "body": { "approve": "no" }
+        },
+        "expires_at": approval.expires_at,
+        "risk_level": approval.risk_level,
+        "payload_preview": if policy.show_payload_preview {
+            approval.request_payload.clone()
+        } else {
+            Value::Null
+        },
+    })
+}

crates/crank-community-mcp/src/catalog.rs

@@ -1,24 +1,27 @@
 use std::{
     collections::HashMap,
-    sync::Arc,
+    sync::{Arc, Weak},
     time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 
 use crank_core::{CacheScope, CoordinationStateStore, CoordinationStateValue};
-use crank_registry::{PostgresRegistry, PublishedAgentTool, RegistryError};
+use crank_registry::{PostgresRegistry, PublishedAgentCatalog, PublishedAgentTool, RegistryError};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
 use serde::{Deserialize, Serialize};
 use tokio::sync::{Mutex, RwLock};
-use tracing::{info, warn};
+use tracing::{Instrument, info, warn};
 
 use crate::manifest::analyze_published_tool_catalog;
 
+const MAX_LOCAL_CATALOGS: usize = 1_024;
+
 #[derive(Clone)]
 pub struct PublishedToolCatalog {
     registry: PostgresRegistry,
     refresh_interval: Duration,
     coordination_store: Arc<dyn CoordinationStateStore>,
     cached: Arc<RwLock<HashMap<CatalogKey, CachedCatalog>>>,
-    refresh_locks: Arc<Mutex<HashMap<CatalogKey, Arc<Mutex<()>>>>>,
+    refresh_locks: Arc<Mutex<HashMap<CatalogKey, Weak<Mutex<()>>>>>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -27,15 +30,22 @@ struct CatalogKey {
     agent_slug: String,
 }
 
-#[derive(Default)]
 struct CachedCatalog {
     loaded_at: Option<Instant>,
-    tools: Vec<PublishedAgentTool>,
+    catalog: PublishedAgentCatalog,
+    metrics: CatalogMetrics,
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct CatalogMetrics {
+    tool_count: usize,
+    estimated_context_tokens: usize,
+    warning_count: usize,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 struct CatalogSnapshot {
-    tools: Vec<PublishedAgentTool>,
+    catalog: PublishedAgentCatalog,
     generated_at_ms: u64,
 }
 
@@ -59,12 +69,37 @@ impl PublishedToolCatalog {
         workspace_slug: &str,
         agent_slug: &str,
     ) -> Result<Vec<PublishedAgentTool>, RegistryError> {
+        Ok(self.get_catalog(workspace_slug, agent_slug).await?.tools)
+    }
+
+    pub async fn get_catalog(
+        &self,
+        workspace_slug: &str,
+        agent_slug: &str,
+    ) -> Result<PublishedAgentCatalog, RegistryError> {
+        let span = Stage::McpCatalogLoad.span();
+        let result = async {
             self.refresh_if_stale(workspace_slug, agent_slug).await?;
             let guard = self.cached.read().await;
-        Ok(guard
+            guard
                 .get(&CatalogKey::new(workspace_slug, agent_slug))
-            .map(|entry| entry.tools.clone())
-            .unwrap_or_default())
+                .map(|entry| entry.catalog.clone())
+                .ok_or_else(|| RegistryError::PublishedAgentNotFound {
+                    workspace_slug: workspace_slug.to_owned(),
+                    agent_slug: agent_slug.to_owned(),
+                })
+        }
+        .instrument(span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&span),
+            Err(_) => {
+                StageOutcome::Error.record(&span);
+                ErrorCategory::Catalog.record(&span);
+            }
+        }
+        drop(span);
+        result
     }
 
     async fn refresh_if_stale(
@@ -88,11 +123,14 @@ impl PublishedToolCatalog {
 
         let refresh_lock = {
             let mut locks = self.refresh_locks.lock().await;
-            Arc::clone(
-                locks
-                    .entry(key.clone())
-                    .or_insert_with(|| Arc::new(Mutex::new(()))),
-            )
+            locks.retain(|_, lock| lock.strong_count() > 0);
+            if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
+                lock
+            } else {
+                let lock = Arc::new(Mutex::new(()));
+                locks.insert(key.clone(), Arc::downgrade(&lock));
+                lock
+            }
         };
         let _refresh_guard = refresh_lock.lock().await;
         let still_stale = {
@@ -106,52 +144,58 @@ impl PublishedToolCatalog {
             return Ok(());
         }
 
-        if let Some((tools, age)) = self.load_shared_snapshot(workspace_slug, agent_slug).await {
-            log_catalog_analysis(workspace_slug, agent_slug, "shared_cache", &tools);
-            let mut guard = self.cached.write().await;
-            guard.insert(
+        if let Some((catalog, age)) = self.load_shared_snapshot(workspace_slug, agent_slug).await {
+            let metrics =
+                log_catalog_analysis(workspace_slug, agent_slug, "shared_cache", &catalog.tools);
+            self.store_local_catalog(
                 key,
                 CachedCatalog {
                     loaded_at: Instant::now().checked_sub(age),
-                    tools,
+                    catalog,
+                    metrics,
                 },
-            );
+            )
+            .await;
             return Ok(());
         }
 
-        let tools = match self
+        let db_span = DbOperation::CatalogLoad.span();
+        let catalog_result = self
             .registry
-            .get_published_agent_tools_by_slug(workspace_slug, agent_slug)
-            .await
-        {
-            Ok(tools) => tools,
-            Err(RegistryError::PublishedAgentNotFound { .. }) => Vec::new(),
-            Err(error) => return Err(error),
-        };
-        log_catalog_analysis(workspace_slug, agent_slug, "postgres", &tools);
-        self.store_shared_snapshot(workspace_slug, agent_slug, &tools)
+            .get_published_agent_catalog_by_slug(workspace_slug, agent_slug)
+            .instrument(db_span.clone())
             .await;
-        let mut guard = self.cached.write().await;
-        let previous_count = guard
-            .get(&key)
-            .map(|entry| entry.tools.len())
-            .unwrap_or_default();
-
-        guard.insert(
+        let catalog = match catalog_result {
+            Ok(catalog) => catalog,
+            Err(error) => {
+                StageOutcome::Error.record(&db_span);
+                ErrorCategory::Database.record(&db_span);
+                drop(db_span);
+                return Err(error);
+            }
+        };
+        StageOutcome::Success.record(&db_span);
+        drop(db_span);
+        let metrics = log_catalog_analysis(workspace_slug, agent_slug, "postgres", &catalog.tools);
+        self.store_shared_snapshot(workspace_slug, agent_slug, &catalog)
+            .await;
+        let published_tool_count = catalog.tools.len();
+        let previous_count = self
+            .store_local_catalog(
                 key,
                 CachedCatalog {
                     loaded_at: Some(Instant::now()),
-                tools,
+                    catalog,
+                    metrics,
                 },
-        );
+            )
+            .await;
 
         info!(
+            name: "mcp.catalog.refreshed",
             workspace_slug,
             agent_slug,
-            published_tool_count = guard
-                .get(&CatalogKey::new(workspace_slug, agent_slug))
-                .map(|entry| entry.tools.len())
-                .unwrap_or_default(),
+            published_tool_count,
             previous_published_tool_count = previous_count,
             "published agent catalog refreshed"
         );
@@ -159,11 +203,32 @@ impl PublishedToolCatalog {
         Ok(())
     }
 
+    async fn store_local_catalog(&self, key: CatalogKey, entry: CachedCatalog) -> usize {
+        let mut guard = self.cached.write().await;
+        let previous_count = guard
+            .get(&key)
+            .map(|current| current.catalog.tools.len())
+            .unwrap_or_default();
+
+        if guard.len() >= MAX_LOCAL_CATALOGS && !guard.contains_key(&key) {
+            let oldest = guard
+                .iter()
+                .min_by_key(|(_, current)| current.loaded_at)
+                .map(|(candidate, _)| candidate.clone());
+            if let Some(oldest) = oldest {
+                guard.remove(&oldest);
+            }
+        }
+        guard.insert(key, entry);
+        record_catalog_metrics(guard.values().map(|entry| entry.metrics));
+        previous_count
+    }
+
     async fn load_shared_snapshot(
         &self,
         workspace_slug: &str,
         agent_slug: &str,
-    ) -> Option<(Vec<PublishedAgentTool>, Duration)> {
+    ) -> Option<(PublishedAgentCatalog, Duration)> {
         if self.refresh_interval.is_zero() {
             return None;
         }
@@ -179,21 +244,21 @@ impl PublishedToolCatalog {
         };
         let snapshot = serde_json::from_value::<CatalogSnapshot>(value.payload).ok()?;
         let age = Duration::from_millis(now_unix_ms().saturating_sub(snapshot.generated_at_ms));
-        (age < self.refresh_interval).then_some((snapshot.tools, age))
+        (age < self.refresh_interval).then_some((snapshot.catalog, age))
     }
 
     async fn store_shared_snapshot(
         &self,
         workspace_slug: &str,
         agent_slug: &str,
-        tools: &[PublishedAgentTool],
+        catalog: &PublishedAgentCatalog,
     ) {
         let Some(ttl) = catalog_snapshot_ttl(self.refresh_interval) else {
             return;
         };
 
         let payload = match serde_json::to_value(CatalogSnapshot {
-            tools: tools.to_vec(),
+            catalog: catalog.clone(),
             generated_at_ms: now_unix_ms(),
         }) {
             Ok(payload) => payload,
@@ -217,12 +282,19 @@ fn log_catalog_analysis(
     agent_slug: &str,
     source: &str,
     tools: &[PublishedAgentTool],
-) {
+) -> CatalogMetrics {
     let analysis = match analyze_published_tool_catalog(tools) {
         Ok(analysis) => analysis,
-        Err(error) => {
-            warn!(workspace_slug, agent_slug, source, %error, "published catalog analysis failed");
-            return;
+        Err(_) => {
+            warn!(
+                name: "mcp.catalog.analysis_failed",
+                workspace_slug,
+                agent_slug,
+                source,
+                error_category = "catalog_validation",
+                "published catalog analysis failed"
+            );
+            return CatalogMetrics::default();
         }
     };
     let warning_count = analysis
@@ -233,6 +305,7 @@ fn log_catalog_analysis(
         .count();
 
     info!(
+        name: "mcp.catalog.analyzed",
         workspace_slug,
         agent_slug,
         source,
@@ -246,6 +319,30 @@ fn log_catalog_analysis(
         catalog_quality_warning_count = warning_count,
         "published agent catalog analyzed"
     );
+
+    CatalogMetrics {
+        tool_count: analysis.budget.tool_count,
+        estimated_context_tokens: analysis.budget.estimated_context_tokens,
+        warning_count,
+    }
+}
+
+fn record_catalog_metrics(metrics: impl Iterator<Item = CatalogMetrics>) {
+    let aggregate = metrics.fold(CatalogMetrics::default(), |mut aggregate, current| {
+        aggregate.tool_count = aggregate.tool_count.saturating_add(current.tool_count);
+        aggregate.estimated_context_tokens = aggregate
+            .estimated_context_tokens
+            .saturating_add(current.estimated_context_tokens);
+        aggregate.warning_count = aggregate
+            .warning_count
+            .saturating_add(current.warning_count);
+        aggregate
+    });
+
+    metrics::gauge!("crank_catalog_tools").set(aggregate.tool_count as f64);
+    metrics::gauge!("crank_catalog_estimated_context_tokens")
+        .set(aggregate.estimated_context_tokens as f64);
+    metrics::gauge!("crank_catalog_warnings").set(aggregate.warning_count as f64);
 }
 
 fn now_unix_ms() -> u64 {

crates/crank-community-mcp/src/lib.rs

@@ -1,13 +1,18 @@
 mod access;
 mod app;
 mod approval_execution;
+mod approval_response;
 pub mod auth;
 pub mod catalog;
 pub mod jsonrpc;
 pub mod manifest;
 mod rate_limit;
+mod request_context;
 pub mod session;
 pub mod tool_error;
+mod tool_search;
 mod transport;
 
-pub use app::{build_app, build_app_with_background_workers};
+pub use app::{
+    build_app, build_app_with_background_workers, build_app_with_background_workers_and_limits,
+};

crates/crank-community-mcp/src/manifest.rs

@@ -1,10 +1,110 @@
 use crank_core::{
-    HttpMethod, OperationSafetyClass, OperationSafetyPolicy, Target, ToolCatalogAnalysis,
-    ToolCatalogAnalysisError, ToolQualityCatalogTool, analyze_tool_catalog,
+    HttpMethod, OperationSafetyClass, OperationSafetyPolicy, SearchableTool, Target,
+    ToolAccessMode, ToolCatalogAnalysis, ToolCatalogAnalysisError, ToolQualityCatalogTool,
+    ToolSelectionPolicy, analyze_tool_catalog,
 };
-use crank_registry::PublishedAgentTool;
+use crank_registry::{PublishedAgentCatalog, PublishedAgentTool};
 use serde_json::{Value, json};
 
+pub const SEARCH_TOOLS_NAME: &str = "search_tools";
+pub const CALL_TOOL_NAME: &str = "call_tool";
+
+pub fn catalog_tool_definitions(catalog: &PublishedAgentCatalog) -> Vec<Value> {
+    match catalog.tool_selection_policy.mode {
+        ToolAccessMode::Direct => catalog.tools.iter().flat_map(tool_definitions).collect(),
+        ToolAccessMode::Search => search_mode_tool_definitions(&catalog.tool_selection_policy),
+    }
+}
+
+pub fn searchable_tools(catalog: &PublishedAgentCatalog) -> Vec<SearchableTool> {
+    catalog
+        .tools
+        .iter()
+        .map(|tool| {
+            let groups = catalog
+                .tool_selection_policy
+                .groups
+                .iter()
+                .filter(|group| group.tool_names.iter().any(|name| name == &tool.tool_name))
+                .collect::<Vec<_>>();
+            SearchableTool {
+                name: tool.tool_name.clone(),
+                title: tool.tool_title.clone(),
+                description: tool.tool_description.clone(),
+                input_schema: tool_definitions(tool)
+                    .into_iter()
+                    .next()
+                    .and_then(|definition| definition.get("inputSchema").cloned())
+                    .unwrap_or_else(|| json!({"type": "object"})),
+                group_ids: groups.iter().map(|group| group.id.clone()).collect(),
+                group_context: groups
+                    .iter()
+                    .map(|group| format!("{} {}", group.name, group.description))
+                    .collect::<Vec<_>>()
+                    .join(" "),
+            }
+        })
+        .collect()
+}
+
+fn search_mode_tool_definitions(policy: &ToolSelectionPolicy) -> Vec<Value> {
+    let group_catalog = if policy.groups.is_empty() {
+        "Разделы каталога не заданы; выполняйте поиск по всему каталогу.".to_owned()
+    } else {
+        policy
+            .groups
+            .iter()
+            .map(|group| format!("{} — {}: {}", group.id, group.name, group.description))
+            .collect::<Vec<_>>()
+            .join("\n")
+    };
+
+    vec![
+        tool_definition(
+            SEARCH_TOOLS_NAME,
+            "Подобрать инструменты",
+            &format!(
+                "Находит подходящие инструменты агента и возвращает их полные входные схемы. Доступные разделы:\n{group_catalog}"
+            ),
+            json!({
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "Краткое описание требуемого действия"
+                    },
+                    "group_ids": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Необязательный список разделов каталога"
+                    },
+                    "max_results": {
+                        "type": "integer",
+                        "minimum": 1,
+                        "maximum": 20,
+                        "default": policy.search.max_results
+                    }
+                },
+                "required": ["query"]
+            }),
+        ),
+        tool_definition(
+            CALL_TOOL_NAME,
+            "Вызвать найденный инструмент",
+            "Вызывает инструмент, ранее найденный через search_tools. Передайте имя, аргументы по полученной схеме и версию каталога из результата поиска.",
+            json!({
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "arguments": {"type": "object"},
+                    "catalog_revision": {"type": "string"}
+                },
+                "required": ["name", "arguments", "catalog_revision"]
+            }),
+        ),
+    ]
+}
+
 pub fn tool_definitions(tool: &PublishedAgentTool) -> Vec<Value> {
     let safety = effective_safety_policy(tool);
     let requires_confirmation = safety.class.requires_confirmation();

crates/crank-community-mcp/src/rate_limit.rs

@@ -4,7 +4,8 @@ use axum::{
     http::{HeaderMap, HeaderValue, StatusCode, header::RETRY_AFTER},
     response::{IntoResponse, Response},
 };
-use crank_runtime::RateLimitRejection;
+use crank_core::PlatformApiKeyKind;
+use crank_runtime::RateLimitCheckError;
 use serde_json::{Value, json};
 
 use crate::{
@@ -14,19 +15,11 @@ use crate::{
     transport::{ResponseMode, session_id_from_headers, transport_response},
 };
 
-pub(super) async fn enforce_post_rate_limit(
-    state: &Arc<AppState>,
-    path: &AgentRoutePath,
-    headers: &HeaderMap,
-) -> Result<(), RateLimitRejection> {
-    enforce_transport_rate_limit(state, path, headers).await
-}
-
 pub(super) async fn enforce_transport_rate_limit(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     headers: &HeaderMap,
-) -> Result<(), RateLimitRejection> {
+) -> Result<(), RateLimitCheckError> {
     let key = rate_limit_key(path, headers);
     state.api_rate_limiter.check(&key).await
 }
@@ -35,8 +28,25 @@ pub(super) fn rate_limited_jsonrpc_response(
     message: &Value,
     response_mode: ResponseMode,
     protocol_version: &str,
-    rejection: RateLimitRejection,
+    error: RateLimitCheckError,
 ) -> Response {
+    let RateLimitCheckError::Rejected(rejection) = error else {
+        return transport_response(
+            StatusCode::SERVICE_UNAVAILABLE,
+            json!({
+                "jsonrpc": "2.0",
+                "id": request_id(message),
+                "error": {
+                    "code": -32603,
+                    "message": "rate limit service unavailable",
+                    "data": { "code": "rate_limit_unavailable" }
+                }
+            }),
+            response_mode,
+            None,
+            Some(protocol_version),
+        );
+    };
     let payload = json!({
         "jsonrpc": "2.0",
         "id": request_id(message),
@@ -61,13 +71,25 @@ pub(super) fn rate_limited_jsonrpc_response(
     response
 }
 
-pub(super) fn rate_limited_status_response(rejection: RateLimitRejection) -> Response {
+pub(super) fn rate_limited_status_response(error: RateLimitCheckError) -> Response {
+    match error {
+        RateLimitCheckError::Rejected(rejection) => {
             let mut response = StatusCode::TOO_MANY_REQUESTS.into_response();
             attach_retry_after_header(&mut response, rejection.retry_after_ms);
             response
+        }
+        RateLimitCheckError::StoreUnavailable => StatusCode::SERVICE_UNAVAILABLE.into_response(),
+    }
 }
 
 fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String {
+    let access_secret = bearer_token(headers);
+    if let Some(secret) = access_secret
+        && secret.starts_with(PlatformApiKeyKind::Approval.secret_marker())
+    {
+        return format!("api_key:{}", hash_access_secret(secret));
+    }
+
     if let Ok(Some(session_id)) = session_id_from_headers(headers) {
         return format!(
             "session:{}:{}:{}",
@@ -75,7 +97,7 @@ fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String {
         );
     }
 
-    if let Some(secret) = bearer_token(headers) {
+    if let Some(secret) = access_secret {
         return format!("api_key:{}", hash_access_secret(secret));
     }
 

crates/crank-community-mcp/src/request_context.rs

@@ -0,0 +1,33 @@
+use axum::{extract::Request, http::HeaderValue, middleware::Next, response::Response};
+use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
+use tracing::{Instrument, info_span};
+
+use crate::transport::HEADER_X_REQUEST_ID;
+
+#[derive(Clone, Debug)]
+pub(super) struct RequestContext {
+    pub(super) request_id: String,
+}
+
+pub(super) async fn apply_request_context(mut request: Request, next: Next) -> Response {
+    let request_id = RequestId::resolve_from_headers(request.headers()).into_string();
+    let context = RequestContext {
+        request_id: request_id.clone(),
+    };
+    let span = info_span!(
+        target: "crank::trace",
+        "mcp.request",
+        request_id = %request_id,
+    );
+    set_remote_trace_parent(&span, request.headers());
+    request.extensions_mut().insert(context);
+
+    with_request_correlation(request_id.clone(), async move {
+        let mut response = next.run(request).instrument(span).await;
+        if let Ok(value) = HeaderValue::from_str(&request_id) {
+            response.headers_mut().insert(HEADER_X_REQUEST_ID, value);
+        }
+        response
+    })
+    .await
+}

crates/crank-community-mcp/src/session.rs

@@ -52,6 +52,8 @@ pub trait TransportSessionStore: Send + Sync {
     ) -> Result<bool, SessionStoreError>;
 
     async fn delete(&self, session_id: &str) -> Result<bool, SessionStoreError>;
+
+    async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError>;
 }
 
 pub type SharedSessionStore = Arc<dyn TransportSessionStore>;
@@ -62,6 +64,11 @@ pub struct PostgresTransportSessionStore {
 }
 
 impl PostgresTransportSessionStore {
+    pub async fn from_pool(pool: PgPool) -> Result<Self, SessionStoreError> {
+        apply_postgres_migrations(&pool).await?;
+        Ok(Self { pool })
+    }
+
     pub async fn connect_with_options_and_pool_config(
         connect_options: PgConnectOptions,
         pool_config: PostgresPoolConfig,
@@ -84,9 +91,7 @@ impl PostgresTransportSessionStore {
                 details: error.to_string(),
             })?;
 
-        apply_postgres_migrations(&pool).await?;
-
-        Ok(Self { pool })
+        Self::from_pool(pool).await
     }
 }
 
@@ -164,6 +169,13 @@ impl TransportSessionStore for InMemorySessionStore {
         let mut guard = self.inner.write().await;
         Ok(guard.remove(session_id).is_some())
     }
+
+    async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError> {
+        let mut guard = self.inner.write().await;
+        let before = guard.len();
+        guard.retain(|_, session| !is_expired(session, now));
+        Ok(u64::try_from(before.saturating_sub(guard.len())).unwrap_or(u64::MAX))
+    }
 }
 
 #[async_trait]
@@ -292,9 +304,68 @@ impl TransportSessionStore for PostgresTransportSessionStore {
 
         Ok(result.rows_affected() > 0)
     }
+
+    async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError> {
+        let result = query(
+            "delete from mcp_transport_sessions
+             where expires_at is not null and expires_at <= $1::timestamptz",
+        )
+        .bind(now)
+        .execute(&self.pool)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+
+        Ok(result.rows_affected())
+    }
 }
 
 async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreError> {
+    let mut transaction = pool.begin().await.map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+    query("select pg_advisory_xact_lock($1)")
+        .bind(0x4352_414E_4B4D_4350_i64)
+        .execute(&mut *transaction)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+    query(
+        "create table if not exists __crank_mcp_migrations (
+            version integer primary key,
+            checksum text not null,
+            applied_at timestamptz not null default now()
+        )",
+    )
+    .execute(&mut *transaction)
+    .await
+    .map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+    let applied = query("select checksum from __crank_mcp_migrations where version = 1")
+        .fetch_optional(&mut *transaction)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+    if let Some(row) = applied {
+        let checksum = row.get::<String, _>("checksum");
+        if checksum != "mcp-transport-sessions-v1" {
+            return Err(SessionStoreError {
+                details: format!("modified MCP migration version 1: {checksum}"),
+            });
+        }
+        transaction
+            .commit()
+            .await
+            .map_err(|error| SessionStoreError {
+                details: error.to_string(),
+            })?;
+        return Ok(());
+    }
+
     query(
         "create table if not exists mcp_transport_sessions (
             id text primary key,
@@ -308,14 +379,14 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
             expires_at timestamptz null
         )",
     )
-    .execute(pool)
+    .execute(&mut *transaction)
     .await
     .map_err(|error| SessionStoreError {
         details: error.to_string(),
     })?;
 
     query("alter table mcp_transport_sessions add column if not exists supports_elicitation boolean not null default false")
-        .execute(pool)
+        .execute(&mut *transaction)
         .await
         .map_err(|error| SessionStoreError {
             details: error.to_string(),
@@ -324,7 +395,7 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
     query(
         "alter table mcp_transport_sessions add column if not exists expires_at timestamptz null",
     )
-    .execute(pool)
+    .execute(&mut *transaction)
     .await
     .map_err(|error| SessionStoreError {
         details: error.to_string(),
@@ -334,7 +405,32 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
         "create index if not exists mcp_transport_sessions_workspace_agent_idx
          on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc)",
     )
-    .execute(pool)
+    .execute(&mut *transaction)
+    .await
+    .map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+
+    query(
+        "create index if not exists mcp_transport_sessions_expires_at_idx
+         on mcp_transport_sessions(expires_at)
+         where expires_at is not null",
+    )
+    .execute(&mut *transaction)
+    .await
+    .map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+
+    query("insert into __crank_mcp_migrations (version, checksum) values (1, $1)")
+        .bind("mcp-transport-sessions-v1")
+        .execute(&mut *transaction)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+    transaction
+        .commit()
         .await
         .map_err(|error| SessionStoreError {
             details: error.to_string(),

crates/crank-community-mcp/src/tool_error.rs

@@ -86,6 +86,10 @@ pub fn runtime_error_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
         RuntimeError::MissingAuthProfile { .. } => "auth_profile_not_found",
         RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
             "secret_not_found"
@@ -146,6 +150,19 @@ fn safe_runtime_error_message(error: &RuntimeError) -> String {
         RuntimeError::ConfirmationStoreUnavailable { .. } => {
             "Хранилище подтверждений временно недоступно.".to_owned()
         }
+        RuntimeError::IdempotencyStoreUnavailable { .. } => {
+            "Хранилище идемпотентности временно недоступно.".to_owned()
+        }
+        RuntimeError::IdempotencyInProgress { .. } => {
+            "Операция с этим ключом идемпотентности уже выполняется.".to_owned()
+        }
+        RuntimeError::IdempotencyConflict { .. } => {
+            "Ключ идемпотентности уже использован с другими параметрами.".to_owned()
+        }
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => {
+            "Результат предыдущего выполнения неизвестен; автоматический повтор заблокирован."
+                .to_owned()
+        }
         RuntimeError::MissingAuthProfile { .. } => "Профиль авторизации не найден.".to_owned(),
         RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
             "Секрет авторизации не найден.".to_owned()
@@ -169,6 +186,8 @@ fn is_recoverable_runtime_error(error: &RuntimeError) -> bool {
             | RuntimeError::ConcurrencyLimitExceeded { .. }
             | RuntimeError::SecretCrypto { .. }
             | RuntimeError::ConfirmationRequired { .. }
+            | RuntimeError::IdempotencyStoreUnavailable { .. }
+            | RuntimeError::IdempotencyInProgress { .. }
     )
 }
 
@@ -198,6 +217,14 @@ fn suggested_action(error: &RuntimeError) -> Option<&'static str> {
             Some("Запросите новый токен подтверждения.")
         }
         RuntimeError::ConfirmationStoreUnavailable { .. } => Some("Повторите запрос позже."),
+        RuntimeError::IdempotencyStoreUnavailable { .. }
+        | RuntimeError::IdempotencyInProgress { .. } => Some("Повторите запрос позже."),
+        RuntimeError::IdempotencyConflict { .. } => {
+            Some("Используйте новый ключ идемпотентности для изменённого запроса.")
+        }
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => {
+            Some("Проверьте результат во внешней системе перед ручным повтором.")
+        }
         RuntimeError::MissingAuthProfile { .. }
         | RuntimeError::MissingSecret { .. }
         | RuntimeError::MissingSecretVersion { .. }

crates/crank-community-mcp/src/tool_search.rs

@@ -0,0 +1,289 @@
+use std::{collections::BTreeSet, sync::Arc};
+
+use axum::{http::StatusCode, response::Response};
+use crank_core::{ToolAccessMode, search_tool_catalog};
+use crank_registry::PublishedAgentCatalog;
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+use crate::{
+    app::{
+        AppState, handle_tool_call, resolve_generated_tool, take_confirmation_token,
+        tool_error_response,
+    },
+    auth::VerifiedMachineCredential,
+    jsonrpc::{jsonrpc_error, jsonrpc_result, request_id},
+    manifest::{CALL_TOOL_NAME, SEARCH_TOOLS_NAME, searchable_tools},
+    session::SessionState,
+    tool_error::generic_tool_error_contract,
+    transport::{ResponseMode, transport_response},
+};
+
+#[derive(Debug, Deserialize)]
+struct SearchToolsArguments {
+    query: String,
+    #[serde(default)]
+    group_ids: Vec<String>,
+    max_results: Option<usize>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ProxyToolCallArguments {
+    name: String,
+    #[serde(default)]
+    arguments: Value,
+    catalog_revision: String,
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(super) async fn handle_catalog_tool_call(
+    state: Arc<AppState>,
+    session: &SessionState,
+    message: &Value,
+    response_mode: ResponseMode,
+    credential: &VerifiedMachineCredential,
+    catalog: &PublishedAgentCatalog,
+    tool_name: &str,
+    arguments: Value,
+    transport_request_id: &str,
+) -> Response {
+    match catalog.tool_selection_policy.mode {
+        ToolAccessMode::Direct => {
+            execute_catalog_tool(
+                state,
+                session,
+                message,
+                response_mode,
+                credential,
+                catalog,
+                tool_name,
+                arguments,
+                transport_request_id,
+            )
+            .await
+        }
+        ToolAccessMode::Search if tool_name == SEARCH_TOOLS_NAME => {
+            handle_search_tools(message, response_mode, session, catalog, arguments)
+        }
+        ToolAccessMode::Search if tool_name == CALL_TOOL_NAME => {
+            let proxy: ProxyToolCallArguments = match serde_json::from_value(arguments) {
+                Ok(proxy) => proxy,
+                Err(error) => {
+                    return invalid_arguments_response(
+                        message,
+                        response_mode,
+                        &session.protocol_version,
+                        error.to_string(),
+                    );
+                }
+            };
+            if proxy.catalog_revision != catalog_revision(catalog) {
+                return tool_error_response(
+                    message,
+                    response_mode,
+                    &session.protocol_version,
+                    generic_tool_error_contract(
+                        "catalog_revision_changed",
+                        format!(
+                            "catalog revision {} is no longer current",
+                            proxy.catalog_revision
+                        ),
+                        transport_request_id,
+                        true,
+                        Some(
+                            "Повторите search_tools и вызовите инструмент с новой версией каталога.",
+                        ),
+                    ),
+                );
+            }
+            execute_catalog_tool(
+                state,
+                session,
+                message,
+                response_mode,
+                credential,
+                catalog,
+                &proxy.name,
+                proxy.arguments,
+                transport_request_id,
+            )
+            .await
+        }
+        ToolAccessMode::Search => {
+            tool_not_found_response(message, response_mode, &session.protocol_version, tool_name)
+        }
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn execute_catalog_tool(
+    state: Arc<AppState>,
+    session: &SessionState,
+    message: &Value,
+    response_mode: ResponseMode,
+    credential: &VerifiedMachineCredential,
+    catalog: &PublishedAgentCatalog,
+    tool_name: &str,
+    mut arguments: Value,
+    transport_request_id: &str,
+) -> Response {
+    let resolve_span = Stage::McpToolsResolve.span();
+    let resolved = resolve_span.in_scope(|| resolve_generated_tool(&catalog.tools, tool_name));
+    let resolved = match resolved {
+        Some(resolved) => {
+            StageOutcome::Success.record(&resolve_span);
+            drop(resolve_span);
+            resolved
+        }
+        None => {
+            StageOutcome::Error.record(&resolve_span);
+            ErrorCategory::Catalog.record(&resolve_span);
+            drop(resolve_span);
+            return tool_not_found_response(
+                message,
+                response_mode,
+                &session.protocol_version,
+                tool_name,
+            );
+        }
+    };
+    let confirmation_token = take_confirmation_token(&mut arguments);
+    handle_tool_call(
+        state,
+        session,
+        message,
+        response_mode,
+        credential,
+        resolved,
+        arguments,
+        confirmation_token,
+        transport_request_id,
+    )
+    .await
+}
+
+fn handle_search_tools(
+    message: &Value,
+    response_mode: ResponseMode,
+    session: &SessionState,
+    catalog: &PublishedAgentCatalog,
+    arguments: Value,
+) -> Response {
+    let search: SearchToolsArguments = match serde_json::from_value(arguments) {
+        Ok(search) => search,
+        Err(error) => {
+            return invalid_arguments_response(
+                message,
+                response_mode,
+                &session.protocol_version,
+                error.to_string(),
+            );
+        }
+    };
+    if search.query.trim().is_empty() {
+        return invalid_arguments_response(
+            message,
+            response_mode,
+            &session.protocol_version,
+            "query must not be empty".to_owned(),
+        );
+    }
+    let known_group_ids = catalog
+        .tool_selection_policy
+        .groups
+        .iter()
+        .map(|group| group.id.as_str())
+        .collect::<BTreeSet<_>>();
+    if let Some(group_id) = search
+        .group_ids
+        .iter()
+        .find(|group_id| !known_group_ids.contains(group_id.as_str()))
+    {
+        return invalid_arguments_response(
+            message,
+            response_mode,
+            &session.protocol_version,
+            format!("unknown tool group {group_id}"),
+        );
+    }
+
+    let configured_limit = catalog.tool_selection_policy.search.max_results;
+    let requested_limit = search.max_results.unwrap_or(configured_limit).clamp(1, 20);
+    let tools = search_tool_catalog(
+        &searchable_tools(catalog),
+        &search.query,
+        &search.group_ids,
+        requested_limit.min(configured_limit),
+    )
+    .into_iter()
+    .map(|found| {
+        json!({
+            "name": found.tool.name,
+            "title": found.tool.title,
+            "description": found.tool.description,
+            "inputSchema": found.tool.input_schema,
+            "group_ids": found.tool.group_ids,
+            "score": found.score,
+        })
+    })
+    .collect::<Vec<_>>();
+    let result = json!({
+        "catalog_revision": catalog_revision(catalog),
+        "tools": tools,
+    });
+    let text = serde_json::to_string_pretty(&result).unwrap_or_else(|_| result.to_string());
+
+    transport_response(
+        StatusCode::OK,
+        jsonrpc_result(
+            request_id(message),
+            json!({
+                "content": [{"type": "text", "text": text}],
+                "structuredContent": result,
+                "isError": false
+            }),
+        ),
+        response_mode,
+        None,
+        Some(&session.protocol_version),
+    )
+}
+
+fn catalog_revision(catalog: &PublishedAgentCatalog) -> String {
+    format!("agent-version-{}", catalog.agent_version)
+}
+
+fn invalid_arguments_response(
+    message: &Value,
+    response_mode: ResponseMode,
+    protocol_version: &str,
+    detail: String,
+) -> Response {
+    transport_response(
+        StatusCode::OK,
+        jsonrpc_error(request_id(message), -32602, detail),
+        response_mode,
+        None,
+        Some(protocol_version),
+    )
+}
+
+fn tool_not_found_response(
+    message: &Value,
+    response_mode: ResponseMode,
+    protocol_version: &str,
+    tool_name: &str,
+) -> Response {
+    transport_response(
+        StatusCode::OK,
+        jsonrpc_error(
+            request_id(message),
+            -32602,
+            format!("tool {tool_name} was not found"),
+        ),
+        response_mode,
+        None,
+        Some(protocol_version),
+    )
+}

crates/crank-community-mcp/src/transport.rs

@@ -22,7 +22,6 @@ use crate::jsonrpc::{
 pub(super) const HEADER_MCP_SESSION_ID: &str = "MCP-Session-Id";
 pub(super) const HEADER_MCP_PROTOCOL_VERSION: &str = "MCP-Protocol-Version";
 pub(super) const HEADER_X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id");
-const MAX_REQUEST_ID_LEN: usize = 128;
 
 #[derive(Clone, Copy)]
 pub(super) enum ResponseMode {
@@ -303,24 +302,6 @@ where
     response
 }
 
-pub(super) fn resolve_request_id(headers: &HeaderMap) -> String {
-    headers
-        .get(&HEADER_X_REQUEST_ID)
-        .and_then(|value| value.to_str().ok())
-        .map(str::trim)
-        .filter(|value| is_valid_request_id(value))
-        .map(ToOwned::to_owned)
-        .unwrap_or_else(|| uuid::Uuid::now_v7().to_string())
-}
-
-pub(super) fn is_valid_request_id(value: &str) -> bool {
-    !value.is_empty()
-        && value.len() <= MAX_REQUEST_ID_LEN
-        && value
-            .bytes()
-            .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
-}
-
 pub(super) fn extract_origin(url: &str) -> Option<String> {
     Some(parse_origin(url, false)?.origin().ascii_serialization())
 }

crates/crank-community-mcp/tests/integration/session.rs

@@ -97,3 +97,41 @@ async fn postgres_transport_sessions_evict_expired_rows_on_read() {
 
     assert_eq!(remaining, 0);
 }
+
+#[tokio::test]
+async fn postgres_transport_session_cleanup_removes_abandoned_expired_rows() {
+    let database_url = crank_test_support::postgres_schema_url("test_mcp_cleanup").await;
+    let store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
+        database_url.parse::<PgConnectOptions>().unwrap(),
+        PostgresPoolConfig::default(),
+    )
+    .await
+    .unwrap();
+    let now = OffsetDateTime::now_utc();
+
+    store
+        .create(
+            "2025-11-25",
+            "default",
+            "sales",
+            false,
+            now - time::Duration::hours(2),
+            Some(now - time::Duration::hours(1)),
+        )
+        .await
+        .unwrap();
+    let active = store
+        .create(
+            "2025-11-25",
+            "default",
+            "sales",
+            false,
+            now,
+            Some(now + time::Duration::hours(1)),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(store.cleanup_expired(now).await.unwrap(), 1);
+    assert!(store.get(&active).await.unwrap().is_some());
+}

crates/crank-community-mcp/tests/unit/session.rs

@@ -72,6 +72,31 @@ async fn drops_expired_in_memory_transport_sessions_on_read() {
     assert!(store.get(&session_id).await.unwrap().is_none());
 }
 
+#[tokio::test]
+async fn cleanup_removes_only_expired_sessions() {
+    let store = InMemorySessionStore::default();
+    let now = time::OffsetDateTime::now_utc();
+    let expired = store
+        .create("2025-11-25", "default", "sales", false, now, Some(now))
+        .await
+        .unwrap();
+    let active = store
+        .create(
+            "2025-11-25",
+            "default",
+            "sales",
+            false,
+            now,
+            Some(now + time::Duration::hours(1)),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(store.cleanup_expired(now).await.unwrap(), 1);
+    assert!(store.get(&expired).await.unwrap().is_none());
+    assert!(store.get(&active).await.unwrap().is_some());
+}
+
 #[test]
 fn formats_transport_session_store_error() {
     let error = SessionStoreError {

crates/crank-core/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-core"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-core/src/access.rs

@@ -42,6 +42,15 @@ pub enum PlatformApiKeyKind {
     Approval,
 }
 
+impl PlatformApiKeyKind {
+    pub const fn secret_marker(self) -> &'static str {
+        match self {
+            Self::McpClient => "crk_",
+            Self::Approval => "crk_appr_",
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum PlatformApiKeyScope {
@@ -123,6 +132,12 @@ mod tests {
     };
     use crate::ids::{AgentId, PlatformApiKeyId, UserId, WorkspaceId};
 
+    #[test]
+    fn api_key_kinds_own_their_secret_markers() {
+        assert_eq!(PlatformApiKeyKind::McpClient.secret_marker(), "crk_");
+        assert_eq!(PlatformApiKeyKind::Approval.secret_marker(), "crk_appr_");
+    }
+
     #[test]
     fn user_serializes_created_at_as_rfc3339() {
         let user = User {

crates/crank-core/src/agent.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
+use thiserror::Error;
 use time::OffsetDateTime;
 
 use crate::ids::{AgentId, OperationId, WorkspaceId};
@@ -36,11 +37,149 @@ pub struct AgentVersion {
     pub version: u32,
     pub status: AgentStatus,
     pub instructions: Value,
-    pub tool_selection_policy: Value,
+    #[serde(default)]
+    pub tool_selection_policy: ToolSelectionPolicy,
     #[serde(with = "time::serde::rfc3339")]
     pub created_at: OffsetDateTime,
 }
 
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ToolAccessMode {
+    #[default]
+    #[serde(alias = "allow_list")]
+    Direct,
+    Search,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ToolGroup {
+    pub id: String,
+    pub name: String,
+    pub description: String,
+    #[serde(default)]
+    pub tool_names: Vec<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ToolSearchSettings {
+    #[serde(default = "default_search_result_limit")]
+    pub max_results: usize,
+}
+
+impl Default for ToolSearchSettings {
+    fn default() -> Self {
+        Self {
+            max_results: default_search_result_limit(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ToolSelectionPolicy {
+    #[serde(default)]
+    pub mode: ToolAccessMode,
+    #[serde(default)]
+    pub groups: Vec<ToolGroup>,
+    #[serde(default)]
+    pub search: ToolSearchSettings,
+}
+
+#[derive(Clone, Debug, Error, PartialEq, Eq)]
+pub enum ToolSelectionPolicyError {
+    #[error("search result limit must be between 1 and 20")]
+    InvalidSearchResultLimit,
+    #[error("tool groups are only allowed in search mode")]
+    GroupsRequireSearchMode,
+    #[error("tool group id {group_id} is invalid")]
+    InvalidGroupId { group_id: String },
+    #[error("tool group id {group_id} is duplicated")]
+    DuplicateGroupId { group_id: String },
+    #[error("tool group {group_id} must have a name and description")]
+    IncompleteGroup { group_id: String },
+    #[error("tool group {group_id} references unknown tool {tool_name}")]
+    UnknownTool { group_id: String, tool_name: String },
+    #[error("tool group {group_id} contains duplicate tool {tool_name}")]
+    DuplicateTool { group_id: String, tool_name: String },
+    #[error("tool name {tool_name} is reserved by search mode")]
+    ReservedToolName { tool_name: String },
+}
+
+impl ToolSelectionPolicy {
+    pub fn validate_for_tools<'a>(
+        &self,
+        tool_names: impl IntoIterator<Item = &'a str>,
+    ) -> Result<(), ToolSelectionPolicyError> {
+        use std::collections::BTreeSet;
+
+        if !(1..=20).contains(&self.search.max_results) {
+            return Err(ToolSelectionPolicyError::InvalidSearchResultLimit);
+        }
+        if self.mode == ToolAccessMode::Direct && !self.groups.is_empty() {
+            return Err(ToolSelectionPolicyError::GroupsRequireSearchMode);
+        }
+
+        let known_tools = tool_names.into_iter().collect::<BTreeSet<_>>();
+        if self.mode == ToolAccessMode::Search
+            && let Some(tool_name) = known_tools
+                .iter()
+                .find(|tool_name| matches!(**tool_name, "search_tools" | "call_tool"))
+        {
+            return Err(ToolSelectionPolicyError::ReservedToolName {
+                tool_name: (*tool_name).to_owned(),
+            });
+        }
+        let mut group_ids = BTreeSet::new();
+        for group in &self.groups {
+            if !valid_group_id(&group.id) {
+                return Err(ToolSelectionPolicyError::InvalidGroupId {
+                    group_id: group.id.clone(),
+                });
+            }
+            if !group_ids.insert(group.id.as_str()) {
+                return Err(ToolSelectionPolicyError::DuplicateGroupId {
+                    group_id: group.id.clone(),
+                });
+            }
+            if group.name.trim().is_empty() || group.description.trim().is_empty() {
+                return Err(ToolSelectionPolicyError::IncompleteGroup {
+                    group_id: group.id.clone(),
+                });
+            }
+
+            let mut grouped_tools = BTreeSet::new();
+            for tool_name in &group.tool_names {
+                if !known_tools.contains(tool_name.as_str()) {
+                    return Err(ToolSelectionPolicyError::UnknownTool {
+                        group_id: group.id.clone(),
+                        tool_name: tool_name.clone(),
+                    });
+                }
+                if !grouped_tools.insert(tool_name.as_str()) {
+                    return Err(ToolSelectionPolicyError::DuplicateTool {
+                        group_id: group.id.clone(),
+                        tool_name: tool_name.clone(),
+                    });
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+const fn default_search_result_limit() -> usize {
+    8
+}
+
+fn valid_group_id(value: &str) -> bool {
+    !value.is_empty()
+        && value.len() <= 64
+        && value
+            .bytes()
+            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct AgentOperationBinding {
     pub operation_id: OperationId,
@@ -56,7 +195,10 @@ mod tests {
     use serde_json::json;
     use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 
-    use super::{Agent, AgentStatus, AgentVersion};
+    use super::{
+        Agent, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup, ToolSelectionPolicy,
+        ToolSelectionPolicyError,
+    };
     use crate::ids::{AgentId, WorkspaceId};
 
     #[test]
@@ -98,5 +240,28 @@ mod tests {
             version.created_at,
             OffsetDateTime::parse("2026-03-25T12:00:00Z", &Rfc3339).unwrap()
         );
+        assert_eq!(version.tool_selection_policy.mode, ToolAccessMode::Direct);
+    }
+
+    #[test]
+    fn search_policy_validates_groups_against_published_tools() {
+        let policy = ToolSelectionPolicy {
+            mode: ToolAccessMode::Search,
+            groups: vec![ToolGroup {
+                id: "finance".to_owned(),
+                name: "Finance".to_owned(),
+                description: "Invoices and payments".to_owned(),
+                tool_names: vec!["create_invoice".to_owned()],
+            }],
+            ..ToolSelectionPolicy::default()
+        };
+
+        assert_eq!(
+            policy.validate_for_tools(["list_invoices"]),
+            Err(ToolSelectionPolicyError::UnknownTool {
+                group_id: "finance".to_owned(),
+                tool_name: "create_invoice".to_owned(),
+            })
+        );
     }
 }

crates/crank-core/src/cache.rs

@@ -74,6 +74,12 @@ pub struct RateLimitBucketState {
     pub last_refill_unix_ms: i64,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RateLimitDecision {
+    Allowed,
+    Rejected { retry_after_ms: u64 },
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum ReplayGuardStatus {
@@ -86,6 +92,12 @@ pub struct CoordinationStateValue {
     pub payload: Value,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum CoordinationStateReservation {
+    Reserved,
+    Existing(CoordinationStateValue),
+}
+
 #[async_trait]
 pub trait ResponseCacheStore: Send + Sync {
     async fn get(&self, key: &str) -> Result<Option<CachedResponse>, CacheStoreError>;
@@ -108,6 +120,14 @@ pub trait RateLimitStateStore: Send + Sync {
         ttl: Duration,
     ) -> Result<(), CacheStoreError>;
     async fn delete_bucket(&self, key: &str) -> Result<(), CacheStoreError>;
+    async fn consume_token(
+        &self,
+        key: &str,
+        burst_tokens_micros: u64,
+        refill_per_second_micros: u64,
+        now_unix_ms: i64,
+        ttl: Duration,
+    ) -> Result<RateLimitDecision, CacheStoreError>;
 }
 
 #[async_trait]
@@ -135,6 +155,26 @@ pub trait CoordinationStateStore: Send + Sync {
         ttl: Duration,
     ) -> Result<(), CacheStoreError>;
     async fn delete_value(&self, scope: CacheScope, key: &str) -> Result<(), CacheStoreError>;
+    async fn take_value(
+        &self,
+        scope: CacheScope,
+        key: &str,
+    ) -> Result<Option<CoordinationStateValue>, CacheStoreError>;
+    async fn reserve_value(
+        &self,
+        scope: CacheScope,
+        key: &str,
+        value: CoordinationStateValue,
+        ttl: Duration,
+    ) -> Result<CoordinationStateReservation, CacheStoreError>;
+    async fn compare_and_set_value(
+        &self,
+        scope: CacheScope,
+        key: &str,
+        expected: &CoordinationStateValue,
+        value: CoordinationStateValue,
+        ttl: Duration,
+    ) -> Result<bool, CacheStoreError>;
 }
 
 #[derive(Debug, Error, PartialEq, Eq)]

crates/crank-core/src/lib.rs

@@ -12,6 +12,7 @@ pub mod protocol;
 pub mod secret;
 pub mod tool_catalog;
 pub mod tool_quality;
+pub mod tool_search;
 pub mod workspace;
 
 pub mod domain {
@@ -19,7 +20,10 @@ pub mod domain {
         InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
         PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
     };
-    pub use crate::agent::{Agent, AgentOperationBinding, AgentStatus, AgentVersion};
+    pub use crate::agent::{
+        Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
+        ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
+    };
     pub use crate::approval::{ApprovalRequest, ApprovalRequestStatus};
     pub use crate::auth::{
         ApiKeyHeaderAuthConfig, ApiKeyQueryAuthConfig, AuthConfig, AuthProfile, BasicAuthConfig,
@@ -56,13 +60,15 @@ pub mod domain {
         ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
         ToolQualityReport, ToolQualitySchemaKind, ToolQualitySchemaNode, ToolQualitySeverity,
     };
+    pub use crate::tool_search::{SearchableTool, ToolSearchMatch, search_tool_catalog};
     pub use crate::workspace::{Workspace, WorkspaceStatus};
 }
 
 pub mod ports {
     pub use crate::cache::{
-        CacheStoreError, CoordinationStateStore, CoordinationStateValue, RateLimitStateStore,
-        ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
+        CacheStoreError, CoordinationStateReservation, CoordinationStateStore,
+        CoordinationStateValue, RateLimitDecision, RateLimitStateStore, ReplayGuardStatus,
+        ReplayGuardStore, ResponseCacheStore,
     };
     pub use crate::ext::access::{
         OwnerOnlyPolicyEngine, PolicyAction, PolicyDecision, PolicyEngine, PolicyScope,
@@ -92,7 +98,10 @@ pub use access::{
     InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
     PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
 };
-pub use agent::{Agent, AgentOperationBinding, AgentStatus, AgentVersion};
+pub use agent::{
+    Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
+    ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
+};
 pub use approval::{ApprovalRequest, ApprovalRequestStatus};
 pub use auth::{
     ApiKeyHeaderAuthConfig, ApiKeyQueryAuthConfig, AuthConfig, AuthProfile, BasicAuthConfig,
@@ -100,8 +109,9 @@ pub use auth::{
 };
 pub use cache::{
     CacheBackend, CacheScope, CacheStoreError, CachedHeader, CachedResponse,
-    CoordinationStateStore, CoordinationStateValue, ParseCacheBackendError, RateLimitBucketState,
-    RateLimitStateStore, ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
+    CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue,
+    ParseCacheBackendError, RateLimitBucketState, RateLimitDecision, RateLimitStateStore,
+    ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
 };
 pub use edition::{
     EditionCapabilities, EditionLimits, MachineAccessMode, OperationSecurityLevel, ProductEdition,
@@ -153,4 +163,5 @@ pub use tool_quality::{
     analyze_agent_tool_catalog_quality, analyze_tool_identity_quality,
     analyze_tool_response_projection_quality, analyze_tool_schema_quality,
 };
+pub use tool_search::{SearchableTool, ToolSearchMatch, search_tool_catalog};
 pub use workspace::{Workspace, WorkspaceStatus};

crates/crank-core/src/tool_search.rs

@@ -0,0 +1,237 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct SearchableTool {
+    pub name: String,
+    pub title: String,
+    pub description: String,
+    pub input_schema: Value,
+    #[serde(default)]
+    pub group_ids: Vec<String>,
+    #[serde(default)]
+    pub group_context: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ToolSearchMatch {
+    pub tool: SearchableTool,
+    pub score: f64,
+}
+
+pub fn search_tool_catalog(
+    tools: &[SearchableTool],
+    query: &str,
+    group_ids: &[String],
+    max_results: usize,
+) -> Vec<ToolSearchMatch> {
+    let query_tokens = tokenize(query);
+    if query_tokens.is_empty() || max_results == 0 {
+        return Vec::new();
+    }
+
+    let requested_groups = group_ids
+        .iter()
+        .map(String::as_str)
+        .collect::<BTreeSet<_>>();
+    let candidates = tools
+        .iter()
+        .filter(|tool| {
+            requested_groups.is_empty()
+                || tool
+                    .group_ids
+                    .iter()
+                    .any(|group_id| requested_groups.contains(group_id.as_str()))
+        })
+        .collect::<Vec<_>>();
+    if candidates.is_empty() {
+        return Vec::new();
+    }
+
+    let documents = candidates
+        .iter()
+        .map(|tool| weighted_tokens(tool))
+        .collect::<Vec<_>>();
+    let average_length =
+        documents.iter().map(Vec::len).sum::<usize>() as f64 / documents.len() as f64;
+    let candidate_count = documents.len();
+    let document_frequency = document_frequency(&documents, &query_tokens);
+    let normalized_query = query.trim().to_lowercase();
+
+    let mut matches = candidates
+        .into_iter()
+        .zip(documents)
+        .filter_map(|(tool, document)| {
+            let score = bm25_score(
+                &document,
+                &query_tokens,
+                &document_frequency,
+                candidate_count,
+                average_length,
+            ) + exact_match_bonus(tool, &normalized_query);
+            (score > 0.0).then(|| ToolSearchMatch {
+                tool: tool.clone(),
+                score,
+            })
+        })
+        .collect::<Vec<_>>();
+
+    matches.sort_by(|left, right| {
+        right
+            .score
+            .total_cmp(&left.score)
+            .then_with(|| left.tool.name.cmp(&right.tool.name))
+    });
+    matches.truncate(max_results);
+    matches
+}
+
+fn weighted_tokens(tool: &SearchableTool) -> Vec<String> {
+    let mut tokens = Vec::new();
+    for _ in 0..3 {
+        tokens.extend(tokenize(&tool.name));
+        tokens.extend(tokenize(&tool.title));
+    }
+    tokens.extend(tokenize(&tool.description));
+    for _ in 0..2 {
+        tokens.extend(tokenize(&tool.group_context));
+    }
+    tokens
+}
+
+fn document_frequency(
+    documents: &[Vec<String>],
+    query_tokens: &[String],
+) -> BTreeMap<String, usize> {
+    let mut frequencies = BTreeMap::new();
+    for query_token in query_tokens {
+        let count = documents
+            .iter()
+            .filter(|document| document.iter().any(|token| token == query_token))
+            .count();
+        frequencies.insert(query_token.clone(), count);
+    }
+    frequencies
+}
+
+fn bm25_score(
+    document: &[String],
+    query_tokens: &[String],
+    document_frequency: &BTreeMap<String, usize>,
+    document_count: usize,
+    average_length: f64,
+) -> f64 {
+    const K1: f64 = 1.2;
+    const B: f64 = 0.75;
+
+    query_tokens.iter().fold(0.0, |score, token| {
+        let term_frequency = document.iter().filter(|term| *term == token).count() as f64;
+        if term_frequency == 0.0 {
+            return score;
+        }
+        let frequency = document_frequency.get(token).copied().unwrap_or_default() as f64;
+        let inverse_document_frequency =
+            ((document_count as f64 - frequency + 0.5) / (frequency + 0.5) + 1.0).ln();
+        let length_ratio = if average_length > 0.0 {
+            document.len() as f64 / average_length
+        } else {
+            1.0
+        };
+        let normalized_frequency =
+            term_frequency * (K1 + 1.0) / (term_frequency + K1 * (1.0 - B + B * length_ratio));
+        score + inverse_document_frequency * normalized_frequency
+    })
+}
+
+fn exact_match_bonus(tool: &SearchableTool, query: &str) -> f64 {
+    if query.is_empty() {
+        return 0.0;
+    }
+    let name = tool.name.to_lowercase();
+    let title = tool.title.to_lowercase();
+    if name == query || title == query {
+        8.0
+    } else if name.contains(query) || title.contains(query) {
+        3.0
+    } else {
+        0.0
+    }
+}
+
+fn tokenize(value: &str) -> Vec<String> {
+    value
+        .to_lowercase()
+        .split(|character: char| !character.is_alphanumeric())
+        .filter(|token| token.chars().count() >= 2)
+        .map(ToOwned::to_owned)
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+
+    use super::{SearchableTool, search_tool_catalog};
+
+    fn tool(name: &str, title: &str, description: &str, group_ids: &[&str]) -> SearchableTool {
+        SearchableTool {
+            name: name.to_owned(),
+            title: title.to_owned(),
+            description: description.to_owned(),
+            input_schema: json!({"type": "object"}),
+            group_ids: group_ids.iter().map(|value| (*value).to_owned()).collect(),
+            group_context: if group_ids.contains(&"finance") {
+                "Finance invoices and payments".to_owned()
+            } else {
+                "Customer support tickets".to_owned()
+            },
+        }
+    }
+
+    #[test]
+    fn ranks_title_and_name_matches_above_description_matches() {
+        let tools = vec![
+            tool(
+                "create_invoice",
+                "Create invoice",
+                "Issue a bill",
+                &["finance"],
+            ),
+            tool(
+                "list_customers",
+                "List customers",
+                "Customers with invoices",
+                &["support"],
+            ),
+        ];
+
+        let matches = search_tool_catalog(&tools, "create invoice", &[], 8);
+
+        assert_eq!(matches[0].tool.name, "create_invoice");
+    }
+
+    #[test]
+    fn filters_candidates_by_group() {
+        let tools = vec![
+            tool(
+                "create_invoice",
+                "Create invoice",
+                "Issue a bill",
+                &["finance"],
+            ),
+            tool(
+                "create_ticket",
+                "Create ticket",
+                "Open a support case",
+                &["support"],
+            ),
+        ];
+
+        let matches = search_tool_catalog(&tools, "create", &["support".to_owned()], 8);
+
+        assert_eq!(matches.len(), 1);
+        assert_eq!(matches[0].tool.name, "create_ticket");
+    }
+}

crates/crank-import/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-import"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-mapping/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-mapping"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-observability/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "crank-observability"
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+publish.workspace = true
+version.workspace = true
+
+[dependencies]
+axum.workspace = true
+metrics.workspace = true
+metrics-exporter-prometheus.workspace = true
+opentelemetry.workspace = true
+opentelemetry-otlp.workspace = true
+opentelemetry_sdk.workspace = true
+percent-encoding.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sentry.workspace = true
+sha2.workspace = true
+subtle.workspace = true
+thiserror.workspace = true
+time.workspace = true
+tokio = { workspace = true, features = ["net"] }
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true
+url.workspace = true
+uuid.workspace = true
+
+[dev-dependencies]
+opentelemetry-proto.workspace = true
+prost.workspace = true
+sentry = { workspace = true, features = ["test"] }
+tower.workspace = true

crates/crank-observability/src/config.rs

@@ -0,0 +1,170 @@
+use std::env;
+
+use thiserror::Error;
+
+use crate::RedactionLimits;
+
+const DEFAULT_ENVIRONMENT: &str = "development";
+const MAX_IDENTITY_LABEL_BYTES: usize = 64;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ServiceIdentity {
+    service: String,
+    version: String,
+    environment: String,
+}
+
+impl ServiceIdentity {
+    pub fn try_new(
+        service: impl Into<String>,
+        version: impl Into<String>,
+        environment: impl Into<String>,
+    ) -> Result<Self, ObservabilityConfigError> {
+        let identity = Self {
+            service: service.into(),
+            version: version.into(),
+            environment: environment.into(),
+        };
+        validate_label("service", &identity.service)?;
+        validate_label("version", &identity.version)?;
+        validate_label("environment", &identity.environment)?;
+        Ok(identity)
+    }
+
+    pub fn service(&self) -> &str {
+        &self.service
+    }
+
+    pub fn version(&self) -> &str {
+        &self.version
+    }
+
+    pub fn environment(&self) -> &str {
+        &self.environment
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct ObservabilityConfig {
+    identity: ServiceIdentity,
+    filter: String,
+    redaction_limits: RedactionLimits,
+}
+
+impl ObservabilityConfig {
+    pub fn new(
+        identity: ServiceIdentity,
+        filter: impl Into<String>,
+        redaction_limits: RedactionLimits,
+    ) -> Self {
+        Self {
+            identity,
+            filter: filter.into(),
+            redaction_limits,
+        }
+    }
+
+    pub fn from_env(
+        service: &'static str,
+        version: &'static str,
+        default_filter: &'static str,
+    ) -> Result<Self, ObservabilityConfigError> {
+        let environment = env_value_or_default(
+            "CRANK_ENVIRONMENT",
+            env::var("CRANK_ENVIRONMENT"),
+            DEFAULT_ENVIRONMENT,
+        )?;
+        let filter = env_value_or_default(
+            "CRANK_LOG_LEVEL",
+            env::var("CRANK_LOG_LEVEL"),
+            default_filter,
+        )?;
+        let identity = ServiceIdentity::try_new(service, version, environment)?;
+
+        Ok(Self::new(identity, filter, RedactionLimits::default()))
+    }
+
+    pub(crate) fn into_parts(self) -> (ServiceIdentity, String, RedactionLimits) {
+        (self.identity, self.filter, self.redaction_limits)
+    }
+
+    pub(crate) fn identity(&self) -> &ServiceIdentity {
+        &self.identity
+    }
+
+    pub(crate) fn redaction_limits(&self) -> RedactionLimits {
+        self.redaction_limits
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum ObservabilityConfigError {
+    #[error("invalid observability identity field: {field}")]
+    InvalidIdentity { field: &'static str },
+    #[error("observability environment variable is not valid UTF-8: {field}")]
+    InvalidEnvironmentEncoding { field: &'static str },
+}
+
+fn env_value_or_default(
+    field: &'static str,
+    value: Result<String, env::VarError>,
+    default: &'static str,
+) -> Result<String, ObservabilityConfigError> {
+    match value {
+        Ok(value) => Ok(value),
+        Err(env::VarError::NotPresent) => Ok(default.to_owned()),
+        Err(env::VarError::NotUnicode(_)) => {
+            Err(ObservabilityConfigError::InvalidEnvironmentEncoding { field })
+        }
+    }
+}
+
+fn validate_label(field: &'static str, value: &str) -> Result<(), ObservabilityConfigError> {
+    let valid = !value.is_empty()
+        && value.len() <= MAX_IDENTITY_LABEL_BYTES
+        && value
+            .bytes()
+            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+'));
+
+    if valid {
+        Ok(())
+    } else {
+        Err(ObservabilityConfigError::InvalidIdentity { field })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::ffi::OsString;
+
+    use super::{ObservabilityConfigError, ServiceIdentity, env_value_or_default};
+
+    #[test]
+    fn accepts_release_and_environment_labels() {
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1+build.7", "production")
+            .expect("identity must be valid");
+
+        assert_eq!(identity.service(), "admin-api");
+        assert_eq!(identity.version(), "0.3.1+build.7");
+        assert_eq!(identity.environment(), "production");
+    }
+
+    #[test]
+    fn rejects_non_utf8_environment_values() {
+        let error = env_value_or_default(
+            "CRANK_ENVIRONMENT",
+            Err(std::env::VarError::NotUnicode(OsString::from(
+                "invalid-environment",
+            ))),
+            "development",
+        )
+        .expect_err("non-UTF-8 values must not be replaced with defaults");
+
+        assert!(matches!(
+            error,
+            ObservabilityConfigError::InvalidEnvironmentEncoding {
+                field: "CRANK_ENVIRONMENT"
+            }
+        ));
+    }
+}

crates/crank-observability/src/correlation.rs

@@ -0,0 +1,51 @@
+use std::fmt;
+
+use axum::http::HeaderMap;
+use uuid::Uuid;
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct RequestId(String);
+
+impl RequestId {
+    pub const MAX_LEN: usize = 128;
+    const HEADER_NAME: &'static str = "x-request-id";
+
+    pub fn resolve(candidate: Option<&str>) -> Self {
+        candidate
+            .filter(|value| Self::is_valid(value))
+            .map(|value| Self(value.to_owned()))
+            .unwrap_or_else(|| Self(Uuid::now_v7().to_string()))
+    }
+
+    pub fn resolve_from_headers(headers: &HeaderMap) -> Self {
+        let mut values = headers.get_all(Self::HEADER_NAME).iter();
+        let candidate = values.next();
+        if values.next().is_some() {
+            return Self::resolve(None);
+        }
+
+        Self::resolve(candidate.and_then(|value| value.to_str().ok()))
+    }
+
+    pub fn is_valid(value: &str) -> bool {
+        !value.is_empty()
+            && value.len() <= Self::MAX_LEN
+            && value
+                .bytes()
+                .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    pub fn into_string(self) -> String {
+        self.0
+    }
+}
+
+impl fmt::Display for RequestId {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str(self.as_str())
+    }
+}

crates/crank-observability/src/error_reporting.rs

@@ -0,0 +1,467 @@
+use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration};
+
+use sentry::{
+    ClientInitGuard, ClientOptions,
+    protocol::{Event, Level},
+    types::Dsn,
+};
+use thiserror::Error;
+
+use crate::{
+    RedactionLimits, ServiceIdentity, propagation::current_trace_id, redaction::truncate_string,
+};
+
+const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
+const CRITICAL_ERROR_MESSAGE: &str = "critical error";
+const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
+
+tokio::task_local! {
+    static REQUEST_ID: String;
+}
+
+pub struct SentryConfig {
+    dsn: Option<Dsn>,
+}
+
+impl SentryConfig {
+    pub fn parse(value: Option<&str>) -> Result<Self, SentryConfigError> {
+        let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
+            return Ok(Self { dsn: None });
+        };
+
+        let dsn = value
+            .parse::<Dsn>()
+            .map_err(|_| SentryConfigError::InvalidDsn)?;
+        Ok(Self { dsn: Some(dsn) })
+    }
+
+    pub fn from_env() -> Result<Self, SentryConfigError> {
+        match env::var(SENTRY_DSN_ENV) {
+            Ok(value) => Self::parse(Some(&value)),
+            Err(env::VarError::NotPresent) => Self::parse(None),
+            Err(env::VarError::NotUnicode(_)) => Err(SentryConfigError::InvalidEnvironmentEncoding),
+        }
+    }
+
+    pub fn enabled(&self) -> bool {
+        self.dsn.is_some()
+    }
+}
+
+impl fmt::Debug for SentryConfig {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("SentryConfig")
+            .field("enabled", &self.enabled())
+            .finish()
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum SentryConfigError {
+    #[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")]
+    InvalidDsn,
+    #[error("CRANK_SENTRY_DSN is not valid UTF-8")]
+    InvalidEnvironmentEncoding,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum CriticalErrorCategory {
+    Panic,
+    Startup,
+    Internal,
+    DataIntegrity,
+}
+
+impl CriticalErrorCategory {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Panic => "panic",
+            Self::Startup => "startup",
+            Self::Internal => "internal",
+            Self::DataIntegrity => "data_integrity",
+        }
+    }
+
+    fn parse(value: &str) -> Option<Self> {
+        match value {
+            "panic" => Some(Self::Panic),
+            "startup" => Some(Self::Startup),
+            "internal" => Some(Self::Internal),
+            "data_integrity" => Some(Self::DataIntegrity),
+            _ => None,
+        }
+    }
+}
+
+pub fn capture_critical_error(category: CriticalErrorCategory) {
+    let mut tags = correlation_tags();
+    tags.insert("category".to_owned(), category.as_str().to_owned());
+    sentry::capture_event(Event {
+        level: Level::Error,
+        message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
+        fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
+        tags,
+        ..Event::default()
+    });
+}
+
+pub async fn with_request_correlation<F>(request_id: String, future: F) -> F::Output
+where
+    F: Future,
+{
+    REQUEST_ID.scope(request_id, future).await
+}
+
+pub(crate) fn init_sentry(
+    identity: &ServiceIdentity,
+    limits: RedactionLimits,
+    config: SentryConfig,
+) -> Option<ClientInitGuard> {
+    let dsn = config.dsn?;
+    let identity = identity.clone();
+    let options = client_options(identity, limits);
+    Some(sentry::init((dsn, options)))
+}
+
+fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
+    let release = identity.version().to_owned();
+    let environment = identity.environment().to_owned();
+    let service = identity.service().to_owned();
+    let sanitizer_identity = identity.clone();
+
+    let mut options = ClientOptions::default();
+    options.release = Some(Cow::Owned(release));
+    options.environment = Some(Cow::Owned(environment));
+    options.server_name = Some(Cow::Owned(service));
+    options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled;
+    options.max_breadcrumbs = 0;
+    options.attach_stacktrace = false;
+    options.send_default_pii = false;
+    options.before_send = Some(std::sync::Arc::new(move |event| {
+        Some(sanitize_event(event, &sanitizer_identity, limits))
+    }));
+    options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT;
+    options.auto_session_tracking = false;
+    options.enable_logs = false;
+    options.enable_metrics = false;
+    options
+}
+
+fn sanitize_event(
+    event: Event<'static>,
+    identity: &ServiceIdentity,
+    limits: RedactionLimits,
+) -> Event<'static> {
+    let category = event
+        .tags
+        .get("category")
+        .and_then(|value| CriticalErrorCategory::parse(value))
+        .unwrap_or_else(|| {
+            if event.exception.is_empty() {
+                CriticalErrorCategory::Internal
+            } else {
+                CriticalErrorCategory::Panic
+            }
+        });
+    let mut tags = correlation_tags()
+        .into_iter()
+        .map(|(key, value)| (key, truncate_string(&value, limits.max_string_bytes)))
+        .collect::<BTreeMap<_, _>>();
+    for key in ["request_id", "trace_id"] {
+        if let Some(value) = event.tags.get(key) {
+            tags.entry(key.to_owned())
+                .or_insert_with(|| truncate_string(value, limits.max_string_bytes));
+        }
+    }
+    tags.insert("service".to_owned(), identity.service().to_owned());
+    tags.insert("category".to_owned(), category.as_str().to_owned());
+
+    enforce_event_budget(
+        Event {
+            event_id: event.event_id,
+            level: Level::Error,
+            fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
+            message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
+            timestamp: event.timestamp,
+            server_name: Some(Cow::Owned(identity.service().to_owned())),
+            release: Some(Cow::Owned(identity.version().to_owned())),
+            environment: Some(Cow::Owned(identity.environment().to_owned())),
+            tags,
+            ..Event::default()
+        },
+        limits.max_event_bytes,
+    )
+}
+
+fn correlation_tags() -> BTreeMap<String, String> {
+    let mut tags = BTreeMap::new();
+    if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) {
+        tags.insert("request_id".to_owned(), request_id);
+    }
+    if let Some(trace_id) = current_trace_id() {
+        tags.insert("trace_id".to_owned(), trace_id);
+    }
+    tags
+}
+
+fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
+    if serialized_event_len(&event) <= max_event_bytes {
+        return event;
+    }
+
+    event.tags.remove("request_id");
+    event.tags.remove("trace_id");
+    event
+}
+
+fn serialized_event_len(event: &Event<'_>) -> usize {
+    serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        collections::BTreeMap,
+        sync::{
+            Arc,
+            atomic::{AtomicUsize, Ordering},
+        },
+    };
+
+    use opentelemetry::trace::TracerProvider as _;
+    use opentelemetry_sdk::trace::SdkTracerProvider;
+    use sentry::{
+        Envelope, Hub,
+        protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values},
+    };
+
+    use super::{
+        CriticalErrorCategory, capture_critical_error, client_options, sanitize_event,
+        with_request_correlation,
+    };
+    use crate::{
+        ObservabilityConfig, RedactionLimits, ServiceIdentity,
+        logging::build_subscriber_with_tracer,
+    };
+
+    fn identity() -> ServiceIdentity {
+        ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity")
+    }
+
+    #[test]
+    fn client_disables_non_error_telemetry() {
+        let options = client_options(identity(), RedactionLimits::default());
+
+        assert_eq!(options.max_breadcrumbs, 0);
+        assert!(!options.attach_stacktrace);
+        assert!(!options.send_default_pii);
+        assert!(!options.auto_session_tracking);
+        assert!(!options.enable_logs);
+        assert!(!options.enable_metrics);
+        assert!(options.before_send.is_some());
+    }
+
+    #[test]
+    fn sanitizer_uses_a_strict_allowlist() {
+        let mut tags = BTreeMap::new();
+        tags.insert("category".to_owned(), "data_integrity".to_owned());
+        tags.insert("secret".to_owned(), "must-not-leak".to_owned());
+        let mut contexts = BTreeMap::new();
+        contexts.insert(
+            "secret".to_owned(),
+            Context::Other(BTreeMap::from([(
+                "token".to_owned(),
+                Value::String("must-not-leak".to_owned()),
+            )])),
+        );
+        let event = sentry::protocol::Event {
+            message: Some("password=must-not-leak".to_owned()),
+            request: Some(Request::default()),
+            user: Some(User::default()),
+            breadcrumbs: Values {
+                values: vec![Breadcrumb::default()],
+            },
+            exception: Values {
+                values: vec![Exception {
+                    value: Some("must-not-leak".to_owned()),
+                    ..Exception::default()
+                }],
+            },
+            contexts,
+            extra: BTreeMap::from([(
+                "payload".to_owned(),
+                Value::String("must-not-leak".to_owned()),
+            )]),
+            tags,
+            ..sentry::protocol::Event::default()
+        };
+
+        let cleaned = sanitize_event(event, &identity(), RedactionLimits::default());
+        let serialized = serde_json::to_string(&cleaned).expect("serialize event");
+
+        assert_eq!(cleaned.message.as_deref(), Some("critical error"));
+        assert_eq!(
+            cleaned.tags.get("category").map(String::as_str),
+            Some(CriticalErrorCategory::DataIntegrity.as_str())
+        );
+        assert!(cleaned.request.is_none());
+        assert!(cleaned.user.is_none());
+        assert!(cleaned.breadcrumbs.is_empty());
+        assert!(cleaned.exception.is_empty());
+        assert!(cleaned.contexts.is_empty());
+        assert!(cleaned.extra.is_empty());
+        assert!(!serialized.contains("must-not-leak"));
+        assert!(!serialized.contains("password"));
+    }
+
+    #[test]
+    fn sanitizer_honours_total_event_budget() {
+        let limits = RedactionLimits {
+            max_string_bytes: 8 * 1024,
+            max_event_bytes: 512,
+            ..RedactionLimits::default()
+        };
+        let event = sentry::protocol::Event {
+            tags: BTreeMap::from([
+                ("category".to_owned(), "internal".to_owned()),
+                ("request_id".to_owned(), "r".repeat(8 * 1024)),
+                ("trace_id".to_owned(), "t".repeat(8 * 1024)),
+            ]),
+            ..sentry::protocol::Event::default()
+        };
+
+        let cleaned = sanitize_event(event, &identity(), limits);
+        let serialized = serde_json::to_vec(&cleaned).expect("serialize event");
+
+        assert!(serialized.len() <= limits.max_event_bytes);
+        assert_eq!(
+            cleaned.tags.get("category").map(String::as_str),
+            Some("internal")
+        );
+        assert_eq!(
+            cleaned.tags.get("service").map(String::as_str),
+            Some("admin-api")
+        );
+        assert!(!cleaned.tags.contains_key("request_id"));
+        assert!(!cleaned.tags.contains_key("trace_id"));
+    }
+
+    #[test]
+    fn expected_application_errors_do_not_create_critical_events() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let events = sentry::test::with_captured_events_options(
+            || tracing::error!("ordinary product error"),
+            options,
+        );
+
+        assert!(events.is_empty());
+    }
+
+    #[test]
+    fn explicit_critical_error_is_correlated_and_sanitized() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .build()
+            .expect("runtime");
+        let provider = SdkTracerProvider::builder().build();
+        let tracer = provider.tracer("critical-error-test");
+        let subscriber = build_subscriber_with_tracer(
+            ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
+            std::io::sink,
+            Some(tracer),
+        )
+        .expect("subscriber");
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let events = sentry::test::with_captured_events_options(
+            || {
+                tracing::dispatcher::with_default(&dispatch, || {
+                    runtime.block_on(with_request_correlation("request-123".to_owned(), async {
+                        let span = tracing::info_span!(target: "crank::trace", "http.request");
+                        let _span_guard = span.enter();
+                        capture_critical_error(CriticalErrorCategory::DataIntegrity);
+                    }));
+                });
+            },
+            options,
+        );
+
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(event.message.as_deref(), Some("critical error"));
+        assert_eq!(
+            event.tags.get("category").map(String::as_str),
+            Some("data_integrity")
+        );
+        assert_eq!(
+            event.tags.get("request_id").map(String::as_str),
+            Some("request-123")
+        );
+        assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
+        assert_eq!(event.release.as_deref(), Some("1.2.3"));
+        assert_eq!(event.environment.as_deref(), Some("test"));
+        assert_eq!(event.server_name.as_deref(), Some("admin-api"));
+    }
+
+    #[test]
+    fn panic_creates_exactly_one_sanitized_critical_event() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let events = sentry::test::with_captured_events_options(
+            || {
+                let result = std::panic::catch_unwind(|| {
+                    panic!("password=must-not-leak");
+                });
+                assert!(result.is_err());
+            },
+            options,
+        );
+
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(
+            event.tags.get("category").map(String::as_str),
+            Some("panic")
+        );
+        let serialized = serde_json::to_string(event).expect("serialize event");
+        assert!(!serialized.contains("must-not-leak"));
+        assert!(!serialized.contains("password"));
+    }
+
+    #[test]
+    fn receiver_failure_does_not_change_product_result_or_recurse() {
+        struct DroppingTransport {
+            attempts: AtomicUsize,
+        }
+
+        impl sentry::Transport for DroppingTransport {
+            fn send_envelope(&self, _envelope: Envelope) {
+                self.attempts.fetch_add(1, Ordering::Relaxed);
+            }
+        }
+
+        let transport = Arc::new(DroppingTransport {
+            attempts: AtomicUsize::new(0),
+        });
+        let mut options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        options.dsn = Some(
+            "https://public@example.invalid/1"
+                .parse()
+                .expect("valid test DSN"),
+        );
+        options.transport = Some(Arc::new(transport.clone()));
+        let client = Arc::new(sentry::Client::from(options));
+        let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default())));
+
+        let product_result = Hub::run(hub, || {
+            capture_critical_error(CriticalErrorCategory::Internal);
+            42
+        });
+
+        assert_eq!(product_result, 42);
+        assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
+    }
+}

crates/crank-observability/src/incidents.rs

@@ -0,0 +1,36 @@
+use std::sync::atomic::{AtomicU64, Ordering};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum OperationalIncident {
+    InvocationHistoryLost,
+}
+
+static INVOCATION_HISTORY_LOST_TOTAL: AtomicU64 = AtomicU64::new(0);
+
+pub fn record_operational_incident(incident: OperationalIncident) {
+    let counter = counter(incident);
+    let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
+        value.checked_add(1)
+    });
+    match incident {
+        OperationalIncident::InvocationHistoryLost => {
+            metrics::counter!("crank_invocation_history_lost_total").increment(1);
+            metrics::counter!(
+                "crank_telemetry_export_failures_total",
+                "signal_type" => "invocation_history",
+                "exporter" => "postgres"
+            )
+            .increment(1);
+        }
+    }
+}
+
+pub fn operational_incident_total(incident: OperationalIncident) -> u64 {
+    counter(incident).load(Ordering::Relaxed)
+}
+
+fn counter(incident: OperationalIncident) -> &'static AtomicU64 {
+    match incident {
+        OperationalIncident::InvocationHistoryLost => &INVOCATION_HISTORY_LOST_TOTAL,
+    }
+}

crates/crank-observability/src/instrumentation.rs

@@ -0,0 +1,133 @@
+use std::time::Instant;
+
+use axum::{
+    extract::{MatchedPath, Request},
+    middleware::Next,
+    response::Response,
+};
+use metrics::{Gauge, Unit};
+
+use crate::{MetricKind, MetricUnit, metric_schema};
+
+pub async fn record_http_request(request: Request, next: Next) -> Response {
+    let route = request
+        .extensions()
+        .get::<MatchedPath>()
+        .map_or("unmatched", MatchedPath::as_str)
+        .to_owned();
+    let method = normalized_http_method(request.method().as_str());
+    let started_at = Instant::now();
+    let _inflight = GaugeGuard::increment("crank_http_inflight");
+
+    let response = next.run(request).await;
+    let status_class = status_class(response.status().as_u16());
+
+    metrics::counter!(
+        "crank_http_requests_total",
+        "route" => route.clone(),
+        "method" => method,
+        "status_class" => status_class
+    )
+    .increment(1);
+    metrics::histogram!(
+        "crank_http_request_duration_seconds",
+        "route" => route,
+        "method" => method
+    )
+    .record(started_at.elapsed().as_secs_f64());
+
+    response
+}
+
+pub fn record_db_pool_connections(total: u32, idle: usize) {
+    let idle = idle.min(total as usize) as f64;
+    metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(idle);
+    metrics::gauge!("crank_db_pool_connections", "state" => "used").set(f64::from(total) - idle);
+}
+
+pub(crate) fn register_metric_schema() {
+    for definition in metric_schema() {
+        let unit = match definition.unit {
+            MetricUnit::Count => Unit::Count,
+            MetricUnit::Seconds => Unit::Seconds,
+        };
+        match definition.kind {
+            MetricKind::Counter => {
+                metrics::describe_counter!(definition.name, unit, definition.description);
+            }
+            MetricKind::Gauge => {
+                metrics::describe_gauge!(definition.name, unit, definition.description);
+            }
+            MetricKind::Histogram => {
+                metrics::describe_histogram!(definition.name, unit, definition.description);
+            }
+        }
+    }
+
+    metrics::gauge!("crank_http_inflight").set(0.0);
+    metrics::gauge!("crank_mcp_active_sessions").set(0.0);
+    metrics::gauge!("crank_runtime_inflight").set(0.0);
+    metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(0.0);
+    metrics::gauge!("crank_db_pool_connections", "state" => "used").set(0.0);
+    metrics::gauge!("crank_catalog_tools").set(0.0);
+    metrics::gauge!("crank_catalog_estimated_context_tokens").set(0.0);
+    metrics::gauge!("crank_catalog_warnings").set(0.0);
+}
+
+fn normalized_http_method(method: &str) -> &'static str {
+    match method {
+        "GET" => "GET",
+        "POST" => "POST",
+        "PUT" => "PUT",
+        "PATCH" => "PATCH",
+        "DELETE" => "DELETE",
+        "OPTIONS" => "OPTIONS",
+        "HEAD" => "HEAD",
+        "CONNECT" => "CONNECT",
+        "TRACE" => "TRACE",
+        _ => "OTHER",
+    }
+}
+
+fn status_class(status: u16) -> &'static str {
+    match status {
+        100..=199 => "1xx",
+        200..=299 => "2xx",
+        300..=399 => "3xx",
+        400..=499 => "4xx",
+        500..=599 => "5xx",
+        _ => "other",
+    }
+}
+
+struct GaugeGuard {
+    gauge: Gauge,
+}
+
+impl GaugeGuard {
+    fn increment(name: &'static str) -> Self {
+        let gauge = metrics::gauge!(name);
+        gauge.increment(1.0);
+        Self { gauge }
+    }
+}
+
+impl Drop for GaugeGuard {
+    fn drop(&mut self) {
+        self.gauge.decrement(1.0);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{normalized_http_method, status_class};
+
+    #[test]
+    fn normalizes_unbounded_http_values() {
+        assert_eq!(normalized_http_method("GET"), "GET");
+        assert_eq!(normalized_http_method("CUSTOM-user-controlled"), "OTHER");
+        assert_eq!(status_class(204), "2xx");
+        assert_eq!(status_class(429), "4xx");
+        assert_eq!(status_class(999), "other");
+    }
+}