наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+jobs = 2

.env.example

@@ -29,7 +29,30 @@ CRANK_RUNTIME_MAX_CONCURRENT_JOBS=16
 CRANK_OUTBOUND_ALLOWED_HOSTS=
 CRANK_OUTBOUND_DENIED_HOSTS=
 CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
+CRANK_ENVIRONMENT=development
 CRANK_LOG_LEVEL=info
+# Пустое значение отключает канал критических ошибок.
+CRANK_SENTRY_DSN=
+# Prometheus endpoints use separate listeners and stay on loopback by default.
+CRANK_METRICS_ENABLED=true
+CRANK_ADMIN_METRICS_BIND=127.0.0.1:9464
+CRANK_MCP_METRICS_BIND=127.0.0.1:9465
+# Required when either metrics listener uses a non-loopback address.
+CRANK_METRICS_BEARER_TOKEN=
+CRANK_INVOCATION_LOG_RETENTION_DAYS=30
+# Пустой endpoint полностью отключает экспорт трасс.
+OTEL_EXPORTER_OTLP_ENDPOINT=
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=
+OTEL_EXPORTER_OTLP_TIMEOUT=10000
+OTEL_EXPORTER_OTLP_TRACES_TIMEOUT=
+OTEL_EXPORTER_OTLP_HEADERS=
+OTEL_EXPORTER_OTLP_TRACES_HEADERS=
+OTEL_BSP_MAX_QUEUE_SIZE=2048
+OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512
+OTEL_BSP_SCHEDULE_DELAY=5000
+OTEL_BSP_EXPORT_TIMEOUT=30000
 CRANK_MASTER_KEY=change-me-master-key
 CRANK_SESSION_SECRET=change-me-session-secret
 CRANK_PASSWORD_PEPPER=change-me-password-pepper

.gitea/workflows/ci.yml

@@ -55,6 +55,9 @@ jobs:
           docker --version
           docker info
 
+      - name: Install dependency policy tool
+        run: cargo install cargo-deny --version 0.20.2 --locked
+
       - name: Run tooling unit tests
         run: python3 -m unittest discover -s tests/unit
 
@@ -67,6 +70,9 @@ jobs:
       - name: Check Rust code health
         run: scripts/check-rust-code-health.sh
 
+      - name: Check dependency licenses and advisories
+        run: cargo deny --locked check advisories bans licenses sources
+
       - name: Check Rust boundaries
         run: scripts/check-rust-boundaries.sh
 
@@ -95,6 +101,10 @@ jobs:
         working-directory: apps/ui
         run: npm ci
 
+      - name: Audit UI dependencies
+        working-directory: apps/ui
+        run: npm audit --audit-level=high
+
       - name: Build UI bundle
         working-directory: apps/ui
         run: npm run build
@@ -179,9 +189,11 @@ jobs:
           find .tmp/ui-e2e/logs -maxdepth 1 -type f -print -exec sed -n '1,220p' {} \; || true
 
   deployment:
-    name: Deployment Manifests
+    name: Community Image Smoke
     runs-on: ubuntu-latest
-    needs: ui
+    needs:
+      - rust
+      - ui
 
     steps:
       - name: Checkout
@@ -190,6 +202,58 @@ jobs:
       - name: Validate Community deployment manifest
         run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
 
+      - name: Build Community images
+        run: |
+          docker build -f apps/admin-api/Dockerfile -t crank/admin-api:ci .
+          docker build -f apps/mcp-server/Dockerfile -t crank/mcp-server:ci .
+          docker build -f apps/ui/Dockerfile -t crank/ui:ci .
+
+      - name: Start Community image stack
+        run: |
+          mkdir -p .tmp
+          cat > .tmp/community-smoke.env <<'EOF'
+          COMPOSE_PROJECT_NAME=crank-ci-smoke
+          POSTGRES_HOST=postgres
+          POSTGRES_PORT=5432
+          POSTGRES_DB=crank
+          POSTGRES_USER=crank
+          POSTGRES_PASSWORD=crank-ci-password
+          CRANK_ADMIN_API_IMAGE=crank/admin-api:ci
+          CRANK_MCP_SERVER_IMAGE=crank/mcp-server:ci
+          CRANK_UI_IMAGE=crank/ui:ci
+          CRANK_MASTER_KEY=0000000000000000000000000000000000000000000000000000000000000000
+          CRANK_SESSION_SECRET=ci-session-secret
+          CRANK_PASSWORD_PEPPER=ci-password-pepper
+          CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
+          CRANK_BOOTSTRAP_ADMIN_PASSWORD=ci-admin-password
+          CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner
+          CRANK_BASE_URL=http://127.0.0.1:3000
+          CRANK_PUBLISH_BIND=127.0.0.1
+          CRANK_DEMO_SEED=true
+          EOF
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db up -d --wait
+
+      - name: Run authenticated Community image smoke
+        env:
+          CRANK_STAGING_ADMIN_EMAIL: owner@crank.test
+          CRANK_STAGING_ADMIN_PASSWORD: ci-admin-password
+        run: scripts/authenticated-product-smoke.sh http://127.0.0.1:3000
+
+      - name: Show Community image logs
+        if: failure()
+        run: |
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db ps || true
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db logs --no-color || true
+
+      - name: Stop Community image stack
+        if: always()
+        run: |
+          docker compose -f deploy/community/docker-compose.images.yml \
+            --env-file .tmp/community-smoke.env --profile local-db down -v --remove-orphans || true
+
   deploy:
     name: Deploy
     runs-on: ubuntu-latest
@@ -245,6 +309,10 @@ jobs:
             -t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \
             -t '${{ env.UI_IMAGE }}:main' \
             .
+          scripts/scan-images.sh \
+            '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:main'
           docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}'
@@ -274,9 +342,14 @@ jobs:
         run: |
           . "$OPENBAO_ENV_FILE"
           ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
-            "mkdir -p '$DEPLOY_PATH'"
+            "mkdir -p '$DEPLOY_PATH' && \
+             if [ -f '$DEPLOY_PATH/docker-compose.yml' ]; then \
+               cp '$DEPLOY_PATH/docker-compose.yml' '$DEPLOY_PATH/docker-compose.previous.yml'; \
+             fi"
           rsync -az -e "ssh -p $DEPLOY_PORT" deploy/community/docker-compose.yml \
             "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/docker-compose.yml"
+          rsync -az -e "ssh -p $DEPLOY_PORT" scripts/deploy-community.sh \
+            "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/deploy-community.sh"
 
       - name: Write environment file
         run: |
@@ -292,6 +365,11 @@ jobs:
           append_if_set POSTGRES_USER "$POSTGRES_USER"
           append_if_set POSTGRES_PASSWORD "$POSTGRES_PASSWORD"
           append_if_set POSTGRES_HOST "$POSTGRES_HOST"
+          append_if_set POSTGRES_MAX_CONNECTIONS "${POSTGRES_MAX_CONNECTIONS:-}"
+          append_if_set POSTGRES_MIN_CONNECTIONS "${POSTGRES_MIN_CONNECTIONS:-}"
+          append_if_set POSTGRES_ACQUIRE_TIMEOUT_MS "${POSTGRES_ACQUIRE_TIMEOUT_MS:-}"
+          append_if_set POSTGRES_IDLE_TIMEOUT_MS "${POSTGRES_IDLE_TIMEOUT_MS:-}"
+          append_if_set POSTGRES_MAX_LIFETIME_MS "${POSTGRES_MAX_LIFETIME_MS:-}"
           if [ -n "${POSTGRES_PORT:-}" ]; then
             append_if_set POSTGRES_PORT "$POSTGRES_PORT"
           elif [ -n "${PGBOUNCER_PORT:-}" ]; then
@@ -302,10 +380,35 @@ jobs:
           append_if_set CRANK_ADMIN_BIND "$CRANK_ADMIN_BIND"
           append_if_set CRANK_MCP_BIND "$CRANK_MCP_BIND"
           append_if_set CRANK_MCP_REFRESH_MS "$CRANK_MCP_REFRESH_MS"
+          append_if_set CRANK_ADMIN_RATE_LIMIT_RPS "${CRANK_ADMIN_RATE_LIMIT_RPS:-}"
+          append_if_set CRANK_ADMIN_RATE_LIMIT_BURST "${CRANK_ADMIN_RATE_LIMIT_BURST:-}"
+          append_if_set CRANK_MCP_RATE_LIMIT_RPS "${CRANK_MCP_RATE_LIMIT_RPS:-}"
+          append_if_set CRANK_MCP_RATE_LIMIT_BURST "${CRANK_MCP_RATE_LIMIT_BURST:-}"
+          append_if_set CRANK_RUNTIME_MAX_CONCURRENT_UNARY "${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-}"
+          append_if_set CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS "${CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS:-}"
           append_if_set CRANK_OUTBOUND_ALLOWED_HOSTS "${CRANK_OUTBOUND_ALLOWED_HOSTS:-}"
           append_if_set CRANK_OUTBOUND_DENIED_HOSTS "${CRANK_OUTBOUND_DENIED_HOSTS:-}"
           append_if_set CRANK_OUTBOUND_MAX_RESPONSE_BYTES "${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-}"
+          append_if_set CRANK_ENVIRONMENT "${CRANK_ENVIRONMENT:-production}"
           append_if_set CRANK_LOG_LEVEL "$CRANK_LOG_LEVEL"
+          append_if_set CRANK_SENTRY_DSN "${CRANK_SENTRY_DSN:-}"
+          append_if_set CRANK_METRICS_ENABLED "${CRANK_METRICS_ENABLED:-}"
+          append_if_set CRANK_ADMIN_METRICS_BIND "${CRANK_ADMIN_METRICS_BIND:-}"
+          append_if_set CRANK_MCP_METRICS_BIND "${CRANK_MCP_METRICS_BIND:-}"
+          append_if_set CRANK_METRICS_BEARER_TOKEN "${CRANK_METRICS_BEARER_TOKEN:-}"
+          append_if_set CRANK_INVOCATION_LOG_RETENTION_DAYS "${CRANK_INVOCATION_LOG_RETENTION_DAYS:-}"
+          append_if_set OTEL_EXPORTER_OTLP_ENDPOINT "${OTEL_EXPORTER_OTLP_ENDPOINT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_PROTOCOL "${OTEL_EXPORTER_OTLP_PROTOCOL:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_PROTOCOL "${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TIMEOUT "${OTEL_EXPORTER_OTLP_TIMEOUT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_TIMEOUT "${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-}"
+          append_if_set OTEL_EXPORTER_OTLP_HEADERS "${OTEL_EXPORTER_OTLP_HEADERS:-}"
+          append_if_set OTEL_EXPORTER_OTLP_TRACES_HEADERS "${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-}"
+          append_if_set OTEL_BSP_MAX_QUEUE_SIZE "${OTEL_BSP_MAX_QUEUE_SIZE:-}"
+          append_if_set OTEL_BSP_MAX_EXPORT_BATCH_SIZE "${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-}"
+          append_if_set OTEL_BSP_SCHEDULE_DELAY "${OTEL_BSP_SCHEDULE_DELAY:-}"
+          append_if_set OTEL_BSP_EXPORT_TIMEOUT "${OTEL_BSP_EXPORT_TIMEOUT:-}"
           append_if_set CRANK_MASTER_KEY "$CRANK_MASTER_KEY"
           append_if_set CRANK_BASE_URL "$CRANK_BASE_URL"
           append_if_set CRANK_CACHE_BACKEND "$CRANK_CACHE_BACKEND"
@@ -325,7 +428,10 @@ jobs:
             printf 'CRANK_UI_IMAGE=%s:%s\n' '${{ env.UI_IMAGE }}' '${{ env.IMAGE_TAG }}'
           } >> "$tmp_env"
           cat "$tmp_env" | ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
-            "mkdir -p '$DEPLOY_PATH' && cat > '$DEPLOY_PATH/.env'"
+            "mkdir -p '$DEPLOY_PATH' && \
+             if [ -f '$DEPLOY_PATH/.env' ]; then \
+               cp '$DEPLOY_PATH/.env' '$DEPLOY_PATH/.env.previous'; \
+             fi && cat > '$DEPLOY_PATH/.env'"
           rm -f "$tmp_env"
 
       - name: Validate required environment variables
@@ -359,48 +465,12 @@ jobs:
       - name: Deploy with Docker Compose
         run: |
           . "$OPENBAO_ENV_FILE"
-          ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" "
+          printf '%s' "$DEPLOY_REGISTRY_TOKEN" | ssh -p "$DEPLOY_PORT" \
-            set -e
+            "$DEPLOY_USER@$DEPLOY_HOST" \
-            cd '$DEPLOY_PATH'
+            "docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin"
-            compose_profiles=''
+          ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
-            cache_backend=\$(grep -E '^CRANK_CACHE_BACKEND=' .env | tail -n1 | cut -d= -f2- || true)
+            "chmod 700 '$DEPLOY_PATH/deploy-community.sh' && \
-            if [ \"\$cache_backend\" = 'valkey' ] || [ \"\$cache_backend\" = 'redis' ]; then
+             '$DEPLOY_PATH/deploy-community.sh' '$DEPLOY_PATH'"
-              compose_profiles='--profile cache'
-            fi
-            echo '$DEPLOY_REGISTRY_TOKEN' | docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin
-            docker compose \$compose_profiles config -q
-            docker compose \$compose_profiles pull
-            docker compose \$compose_profiles down --remove-orphans
-            for container in \
-              crank-ui-1 \
-              crank-admin-api-1 \
-              crank-mcp-server-1 \
-              crank-postgres-1 \
-              crank-valkey-1 \
-              crank-community-ui-1 \
-              crank-community-admin-api-1 \
-              crank-community-mcp-server-1 \
-              crank-community-postgres-1 \
-              crank-community-valkey-1; do
-              if docker ps -a --format '{{.Names}}' | grep -Fx \"\$container\" >/dev/null; then
-                docker rm -f \"\$container\"
-              fi
-            done
-            echo 'Docker containers before freeing required ports:'
-            docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Ports}}'
-            for port in 3000 3001 3002; do
-              container_ids=\$(docker ps -aq --filter \"publish=\$port\")
-              if [ -n \"\$container_ids\" ]; then
-                echo \"Removing containers publishing port \$port\"
-                docker inspect --format '{{.Name}} {{json .NetworkSettings.Ports}}' \$container_ids || true
-                docker rm -f \$container_ids
-              fi
-            done
-            if command -v ss >/dev/null 2>&1; then
-              ss -ltnp '( sport = :3000 or sport = :3001 or sport = :3002 )' || true
-            fi
-            docker compose \$compose_profiles up -d --remove-orphans
-          "
 
       - name: Verify health endpoints
         run: |
@@ -410,8 +480,8 @@ jobs:
             cd '$DEPLOY_PATH'
             for attempt in \$(seq 1 30); do
               if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
-                && curl --fail --silent http://127.0.0.1:3001/health >/dev/null \
+                && curl --fail --silent http://127.0.0.1:3001/ready >/dev/null \
-                && curl --fail --silent http://127.0.0.1:3002/health >/dev/null; then
+                && curl --fail --silent http://127.0.0.1:3002/ready >/dev/null; then
                 exit 0
               fi
               sleep 2

.gitea/workflows/release.yml

@@ -49,6 +49,16 @@ jobs:
           command -v bao
           bao version
 
+      - name: Install dependency policy tool
+        run: cargo install cargo-deny --version 0.20.2 --locked
+
+      - name: Run release quality gates
+        run: |
+          cargo fmt --all --check
+          cargo clippy --workspace --all-targets --all-features -- -D warnings
+          cargo test --workspace --all-targets
+          cargo deny --locked check advisories bans licenses sources
+
       - name: Build release binaries
         run: cargo build --release -p admin-api -p mcp-server
 
@@ -56,10 +66,25 @@ jobs:
         working-directory: apps/ui
         run: npm ci
 
+      - name: Audit UI dependencies
+        working-directory: apps/ui
+        run: npm audit --audit-level=high
+
       - name: Build UI dist
         working-directory: apps/ui
         run: npm run build
 
+      - name: Install Playwright browser
+        working-directory: apps/ui
+        run: npx playwright install --with-deps chromium
+
+      - name: Run release end-to-end tests
+        working-directory: apps/ui
+        run: npm run e2e
+
+      - name: Validate deployment manifest
+        run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
+
       - name: Package release artifacts
         run: |
           mkdir -p dist/release
@@ -105,6 +130,10 @@ jobs:
           docker build -f apps/ui/Dockerfile \
             -t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \
             -t '${{ env.UI_IMAGE }}:latest' .
+          scripts/scan-images.sh \
+            '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
+            '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}'
           docker push '${{ env.ADMIN_API_IMAGE }}:latest'
           docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}'

Cargo.toml

@@ -8,9 +8,11 @@ members = [
     "crates/crank-import",
     "crates/crank-schema",
     "crates/crank-mapping",
+    "crates/crank-observability",
     "crates/crank-registry",
     "crates/crank-runtime",
     "crates/crank-test-support",
+    "crates/crank-trace",
     "crates/crank-adapter-rest",
 ]
 resolver = "3"
@@ -20,6 +22,7 @@ edition = "2024"
 license = "AGPL-3.0-only"
 rust-version = "1.96"
 version = "0.3.1"
+publish = false
 
 [workspace.dependencies]
 aes-gcm = "0.10"
@@ -28,18 +31,43 @@ axum = "0.8"
 axum-extra = { version = "0.12", features = ["cookie"] }
 base64 = "0.22"
 hkdf = "0.12"
+metrics = "0.24.6"
+metrics-exporter-prometheus = { version = "0.18.3", default-features = false }
+opentelemetry = { version = "0.32.0", default-features = false, features = ["trace"] }
+opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["http-proto", "reqwest-blocking-client", "reqwest-rustls", "trace"] }
+opentelemetry-proto = { version = "0.32.0", default-features = false, features = ["gen-tonic-messages", "trace"] }
+opentelemetry_sdk = { version = "0.32.1", default-features = false, features = ["trace"] }
+percent-encoding = "2"
+prost = "0.14"
 rand = "0.10"
 reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "rustls-tls"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
+sentry = { version = "0.49.0", default-features = false, features = ["backtrace", "panic", "rustls", "ureq"] }
 sha2 = "0.10"
 sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "macros", "json", "time", "uuid"] }
+subtle = "2.6"
 thiserror = "2"
 time = { version = "0.3.53", features = ["formatting", "parsing", "serde"] }
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+tower = "0.5"
 tracing = "0.1"
+tracing-opentelemetry = { version = "0.33.0", default-features = false }
 tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+url = "2"
 uuid = { version = "1", features = ["serde", "v7"] }
 testcontainers = { version = "0.27", features = ["blocking"] }
 testcontainers-modules = { version = "0.15", features = ["postgres", "blocking"] }
+
+[profile.dev]
+debug = "line-tables-only"
+
+[profile.dev.package."*"]
+debug = false
+
+[profile.test]
+debug = "line-tables-only"
+
+[profile.test.package."*"]
+debug = false

apps/admin-api/Cargo.toml

@@ -3,6 +3,7 @@ name = "admin-api"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [[bin]]
@@ -18,9 +19,12 @@ crank-community-auth = { path = "../../crates/crank-community-auth" }
 crank-core = { path = "../../crates/crank-core" }
 crank-import = { path = "../../crates/crank-import" }
 crank-mapping = { path = "../../crates/crank-mapping" }
+crank-observability = { path = "../../crates/crank-observability" }
 crank-registry = { path = "../../crates/crank-registry" }
 crank-runtime = { path = "../../crates/crank-runtime" }
 crank-schema = { path = "../../crates/crank-schema" }
+crank-trace = { path = "../../crates/crank-trace" }
+metrics.workspace = true
 rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -37,5 +41,9 @@ uuid.workspace = true
 [dev-dependencies]
 async-trait = "0.1"
 crank-test-support = { path = "../../crates/crank-test-support" }
+opentelemetry.workspace = true
+opentelemetry_sdk.workspace = true
 reqwest.workspace = true
 serial_test = "3"
+tower.workspace = true
+tracing-opentelemetry.workspace = true

apps/admin-api/src/app.rs

@@ -166,6 +166,7 @@ pub fn build_app(state: AppState) -> Router {
 
     Router::new()
         .route("/health", get(crate::routes::health))
+        .route("/ready", get(crate::routes::readiness))
         .nest(
             "/api/auth",
             Router::new()
@@ -178,6 +179,9 @@ pub fn build_app(state: AppState) -> Router {
             apply_api_rate_limit,
         ))
         .layer(middleware::from_fn(apply_request_context))
+        .layer(middleware::from_fn(
+            crank_observability::record_http_request,
+        ))
         .with_state(state)
 }
 

apps/admin-api/src/dto.rs

@@ -269,7 +269,12 @@ pub struct CreatedPlatformApiKeyResponse {
 }
 
 #[derive(Clone, Debug, Serialize)]
-pub struct WorkspaceExportResponse {
+pub struct WorkspaceCatalogSnapshotResponse {
+    pub kind: String,
+    pub format_version: String,
+    pub restorable: bool,
+    pub included: Vec<String>,
+    pub excluded: Vec<String>,
     pub workspace: WorkspaceRecord,
     pub operations: Vec<OperationSummaryView>,
     pub agents: Vec<AgentSummaryView>,

apps/admin-api/src/error.rs

@@ -137,16 +137,24 @@ impl ApiError {
 impl IntoResponse for ApiError {
     fn into_response(self) -> Response {
         match &self {
-            Self::Internal { message, .. } => {
+            Self::Internal { .. } => {
-                error!(error_code = self.code(), error_message = %message)
+                error!(
+                    name: "admin.response.internal_error",
+                    error_code = self.code(),
+                    "internal API error response"
+                )
             }
-            Self::Unauthorized { message, .. }
+            Self::Unauthorized { .. }
-            | Self::Forbidden { message, .. }
+            | Self::Forbidden { .. }
-            | Self::Validation { message, .. }
+            | Self::Validation { .. }
-            | Self::NotFound { message, .. }
+            | Self::NotFound { .. }
-            | Self::Conflict { message, .. }
+            | Self::Conflict { .. }
-            | Self::RateLimited { message, .. } => {
+            | Self::RateLimited { .. } => {
-                warn!(error_code = self.code(), error_message = %message)
+                warn!(
+                    name: "admin.response.rejected",
+                    error_code = self.code(),
+                    "API request rejected"
+                )
             }
         }
 
@@ -351,6 +359,10 @@ impl From<RegistryError> for ApiError {
                 format!("import job {job_id} was not found"),
                 json!({ "job_id": job_id }),
             ),
+            RegistryError::ImportJobAlreadyApplied { job_id } => Self::conflict_with_context(
+                format!("import job {job_id} was already applied with different parameters"),
+                json!({ "job_id": job_id }),
+            ),
             RegistryError::Storage(_) | RegistryError::Serialization(_) => {
                 Self::internal(value.to_string())
             }
@@ -407,6 +419,10 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "runtime_confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "runtime_confirmation_error",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "runtime_confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "runtime_idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "runtime_idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "runtime_idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "runtime_idempotency_outcome_unknown",
         RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error",
         RuntimeError::MissingAuthProfile { .. } => "runtime_auth_profile_error",
         RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
@@ -434,7 +450,11 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
             "safety_class": safety_class,
         })),
         RuntimeError::InvalidConfirmationToken { operation_id }
-        | RuntimeError::ConfirmationStoreUnavailable { operation_id } => Some(json!({
+        | RuntimeError::ConfirmationStoreUnavailable { operation_id }
+        | RuntimeError::IdempotencyStoreUnavailable { operation_id }
+        | RuntimeError::IdempotencyInProgress { operation_id }
+        | RuntimeError::IdempotencyConflict { operation_id }
+        | RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({
             "operation_id": operation_id,
         })),
         RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({

apps/admin-api/src/main.rs

@@ -1,4 +1,4 @@
-use std::{env, net::SocketAddr, path::PathBuf};
+use std::{env, net::SocketAddr, path::PathBuf, time::Duration};
 
 use admin_api::{
     app::build_app,
@@ -7,22 +7,50 @@ use admin_api::{
     state::AppState,
 };
 use crank_community_auth::PasswordIdentityProvider;
+use crank_observability::{
+    CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
+    capture_critical_error,
+};
 use crank_registry::{PostgresPoolConfig, PostgresRegistry};
 use crank_runtime::{
     RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
     RuntimeLimits, SecretCrypto,
 };
-use sqlx::postgres::PgConnectOptions;
+use sqlx::{PgPool, postgres::PgConnectOptions};
 use tokio::net::TcpListener;
-use tracing::info;
+use tracing::{info, warn};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    tracing_subscriber::fmt()
+    let observability = crank_observability::init(ObservabilityConfig::from_env(
-        .with_env_filter(
+        "admin-api",
-            env::var("CRANK_LOG_LEVEL").unwrap_or_else(|_| "admin_api=info,tower_http=info".into()),
+        env!("CARGO_PKG_VERSION"),
-        )
+        "admin_api=info,tower_http=info",
-        .init();
+    )?)?;
+    let mut startup_completed = false;
+    let result = run(&observability, &mut startup_completed).await;
+    if result.is_err() {
+        capture_critical_error(if startup_completed {
+            CriticalErrorCategory::Internal
+        } else {
+            CriticalErrorCategory::Startup
+        });
+    }
+    result
+}
+
+async fn run(
+    observability: &ObservabilityLifecycle,
+    startup_completed: &mut bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let metrics_config =
+        MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?;
+    let metrics_enabled = metrics_config.enabled();
+    let metrics_server = if metrics_config.enabled() {
+        Some(observability.metrics_surface(metrics_config).bind().await?)
+    } else {
+        None
+    };
 
     let storage_root = PathBuf::from(
         env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()),
@@ -36,6 +64,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         pool_config,
     )
     .await?;
+    if metrics_enabled {
+        spawn_postgres_pool_metrics(registry.pool().clone());
+    }
     let auth_settings = AuthSettings {
         session_secret: env::var("CRANK_SESSION_SECRET")?,
         password_pepper: env::var("CRANK_PASSWORD_PEPPER")?,
@@ -74,10 +105,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     .with_outbound_http_policy(outbound_http_policy)
     .with_identity_provider(std::sync::Arc::new(identity_provider))
     .build();
+    let invocation_log_retention_days =
+        positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
     service.bootstrap_admin_user().await?;
     if env_flag("CRANK_DEMO_SEED") {
         service.seed_demo_assets().await?;
     }
+    spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days);
     let state = AppState {
         service,
         api_rate_limiter: if cache_config.backend.is_external() {
@@ -92,6 +126,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
 
     info!(
+        name: "admin.postgres_pool.configured",
         runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
         admin_rate_limit_rps = api_rate_limit.requests_per_second,
         admin_rate_limit_burst = api_rate_limit.burst,
@@ -101,15 +136,76 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         acquire_timeout_ms = pool_config.acquire_timeout_ms,
         idle_timeout_ms = pool_config.idle_timeout_ms,
         max_lifetime_ms = pool_config.max_lifetime_ms,
+        invocation_log_retention_days,
         "postgres pool configured"
     );
-    info!("admin-api listening on {}", socket_addr);
+    info!(
+        name: "admin.server.listening",
+        bind_address = %socket_addr,
+        "admin-api listening"
+    );
+    *startup_completed = true;
 
+    if let Some(metrics_server) = metrics_server {
+        tokio::select! {
+            result = axum::serve(listener, make_service) => result?,
+            result = metrics_server.serve() => result?,
+        }
+    } else {
         axum::serve(listener, make_service).await?;
+    }
 
     Ok(())
 }
 
+fn positive_i64_from_env(
+    name: &'static str,
+    default: i64,
+) -> Result<i64, Box<dyn std::error::Error>> {
+    let value = match env::var(name) {
+        Ok(raw) => raw.parse::<i64>()?,
+        Err(env::VarError::NotPresent) => default,
+        Err(error) => return Err(error.into()),
+    };
+    if value <= 0 {
+        return Err(format!("{name} must be greater than zero").into());
+    }
+    Ok(value)
+}
+
+fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(60 * 60));
+        loop {
+            interval.tick().await;
+            let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days);
+            match service.cleanup_invocation_logs_before(cutoff).await {
+                Ok(removed) if removed > 0 => info!(
+                    name: "admin.invocation_log_cleanup.completed",
+                    removed,
+                    "expired invocation logs removed"
+                ),
+                Ok(_) => {}
+                Err(_) => warn!(
+                    name: "admin.invocation_log_cleanup.failed",
+                    error_category = "registry_cleanup",
+                    "failed to remove expired invocation logs"
+                ),
+            }
+        }
+    });
+}
+
+fn spawn_postgres_pool_metrics(pool: PgPool) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(5));
+        loop {
+            interval.tick().await;
+            crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
+        }
+    });
+}
+
 fn env_flag(name: &str) -> bool {
     matches!(
         env::var(name)

apps/admin-api/src/rate_limit.rs

@@ -6,7 +6,7 @@ use axum::{
     middleware::Next,
     response::Response,
 };
-use crank_runtime::RateLimitRejection;
+use crank_runtime::{RateLimitCheckError, RateLimitRejection};
 
 use crate::{error::ApiError, state::AppState};
 
@@ -25,11 +25,16 @@ pub async fn apply_api_rate_limit(
         peer_ip,
         state.trust_forwarded_headers,
     );
-    if let Err(rejection) = state.api_rate_limiter.check(&key).await {
+    if let Err(error) = state.api_rate_limiter.check(&key).await {
-        return Err(ApiError::rate_limited_with_context(
+        return match error {
+            RateLimitCheckError::Rejected(rejection) => Err(ApiError::rate_limited_with_context(
                 "request rate limit exceeded",
                 rejection_context(rejection),
-        ));
+            )),
+            RateLimitCheckError::StoreUnavailable => {
+                Err(ApiError::internal("rate limit service unavailable"))
+            }
+        };
     }
 
     Ok(next.run(request).await)

apps/admin-api/src/request_context.rs

@@ -4,11 +4,10 @@ use axum::{
     middleware::Next,
     response::Response,
 };
-use tracing::info;
+use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
-use uuid::Uuid;
+use tracing::{Instrument, info, info_span};
 
 pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
-const MAX_REQUEST_ID_LEN: usize = 128;
 
 #[derive(Clone, Debug)]
 pub struct RequestContext {
@@ -21,10 +20,18 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
     };
     let method = request.method().clone();
     let path = request.uri().path().to_owned();
+    let span = info_span!(
+        target: "crank::trace",
+        "http.request",
+        request_id = %context.request_id,
+    );
+    set_remote_trace_parent(&span, request.headers());
     request.extensions_mut().insert(context.clone());
 
-    let mut response = next.run(request).await;
+    with_request_correlation(context.request_id.clone(), async move {
+        let mut response = next.run(request).instrument(span).await;
         info!(
+            name: "admin.request.completed",
             request_id = %context.request_id,
             method = %method,
             path,
@@ -35,131 +42,31 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
             response.headers_mut().insert(REQUEST_ID_HEADER, value);
         }
         response
+    })
+    .await
 }
 
 fn resolve_request_id(headers: &HeaderMap) -> String {
+    RequestId::resolve(
         headers
             .get(&REQUEST_ID_HEADER)
-        .and_then(|value| value.to_str().ok())
+            .and_then(|value| value.to_str().ok()),
-        .map(str::trim)
+    )
-        .filter(|value| is_valid_request_id(value))
+    .into_string()
-        .map(ToOwned::to_owned)
-        .unwrap_or_else(|| Uuid::now_v7().to_string())
-}
-
-fn is_valid_request_id(value: &str) -> bool {
-    !value.is_empty()
-        && value.len() <= MAX_REQUEST_ID_LEN
-        && value
-            .bytes()
-            .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
 }
 
 #[cfg(test)]
 mod tests {
-    use std::io;
-    use std::sync::{Arc, Mutex};
-
-    use axum::{Router, routing::get};
-    use reqwest::Client;
-    use tokio::net::TcpListener;
-    use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
-
-    use super::{REQUEST_ID_HEADER, apply_request_context, is_valid_request_id};
-
     #[test]
     fn accepts_visible_ascii_request_ids() {
-        assert!(is_valid_request_id("req_test_123"));
+        assert!(crank_observability::RequestId::is_valid("req_test_123"));
-        assert!(is_valid_request_id("trace-123/abc"));
+        assert!(crank_observability::RequestId::is_valid("trace-123/abc"));
     }
 
     #[test]
     fn rejects_empty_or_control_request_ids() {
-        assert!(!is_valid_request_id(""));
+        assert!(!crank_observability::RequestId::is_valid(""));
-        assert!(!is_valid_request_id("bad value"));
+        assert!(!crank_observability::RequestId::is_valid("bad value"));
-        assert!(!is_valid_request_id("bad\nvalue"));
+        assert!(!crank_observability::RequestId::is_valid("bad\nvalue"));
-    }
-
-    #[derive(Clone, Default)]
-    struct SharedLogWriter {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    impl SharedLogWriter {
-        fn output(&self) -> String {
-            String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
-        }
-    }
-
-    impl<'a> MakeWriter<'a> for SharedLogWriter {
-        type Writer = SharedLogGuard;
-
-        fn make_writer(&'a self) -> Self::Writer {
-            SharedLogGuard {
-                buffer: Arc::clone(&self.buffer),
-            }
-        }
-    }
-
-    struct SharedLogGuard {
-        buffer: Arc<Mutex<Vec<u8>>>,
-    }
-
-    impl io::Write for SharedLogGuard {
-        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
-            self.buffer.lock().unwrap().extend_from_slice(bytes);
-            Ok(bytes.len())
-        }
-
-        fn flush(&mut self) -> io::Result<()> {
-            Ok(())
-        }
-    }
-
-    #[tokio::test]
-    async fn logs_request_completion_with_request_id() {
-        let writer = SharedLogWriter::default();
-        let subscriber = tracing_subscriber::registry().with(
-            tracing_subscriber::fmt::layer()
-                .with_writer(writer.clone())
-                .without_time()
-                .with_ansi(false)
-                .with_target(false)
-                .compact()
-                .with_filter(LevelFilter::INFO),
-        );
-        let dispatch = tracing::Dispatch::new(subscriber);
-        let app = Router::new()
-            .route("/probe", get(|| async { "ok" }))
-            .layer(axum::middleware::from_fn(apply_request_context));
-        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let address = listener.local_addr().unwrap();
-
-        let _guard = tracing::dispatcher::set_default(&dispatch);
-        tokio::spawn(async move {
-            axum::serve(listener, app).await.unwrap();
-        });
-
-        let response = Client::new()
-            .get(format!("http://{address}/probe"))
-            .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
-            .send()
-            .await
-            .unwrap();
-
-        assert_eq!(response.status(), reqwest::StatusCode::OK);
-        assert_eq!(
-            response.headers()[REQUEST_ID_HEADER.as_str()]
-                .to_str()
-                .unwrap(),
-            "req_admin_trace_123"
-        );
-
-        let logs = writer.output();
-        assert!(logs.contains("admin request completed"));
-        assert!(logs.contains("req_admin_trace_123"));
-        assert!(logs.contains("GET"));
-        assert!(logs.contains("/probe"));
-        assert!(logs.contains("status=200"));
     }
 }

apps/admin-api/src/routes.rs

@@ -10,12 +10,36 @@ pub mod secrets;
 pub mod upstreams;
 pub mod workspaces;
 
-use axum::Json;
+use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
 use serde_json::json;
 
+use crate::state::AppState;
+
 pub async fn health() -> Json<serde_json::Value> {
     Json(json!({
         "service": "admin-api",
         "status": "ok"
     }))
 }
+
+pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
+    match state.service.readiness().await {
+        Ok(()) => (
+            StatusCode::OK,
+            Json(json!({
+                "service": "admin-api",
+                "status": "ready",
+                "checks": { "postgres": "ready" }
+            })),
+        ),
+        Err(error) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "service": "admin-api",
+                "status": "not_ready",
+                "checks": { "postgres": "not_ready" },
+                "error": error.to_string()
+            })),
+        ),
+    }
+}

apps/admin-api/src/routes/access.rs

@@ -18,7 +18,7 @@ pub async fn export_workspace(
 ) -> Result<Json<Value>, ApiError> {
     let exported = state
         .service
-        .export_workspace(&path.workspace_id.as_str().into())
+        .export_workspace_catalog_snapshot(&path.workspace_id.as_str().into())
         .await?;
     Ok(Json(json!(exported)))
 }

apps/admin-api/src/routes/auth.rs

@@ -103,7 +103,7 @@ pub async fn change_password(
 ) -> Result<StatusCode, ApiError> {
     state
         .service
-        .change_password(&session.user.id, payload)
+        .change_password(&session.user.id, &session.session_id, payload)
         .await?;
     Ok(StatusCode::NO_CONTENT)
 }

apps/admin-api/src/service.rs

@@ -12,16 +12,18 @@ use crank_core::{
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_registry::{
-    AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary,
+    AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef,
-    OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
+    OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
 };
 use crank_runtime::{
     OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
 };
 use crank_schema::{Schema, SchemaKind};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use serde_json::{Value, json};
 use sha2::{Digest, Sha256};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tracing::Instrument;
 use uuid::Uuid;
 
 mod agents;
@@ -74,6 +76,11 @@ pub struct AdminServiceBuilder {
 pub use crate::dto::*;
 
 impl AdminService {
+    pub async fn readiness(&self) -> Result<(), ApiError> {
+        self.registry.ping().await?;
+        Ok(())
+    }
+
     #[cfg(test)]
     pub fn new(
         registry: PostgresRegistry,
@@ -199,16 +206,33 @@ impl AdminServiceBuilder {
 }
 
 impl AdminService {
-    pub async fn export_workspace(
+    pub async fn export_workspace_catalog_snapshot(
         &self,
         workspace_id: &WorkspaceId,
-    ) -> Result<WorkspaceExportResponse, ApiError> {
+    ) -> Result<WorkspaceCatalogSnapshotResponse, ApiError> {
         let workspace = self.get_workspace(workspace_id).await?;
         let operations = self.list_operations(workspace_id).await?;
         let agents = self.list_agents(workspace_id).await?;
         let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?;
 
-        Ok(WorkspaceExportResponse {
+        Ok(WorkspaceCatalogSnapshotResponse {
+            kind: "workspace_catalog_snapshot".to_owned(),
+            format_version: "1".to_owned(),
+            restorable: false,
+            included: vec![
+                "workspace_settings".to_owned(),
+                "operation_summaries".to_owned(),
+                "agent_summaries".to_owned(),
+                "platform_api_key_metadata".to_owned(),
+            ],
+            excluded: vec![
+                "operation_versions_and_samples".to_owned(),
+                "agent_versions_and_bindings".to_owned(),
+                "secret_metadata_and_values".to_owned(),
+                "secret_values".to_owned(),
+                "invocation_logs_and_usage".to_owned(),
+                "authentication_sessions".to_owned(),
+            ],
             workspace,
             operations,
             agents,
@@ -239,9 +263,13 @@ impl AdminService {
             return Ok(None);
         };
 
-        let auth_profile = self
+        let span = Stage::AuthResolve.span();
-            .registry
+        let result = async {
-            .get_auth_profile(workspace_id, auth_profile_id)
+            let auth_profile = observe_db_query(
+                DbOperation::AuthProfileRead,
+                self.registry
+                    .get_auth_profile(workspace_id, auth_profile_id),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "load auth profile",
@@ -255,6 +283,17 @@ impl AdminService {
                 .await
                 .map(Some)
         }
+        .instrument(span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&span),
+            Err(_) => {
+                StageOutcome::Error.record(&span);
+                ErrorCategory::Configuration.record(&span);
+            }
+        }
+        result
+    }
 
     async fn resolve_auth_profile(
         &self,
@@ -265,9 +304,10 @@ impl AdminService {
         let used_at = OffsetDateTime::now_utc();
 
         for secret_id in auth_profile.config.secret_ids() {
-            let secret = self
+            let secret = observe_db_query(
-                .registry
+                DbOperation::SecretRead,
-                .get_secret(workspace_id, secret_id)
+                self.registry.get_secret(workspace_id, secret_id),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "load secret",
@@ -276,9 +316,11 @@ impl AdminService {
             .ok_or_else(|| RuntimeError::MissingSecret {
                 secret_id: secret_id.as_str().to_owned(),
             })?;
-            let version = self
+            let version = observe_db_query(
-                .registry
+                DbOperation::SecretRead,
-                .get_current_secret_version(workspace_id, secret_id)
+                self.registry
+                    .get_current_secret_version(workspace_id, secret_id),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "load current secret version",
@@ -292,8 +334,11 @@ impl AdminService {
                 &version.secret_version.key_version,
                 &version.secret_version.ciphertext,
             )?;
+            observe_db_query(
+                DbOperation::SecretTouch,
                 self.registry
-                .touch_secret(workspace_id, secret_id, &used_at)
+                    .touch_secret(workspace_id, secret_id, &used_at),
+            )
             .await
             .map_err(|error| RuntimeError::SecretCrypto {
                 operation: "touch secret",
@@ -412,7 +457,7 @@ impl AdminService {
     async fn record_invocation(
         &self,
         request: InvocationRecordRequest<'_>,
-    ) -> Result<(), ApiError> {
+    ) -> InvocationHistoryWriteOutcome {
         let log = InvocationLog {
             id: InvocationLogId::new(new_prefixed_id("log")),
             workspace_id: request.workspace_id.clone(),
@@ -432,11 +477,70 @@ impl AdminService {
             created_at: OffsetDateTime::now_utc(),
         };
 
-        self.registry
+        let history_span = crank_trace::Stage::HistoryWrite.span();
+        let (outcome, db_span) = async {
+            let db_span = crank_trace::Stage::DbQuery
+                .db_span(crank_trace::DbOperation::InvocationHistoryWrite)
+                .expect("database stage");
+            let outcome = self
+                .registry
                 .create_invocation_log(CreateInvocationLogRequest { log: &log })
-            .await?;
+                .instrument(db_span.clone())
+                .await;
+            (outcome, db_span)
+        }
+        .instrument(history_span.clone())
+        .await;
+        match outcome {
+            InvocationHistoryWriteOutcome::Recorded => {
+                crank_trace::StageOutcome::Success.record(&db_span);
+                crank_trace::StageOutcome::Success.record(&history_span);
+            }
+            InvocationHistoryWriteOutcome::Lost(_) => {
+                crank_trace::StageOutcome::Error.record(&db_span);
+                crank_trace::ErrorCategory::Database.record(&db_span);
+                crank_trace::StageOutcome::Error.record(&history_span);
+                crank_trace::ErrorCategory::History.record(&history_span);
+            }
+        }
+        drop(db_span);
+        drop(history_span);
+        observe_invocation_history_outcome(
+            outcome,
+            request.request_id,
+            request.status,
+            "admin_test_run",
+        );
+        outcome
+    }
+}
 
-        Ok(())
+fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: crank_core::InvocationStatus,
+    source: &'static str,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    tracing::warn!(
+        name: "admin.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source,
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
+    match status {
+        crank_core::InvocationStatus::Ok => "ok",
+        crank_core::InvocationStatus::Error => "error",
     }
 }
 
@@ -543,6 +647,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
         RuntimeError::RestAdapter(_) => "rest_error",
         RuntimeError::ProtocolAdapter(_) => "adapter_error",
         RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
@@ -761,7 +869,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule {
 #[cfg(test)]
 #[allow(clippy::items_after_test_module)]
 mod tests {
-    use super::{validate_profile_display_name, validate_profile_email};
+    use std::{
+        io,
+        sync::{Arc, Mutex},
+    };
+
+    use crank_core::InvocationStatus;
+    use crank_observability::{
+        ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+        operational_incident_total,
+    };
+    use crank_registry::{
+        InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
+    };
+    use serde_json::Value;
+    use tracing_subscriber::fmt::MakeWriter;
+
+    use super::{
+        observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email,
+    };
 
     #[test]
     fn validates_profile_identity_fields() {
@@ -782,6 +908,81 @@ mod tests {
         assert!(validate_profile_display_name(&"x".repeat(81)).is_err());
         assert!(validate_profile_email("owner <html>@crank.local").is_err());
     }
+
+    #[test]
+    fn emits_bounded_history_loss_incident() {
+        let writer = SharedLogWriter::default();
+        let subscriber = crank_observability::build_subscriber(
+            ObservabilityConfig::new(
+                ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+                "info",
+                RedactionLimits::default(),
+            ),
+            writer.clone(),
+        )
+        .unwrap();
+        let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let _guard = tracing::dispatcher::set_default(&dispatch);
+
+        observe_invocation_history_outcome(
+            InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
+                category: InvocationHistoryLossCategory::InvalidRecord,
+            }),
+            Some("req_admin_dc08"),
+            InvocationStatus::Error,
+            "admin_test_run",
+        );
+
+        let output = writer.output();
+        assert!(!output.contains("dc08-canary-secret"));
+        let event: Value = output
+            .lines()
+            .map(|line| serde_json::from_str(line).unwrap())
+            .find(|event: &Value| event["event"] == "admin.invocation_history.lost")
+            .unwrap();
+        assert_eq!(event["request_id"], "req_admin_dc08");
+        assert_eq!(event["fields"]["source"], "admin_test_run");
+        assert_eq!(event["fields"]["invocation_status"], "error");
+        assert_eq!(event["fields"]["error_category"], "invalid_record");
+        assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+    }
+
+    #[derive(Clone, Default)]
+    struct SharedLogWriter {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl SharedLogWriter {
+        fn output(&self) -> String {
+            String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+        }
+    }
+
+    impl<'a> MakeWriter<'a> for SharedLogWriter {
+        type Writer = SharedLogGuard;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            SharedLogGuard {
+                buffer: Arc::clone(&self.buffer),
+            }
+        }
+    }
+
+    struct SharedLogGuard {
+        buffer: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl io::Write for SharedLogGuard {
+        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+            self.buffer.lock().unwrap().extend_from_slice(bytes);
+            Ok(bytes.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
 }
 
 fn enrich_operation_summary(

apps/admin-api/src/service/agents.rs

@@ -296,7 +296,12 @@ impl AdminService {
                 bindings: &[],
             })
             .await?;
-        info!(agent_id = %agent_id.as_str(), version = 1, "agent created");
+        info!(
+            name: "admin.agent.created",
+            agent_id = %agent_id.as_str(),
+            version = 1,
+            "agent created"
+        );
 
         Ok(CreatedAgentResponse {
             agent_id: agent_id.as_str().to_owned(),
@@ -417,6 +422,7 @@ impl AdminService {
             })
             .await?;
         info!(
+            name: "admin.agent.bindings_saved",
             agent_id = %agent_id.as_str(),
             version = current_version.version,
             binding_count = bindings.len(),
@@ -531,7 +537,12 @@ impl AdminService {
                 published_by: None,
             })
             .await?;
-        info!(agent_id = %agent_id.as_str(), version, "agent published");
+        info!(
+            name: "admin.agent.published",
+            agent_id = %agent_id.as_str(),
+            version,
+            "agent published"
+        );
 
         Ok(PublishAgentResponse {
             agent_id: agent_id.as_str().to_owned(),
@@ -589,7 +600,11 @@ impl AdminService {
         self.registry
             .unpublish_agent(workspace_id, agent_id, &updated_at)
             .await?;
-        info!(agent_id = %agent_id.as_str(), "agent moved to draft");
+        info!(
+            name: "admin.agent.unpublished",
+            agent_id = %agent_id.as_str(),
+            "agent moved to draft"
+        );
 
         Ok(AgentMutationResult {
             agent_id: agent_id.as_str().to_owned(),
@@ -609,7 +624,11 @@ impl AdminService {
         self.registry
             .archive_agent(workspace_id, agent_id, &updated_at)
             .await?;
-        info!(agent_id = %agent_id.as_str(), "agent archived");
+        info!(
+            name: "admin.agent.archived",
+            agent_id = %agent_id.as_str(),
+            "agent archived"
+        );
 
         Ok(AgentMutationResult {
             agent_id: agent_id.as_str().to_owned(),

apps/admin-api/src/service/auth.rs

@@ -22,7 +22,7 @@ impl AdminService {
         )?;
         let user_id = self
             .registry
-            .upsert_bootstrap_user(
+            .ensure_bootstrap_user(
                 &self.auth_settings.bootstrap_admin.email,
                 &self.auth_settings.bootstrap_admin.display_name,
                 &password_hash,
@@ -227,6 +227,7 @@ impl AdminService {
     pub async fn change_password(
         &self,
         user_id: &crank_core::UserId,
+        current_session_id: &UserSessionId,
         payload: ChangePasswordPayload,
     ) -> Result<(), ApiError> {
         if payload.new_password.len() < 12 {
@@ -257,7 +258,11 @@ impl AdminService {
         let password_hash =
             hash_password(&payload.new_password, &self.auth_settings.password_pepper)?;
         self.registry
-            .update_user_password(user_id, &password_hash)
+            .update_user_password_and_revoke_other_sessions(
+                user_id,
+                current_session_id,
+                &password_hash,
+            )
             .await?;
 
         Ok(())

apps/admin-api/src/service/demo.rs

@@ -131,6 +131,7 @@ impl AdminService {
             Ok(()) => Ok(()),
             Err(RegistryError::OperationHasPublishedAgentBindings { .. }) => {
                 tracing::warn!(
+                    name: "admin.demo_operation.cleanup_skipped",
                     operation_id = %operation_id.as_str(),
                     "legacy demo operation is still bound to a published agent; leaving it in place"
                 );
@@ -335,7 +336,7 @@ impl AdminService {
             }),
             response_preview: demo_rest_response_sample(),
         })
-        .await?;
+        .await;
         Ok(())
     }
 }

apps/admin-api/src/service/import_export.rs

@@ -110,6 +110,7 @@ impl AdminService {
                         warnings,
                     };
                     info!(
+                        name: "admin.operation.imported",
                         operation_id = %response.operation_id,
                         version = response.version,
                         "operation imported by upsert"
@@ -125,6 +126,7 @@ impl AdminService {
                         warnings,
                     };
                     info!(
+                        name: "admin.operation.imported",
                         operation_id = %response.operation_id,
                         version = response.version,
                         "operation imported by upsert"

apps/admin-api/src/service/imports.rs

@@ -8,9 +8,11 @@ use crank_import::rest::{
     ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
 };
 use crank_registry::{
-    CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus,
+    ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind,
+    ImportJobStatus, ImportOperationDraft,
 };
 use serde_json::json;
+use sha2::{Digest, Sha256};
 use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
 use tracing::{info, instrument};
 
@@ -50,7 +52,7 @@ impl AdminService {
                 kind: ImportJobKind::OpenApi,
                 source_format: &preview.source.format,
                 source_version: preview.source.version.as_deref(),
-                status: ImportJobStatus::Completed,
+                status: ImportJobStatus::Pending,
                 preview_payload: &preview_payload,
                 created_at: &now,
                 expires_at: &expires_at,
@@ -99,8 +101,12 @@ impl AdminService {
             return Err(ApiError::validation("import job kind is not openapi"));
         }
 
-        let preview: crank_import::rest::ImportPreview =
+        let stored_preview = job
-            serde_json::from_value(job.preview_payload.clone())
+            .preview_payload
+            .get("preview")
+            .cloned()
+            .unwrap_or_else(|| job.preview_payload.clone());
+        let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview)
             .map_err(|error| ApiError::internal(error.to_string()))?;
         let selected = payload
             .selected_operation_keys
@@ -120,10 +126,8 @@ impl AdminService {
             }
         }
 
-        let mut created = Vec::new();
         let mut skipped = Vec::new();
-        let mut findings = Vec::new();
+        let mut operations = Vec::new();
-        let mut created_ids = Vec::new();
 
         for operation_key in selected {
             let Some(candidate) = candidates.get(&operation_key) else {
@@ -137,44 +141,7 @@ impl AdminService {
             let mut draft =
                 operation_draft_from_candidate(candidate, payload.server_url.as_deref());
             attach_import_findings(&mut draft, candidate);
-            if let Some(existing_name) = self
+            let operation = self.new_operation_snapshot(OperationPayload {
-                .find_operation_by_name(workspace_id, &draft.name)
-                .await?
-                .map(|operation| operation.name)
-            {
-                if payload.conflict_mode == "skip" {
-                    skipped.push(OpenApiImportSkippedOperation {
-                        operation_key: candidate.key.clone(),
-                        name: draft.name.clone(),
-                        reason: "operation with this name already exists".to_owned(),
-                    });
-                    findings.push(ImportFinding {
-                        code: "operation_name_conflict".to_owned(),
-                        severity: ImportFindingSeverity::Warning,
-                        message: format!(
-                            "Операция {} уже существует и была пропущена.",
-                            draft.name
-                        ),
-                        operation_key: Some(candidate.key.clone()),
-                    });
-                    continue;
-                }
-
-                let renamed = self
-                    .next_available_operation_name(workspace_id, &draft.name)
-                    .await?;
-                findings.push(ImportFinding {
-                    code: "operation_name_renamed".to_owned(),
-                    severity: ImportFindingSeverity::Info,
-                    message: format!(
-                        "Операция {existing_name} уже существует, новый черновик создан как {renamed}."
-                    ),
-                    operation_key: Some(candidate.key.clone()),
-                });
-                draft.name = renamed;
-            }
-
-            let payload = OperationPayload {
                 name: draft.name.clone(),
                 display_name: draft.display_name.clone(),
                 category: draft.category,
@@ -197,27 +164,74 @@ impl AdminService {
                 },
                 tool_description: draft.tool_description,
                 wizard_state: draft.wizard_state,
-            };
+            })?;
-            let result = self.create_operation(workspace_id, payload).await?;
+            operations.push(ImportOperationDraft {
-            created_ids.push(result.operation_id.clone());
+                operation_key: candidate.key.clone(),
-            created.push(OpenApiImportCreatedOperation {
+                operation,
-                operation_id: result.operation_id,
-                name: draft.name,
-                version: result.version,
             });
         }
 
         let finished_at = OffsetDateTime::now_utc();
-        self.registry
+        let application_key = openapi_application_key(&payload)?;
-            .finish_import_job(FinishImportJobRequest {
+        let conflict_mode = if payload.conflict_mode == "skip" {
+            ImportConflictMode::Skip
+        } else {
+            ImportConflictMode::Rename
+        };
+        let applied = self
+            .registry
+            .apply_import_job(ApplyImportJobRequest {
                 id: job_id,
-                status: ImportJobStatus::Completed,
+                workspace_id,
-                created_operation_ids: &json!(created_ids),
+                application_key: &application_key,
-                error_text: None,
+                conflict_mode,
+                operations: &operations,
                 finished_at: &finished_at,
             })
             .await?;
+
+        let created = applied
+            .created
+            .iter()
+            .map(|operation| OpenApiImportCreatedOperation {
+                operation_id: operation.operation_id.as_str().to_owned(),
+                name: operation.name.clone(),
+                version: operation.version,
+            })
+            .collect::<Vec<_>>();
+        let mut findings = applied
+            .created
+            .iter()
+            .filter_map(|operation| {
+                operation.renamed_from.as_ref().map(|previous_name| ImportFinding {
+                    code: "operation_name_renamed".to_owned(),
+                    severity: ImportFindingSeverity::Info,
+                    message: format!(
+                        "Операция {previous_name} уже существует, новый черновик создан как {}.",
+                        operation.name
+                    ),
+                    operation_key: Some(operation.operation_key.clone()),
+                })
+            })
+            .collect::<Vec<_>>();
+        for operation in applied.skipped {
+            skipped.push(OpenApiImportSkippedOperation {
+                operation_key: operation.operation_key.clone(),
+                name: operation.name.clone(),
+                reason: "operation with this name already exists".to_owned(),
+            });
+            findings.push(ImportFinding {
+                code: operation.reason,
+                severity: ImportFindingSeverity::Warning,
+                message: format!(
+                    "Операция {} уже существует и была пропущена.",
+                    operation.name
+                ),
+                operation_key: Some(operation.operation_key),
+            });
+        }
         info!(
+            name: "admin.openapi_import.completed",
             created = created.len(),
             skipped = skipped.len(),
             "openapi import created drafts"
@@ -229,25 +243,21 @@ impl AdminService {
             findings,
         })
     }
-
-    async fn next_available_operation_name(
-        &self,
-        workspace_id: &WorkspaceId,
-        base_name: &str,
-    ) -> Result<String, ApiError> {
-        for index in 2.. {
-            let candidate = format!("{base_name}_{index}");
-            if self
-                .find_operation_by_name(workspace_id, &candidate)
-                .await?
-                .is_none()
-            {
-                return Ok(candidate);
-            }
 }
 
-        unreachable!()
+fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
-    }
+    let selected_operation_keys = payload
+        .selected_operation_keys
+        .iter()
+        .cloned()
+        .collect::<BTreeSet<_>>();
+    let canonical = serde_json::to_vec(&json!({
+        "selected_operation_keys": selected_operation_keys,
+        "server_url": payload.server_url.as_deref(),
+        "conflict_mode": payload.conflict_mode.as_str(),
+    }))
+    .map_err(|error| ApiError::internal(error.to_string()))?;
+    Ok(format!("{:x}", Sha256::digest(canonical)))
 }
 
 fn attach_import_findings(

apps/admin-api/src/service/observability.rs

@@ -19,6 +19,16 @@ use crate::{
 };
 
 impl AdminService {
+    pub async fn cleanup_invocation_logs_before(
+        &self,
+        cutoff: OffsetDateTime,
+    ) -> Result<u64, ApiError> {
+        self.registry
+            .delete_invocation_logs_before(cutoff)
+            .await
+            .map_err(ApiError::from)
+    }
+
     #[instrument(skip(self))]
     pub async fn list_logs(
         &self,

apps/admin-api/src/service/operations.rs

@@ -141,7 +141,6 @@ impl AdminService {
         workspace_id: &WorkspaceId,
         payload: OperationPayload,
     ) -> Result<CreatedOperationResponse, ApiError> {
-        self.validate_operation_payload(&payload)?;
         self.ensure_workspace_exists(workspace_id).await?;
 
         if self
@@ -155,10 +154,36 @@ impl AdminService {
             ));
         }
 
+        let snapshot = self.new_operation_snapshot(payload)?;
+        let operation_id = snapshot.id.clone();
+
+        self.registry
+            .create_operation(workspace_id, &snapshot, None)
+            .await?;
+        info!(
+            name: "admin.operation.created",
+            operation_id = %operation_id.as_str(),
+            version = 1,
+            "operation created"
+        );
+
+        Ok(CreatedOperationResponse {
+            operation_id: operation_id.as_str().to_owned(),
+            workspace_id: workspace_id.as_str().to_owned(),
+            version: 1,
+            status: OperationStatus::Draft,
+            updated_at: format_timestamp(snapshot.updated_at),
+        })
+    }
+
+    pub(super) fn new_operation_snapshot(
+        &self,
+        payload: OperationPayload,
+    ) -> Result<RegistryOperation, ApiError> {
+        self.validate_operation_payload(&payload)?;
         let now = OffsetDateTime::now_utc();
-        let operation_id = OperationId::new(new_prefixed_id("op"));
+        Ok(RegistryOperation {
-        let snapshot = RegistryOperation {
+            id: OperationId::new(new_prefixed_id("op")),
-            id: operation_id.clone(),
             name: payload.name,
             display_name: payload.display_name,
             category: payload.category,
@@ -183,19 +208,6 @@ impl AdminService {
             created_at: now,
             updated_at: now,
             published_at: None,
-        };
-
-        self.registry
-            .create_operation(workspace_id, &snapshot, None)
-            .await?;
-        info!(operation_id = %operation_id.as_str(), version = 1, "operation created");
-
-        Ok(CreatedOperationResponse {
-            operation_id: operation_id.as_str().to_owned(),
-            workspace_id: workspace_id.as_str().to_owned(),
-            version: 1,
-            status: OperationStatus::Draft,
-            updated_at: format_timestamp(snapshot.updated_at),
         })
     }
 
@@ -279,7 +291,12 @@ impl AdminService {
                 created_by: None,
             })
             .await?;
-        info!(operation_id = %operation_id.as_str(), version, "operation version created");
+        info!(
+            name: "admin.operation.version_created",
+            operation_id = %operation_id.as_str(),
+            version,
+            "operation version created"
+        );
 
         Ok(CreatedOperationResponse {
             operation_id: operation_id.as_str().to_owned(),
@@ -364,7 +381,12 @@ impl AdminService {
                 published_by: None,
             })
             .await?;
-        info!(operation_id = %operation_id.as_str(), version, "operation published");
+        info!(
+            name: "admin.operation.published",
+            operation_id = %operation_id.as_str(),
+            version,
+            "operation published"
+        );
 
         Ok(PublishResponse {
             operation_id: operation_id.as_str().to_owned(),
@@ -431,10 +453,15 @@ impl AdminService {
             .await?;
         let runtime = RuntimeOperation::from(record.snapshot.clone());
         let mode = ExecutionMode::Unary;
-        let request_preview =
+        let preview_span = crank_trace::Stage::RuntimeArgumentsMap.span();
-            match build_request_preview(&record.snapshot.input_mapping, &payload.input) {
+        let preview_result = preview_span
+            .in_scope(|| build_request_preview(&record.snapshot.input_mapping, &payload.input));
+        let request_preview = match preview_result {
             Ok(preview) => preview,
             Err(error) => {
+                crank_trace::StageOutcome::Error.record(&preview_span);
+                crank_trace::ErrorCategory::Mapping.record(&preview_span);
+                drop(preview_span);
                 self.record_invocation(InvocationRecordRequest {
                     workspace_id,
                     agent_id: None,
@@ -450,7 +477,7 @@ impl AdminService {
                     request_preview: Value::Null,
                     response_preview: Value::Null,
                 })
-                    .await?;
+                .await;
                 return Ok(TestRunResult {
                     ok: false,
                     mode,
@@ -462,6 +489,8 @@ impl AdminService {
                 });
             }
         };
+        crank_trace::StageOutcome::Success.record(&preview_span);
+        drop(preview_span);
 
         let resolved_auth = self
             .resolve_operation_auth(workspace_id, &runtime.execution_config)
@@ -497,7 +526,7 @@ impl AdminService {
                     request_preview: request_preview.clone(),
                     response_preview: response_preview.clone(),
                 })
-                .await?;
+                .await;
                 Ok(TestRunResult {
                     ok: true,
                     mode,
@@ -524,7 +553,7 @@ impl AdminService {
                     request_preview: request_preview.clone(),
                     response_preview: Value::Null,
                 })
-                .await?;
+                .await;
                 Ok(TestRunResult {
                     ok: false,
                     mode,

apps/admin-api/src/service/samples.rs

@@ -50,6 +50,7 @@ impl AdminService {
             .save_sample_metadata(SaveSampleMetadataRequest { sample: &metadata })
             .await?;
         info!(
+            name: "admin.sample.saved",
             operation_id = %operation_id.as_str(),
             sample_id = %metadata.id.as_str(),
             version,
@@ -119,7 +120,11 @@ impl AdminService {
             input_mapping,
             output_mapping,
         };
-        info!(operation_id = %operation_id.as_str(), "draft generated from samples");
+        info!(
+            name: "admin.operation_draft.generated",
+            operation_id = %operation_id.as_str(),
+            "draft generated from samples"
+        );
 
         Ok(result)
     }

apps/admin-api/src/service/secrets.rs

@@ -92,7 +92,11 @@ impl AdminService {
                 created_by,
             })
             .await?;
-        info!(secret_id = %secret.id.as_str(), "secret created");
+        info!(
+            name: "admin.secret.created",
+            secret_id = %secret.id.as_str(),
+            "secret created"
+        );
 
         Ok(secret)
     }
@@ -126,7 +130,11 @@ impl AdminService {
                 created_by,
             })
             .await?;
-        info!(secret_id = %secret_id.as_str(), "secret rotated");
+        info!(
+            name: "admin.secret.rotated",
+            secret_id = %secret_id.as_str(),
+            "secret rotated"
+        );
 
         self.get_secret(workspace_id, secret_id).await
     }
@@ -152,7 +160,11 @@ impl AdminService {
             .into());
         }
         self.registry.delete_secret(workspace_id, secret_id).await?;
-        info!(secret_id = %secret_id.as_str(), "secret deleted");
+        info!(
+            name: "admin.secret.deleted",
+            secret_id = %secret_id.as_str(),
+            "secret deleted"
+        );
         Ok(())
     }
 
@@ -201,7 +213,11 @@ impl AdminService {
                 profile: &profile,
             })
             .await?;
-        info!(auth_profile_id = %profile.id.as_str(), "auth profile created");
+        info!(
+            name: "admin.auth_profile.created",
+            auth_profile_id = %profile.id.as_str(),
+            "auth profile created"
+        );
 
         Ok(profile)
     }

apps/admin-api/src/service/upstreams.rs

@@ -75,7 +75,11 @@ impl AdminService {
                 upstream: &upstream,
             })
             .await?;
-        info!(upstream_id = %upstream.id.as_str(), "workspace upstream saved");
+        info!(
+            name: "admin.upstream.saved",
+            upstream_id = %upstream.id.as_str(),
+            "workspace upstream saved"
+        );
 
         Ok(upstream)
     }

apps/admin-api/tests/integration/community_access_usage.rs

@@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind};
 use serde_json::{Value, json};
 use serial_test::serial;
 use tokio::net::TcpListener;
+use uuid::Version;
 
 use admin_api::{
     app::build_app,
@@ -40,6 +41,8 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
 const TEST_SESSION_SECRET: &str = "test-session-secret";
 const TEST_MASTER_KEY: &str = "test-master-key";
 
+mod history_loss;
+
 struct TestServer {
     base_url: String,
     shutdown: Option<tokio::sync::oneshot::Sender<()>>,
@@ -385,6 +388,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() {
         exported["workspace"]["workspace"]["id"],
         DEFAULT_WORKSPACE_ID
     );
+    assert_eq!(exported["kind"], "workspace_catalog_snapshot");
+    assert_eq!(exported["format_version"], "1");
+    assert_eq!(exported["restorable"], false);
+    assert_eq!(
+        exported["included"],
+        json!([
+            "workspace_settings",
+            "operation_summaries",
+            "agent_summaries",
+            "platform_api_key_metadata"
+        ])
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("secret_values"))
+    );
+    assert!(
+        exported["excluded"]
+            .as_array()
+            .unwrap()
+            .contains(&json!("invocation_logs_and_usage"))
+    );
     assert!(exported.get("memberships").is_none());
     assert!(exported.get("invitations").is_none());
 
@@ -574,6 +601,25 @@ async fn updates_profile_and_changes_password() {
         .unwrap()
         .to_owned();
     let client = authorized_client(&base_url).await;
+    let second_client = reqwest::Client::builder()
+        .cookie_store(true)
+        .build()
+        .unwrap();
+    let second_login = second_client
+        .post(format!("{root_url}/api/auth/login"))
+        .json(&json!({
+            "email": TEST_AUTH_EMAIL,
+            "password": TEST_AUTH_PASSWORD,
+        }))
+        .send()
+        .await
+        .unwrap();
+    let second_login_status = second_login.status();
+    let second_login_body = second_login.text().await.unwrap();
+    assert!(
+        second_login_status.is_success(),
+        "second login failed with {second_login_status}: {second_login_body}"
+    );
 
     let profile = assert_success_json(
         client
@@ -615,6 +661,21 @@ async fn updates_profile_and_changes_password() {
         .status();
     assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);
 
+    let current_session_status = client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    let other_session_status = second_client
+        .get(format!("{root_url}/api/auth/profile"))
+        .send()
+        .await
+        .unwrap()
+        .status();
+    assert_eq!(current_session_status, reqwest::StatusCode::OK);
+    assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
+
     let relogin_client = reqwest::Client::builder()
         .cookie_store(true)
         .build()
@@ -855,7 +916,10 @@ async fn generates_request_id_for_test_run_invocations() {
         .unwrap()
         .to_owned();
 
-    assert!(!request_id.is_empty());
+    assert_eq!(
+        uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
     response.error_for_status().unwrap();
 
     let logs = client

apps/admin-api/tests/integration/community_access_usage/history_loss.rs

@@ -0,0 +1,141 @@
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+use axum::{Json, Router, extract::State, routing::post};
+use tokio::{net::TcpListener, sync::Notify};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn preserves_external_success_when_invocation_history_is_lost() {
+    let registry = test_registry().await;
+    let registry_for_failure = registry.clone();
+    let storage_root = test_storage_root("observability_history_loss");
+    let upstream = spawn_blocking_upstream_server().await;
+    let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
+    let client = authorized_client(&base_url).await;
+
+    let created = client
+        .post(format!("{base_url}/operations"))
+        .json(&test_operation_payload(
+            &upstream.base_url,
+            "crm_history_loss",
+        ))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    let operation_id = created["operation_id"].as_str().unwrap().to_owned();
+    let request_client = client.clone();
+    let request_url = format!("{base_url}/operations/{operation_id}/test-runs");
+    let before = crank_observability::operational_incident_total(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+
+    let request = tokio::spawn(async move {
+        request_client
+            .post(request_url)
+            .header("x-request-id", "req_dc08_admin")
+            .json(&json!({
+                "version": 1,
+                "input": { "email": "dc08-canary-secret@example.com" }
+            }))
+            .send()
+            .await
+            .unwrap()
+    });
+
+    upstream.started.notified().await;
+    registry_for_failure
+        .delete_operation(
+            &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
+            &OperationId::new(operation_id.clone()),
+        )
+        .await
+        .unwrap();
+    upstream.release.notify_one();
+
+    let response = request.await.unwrap();
+    assert_eq!(response.status(), reqwest::StatusCode::OK);
+    assert_eq!(
+        response.headers()["x-request-id"].to_str().unwrap(),
+        "req_dc08_admin"
+    );
+    let body = response.json::<Value>().await.unwrap();
+    assert_eq!(body["ok"], true);
+    assert_eq!(body["response_preview"]["id"], "lead_123");
+    assert_eq!(upstream.calls.load(Ordering::SeqCst), 1);
+    assert!(
+        crank_observability::operational_incident_total(
+            crank_observability::OperationalIncident::InvocationHistoryLost
+        ) > before
+    );
+
+    let logs = client
+        .get(format!("{base_url}/logs?period=7d"))
+        .send()
+        .await
+        .unwrap()
+        .json::<Value>()
+        .await
+        .unwrap();
+    assert!(logs["items"].as_array().unwrap().is_empty());
+}
+
+struct BlockingUpstream {
+    base_url: String,
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+#[derive(Clone)]
+struct BlockingUpstreamState {
+    started: Arc<Notify>,
+    release: Arc<Notify>,
+    calls: Arc<AtomicUsize>,
+}
+
+async fn spawn_blocking_upstream_server() -> BlockingUpstream {
+    let state = BlockingUpstreamState {
+        started: Arc::new(Notify::new()),
+        release: Arc::new(Notify::new()),
+        calls: Arc::new(AtomicUsize::new(0)),
+    };
+    let app = Router::new()
+        .route("/crm/leads", post(blocking_create_lead))
+        .with_state(state.clone());
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    BlockingUpstream {
+        base_url: format!("http://{address}"),
+        started: state.started,
+        release: state.release,
+        calls: state.calls,
+    }
+}
+
+async fn blocking_create_lead(
+    State(state): State<BlockingUpstreamState>,
+    Json(payload): Json<Value>,
+) -> Json<Value> {
+    state.calls.fetch_add(1, Ordering::SeqCst);
+    state.started.notify_one();
+    state.release.notified().await;
+
+    Json(json!({
+        "id": "lead_123",
+        "status": "created",
+        "email": payload["email"]
+    }))
+}

apps/admin-api/tests/integration/openapi_import.rs

@@ -1,5 +1,6 @@
 use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
 use crank_core::WorkspaceId;
+use crank_registry::ImportJobStatus;
 use serial_test::serial;
 
 use super::common::{
@@ -42,7 +43,7 @@ paths:
 async fn previews_openapi_and_creates_draft_operations() {
     let registry = test_registry().await;
     let service = test_service(
-        registry,
+        registry.clone(),
         test_storage_root("openapi_import"),
         test_auth_settings(),
         test_secret_crypto(),
@@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() {
         preview.preview.groups[0].operations[0].suggested_name,
         "latest_rates"
     );
+    let preview_job = registry
+        .get_import_job(&workspace_id, &preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(preview_job.status, ImportJobStatus::Pending);
 
     let created = service
         .create_openapi_import(
@@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() {
             .any(|finding| finding.code == "openapi_import.weak_tool_description")
     );
 
+    let skip_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let skipped = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &skip_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(skipped.skipped[0].name, "latest_rates");
     assert_eq!(skipped.findings[0].code, "operation_name_conflict");
 
+    let rename_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
     let renamed = service
         .create_openapi_import(
             &workspace_id,
-            &preview.job_id.as_str().into(),
+            &rename_preview.job_id.as_str().into(),
             OpenApiImportCreatePayload {
                 selected_operation_keys: vec!["GET /v2/latest".to_owned()],
                 server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() {
     assert_eq!(renamed.created[0].name, "latest_rates_2");
     assert_eq!(renamed.findings[0].code, "operation_name_renamed");
 }
+
+#[tokio::test]
+#[serial]
+async fn concurrent_openapi_import_replays_the_same_atomic_result() {
+    let registry = test_registry().await;
+    let service = test_service(
+        registry,
+        test_storage_root("openapi_import_replay"),
+        test_auth_settings(),
+        test_secret_crypto(),
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiImportPreviewPayload {
+                document: OPENAPI3.to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    let job_id = preview.job_id.as_str().into();
+    let payload = OpenApiImportCreatePayload {
+        selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+        server_url: Some("https://api.frankfurter.dev".to_owned()),
+        conflict_mode: "rename".to_owned(),
+    };
+
+    let (first, second) = tokio::join!(
+        service.create_openapi_import(&workspace_id, &job_id, payload.clone()),
+        service.create_openapi_import(&workspace_id, &job_id, payload),
+    );
+    let first = first.unwrap();
+    let second = second.unwrap();
+
+    assert_eq!(first.created.len(), 1);
+    assert_eq!(second.created.len(), 1);
+    assert_eq!(
+        first.created[0].operation_id,
+        second.created[0].operation_id
+    );
+    assert_eq!(first.created[0].name, second.created[0].name);
+    assert_eq!(
+        service.list_operations(&workspace_id).await.unwrap().len(),
+        1
+    );
+
+    let conflicting_replay = service
+        .create_openapi_import(
+            &workspace_id,
+            &job_id,
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+                server_url: Some("https://api.frankfurter.dev".to_owned()),
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await;
+    assert!(conflicting_replay.is_err());
+}

apps/admin-api/tests/integration/request_context.rs

@@ -0,0 +1,211 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
+use axum::{
+    Router,
+    body::Body,
+    http::{HeaderMap, Request, StatusCode},
+    routing::get,
+};
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
+};
+use opentelemetry::{global, trace::TracerProvider as _};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
+use tower::ServiceExt;
+use tracing::instrument::WithSubscriber;
+use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
+use uuid::Version;
+
+#[tokio::test(flavor = "current_thread")]
+async fn logs_request_completion_and_rejects_untrusted_values() {
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = probe_app();
+
+    let response = app
+        .clone()
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch.clone())
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(
+        response.headers()[REQUEST_ID_HEADER.as_str()]
+            .to_str()
+            .unwrap(),
+        "req_admin_trace_123"
+    );
+    let event: serde_json::Value = writer
+        .output()
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
+        .unwrap();
+    assert_eq!(event["request_id"], "req_admin_trace_123");
+    assert_eq!(event["fields"]["status"], 200);
+
+    let invalid_response = app
+        .oneshot(
+            Request::builder()
+                .uri("/probe")
+                .header(REQUEST_ID_HEADER.as_str(), "bad,value")
+                .header("traceparent", "canary-invalid-traceparent")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch)
+        .await
+        .unwrap();
+    let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
+        .to_str()
+        .unwrap();
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
+    assert!(!writer.output().contains("canary-invalid-traceparent"));
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn covers_valid_invalid_and_absent_traceparent() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("admin-request-context-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = trace_probe_app();
+
+    let valid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header(
+                        "traceparent",
+                        "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+                    )
+                    .header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .with_subscriber(dispatch.clone())
+            .await
+            .unwrap(),
+    );
+    let invalid = observed_trace_id(
+        app.clone()
+            .oneshot(
+                Request::builder()
+                    .uri("/trace")
+                    .header("traceparent", "canary-invalid-traceparent")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .with_subscriber(dispatch.clone())
+            .await
+            .unwrap(),
+    );
+    let absent = observed_trace_id(
+        app.oneshot(
+            Request::builder()
+                .uri("/trace")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .with_subscriber(dispatch)
+        .await
+        .unwrap(),
+    );
+
+    assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c");
+    assert_ne!(invalid, valid);
+    assert_ne!(absent, valid);
+    assert_ne!(invalid, absent);
+    provider.shutdown().unwrap();
+}
+
+fn probe_app() -> Router {
+    Router::new()
+        .route("/probe", get(|| async { "ok" }))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+fn trace_probe_app() -> Router {
+    Router::new()
+        .route("/trace", get(observed_traceparent))
+        .layer(axum::middleware::from_fn(apply_request_context))
+}
+
+async fn observed_traceparent() -> HeaderMap {
+    let mut trace_headers = HeaderMap::new();
+    inject_current_trace_context(&mut trace_headers);
+    let mut response_headers = HeaderMap::new();
+    if let Some(traceparent) = trace_headers.remove("traceparent") {
+        response_headers.insert("x-observed-traceparent", traceparent);
+    }
+    response_headers
+}
+
+fn observed_trace_id(response: axum::response::Response) -> String {
+    let traceparent = response.headers()["x-observed-traceparent"]
+        .to_str()
+        .unwrap();
+    traceparent[3..35].to_owned()
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

apps/admin-api/tests/request_context.rs

@@ -0,0 +1,2 @@
+#[path = "integration/request_context.rs"]
+mod request_context;

apps/mcp-server/Cargo.toml

@@ -3,6 +3,7 @@ name = "mcp-server"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [[bin]]
@@ -15,10 +16,12 @@ axum.workspace = true
 base64.workspace = true
 crank-community-mcp = { path = "../../crates/crank-community-mcp" }
 crank-core = { path = "../../crates/crank-core" }
+crank-observability = { path = "../../crates/crank-observability" }
 crank-registry = { path = "../../crates/crank-registry" }
 crank-runtime = { path = "../../crates/crank-runtime" }
 crank-schema = { path = "../../crates/crank-schema" }
 futures-util = "0.3"
+metrics.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
@@ -34,4 +37,10 @@ uuid.workspace = true
 crank-mapping = { path = "../../crates/crank-mapping" }
 crank-schema = { path = "../../crates/crank-schema" }
 crank-test-support = { path = "../../crates/crank-test-support" }
+opentelemetry.workspace = true
+opentelemetry-proto.workspace = true
+opentelemetry_sdk.workspace = true
+prost.workspace = true
 reqwest.workspace = true
+tower.workspace = true
+tracing-opentelemetry.workspace = true

apps/mcp-server/src/main.rs

@@ -1,26 +1,53 @@
 use std::{env, net::SocketAddr, time::Duration};
 
 use crank_community_mcp::{
-    auth::CommunityMachineCredentialVerifier, build_app_with_background_workers,
+    auth::CommunityMachineCredentialVerifier, build_app_with_background_workers_and_limits,
     session::PostgresTransportSessionStore,
 };
+use crank_observability::{
+    CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
+    capture_critical_error,
+};
 use crank_registry::{PostgresPoolConfig, PostgresRegistry};
 use crank_runtime::{
     RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
     RuntimeLimits, SecretCrypto,
 };
-use sqlx::postgres::PgConnectOptions;
+use sqlx::{PgPool, postgres::PgConnectOptions};
 use tokio::net::TcpListener;
 use tracing::info;
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    tracing_subscriber::fmt()
+    let observability = crank_observability::init(ObservabilityConfig::from_env(
-        .with_env_filter(
+        "mcp-server",
-            env::var("CRANK_LOG_LEVEL")
+        env!("CARGO_PKG_VERSION"),
-                .unwrap_or_else(|_| "mcp_server=info,tower_http=info".into()),
+        "mcp_server=info,tower_http=info",
-        )
+    )?)?;
-        .init();
+    let mut startup_completed = false;
+    let result = run(&observability, &mut startup_completed).await;
+    if result.is_err() {
+        capture_critical_error(if startup_completed {
+            CriticalErrorCategory::Internal
+        } else {
+            CriticalErrorCategory::Startup
+        });
+    }
+    result
+}
+
+async fn run(
+    observability: &ObservabilityLifecycle,
+    startup_completed: &mut bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let metrics_config =
+        MetricsConfig::from_env("CRANK_MCP_METRICS_BIND", "127.0.0.1:9465".parse()?)?;
+    let metrics_enabled = metrics_config.enabled();
+    let metrics_server = if metrics_config.enabled() {
+        Some(observability.metrics_surface(metrics_config).bind().await?)
+    } else {
+        None
+    };
 
     let bind_addr = env::var("CRANK_MCP_BIND").unwrap_or_else(|_| "0.0.0.0:3002".into());
     let base_url = env::var("CRANK_BASE_URL").ok();
@@ -36,23 +63,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
     let api_rate_limit = mcp_api_rate_limit_config_from_env()?;
     let database_options = database_options_from_env()?;
-    let registry = PostgresRegistry::connect_with_options_and_pool_config(
+    let registry =
-        database_options.clone(),
+        PostgresRegistry::connect_with_options_and_pool_config(database_options, pool_config)
-        pool_config,
-    )
-    .await?;
-    let session_store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
-        database_options,
-        pool_config,
-    )
             .await?;
+    if metrics_enabled {
+        spawn_postgres_pool_metrics(registry.pool().clone());
+    }
+    let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
     let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?;
     let runtime = crank_runtime::community_from_env()?
         .with_limits(runtime_limits)
         .with_response_cache(cache_stores.response.clone())
         .with_coordination_store(cache_stores.coordination.clone())
         .build();
-    let app = build_app_with_background_workers(
+    let app = build_app_with_background_workers_and_limits(
         registry,
         refresh_interval,
         base_url,
@@ -66,11 +90,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         cache_stores.coordination.clone(),
         std::sync::Arc::new(session_store),
         std::sync::Arc::new(CommunityMachineCredentialVerifier),
+        runtime_limits.max_concurrent_sessions,
     );
     let listener = TcpListener::bind(socket_addr).await?;
 
     info!(
+        name: "mcp.postgres_pool.configured",
         runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
+        runtime_max_concurrent_sessions = runtime_limits.max_concurrent_sessions,
         mcp_rate_limit_rps = api_rate_limit.requests_per_second,
         mcp_rate_limit_burst = api_rate_limit.burst,
         cache_backend = %cache_config.backend,
@@ -81,9 +108,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         max_lifetime_ms = pool_config.max_lifetime_ms,
         "postgres pool configured"
     );
-    info!("mcp-server listening on {}", socket_addr);
+    info!(
+        name: "mcp.server.listening",
+        bind_address = %socket_addr,
+        "mcp-server listening"
+    );
+    *startup_completed = true;
 
+    if let Some(metrics_server) = metrics_server {
+        tokio::select! {
+            result = axum::serve(listener, app) => result?,
+            result = metrics_server.serve() => result?,
+        }
+    } else {
         axum::serve(listener, app).await?;
+    }
 
     Ok(())
 }
@@ -123,3 +162,13 @@ fn mcp_api_rate_limit_config_from_env() -> Result<RequestRateLimitConfig, Box<dy
 
     Ok(RequestRateLimitConfig::new(requests_per_second, burst)?)
 }
+
+fn spawn_postgres_pool_metrics(pool: PgPool) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(5));
+        loop {
+            interval.tick().await;
+            crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
+        }
+    });
+}

apps/mcp-server/tests/execution_stages.rs

@@ -0,0 +1,4 @@
+#[path = "integration/common.rs"]
+mod common;
+#[path = "integration/execution_stages.rs"]
+mod execution_stages;

apps/mcp-server/tests/integration/catalog_access/approval_access.rs

@@ -89,11 +89,16 @@ async fn approval_key_lists_and_decides_pending_requests() {
     let approved = client
         .post(&approve_url)
         .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .header("x-request-id", "req_approval_execute_123")
         .json(&json!({ "approve": "yes", "note": "confirmed by test" }))
         .send()
         .await
         .unwrap();
     assert_eq!(approved.status(), reqwest::StatusCode::OK);
+    assert_eq!(
+        approved.headers()["x-request-id"].to_str().unwrap(),
+        "req_approval_execute_123"
+    );
     let approved_body = approved.json::<Value>().await.unwrap();
     assert_eq!(
         approved_body["approval"]["status"],
@@ -169,6 +174,10 @@ async fn approval_key_lists_and_decides_pending_requests() {
         .await
         .unwrap();
     assert_eq!(logs.len(), 1);
+    assert_eq!(
+        logs[0].log.request_id.as_deref(),
+        Some("req_approval_execute_123")
+    );
 }
 
 #[tokio::test]
@@ -401,12 +410,7 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
     let mcp_url = agent_mcp_url(&base_url, "sales-gated");
     let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
 
-    let tool_result = post_jsonrpc(
+    let tool_call = json!({
-        &client,
-        &mcp_url,
-        &api_key,
-        Some(&initialized_session),
-        json!({
         "jsonrpc": "2.0",
         "id": 9,
         "method": "tools/call",
@@ -416,7 +420,13 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
                 "email": "ada@example.com"
             }
         }
-        }),
+    });
+    let tool_result = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&initialized_session),
+        tool_call.clone(),
     )
     .await;
 
@@ -430,6 +440,19 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
         .unwrap();
     assert!(approval_id.starts_with("approval_"));
 
+    let repeated_tool_result = post_jsonrpc(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&initialized_session),
+        tool_call,
+    )
+    .await;
+    assert_eq!(
+        repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
+        "deduplicated tools/call must return the persisted approval id",
+    );
+
     let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
     let pending = client
         .get(&approvals_url)
@@ -448,6 +471,189 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
     );
 }
 
+#[tokio::test]
+async fn approval_http_endpoints_enforce_request_rate_limit() {
+    let registry = test_registry().await;
+    let upstream_base_url = spawn_upstream_server().await;
+    let operation = test_operation(&upstream_base_url, "crm_approval_rate_limit");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-approval-rate-limit").await;
+    let approval_key = create_approval_platform_api_key(
+        &registry,
+        "sales-approval-rate-limit",
+        "approval-rate-limit",
+    )
+    .await;
+    let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+        RequestRateLimitConfig::new(1, 1).unwrap(),
+    ))
+    .await;
+    let approvals_url = format!(
+        "{}/approvals",
+        agent_mcp_url(&base_url, "sales-approval-rate-limit")
+    );
+    let client = reqwest::Client::new();
+
+    let allowed = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(allowed.status(), reqwest::StatusCode::OK);
+
+    let limited = client
+        .get(&approvals_url)
+        .header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
+    assert!(limited.headers().contains_key(header::RETRY_AFTER));
+}
+
+#[tokio::test]
+async fn recovery_does_not_repeat_interrupted_mutating_approval() {
+    let registry = test_registry().await;
+    let (upstream_base_url, upstream_calls) = spawn_counted_approval_upstream().await;
+    let operation = test_operation(&upstream_base_url, "crm_interrupted_approval");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-interrupted-approval").await;
+    let approval_key_name = "approval-interrupted";
+    create_approval_platform_api_key(&registry, "sales-interrupted-approval", approval_key_name)
+        .await;
+    let now = OffsetDateTime::now_utc();
+    let approval = ApprovalRequest {
+        id: ApprovalRequestId::new("approval_interrupted_mutation"),
+        workspace_id: test_workspace_id(),
+        agent_id: test_agent_id("sales-interrupted-approval"),
+        operation_id: operation.id.clone(),
+        operation_version: operation.version,
+        status: ApprovalRequestStatus::Pending,
+        risk_level: OperationApprovalRiskLevel::Dangerous,
+        request_payload: json!({"email": "interrupted@example.com"}),
+        response_payload: None,
+        created_at: now - time::Duration::minutes(10),
+        expires_at: now + time::Duration::minutes(5),
+        decided_at: None,
+        decided_by_key_id: None,
+        decision_note: None,
+    };
+    registry
+        .create_approval_request(CreateApprovalRequest {
+            approval: &approval,
+        })
+        .await
+        .unwrap();
+    let approval_key_id = PlatformApiKeyId::new(format!("pk_{approval_key_name}"));
+    registry
+        .decide_approval_request(crank_registry::DecideApprovalRequest {
+            workspace_id: &approval.workspace_id,
+            agent_id: &approval.agent_id,
+            approval_id: &approval.id,
+            status: ApprovalRequestStatus::Approved,
+            decided_at: now - time::Duration::minutes(10),
+            decided_by_key_id: &approval_key_id,
+            response_payload: Some(json!({"approve": "yes"})),
+            decision_note: None,
+        })
+        .await
+        .unwrap()
+        .unwrap();
+    registry
+        .claim_approval_request(
+            &approval.workspace_id,
+            &approval.agent_id,
+            &approval.id,
+            now - time::Duration::minutes(7),
+        )
+        .await
+        .unwrap()
+        .unwrap();
+
+    let _app = build_test_app_with_approval_recovery(registry.clone());
+    let failed = tokio::time::timeout(Duration::from_secs(2), async {
+        loop {
+            let current = registry
+                .get_approval_request_for_agent(
+                    &approval.workspace_id,
+                    &approval.agent_id,
+                    &approval.id,
+                )
+                .await
+                .unwrap()
+                .unwrap();
+            if current.approval.status == ApprovalRequestStatus::Failed {
+                break current;
+            }
+            tokio::time::sleep(Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("recovery must quarantine interrupted execution");
+
+    assert_eq!(
+        failed.approval.response_payload.unwrap()["error"]["code"],
+        "approval_execution_outcome_unknown"
+    );
+    assert_eq!(
+        upstream_calls.load(std::sync::atomic::Ordering::SeqCst),
+        0,
+        "recovery must not repeat a mutating upstream request"
+    );
+}
+
+fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router {
+    crank_community_mcp::build_app_with_background_workers(
+        registry,
+        Duration::from_millis(0),
+        Some("https://crank.example.com".to_owned()),
+        SecretCrypto::new("test-master-key").unwrap(),
+        crank_runtime::community_with_outbound_policy(
+            crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
+        )
+        .build(),
+        RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
+        Arc::new(InMemoryCoordinationStateStore::default()),
+        Arc::new(InMemorySessionStore::default()),
+        Arc::new(CommunityMachineCredentialVerifier),
+    )
+}
+
+async fn spawn_counted_approval_upstream() -> (String, Arc<std::sync::atomic::AtomicUsize>) {
+    let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let handler_calls = Arc::clone(&calls);
+    let app = Router::new().route(
+        "/crm/leads",
+        post(move |Json(payload): Json<Value>| {
+            let calls = Arc::clone(&handler_calls);
+            async move {
+                calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                Json(json!({
+                    "id": "lead_123",
+                    "email": payload["email"]
+                }))
+            }
+        }),
+    );
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    (format!("http://{address}"), calls)
+}
+
 #[tokio::test]
 async fn elicitation_approval_requires_client_capability() {
     let registry = test_registry().await;

apps/mcp-server/tests/integration/execution_stages.rs

@@ -0,0 +1,376 @@
+use std::{
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use axum::{
+    Json, Router,
+    body::{Body, Bytes, to_bytes},
+    extract::State,
+    http::{HeaderMap, Request, StatusCode, header},
+    routing::post,
+};
+use crank_core::{InvocationSource, PlatformApiKeyScope};
+use crank_observability::{
+    OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider,
+};
+use crank_registry::{ListInvocationLogsQuery, PublishRequest};
+use opentelemetry::global;
+use opentelemetry_proto::tonic::{
+    collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value, trace::v1::Span,
+};
+use opentelemetry_sdk::propagation::TraceContextPropagator;
+use prost::Message;
+use serde_json::{Value, json};
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tokio::net::TcpListener;
+use tower::ServiceExt;
+use tracing::instrument::WithSubscriber;
+use tracing_subscriber::{Layer, filter::filter_fn, layer::SubscriberExt};
+
+use super::common::{
+    build_test_app, create_platform_api_key, publish_agent_for_operation, test_operation,
+    test_registry, test_workspace_id,
+};
+
+const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
+const REQUEST_ID: &str = "req_stage_end_to_end";
+const CANARY: &str = "dc-stage-canary-secret";
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn exports_real_tool_stages_without_sensitive_data() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let (otlp_endpoint, request_rx, collector) = spawn_otlp_collector().await;
+    let trace_config = OtlpTraceConfig::try_new(
+        Some(otlp_endpoint),
+        Some("http/protobuf".to_owned()),
+        Duration::from_secs(2),
+        OtlpBatchConfig::try_new(128, 128, Duration::from_secs(300), Duration::from_secs(2))
+            .unwrap(),
+    )
+    .unwrap();
+    let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "integration-test").unwrap();
+    let (provider, tracer) = build_tracer_provider(&identity, &trace_config)
+        .unwrap()
+        .expect("enabled OTLP provider");
+    let subscriber = tracing_subscriber::registry().with(
+        tracing_opentelemetry::layer()
+            .with_tracer(tracer)
+            .with_filter(filter_fn(|metadata| {
+                metadata.is_span() && metadata.target() == "crank::trace"
+            })),
+    );
+    let dispatch = tracing::Dispatch::new(subscriber);
+
+    let registry = test_registry().await;
+    let observed_traceparent = Arc::new(Mutex::new(None));
+    let upstream_base_url = spawn_upstream(Arc::clone(&observed_traceparent)).await;
+    let operation = test_operation(&upstream_base_url, "stage_end_to_end");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("test"))
+        .await
+        .unwrap();
+    registry
+        .publish_operation(PublishRequest {
+            workspace_id: &test_workspace_id(),
+            operation_id: &operation.id,
+            version: 1,
+            published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
+            published_by: Some("test"),
+        })
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "stage-agent").await;
+    let api_key = create_platform_api_key(
+        &registry,
+        "stage-agent",
+        "stage-key",
+        &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
+    )
+    .await;
+    let app = build_test_app(registry.clone(), Duration::ZERO, None);
+
+    let call_result = async {
+        let initialized = send_jsonrpc(
+            app.clone(),
+            &api_key,
+            None,
+            None,
+            None,
+            json!({
+                "jsonrpc": "2.0",
+                "id": 1,
+                "method": "initialize",
+                "params": {
+                    "protocolVersion": "2025-11-25",
+                    "capabilities": {}
+                }
+            }),
+        )
+        .await;
+        assert_eq!(initialized.status(), StatusCode::OK);
+        let session_id = initialized
+            .headers()
+            .get("MCP-Session-Id")
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .to_owned();
+
+        let notification = send_jsonrpc(
+            app.clone(),
+            &api_key,
+            Some(&session_id),
+            None,
+            None,
+            json!({
+                "jsonrpc": "2.0",
+                "method": "notifications/initialized",
+                "params": {}
+            }),
+        )
+        .await;
+        assert_eq!(notification.status(), StatusCode::ACCEPTED);
+
+        send_jsonrpc(
+            app,
+            &api_key,
+            Some(&session_id),
+            Some(REQUEST_ID),
+            Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+            json!({
+                "jsonrpc": "2.0",
+                "id": 2,
+                "method": "tools/call",
+                "params": {
+                    "name": "stage_end_to_end",
+                    "arguments": { "email": CANARY }
+                }
+            }),
+        )
+        .await
+    }
+    .with_subscriber(dispatch)
+    .await;
+
+    assert_eq!(call_result.status(), StatusCode::OK);
+    let body = to_bytes(call_result.into_body(), 1024 * 1024)
+        .await
+        .unwrap();
+    let body: Value = serde_json::from_slice(&body).unwrap();
+    assert_eq!(body["result"]["isError"], false);
+    provider.force_flush().unwrap();
+    let request = tokio::time::timeout(Duration::from_secs(2), request_rx)
+        .await
+        .unwrap()
+        .unwrap();
+    collector.abort();
+    assert_eq!(
+        request
+            .headers
+            .get(header::CONTENT_TYPE)
+            .and_then(|value| value.to_str().ok()),
+        Some("application/x-protobuf")
+    );
+    assert!(
+        !request
+            .body
+            .windows(CANARY.len())
+            .any(|window| window == CANARY.as_bytes())
+    );
+    let export = ExportTraceServiceRequest::decode(request.body).unwrap();
+    let spans = export
+        .resource_spans
+        .iter()
+        .flat_map(|resource| &resource.scope_spans)
+        .flat_map(|scope| &scope.spans)
+        .collect::<Vec<_>>();
+
+    let traceparent = observed_traceparent
+        .lock()
+        .unwrap()
+        .clone()
+        .expect("upstream traceparent");
+    assert_eq!(&traceparent[3..35], REMOTE_TRACE_ID);
+
+    let expected_trace_id = decode_trace_id(REMOTE_TRACE_ID);
+    let trace_spans = spans
+        .iter()
+        .copied()
+        .filter(|span| span.trace_id.as_slice() == expected_trace_id)
+        .collect::<Vec<_>>();
+    for expected in [
+        "mcp.request",
+        "mcp.rate_limit",
+        "mcp.access.check",
+        "mcp.catalog.load",
+        "mcp.tools.resolve",
+        "runtime.execute",
+        "runtime.arguments.map",
+        "upstream.http",
+        "runtime.response.transform",
+        "history.write",
+        "db.query",
+    ] {
+        assert!(
+            trace_spans.iter().any(|span| span.name == expected),
+            "missing span {expected}"
+        );
+    }
+    assert!(!trace_spans.iter().any(|span| span.name == "approval.check"));
+    assert!(
+        !trace_spans
+            .iter()
+            .any(|span| span.name == "runtime.idempotency")
+    );
+
+    let root = trace_spans
+        .iter()
+        .find(|span| span.name == "mcp.request")
+        .expect("mcp root");
+    let runtime = trace_spans
+        .iter()
+        .find(|span| span.name == "runtime.execute")
+        .expect("runtime");
+    assert!(!runtime.parent_span_id.is_empty());
+    assert_eq!(runtime.parent_span_id, root.span_id);
+
+    let history = trace_spans
+        .iter()
+        .find(|span| span.name == "history.write")
+        .expect("history write");
+    let history_db = trace_spans
+        .iter()
+        .find(|span| {
+            span.name == "db.query"
+                && string_attribute(span, "db.operation") == Some("invocation_history.write")
+        })
+        .expect("history PostgreSQL write");
+    assert_eq!(history_db.parent_span_id, history.span_id);
+
+    let logs = registry
+        .list_invocation_logs(ListInvocationLogsQuery {
+            workspace_id: &test_workspace_id(),
+            level: None,
+            search_text: None,
+            source: Some(InvocationSource::AgentToolCall),
+            operation_id: Some(&operation.id),
+            agent_id: None,
+            created_after: None,
+            limit: 10,
+        })
+        .await
+        .unwrap();
+    assert_eq!(logs.len(), 1);
+    assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID));
+
+    provider.shutdown().unwrap();
+}
+
+struct OtlpRequest {
+    headers: HeaderMap,
+    body: Bytes,
+}
+
+async fn spawn_otlp_collector() -> (
+    String,
+    tokio::sync::oneshot::Receiver<OtlpRequest>,
+    tokio::task::JoinHandle<()>,
+) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    let (request_tx, request_rx) = tokio::sync::oneshot::channel();
+    let sender = Arc::new(Mutex::new(Some(request_tx)));
+    let app = Router::new().route(
+        "/v1/traces",
+        post({
+            let sender = Arc::clone(&sender);
+            move |headers: HeaderMap, body: Bytes| {
+                let sender = Arc::clone(&sender);
+                async move {
+                    if let Some(sender) = sender.lock().unwrap().take() {
+                        let _ = sender.send(OtlpRequest { headers, body });
+                    }
+                    StatusCode::OK
+                }
+            }
+        }),
+    );
+    let collector = tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+    (format!("http://{address}/v1/traces"), request_rx, collector)
+}
+
+fn decode_trace_id(value: &str) -> [u8; 16] {
+    let mut bytes = [0_u8; 16];
+    for (index, byte) in bytes.iter_mut().enumerate() {
+        *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap();
+    }
+    bytes
+}
+
+fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> {
+    span.attributes.iter().find_map(|attribute| {
+        let value = attribute.value.as_ref()?.value.as_ref()?;
+        (attribute.key == key)
+            .then_some(value)
+            .and_then(|value| match value {
+                any_value::Value::StringValue(value) => Some(value.as_str()),
+                _ => None,
+            })
+    })
+}
+
+async fn send_jsonrpc(
+    app: Router,
+    api_key: &str,
+    session_id: Option<&str>,
+    request_id: Option<&str>,
+    traceparent: Option<&str>,
+    payload: Value,
+) -> axum::response::Response {
+    let mut request = Request::builder()
+        .method("POST")
+        .uri("/v1/default/stage-agent")
+        .header(header::CONTENT_TYPE, "application/json")
+        .header(header::ACCEPT, "application/json, text/event-stream")
+        .header(header::AUTHORIZATION, format!("Bearer {api_key}"))
+        .header("MCP-Protocol-Version", "2025-11-25");
+    if let Some(session_id) = session_id {
+        request = request.header("MCP-Session-Id", session_id);
+    }
+    if let Some(request_id) = request_id {
+        request = request.header("x-request-id", request_id);
+    }
+    if let Some(traceparent) = traceparent {
+        request = request.header("traceparent", traceparent);
+    }
+    app.oneshot(request.body(Body::from(payload.to_string())).unwrap())
+        .await
+        .unwrap()
+}
+
+async fn spawn_upstream(observed: Arc<Mutex<Option<String>>>) -> String {
+    async fn create_lead(
+        State(observed): State<Arc<Mutex<Option<String>>>>,
+        headers: HeaderMap,
+        Json(_payload): Json<Value>,
+    ) -> Json<Value> {
+        *observed.lock().unwrap() = headers
+            .get("traceparent")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned);
+        Json(json!({ "id": "lead_123" }))
+    }
+
+    let app = Router::new()
+        .route("/crm/leads", post(create_lead))
+        .with_state(observed);
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+    format!("http://{address}")
+}

apps/mcp-server/tests/integration/request_context.rs

@@ -0,0 +1,130 @@
+use std::{
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+
+use axum::{
+    body::Body,
+    http::{Request, StatusCode},
+};
+use opentelemetry::{
+    global,
+    trace::{TraceId, TracerProvider as _},
+};
+use opentelemetry_sdk::{
+    error::OTelSdkResult,
+    propagation::TraceContextPropagator,
+    trace::{SdkTracerProvider, SpanData, SpanExporter},
+};
+use tower::ServiceExt;
+use tracing::instrument::WithSubscriber;
+use tracing_subscriber::layer::SubscriberExt;
+
+use super::common::{build_test_app, test_registry};
+
+const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
+
+#[tokio::test(flavor = "current_thread")]
+async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let exported = Arc::new(Mutex::new(Vec::new()));
+    let provider = SdkTracerProvider::builder()
+        .with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
+        .build();
+    let tracer = provider.tracer("mcp-request-context-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let app = build_test_app(test_registry().await, Duration::ZERO, None);
+
+    let (valid, invalid, absent) = async {
+        let valid = send_health(
+            app.clone(),
+            Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+            Some("request-id-is-separate"),
+        )
+        .await;
+        let invalid = send_health(
+            app.clone(),
+            Some("canary-invalid-traceparent"),
+            Some("bad,value"),
+        )
+        .await;
+        let absent = send_health(app, None, None).await;
+        (valid, invalid, absent)
+    }
+    .with_subscriber(dispatch)
+    .await;
+    provider.force_flush().unwrap();
+
+    assert_eq!(valid.status, StatusCode::OK);
+    assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate"));
+    assert_eq!(invalid.status, StatusCode::OK);
+    assert_eq!(absent.status, StatusCode::OK);
+    assert!(valid.traceparent_response.is_none());
+    assert!(invalid.traceparent_response.is_none());
+    assert!(absent.traceparent_response.is_none());
+
+    let trace_ids: Vec<_> = exported
+        .lock()
+        .unwrap()
+        .iter()
+        .filter(|span| span.name.as_ref() == "mcp.request")
+        .map(|span| span.span_context.trace_id())
+        .collect();
+    assert_eq!(trace_ids.len(), 3);
+    assert_eq!(trace_ids[0].to_string(), REMOTE_TRACE_ID);
+    assert_ne!(trace_ids[1], trace_ids[0]);
+    assert_ne!(trace_ids[2], trace_ids[0]);
+    assert_ne!(trace_ids[1], trace_ids[2]);
+    assert!(!trace_ids.contains(&TraceId::INVALID));
+    provider.shutdown().unwrap();
+}
+
+async fn send_health(
+    app: axum::Router,
+    traceparent: Option<&str>,
+    request_id: Option<&str>,
+) -> ProbeResponse {
+    let mut request = Request::builder().uri("/health");
+    if let Some(traceparent) = traceparent {
+        request = request.header("traceparent", traceparent);
+    }
+    if let Some(request_id) = request_id {
+        request = request.header("x-request-id", request_id);
+    }
+    let response = app
+        .oneshot(request.body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+
+    ProbeResponse {
+        status: response.status(),
+        request_id: response
+            .headers()
+            .get("x-request-id")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned),
+        traceparent_response: response
+            .headers()
+            .get("traceparent")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned),
+    }
+}
+
+struct ProbeResponse {
+    status: StatusCode,
+    request_id: Option<String>,
+    traceparent_response: Option<String>,
+}
+
+#[derive(Clone, Debug)]
+struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
+
+impl SpanExporter for CapturingExporter {
+    async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
+        self.0.lock().unwrap().extend(batch);
+        Ok(())
+    }
+}

apps/mcp-server/tests/integration/transport_protocol.rs

@@ -38,7 +38,8 @@ use sha2::{Digest, Sha256};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::net::TcpListener;
 use tokio::time::sleep;
-use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
+use tracing_subscriber::fmt::MakeWriter;
+use uuid::Version;
 
 use crank_community_mcp::{
     auth::{CommunityMachineCredentialVerifier, SharedMachineCredentialVerifier},
@@ -46,6 +47,9 @@ use crank_community_mcp::{
     catalog::PublishedToolCatalog,
     session::{InMemorySessionStore, SharedSessionStore, TransportSessionStore},
 };
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber,
+};
 
 fn test_workspace_id() -> WorkspaceId {
     WorkspaceId::new("ws_default")
@@ -404,7 +408,10 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
         .to_str()
         .unwrap()
         .to_owned();
-    assert!(!request_id.is_empty());
+    assert_eq!(
+        uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
 
     let call_result = response.json::<Value>().await.unwrap();
     assert_eq!(call_result["result"]["isError"], false);
@@ -427,7 +434,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
     assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str()));
 }
 
-#[tokio::test]
+#[tokio::test(flavor = "current_thread")]
 async fn emits_request_id_in_mcp_ingress_logs() {
     let registry = test_registry().await;
     let upstream_base_url = spawn_upstream_server().await;
@@ -456,6 +463,18 @@ async fn emits_request_id_in_mcp_ingress_logs() {
     )
     .await;
 
+    let writer = SharedLogWriter::default();
+    let subscriber = build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _guard = tracing::dispatcher::set_default(&dispatch);
     let base_url = spawn_mcp_server(build_test_app(
         registry,
         Duration::from_millis(0),
@@ -464,18 +483,7 @@ async fn emits_request_id_in_mcp_ingress_logs() {
     .await;
     let client = reqwest::Client::new();
     let mcp_url = agent_mcp_url(&base_url, "sales-request-trace");
-    let writer = SharedLogWriter::default();
-    let subscriber = tracing_subscriber::registry().with(
-        tracing_subscriber::fmt::layer()
-            .with_writer(writer.clone())
-            .without_time()
-            .with_ansi(false)
-            .with_target(false)
-            .compact()
-            .with_filter(LevelFilter::INFO),
-    );
 
-    let _ = tracing::subscriber::set_global_default(subscriber);
     let response = post_jsonrpc_response(
         &client,
         &mcp_url,
@@ -500,14 +508,16 @@ async fn emits_request_id_in_mcp_ingress_logs() {
     assert_eq!(response.status(), reqwest::StatusCode::OK);
 
     let logs = writer.output();
-    assert!(
+    let event = logs
-        logs.contains("mcp request received"),
+        .lines()
-        "captured logs did not include ingress marker: {logs}"
+        .filter_map(|line| serde_json::from_str::<Value>(line).ok())
-    );
+        .find(|event| event["event"] == "mcp.request.received")
-    assert!(logs.contains("req_mcp_trace_123"));
+        .unwrap();
-    assert!(logs.contains("sales-request-trace"));
+    assert_eq!(event["service"], "mcp-server");
-    assert!(logs.contains("default"));
+    assert_eq!(event["request_id"], "req_mcp_trace_123");
-    assert!(logs.contains("initialize"));
+    assert_eq!(event["fields"]["agent_slug"], "sales-request-trace");
+    assert_eq!(event["fields"]["workspace_slug"], "default");
+    assert_eq!(event["fields"]["jsonrpc_method"], "initialize");
 }
 
 #[tokio::test]
@@ -803,11 +813,30 @@ async fn get_requires_session_header() {
         .get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
         .header(header::ACCEPT, "text/event-stream")
         .header(header::AUTHORIZATION, format!("Bearer {api_key}"))
+        .header("x-request-id", "req_early_mcp_error")
         .send()
         .await
         .unwrap();
 
     assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
+    assert_eq!(
+        response.headers()["x-request-id"].to_str().unwrap(),
+        "req_early_mcp_error"
+    );
+
+    let invalid_response = client
+        .get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
+        .header(header::ACCEPT, "text/event-stream")
+        .header(header::AUTHORIZATION, format!("Bearer {api_key}"))
+        .header("x-request-id", "bad,value")
+        .send()
+        .await
+        .unwrap();
+    let generated = invalid_response.headers()["x-request-id"].to_str().unwrap();
+    assert_eq!(
+        uuid::Uuid::parse_str(generated).unwrap().get_version(),
+        Some(Version::SortRand)
+    );
 }
 
 #[tokio::test]

apps/mcp-server/tests/request_context.rs

@@ -0,0 +1,4 @@
+#[path = "integration/common.rs"]
+mod common;
+#[path = "integration/request_context.rs"]
+mod request_context;

apps/ui/html/workspace-setup.html

@@ -97,8 +97,8 @@
           </div>
           <div class="danger-zone-action">
             <div class="danger-zone-text">
-              <div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export all data</div>
+              <div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export workspace catalog</div>
-              <div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.</div>
+              <div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.</div>
             </div>
             <button class="btn-danger" id="export-workspace-btn" type="button" data-i18n="workspace_setup.export">Export</button>
           </div>

apps/ui/js/i18n.js

@@ -442,8 +442,8 @@ var TRANSLATIONS = {
     'workspace_setup.create.subtitle': 'This Community installation uses one workspace for MCP operations and agents.',
     'workspace_setup.create.footer': 'This Community installation uses one workspace.',
     'workspace_setup.danger.title': 'Danger zone',
-    'workspace_setup.danger.export_title': 'Export all data',
+    'workspace_setup.danger.export_title': 'Export workspace catalog',
-    'workspace_setup.danger.export_body': 'Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.',
+    'workspace_setup.danger.export_body': 'Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.',
     'workspace_setup.export': 'Export',
     'workspace_setup.role.owner': 'Owner',
     'workspace_setup.role.admin': 'Admin',
@@ -1334,8 +1334,8 @@ var TRANSLATIONS = {
     'workspace_setup.create.subtitle': 'В Community используется один воркспейс для MCP-операций и агентов.',
     'workspace_setup.create.footer': 'В Community используется один воркспейс.',
     'workspace_setup.danger.title': 'Опасная зона',
-    'workspace_setup.danger.export_title': 'Экспортировать все данные',
+    'workspace_setup.danger.export_title': 'Экспорт каталога рабочего пространства',
-    'workspace_setup.danger.export_body': 'Скачать JSON-снимок настроек воркспейса, операций, агентов, секретов, данных использования и ключей доступа агентов.',
+    'workspace_setup.danger.export_body': 'Скачать невосстанавливаемый JSON-каталог настроек рабочего пространства, сводок операций и агентов, а также метаданных ключей API.',
     'workspace_setup.export': 'Экспорт',
     'workspace_setup.role.owner': 'Владелец',
     'workspace_setup.role.admin': 'Администратор',

apps/ui/js/logs.js

@@ -8,6 +8,8 @@ document.addEventListener('DOMContentLoaded', function () {
     openId: null,
     liveMode: true,
     timer: null,
+    searchTimer: null,
+    refreshPromise: null,
     workspaceId: null,
     loading: false,
     loadError: '',
@@ -457,7 +459,13 @@ document.addEventListener('DOMContentLoaded', function () {
   }
 
   async function refreshOperationalData() {
-    await Promise.all([loadLogs(), loadApprovals()]);
+    if (state.refreshPromise) {
+      return state.refreshPromise;
+    }
+    state.refreshPromise = Promise.all([loadLogs(), loadApprovals()]).finally(function () {
+      state.refreshPromise = null;
+    });
+    return state.refreshPromise;
   }
 
   async function loadLogDetail(logId) {
@@ -494,10 +502,14 @@ document.addEventListener('DOMContentLoaded', function () {
 
   function startPolling() {
     stopPolling();
-    if (!state.liveMode) {
+    if (!state.liveMode || document.hidden) {
       return;
     }
-    state.timer = setInterval(refreshOperationalData, 4000);
+    state.timer = setTimeout(async function poll() {
+      state.timer = null;
+      await refreshOperationalData();
+      startPolling();
+    }, 4000);
   }
 
   function toggleLive() {
@@ -526,7 +538,13 @@ document.addEventListener('DOMContentLoaded', function () {
   if (logSearch) {
     logSearch.addEventListener('input', function () {
       state.search = this.value.trim();
+      if (state.searchTimer) {
+        clearTimeout(state.searchTimer);
+      }
+      state.searchTimer = setTimeout(function () {
+        state.searchTimer = null;
         loadLogs();
+      }, 250);
     });
   }
 
@@ -572,6 +590,24 @@ document.addEventListener('DOMContentLoaded', function () {
     refreshOperationalData();
   });
 
+  document.addEventListener('visibilitychange', function () {
+    if (document.hidden) {
+      stopPolling();
+      return;
+    }
+    if (state.liveMode) {
+      refreshOperationalData().finally(startPolling);
+    }
+  });
+
+  window.addEventListener('pagehide', function () {
+    stopPolling();
+    if (state.searchTimer) {
+      clearTimeout(state.searchTimer);
+      state.searchTimer = null;
+    }
+  });
+
   setLiveState();
   startPolling();
 

apps/ui/js/workspace-setup.js

@@ -196,7 +196,7 @@ async function exportWorkspaceSnapshot() {
     var slug = workspaceFormState.workspaceRecord && workspaceFormState.workspaceRecord.workspace
       ? workspaceFormState.workspaceRecord.workspace.slug
       : tKey('settings.nav.workspace');
-    downloadJsonFile(slug + '-snapshot.json', snapshot);
+    downloadJsonFile(slug + '-catalog-snapshot.json', snapshot);
   } catch (error) {
     if (window.CrankUi) {
       window.CrankUi.error(error.message || tKey('workspace_setup.export_error'), tKey('workspace_setup.export_error_title'));

apps/ui/package-lock.json

@@ -9,7 +9,7 @@
         "@fontsource/inter": "5.2.8",
         "@fontsource/jetbrains-mono": "5.2.8",
         "alpinejs": "3.15.12",
-        "js-yaml": "5.2.1"
+        "js-yaml": "5.2.2"
       },
       "devDependencies": {
         "@playwright/test": "1.61.1",
@@ -580,9 +580,9 @@
       }
     },
     "node_modules/js-yaml": {
-      "version": "5.2.1",
+      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz",
-      "integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==",
+      "integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==",
       "funding": [
         {
           "type": "github",

apps/ui/package.json

@@ -11,7 +11,7 @@
     "@fontsource/inter": "5.2.8",
     "@fontsource/jetbrains-mono": "5.2.8",
     "alpinejs": "3.15.12",
-    "js-yaml": "5.2.1"
+    "js-yaml": "5.2.2"
   },
   "devDependencies": {
     "@playwright/test": "1.61.1",

apps/ui/tests/e2e/wizard.spec.js

@@ -161,15 +161,15 @@ test('wizard builds visual request mappings from JSON sample and path params', a
 
   await page.goto('/wizard/');
   await page.locator('[data-testid="wizard-protocol-rest"]').click();
-  await page.evaluate(() => window.CrankWizardShell.doGoToStep(2));
+  await page.evaluate(() => window.CrankWizardShell.goToStep(2));
   await expect(page.locator('#step-panel-2')).toBeVisible();
   await page.locator('#endpoint-path').fill('/rates/{date}');
 
-  await page.evaluate(() => window.CrankWizardShell.doGoToStep(3));
+  await page.evaluate(() => window.CrankWizardShell.goToStep(3));
   await expect(page.locator('#step-panel-3-rest')).toBeVisible();
   await page.locator('.method-card[data-method="GET"]').click();
 
-  await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
+  await page.evaluate(() => window.CrankWizardShell.goToStep(5));
   await expect(page.locator('#step-panel-5')).toBeVisible();
 
   await page.locator('#wizard-input-sample').fill(JSON.stringify({

apps/ui/tests/e2e/workspace-settings.spec.js

@@ -11,6 +11,12 @@ test('workspace and settings pages show live session data', async ({ page }) =>
   await expect(page.locator('#section-members')).toHaveCount(0);
   await expect(page.locator('#section-invite')).toHaveCount(0);
   await expect(page.locator('#delete-workspace-btn')).toHaveCount(0);
+  await expect(page.locator('[data-i18n="workspace_setup.danger.export_title"]')).toHaveText(
+    localized('Export workspace catalog', 'Экспорт каталога рабочего пространства'),
+  );
+  await expect(page.locator('[data-i18n="workspace_setup.danger.export_body"]')).not.toContainText(
+    localized('all data', 'все данные'),
+  );
 
   await page.goto('/settings');
   await expect(page.locator('.page-title')).toHaveText(localized('Account settings', 'Настройки аккаунта'));

crates/crank-adapter-rest/Cargo.toml

@@ -3,18 +3,26 @@ name = "crank-adapter-rest"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]
 async-trait = "0.1"
 crank-core = { path = "../crank-core" }
+crank-trace = { path = "../crank-trace" }
 futures-util = "0.3"
+metrics.workspace = true
+opentelemetry.workspace = true
 reqwest = { workspace = true, features = ["stream"] }
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
 
 [dev-dependencies]
 axum.workspace = true
+opentelemetry_sdk.workspace = true
 tokio.workspace = true
+tracing-subscriber.workspace = true

crates/crank-adapter-rest/src/client.rs

@@ -7,7 +7,9 @@ use std::{
 };
 
 use crank_core::{HttpMethod, RestTarget};
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
 use futures_util::StreamExt;
+use opentelemetry::{global, propagation::Injector, trace::TraceContextExt};
 use reqwest::{
     Client,
     dns::{Addrs, Name, Resolve, Resolving},
@@ -15,6 +17,8 @@ use reqwest::{
     redirect,
 };
 use serde_json::Value;
+use tracing::{Instrument, Span};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 use crate::{RestAdapterError, RestRequest, RestResponse};
 
@@ -66,10 +70,37 @@ impl RestAdapter {
         &self,
         target: &RestTarget,
         request: &RestRequest,
+    ) -> Result<RestResponse, RestAdapterError> {
+        let started_at = std::time::Instant::now();
+        let result = self.execute_inner(target, request).await;
+        let outcome = match &result {
+            Ok(_) => "success",
+            Err(error) => upstream_outcome(error),
+        };
+        metrics::counter!(
+            "crank_upstream_requests_total",
+            "operation_kind" => "rest",
+            "outcome" => outcome
+        )
+        .increment(1);
+        metrics::histogram!(
+            "crank_upstream_request_duration_seconds",
+            "operation_kind" => "rest",
+            "outcome" => outcome
+        )
+        .record(started_at.elapsed().as_secs_f64());
+        result
+    }
+
+    async fn execute_inner(
+        &self,
+        target: &RestTarget,
+        request: &RestRequest,
     ) -> Result<RestResponse, RestAdapterError> {
         let url = build_url(target, request)?;
         self.policy.validate_url(&url)?;
-        let headers = build_headers(target, request)?;
+        let mut headers = build_headers(target, request)?;
+        apply_current_trace_context(&mut headers);
         let client =
             self.client
                 .as_ref()
@@ -85,6 +116,8 @@ impl RestAdapter {
             builder = builder.json(body);
         }
 
+        let upstream_span = Stage::UpstreamHttp.span();
+        let result = async {
             let response = builder.send().await?;
             let status = response.status();
             let headers = normalize_headers(response.headers());
@@ -103,6 +136,41 @@ impl RestAdapter {
                 body,
             })
         }
+        .instrument(upstream_span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&upstream_span),
+            Err(_) => {
+                StageOutcome::Error.record(&upstream_span);
+                ErrorCategory::Upstream.record(&upstream_span);
+            }
+        }
+        result
+    }
+}
+
+fn upstream_outcome(error: &RestAdapterError) -> &'static str {
+    match error {
+        RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
+            "client_error"
+        }
+        RestAdapterError::UnexpectedStatus { status, .. } if (500..600).contains(status) => {
+            "server_error"
+        }
+        RestAdapterError::UnexpectedStatus { .. } => "unexpected_status",
+        RestAdapterError::Transport(error) if error.is_timeout() => "timeout",
+        RestAdapterError::Transport(_) => "transport_error",
+        RestAdapterError::ResponseTooLarge { .. } => "response_too_large",
+        RestAdapterError::TargetNotAllowed { .. } => "rejected",
+        RestAdapterError::WindowExpired => "window_expired",
+        RestAdapterError::InvalidSseEvent => "invalid_response",
+        RestAdapterError::InvalidBaseUrl { .. }
+        | RestAdapterError::InvalidPathParameter { .. }
+        | RestAdapterError::InvalidQueryParameter { .. }
+        | RestAdapterError::InvalidHeaderName { .. }
+        | RestAdapterError::InvalidHeaderValue { .. } => "invalid_request",
+        RestAdapterError::InvalidConfiguration { .. } => "configuration",
+    }
 }
 
 impl Default for OutboundHttpPolicy {
@@ -407,6 +475,9 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
         HeaderName::try_from(name).map_err(|_| RestAdapterError::InvalidHeaderName {
             header: name.to_owned(),
         })?;
+    if is_trace_propagation_header(&header_name) {
+        return Ok(());
+    }
     let header_value =
         HeaderValue::try_from(value).map_err(|_| RestAdapterError::InvalidHeaderValue {
             header: name.to_owned(),
@@ -416,6 +487,38 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
     Ok(())
 }
 
+fn is_trace_propagation_header(name: &HeaderName) -> bool {
+    matches!(name.as_str(), "traceparent" | "tracestate" | "baggage")
+}
+
+fn apply_current_trace_context(headers: &mut HeaderMap) {
+    for header in ["traceparent", "tracestate", "baggage"] {
+        headers.remove(header);
+    }
+
+    let context = Span::current().context();
+    if !context.span().span_context().is_valid() {
+        return;
+    }
+    global::get_text_map_propagator(|propagator| {
+        propagator.inject_context(&context, &mut ReqwestHeaderInjector(headers));
+    });
+}
+
+struct ReqwestHeaderInjector<'a>(&'a mut HeaderMap);
+
+impl Injector for ReqwestHeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(name) = HeaderName::try_from(key) else {
+            return;
+        };
+        let Ok(value) = HeaderValue::try_from(value) else {
+            return;
+        };
+        self.0.insert(name, value);
+    }
+}
+
 async fn decode_body(
     response: reqwest::Response,
     max_response_bytes: usize,

crates/crank-adapter-rest/src/lib.rs

@@ -26,13 +26,15 @@ impl ProtocolAdapter for RestAdapter {
         &self,
         target: &Target,
         prepared: &PreparedRequest,
-        _context: &RuntimeRequestContext,
+        context: &RuntimeRequestContext,
     ) -> Result<AdapterResponse, ProtocolAdapterError> {
         let target = rest_target(target)?;
+        let mut headers = prepared.headers.clone();
+        headers.extend(context.outbound_headers());
         let request = RestRequest {
             path_params: prepared.path_params.clone(),
             query_params: prepared.query_params.clone(),
-            headers: prepared.headers.clone(),
+            headers,
             body: prepared.body.clone(),
             timeout_ms: prepared.timeout_ms,
         };

crates/crank-adapter-rest/tests/integration/client.rs

@@ -8,9 +8,19 @@ use axum::{
     routing::{get, post},
 };
 use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
-use crank_core::{HttpMethod, RestTarget};
+use crank_core::{
+    HttpMethod, PreparedRequest, ProtocolAdapter, RestTarget, RuntimeRequestContext, Target,
+};
+use opentelemetry::{
+    global,
+    trace::{TraceContextExt, TracerProvider as _},
+};
+use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
 use serde_json::{Value, json};
 use tokio::net::TcpListener;
+use tracing::Instrument;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+use tracing_subscriber::layer::SubscriberExt;
 
 #[tokio::test]
 async fn executes_rest_request_and_normalizes_json_response() {
@@ -45,6 +55,120 @@ async fn executes_rest_request_and_normalizes_json_response() {
     );
 }
 
+#[tokio::test]
+async fn protocol_context_overrides_mapped_correlation_headers() {
+    let base_url = spawn_test_server().await;
+    let adapter = test_adapter();
+    let target = Target::Rest(RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::from([
+            ("x-request-id".to_owned(), "static-request".to_owned()),
+            (
+                "x-correlation-id".to_owned(),
+                "static-correlation".to_owned(),
+            ),
+        ]),
+    });
+    let prepared = PreparedRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        headers: BTreeMap::from([
+            ("x-request-id".to_owned(), "mapped-request".to_owned()),
+            (
+                "x-correlation-id".to_owned(),
+                "mapped-correlation".to_owned(),
+            ),
+        ]),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+        ..PreparedRequest::default()
+    };
+    let context = RuntimeRequestContext::new("req-runtime", "corr-runtime");
+
+    let response = adapter
+        .invoke_unary(&target, &prepared, &context)
+        .await
+        .unwrap();
+
+    assert_eq!(response.body["request_id"], "req-runtime");
+    assert_eq!(response.body["correlation_id"], "corr-runtime");
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn current_trace_context_overrides_mapped_traceparent() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder().build();
+    let tracer = provider.tracer("rest-propagation-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+    let span = tracing::info_span!("runtime.execute");
+    let context = span.context();
+    let expected_trace_id = context.span().span_context().trace_id().to_string();
+    let base_url = spawn_test_server().await;
+    let target = RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::new(),
+    };
+    let request = RestRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        query_params: BTreeMap::new(),
+        headers: BTreeMap::from([(
+            "traceparent".to_owned(),
+            "00-11111111111111111111111111111111-2222222222222222-01".to_owned(),
+        )]),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+    };
+
+    let response = test_adapter()
+        .execute(&target, &request)
+        .instrument(span)
+        .await
+        .unwrap();
+
+    assert_eq!(
+        &response.body["traceparent"].as_str().unwrap()[3..35],
+        expected_trace_id
+    );
+    provider.shutdown().unwrap();
+}
+
+#[tokio::test]
+async fn user_configured_propagation_headers_are_removed_without_trusted_context() {
+    let base_url = spawn_test_server().await;
+    let target = RestTarget {
+        base_url,
+        method: HttpMethod::Post,
+        path_template: "/users/{user_id}".to_owned(),
+        static_headers: BTreeMap::from([
+            (
+                "traceparent".to_owned(),
+                "untrusted\ninvalid-value".to_owned(),
+            ),
+            ("tracestate".to_owned(), "vendor=value".to_owned()),
+            ("baggage".to_owned(), "secret=must-not-leave".to_owned()),
+        ]),
+    };
+    let request = RestRequest {
+        path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
+        query_params: BTreeMap::new(),
+        headers: BTreeMap::new(),
+        body: Some(json!({ "name": "Ada" })),
+        timeout_ms: 1_000,
+    };
+
+    let response = test_adapter().execute(&target, &request).await.unwrap();
+
+    assert!(response.body.get("traceparent").is_none());
+    assert!(response.body.get("tracestate").is_none());
+    assert!(response.body.get("baggage").is_none());
+}
+
 #[tokio::test]
 async fn returns_unexpected_status_with_normalized_body() {
     let base_url = spawn_test_server().await;
@@ -186,14 +310,57 @@ async fn create_user(
         .get("x-static")
         .and_then(|value| value.to_str().ok())
         .unwrap_or_default();
+    let request_id = headers
+        .get("x-request-id")
+        .and_then(|value| value.to_str().ok());
+    let correlation_id = headers
+        .get("x-correlation-id")
+        .and_then(|value| value.to_str().ok());
+    let traceparent = headers
+        .get("traceparent")
+        .and_then(|value| value.to_str().ok());
+    let tracestate = headers
+        .get("tracestate")
+        .and_then(|value| value.to_str().ok());
+    let baggage = headers.get("baggage").and_then(|value| value.to_str().ok());
 
-    Json(json!({
+    let mut response = json!({
         "id": user_id,
         "query": query.get("expand").cloned().unwrap_or_default(),
         "trace": trace,
         "static": static_header,
         "payload": payload
-    }))
+    });
+    let response = response.as_object_mut().unwrap();
+    if let Some(request_id) = request_id {
+        response.insert(
+            "request_id".to_owned(),
+            Value::String(request_id.to_owned()),
+        );
+    }
+    if let Some(correlation_id) = correlation_id {
+        response.insert(
+            "correlation_id".to_owned(),
+            Value::String(correlation_id.to_owned()),
+        );
+    }
+    if let Some(traceparent) = traceparent {
+        response.insert(
+            "traceparent".to_owned(),
+            Value::String(traceparent.to_owned()),
+        );
+    }
+    if let Some(tracestate) = tracestate {
+        response.insert(
+            "tracestate".to_owned(),
+            Value::String(tracestate.to_owned()),
+        );
+    }
+    if let Some(baggage) = baggage {
+        response.insert("baggage".to_owned(), Value::String(baggage.to_owned()));
+    }
+
+    Json(Value::Object(response.clone()))
 }
 
 async fn fail() -> (axum::http::StatusCode, Json<Value>) {

crates/crank-community-auth/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-community-auth"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-community-auth/src/password_provider.rs

@@ -52,7 +52,11 @@ impl IdentityProvider for PasswordIdentityProvider {
             &self.password_pepper,
             &user.password_hash,
         ) {
-            debug!(email = %payload.email, "password identity provider rejected credentials");
+            debug!(
+                name: "auth.password.rejected",
+                identity_provider = "password",
+                "password identity provider rejected credentials"
+            );
             return Err(IdentityError::BadCredentials);
         }
 

crates/crank-community-mcp/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-community-mcp"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]
@@ -11,10 +12,13 @@ axum.workspace = true
 base64.workspace = true
 crank-adapter-rest = { path = "../crank-adapter-rest" }
 crank-core = { path = "../crank-core" }
+crank-observability = { path = "../crank-observability" }
 crank-registry = { path = "../crank-registry" }
 crank-runtime = { path = "../crank-runtime" }
 crank-schema = { path = "../crank-schema" }
+crank-trace = { path = "../crank-trace" }
 futures-util = "0.3"
+metrics.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -29,3 +33,7 @@ uuid.workspace = true
 [dev-dependencies]
 crank-mapping = { path = "../crank-mapping" }
 crank-test-support = { path = "../crank-test-support" }
+opentelemetry.workspace = true
+opentelemetry_sdk.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true

crates/crank-community-mcp/src/access.rs

@@ -1,27 +1,54 @@
 use std::sync::Arc;
 
-use axum::http::{HeaderMap, StatusCode, header::AUTHORIZATION};
+use axum::{
+    http::{HeaderMap, StatusCode, header::AUTHORIZATION},
+    response::{IntoResponse, Response},
+};
 use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
 use crank_core::{OperationSecurityLevel, PlatformApiKeyScope};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use sha2::{Digest, Sha256};
 use time::OffsetDateTime;
+use tracing::Instrument;
 
 use crate::{
     app::{AgentRoutePath, AppState},
     auth::VerifiedMachineCredential,
 };
 
+#[derive(Clone, Copy, Debug)]
+pub(super) enum MachineAccessError {
+    Denied(StatusCode),
+    Unavailable,
+}
+
+impl MachineAccessError {
+    pub(super) fn is_denied(self) -> bool {
+        matches!(self, Self::Denied(_))
+    }
+}
+
+impl IntoResponse for MachineAccessError {
+    fn into_response(self) -> Response {
+        match self {
+            Self::Denied(status) => status.into_response(),
+            Self::Unavailable => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
+        }
+    }
+}
+
 pub(super) async fn require_machine_access(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     headers: &HeaderMap,
     required_scope: PlatformApiKeyScope,
-) -> Result<VerifiedMachineCredential, StatusCode> {
+) -> Result<VerifiedMachineCredential, MachineAccessError> {
-    let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?;
+    let secret =
+        bearer_token(headers).ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))?;
     let credential = resolve_machine_credential(state, path, secret).await?;
 
     if !allows_scope(&credential.scopes, required_scope) {
-        return Err(StatusCode::FORBIDDEN);
+        return Err(MachineAccessError::Denied(StatusCode::FORBIDDEN));
     }
 
     Ok(credential)
@@ -35,12 +62,15 @@ pub(super) async fn require_approval_access(
 ) -> Result<crank_registry::PlatformApiKeyRecord, StatusCode> {
     let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?;
     let secret_hash = hash_access_secret(secret);
-    let Some(api_key) = state
+    let Some(api_key) = observe_db_query(
+        DbOperation::MachineAccessRead,
+        state
             .registry
             .get_approval_api_key_by_secret_for_agent_slug(
                 &path.workspace_slug,
                 &path.agent_slug,
                 &secret_hash,
+            ),
     )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
@@ -53,9 +83,14 @@ pub(super) async fn require_approval_access(
     }
 
     let used_at = OffsetDateTime::now_utc();
-    state
+    observe_db_query(
-        .registry
+        DbOperation::MachineAccessTouch,
-        .touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
+        state.registry.touch_platform_api_key(
+            &api_key.api_key.workspace_id,
+            &api_key.api_key.id,
+            &used_at,
+        ),
+    )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
 
@@ -100,7 +135,7 @@ async fn resolve_machine_credential(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     token: &str,
-) -> Result<VerifiedMachineCredential, StatusCode> {
+) -> Result<VerifiedMachineCredential, MachineAccessError> {
     if let Some(credential) = verify_static_agent_key(state, path, token).await? {
         return Ok(credential);
     }
@@ -109,35 +144,68 @@ async fn resolve_machine_credential(
         .credential_verifier
         .verify_bearer_token(&path.workspace_slug, &path.agent_slug, token)
         .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
+        .map_err(|_| MachineAccessError::Unavailable)?
-        .ok_or(StatusCode::UNAUTHORIZED)
+        .ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))
 }
 
 async fn verify_static_agent_key(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     secret: &str,
-) -> Result<Option<VerifiedMachineCredential>, StatusCode> {
+) -> Result<Option<VerifiedMachineCredential>, MachineAccessError> {
     let secret_hash = hash_access_secret(secret);
-    let Some(api_key) = state
+    let read_span = Stage::DbQuery
+        .db_span(DbOperation::MachineAccessRead)
+        .expect("database stage");
+    let api_key_result = state
         .registry
         .get_platform_api_key_by_secret_for_agent_slug(
             &path.workspace_slug,
             &path.agent_slug,
             &secret_hash,
         )
+        .instrument(read_span.clone())
         .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
+        .map_err(|_| MachineAccessError::Unavailable);
-    else {
+    let api_key = match api_key_result {
+        Ok(api_key) => {
+            StageOutcome::Success.record(&read_span);
+            drop(read_span);
+            api_key
+        }
+        Err(status) => {
+            StageOutcome::Error.record(&read_span);
+            ErrorCategory::Database.record(&read_span);
+            drop(read_span);
+            return Err(status);
+        }
+    };
+    let Some(api_key) = api_key else {
         return Ok(None);
     };
 
     let used_at = OffsetDateTime::now_utc();
-    state
+    let touch_span = Stage::DbQuery
+        .db_span(DbOperation::MachineAccessTouch)
+        .expect("database stage");
+    let touch_result = state
         .registry
         .touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
+        .instrument(touch_span.clone())
         .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+        .map_err(|_| MachineAccessError::Unavailable);
+    match touch_result {
+        Ok(()) => {
+            StageOutcome::Success.record(&touch_span);
+            drop(touch_span);
+        }
+        Err(status) => {
+            StageOutcome::Error.record(&touch_span);
+            ErrorCategory::Database.record(&touch_span);
+            drop(touch_span);
+            return Err(status);
+        }
+    }
 
     Ok(Some(VerifiedMachineCredential {
         machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,

crates/crank-community-mcp/src/app.rs

@@ -7,36 +7,38 @@ use std::{
 
 use axum::{
     Json, Router,
-    extract::{Path, State},
+    extract::{Extension, Path, State},
     http::{HeaderMap, StatusCode},
     response::{IntoResponse, Response, sse::Event},
     routing::{get, post},
 };
 use crank_core::{
     ApprovalRequest, ApprovalRequestId, ApprovalRequestStatus, AuthProfile, CoordinationStateStore,
-    InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus,
+    InvocationLevel, InvocationSource, InvocationStatus, OperationApprovalMode,
-    OperationApprovalMode, PlatformApiKeyScope, SecretId,
+    PlatformApiKeyScope, SecretId,
 };
 use crank_registry::{
-    CreateApprovalRequest, CreateInvocationLogRequest, DecideApprovalRequest,
+    CreateApprovalRequest, DecideApprovalRequest, ExpireApprovalRequest, PostgresRegistry,
-    ExpireApprovalRequest, PostgresRegistry, PublishedAgentTool,
+    PublishedAgentTool,
 };
 use crank_runtime::{
     RequestRateLimiter, ResolvedAuth, RuntimeError, RuntimeExecutionRequest, RuntimeExecutor,
     RuntimeOperation, RuntimeRequestContext, SecretCrypto,
 };
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use futures_util::stream;
 use serde::{Deserialize, Serialize};
 use serde_json::{Value, json};
 use time::OffsetDateTime;
-use tracing::{info, warn};
+use tokio::sync::Semaphore;
+use tracing::{Instrument, info, warn};
 
 use crate::{
     access::{
-        credential_allows_security_level, require_approval_access, require_machine_access,
+        credential_allows_security_level, serialize_machine_access_mode, serialize_security_level,
-        serialize_machine_access_mode, serialize_security_level,
     },
     approval_execution::{execute_approved_request, spawn_approval_recovery},
+    approval_response::approval_required_response,
     auth::{SharedMachineCredentialVerifier, VerifiedMachineCredential},
     catalog::PublishedToolCatalog,
     jsonrpc::{
@@ -44,10 +46,8 @@ use crate::{
         jsonrpc_result, method_name, negotiated_protocol_version, params, request_id,
     },
     manifest::catalog_tool_definitions,
-    rate_limit::{
+    rate_limit::{rate_limited_jsonrpc_response, rate_limited_status_response},
-        enforce_post_rate_limit, enforce_transport_rate_limit, rate_limited_jsonrpc_response,
+    request_context::{RequestContext, apply_request_context},
-        rate_limited_status_response,
-    },
     session::{SessionState, SharedSessionStore},
     tool_error::{
         ToolErrorContract, generic_tool_error_contract, runtime_error_code,
@@ -56,13 +56,25 @@ use crate::{
     tool_search::handle_catalog_tool_call,
     transport::{
         AllowedOrigins, HEADER_MCP_SESSION_ID, ResponseMode, json_response,
-        negotiate_post_response_mode, protocol_version_from_headers, resolve_request_id,
+        negotiate_post_response_mode, protocol_version_from_headers, session_id_from_headers,
-        session_id_from_headers, sse_response, transport_response, validate_get_accept_header,
+        sse_response, transport_response, validate_get_accept_header, validate_origin,
-        validate_origin, validate_session_protocol_version, with_request_id_header,
+        validate_session_protocol_version, with_request_id_header,
     },
 };
+mod invocation_history;
+mod metrics;
+mod stages;
+use self::metrics::{ActiveSessionGuard, McpRequestMetrics};
+use self::stages::{
+    enforce_traced_rate_limit, require_traced_approval_access, require_traced_machine_access,
+};
+#[cfg(test)]
+use invocation_history::observe_invocation_history_outcome;
+pub(super) use invocation_history::{InvocationRecord, persist_invocation};
 
 const TRANSPORT_SESSION_TTL_MS: u64 = 86_400_000;
+const DEFAULT_MAX_CONCURRENT_SESSIONS: usize = 16;
+const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60);
 
 #[derive(Clone)]
 pub(super) struct AppState {
@@ -72,6 +84,7 @@ pub(super) struct AppState {
     pub(super) api_rate_limiter: RequestRateLimiter,
     secret_crypto: SecretCrypto,
     sessions: SharedSessionStore,
+    session_slots: Arc<Semaphore>,
     pub(super) credential_verifier: SharedMachineCredentialVerifier,
     allowed_origins: AllowedOrigins,
 }
@@ -144,6 +157,7 @@ pub fn build_app(
         coordination_store,
         sessions,
         credential_verifier,
+        DEFAULT_MAX_CONCURRENT_SESSIONS,
         false,
     )
 }
@@ -159,6 +173,33 @@ pub fn build_app_with_background_workers(
     coordination_store: Arc<dyn CoordinationStateStore>,
     sessions: SharedSessionStore,
     credential_verifier: SharedMachineCredentialVerifier,
+) -> Router {
+    build_app_with_background_workers_and_limits(
+        registry,
+        refresh_interval,
+        public_base_url,
+        secret_crypto,
+        runtime,
+        api_rate_limiter,
+        coordination_store,
+        sessions,
+        credential_verifier,
+        DEFAULT_MAX_CONCURRENT_SESSIONS,
+    )
+}
+
+#[allow(clippy::too_many_arguments)]
+pub fn build_app_with_background_workers_and_limits(
+    registry: PostgresRegistry,
+    refresh_interval: Duration,
+    public_base_url: Option<String>,
+    secret_crypto: SecretCrypto,
+    runtime: RuntimeExecutor,
+    api_rate_limiter: RequestRateLimiter,
+    coordination_store: Arc<dyn CoordinationStateStore>,
+    sessions: SharedSessionStore,
+    credential_verifier: SharedMachineCredentialVerifier,
+    max_concurrent_sessions: usize,
 ) -> Router {
     build_app_inner(
         registry,
@@ -170,6 +211,7 @@ pub fn build_app_with_background_workers(
         coordination_store,
         sessions,
         credential_verifier,
+        max_concurrent_sessions,
         true,
     )
 }
@@ -185,6 +227,7 @@ fn build_app_inner(
     coordination_store: Arc<dyn CoordinationStateStore>,
     sessions: SharedSessionStore,
     credential_verifier: SharedMachineCredentialVerifier,
+    max_concurrent_sessions: usize,
     start_background_workers: bool,
 ) -> Router {
     let state = Arc::new(AppState {
@@ -194,15 +237,18 @@ fn build_app_inner(
         api_rate_limiter,
         secret_crypto,
         sessions,
+        session_slots: Arc::new(Semaphore::new(max_concurrent_sessions)),
         credential_verifier,
         allowed_origins: AllowedOrigins::new(public_base_url),
     });
     if start_background_workers {
         spawn_approval_recovery(Arc::clone(&state));
+        spawn_session_cleanup(Arc::clone(&state));
     }
 
     Router::new()
         .route("/health", get(health))
+        .route("/ready", get(readiness))
         .route(
             "/v1/{workspace_slug}/{agent_slug}",
             get(mcp_get).post(mcp_post).delete(mcp_delete),
@@ -224,6 +270,10 @@ fn build_app_inner(
             post(deny_request),
         )
         .with_state(state)
+        .layer(axum::middleware::from_fn(apply_request_context))
+        .layer(axum::middleware::from_fn(
+            crank_observability::record_http_request,
+        ))
 }
 
 async fn health() -> Json<Value> {
@@ -233,13 +283,64 @@ async fn health() -> Json<Value> {
     }))
 }
 
+fn spawn_session_cleanup(state: Arc<AppState>) {
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(SESSION_CLEANUP_INTERVAL);
+        loop {
+            interval.tick().await;
+            match state
+                .sessions
+                .cleanup_expired(OffsetDateTime::now_utc())
+                .await
+            {
+                Ok(removed) if removed > 0 => {
+                    info!(name: "mcp.session_cleanup.completed", removed);
+                }
+                Ok(_) => {}
+                Err(_) => {
+                    warn!(name: "mcp.session_cleanup.failed", error_category = "session_store");
+                }
+            }
+        }
+    });
+}
+
+async fn readiness(State(state): State<Arc<AppState>>) -> Response {
+    match state.registry.ping().await {
+        Ok(()) => Json(json!({
+            "service": "mcp-server",
+            "status": "ready",
+            "checks": { "postgres": "ready" }
+        }))
+        .into_response(),
+        Err(error) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({
+                "service": "mcp-server",
+                "status": "not_ready",
+                "checks": { "postgres": "not_ready" },
+                "error": error.to_string()
+            })),
+        )
+            .into_response(),
+    }
+}
+
 async fn list_pending_approvals(
     Path(path): Path<AgentRoutePath>,
     State(state): State<Arc<AppState>>,
     headers: HeaderMap,
 ) -> Response {
-    let key =
+    if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await {
-        match require_approval_access(&state, &path, &headers, PlatformApiKeyScope::ReadPending)
+        return rate_limited_status_response(rejection);
+    }
+
+    let key = match require_traced_approval_access(
+        &state,
+        &path,
+        &headers,
+        PlatformApiKeyScope::ReadPending,
+    )
     .await
     {
         Ok(key) => key,
@@ -250,9 +351,12 @@ async fn list_pending_approvals(
         return StatusCode::FORBIDDEN.into_response();
     };
 
-    match state
+    match observe_db_query(
+        DbOperation::ApprovalRead,
+        state
             .registry
-        .list_pending_approval_requests_for_agent(&key.api_key.workspace_id, agent_id)
+            .list_pending_approval_requests_for_agent(&key.api_key.workspace_id, agent_id),
+    )
     .await
     {
         Ok(items) => Json(json!({ "items": items })).into_response(),
@@ -263,6 +367,7 @@ async fn list_pending_approvals(
 async fn approve_request(
     Path(path): Path<ApprovalRoutePath>,
     State(state): State<Arc<AppState>>,
+    Extension(request_context): Extension<RequestContext>,
     headers: HeaderMap,
     Json(payload): Json<ApprovalDecisionPayload>,
 ) -> Response {
@@ -273,6 +378,7 @@ async fn approve_request(
         payload,
         PlatformApiKeyScope::Approve,
         ApprovalRequestStatus::Approved,
+        Some(request_context.request_id),
     )
     .await
 }
@@ -286,7 +392,11 @@ async fn get_approval_request(
         workspace_slug: path.workspace_slug,
         agent_slug: path.agent_slug,
     };
-    let key = match require_approval_access(
+    if let Err(rejection) = enforce_traced_rate_limit(&state, &agent_path, &headers).await {
+        return rate_limited_status_response(rejection);
+    }
+
+    let key = match require_traced_approval_access(
         &state,
         &agent_path,
         &headers,
@@ -318,6 +428,7 @@ async fn deny_request(
         payload,
         PlatformApiKeyScope::Deny,
         ApprovalRequestStatus::Denied,
+        None,
     )
     .await
 }
@@ -329,12 +440,18 @@ async fn decide_approval_request(
     payload: ApprovalDecisionPayload,
     required_scope: PlatformApiKeyScope,
     status: ApprovalRequestStatus,
+    execution_request_id: Option<String>,
 ) -> Response {
     let agent_path = AgentRoutePath {
         workspace_slug: path.workspace_slug,
         agent_slug: path.agent_slug,
     };
-    let key = match require_approval_access(&state, &agent_path, &headers, required_scope).await {
+    if let Err(rejection) = enforce_traced_rate_limit(&state, &agent_path, &headers).await {
+        return rate_limited_status_response(rejection);
+    }
+
+    let key =
+        match require_traced_approval_access(&state, &agent_path, &headers, required_scope).await {
             Ok(key) => key,
             Err(status) => return status.into_response(),
         };
@@ -357,7 +474,9 @@ async fn decide_approval_request(
     };
     let approval_id = ApprovalRequestId::new(path.approval_id);
 
-    match state
+    match observe_db_query(
+        DbOperation::ApprovalWrite,
+        state
             .registry
             .decide_approval_request(DecideApprovalRequest {
                 workspace_id: &key.api_key.workspace_id,
@@ -368,17 +487,19 @@ async fn decide_approval_request(
                 decided_by_key_id: &key.api_key.id,
                 response_payload: Some(json!({ "approve": payload.approve })),
                 decision_note: payload.note.as_deref(),
-        })
+            }),
+    )
     .await
     {
         Ok(Some(record)) if status == ApprovalRequestStatus::Approved => {
-            let claimed = match state
+            let claimed = match observe_db_query(
-                .registry
+                DbOperation::ApprovalWrite,
-                .claim_approval_request(
+                state.registry.claim_approval_request(
                     &record.approval.workspace_id,
                     &record.approval.agent_id,
                     &record.approval.id,
                     OffsetDateTime::now_utc(),
+                ),
             )
             .await
             {
@@ -386,7 +507,14 @@ async fn decide_approval_request(
                 Ok(None) => return StatusCode::CONFLICT.into_response(),
                 Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(),
             };
-            match execute_approved_request(&state, &agent_path, claimed).await {
+            match execute_approved_request(
+                &state,
+                &agent_path,
+                claimed,
+                execution_request_id.as_deref(),
+            )
+            .await
+            {
                 Ok(record) => Json(json!(record)).into_response(),
                 Err(response) => response,
             }
@@ -406,9 +534,12 @@ async fn approval_record_response(
     agent_id: &crank_core::AgentId,
     approval_id: &ApprovalRequestId,
 ) -> Response {
-    match state
+    match observe_db_query(
+        DbOperation::ApprovalRead,
+        state
             .registry
-        .get_approval_request_for_agent(workspace_id, agent_id, approval_id)
+            .get_approval_request_for_agent(workspace_id, agent_id, approval_id),
+    )
     .await
     {
         Ok(Some(record))
@@ -429,9 +560,12 @@ async fn terminal_decision_response(
     agent_id: &crank_core::AgentId,
     approval_id: &ApprovalRequestId,
 ) -> Response {
-    match state
+    match observe_db_query(
+        DbOperation::ApprovalRead,
+        state
             .registry
-        .get_approval_request_for_agent(workspace_id, agent_id, approval_id)
+            .get_approval_request_for_agent(workspace_id, agent_id, approval_id),
+    )
     .await
     {
         Ok(Some(record))
@@ -463,14 +597,17 @@ async fn expire_approval_response(
     agent_id: &crank_core::AgentId,
     approval_id: &ApprovalRequestId,
 ) -> Response {
-    match state
+    match observe_db_query(
+        DbOperation::ApprovalWrite,
+        state
             .registry
             .expire_approval_request(ExpireApprovalRequest {
                 workspace_id,
                 agent_id,
                 approval_id,
                 expired_at: OffsetDateTime::now_utc(),
-        })
+            }),
+    )
     .await
     {
         Ok(Some(record)) => Json(json!(record)).into_response(),
@@ -493,12 +630,12 @@ async fn mcp_get(
     }
 
     if let Err(status) =
-        require_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
+        require_traced_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
     {
         return status.into_response();
     }
 
-    if let Err(rejection) = enforce_transport_rate_limit(&state, &path, &headers).await {
+    if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await {
         return rate_limited_status_response(rejection);
     }
 
@@ -527,9 +664,19 @@ async fn mcp_get(
         return status.into_response();
     }
 
+    let Ok(permit) = ActiveSessionGuard::try_acquire(&state.session_slots) else {
+        return StatusCode::TOO_MANY_REQUESTS.into_response();
+    };
+
+    let stream = stream::unfold(permit, |permit| async move {
+        tokio::time::sleep(Duration::from_millis(TRANSPORT_SESSION_TTL_MS)).await;
+        drop(permit);
+        None::<(Result<Event, Infallible>, _)>
+    });
+
     sse_response(
         StatusCode::OK,
-        stream::pending::<Result<Event, Infallible>>(),
+        stream,
         Some(&session_id),
         Some(&session.protocol_version),
     )
@@ -541,12 +688,12 @@ async fn mcp_delete(
     headers: HeaderMap,
 ) -> Response {
     if let Err(status) =
-        require_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
+        require_traced_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
     {
         return status.into_response();
     }
 
-    if let Err(rejection) = enforce_transport_rate_limit(&state, &path, &headers).await {
+    if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await {
         return rate_limited_status_response(rejection);
     }
 
@@ -574,11 +721,14 @@ async fn mcp_delete(
 async fn mcp_post(
     Path(path): Path<AgentRoutePath>,
     State(state): State<Arc<AppState>>,
+    Extension(request_context): Extension<RequestContext>,
     headers: HeaderMap,
     Json(message): Json<Value>,
 ) -> Response {
-    let transport_request_id = resolve_request_id(&headers);
+    let mut request_metrics = McpRequestMetrics::new(&message);
+    let transport_request_id = request_context.request_id;
     info!(
+        name: "mcp.request.received",
         request_id = %transport_request_id,
         workspace_slug = %path.workspace_slug,
         agent_slug = %path.agent_slug,
@@ -591,7 +741,7 @@ async fn mcp_post(
     }
 
     let response_mode = match negotiate_post_response_mode(&headers) {
-        Ok(mode) => mode,
+        Ok(mode) => request_metrics.set_response_mode(mode),
         Err(status) => {
             return with_request_id_header(status.into_response(), &transport_request_id);
         }
@@ -608,13 +758,13 @@ async fn mcp_post(
         }
     };
 
-    if let Err(rejection) = enforce_post_rate_limit(&state, &path, &headers).await {
+    let rate_limit_result = enforce_traced_rate_limit(&state, &path, &headers).await;
+    if let Err(error) = rate_limit_result {
         return with_request_id_header(
-            rate_limited_jsonrpc_response(&message, response_mode, &protocol_version, rejection),
+            rate_limited_jsonrpc_response(&message, response_mode, &protocol_version, error),
             &transport_request_id,
         );
     }
-
     if let Some(session_id) = headers.get(HEADER_MCP_SESSION_ID)
         && let Ok(session_id) = session_id.to_str()
     {
@@ -639,10 +789,12 @@ async fn mcp_post(
         Some("tools/call") => PlatformApiKeyScope::Write,
         _ => PlatformApiKeyScope::Read,
     };
-    let credential = match require_machine_access(&state, &path, &headers, required_scope).await {
+    let access_result =
+        require_traced_machine_access(&state, &path, &headers, required_scope).await;
+    let credential = match access_result {
         Ok(credential) => credential,
-        Err(status) => {
+        Err(error) => {
-            return with_request_id_header(status.into_response(), &transport_request_id);
+            return with_request_id_header(error.into_response(), &transport_request_id);
         }
     };
 
@@ -768,6 +920,7 @@ async fn mcp_post(
         ),
     };
 
+    request_metrics.complete(response.status());
     with_request_id_header(response, &transport_request_id)
 }
 
@@ -822,13 +975,26 @@ pub(super) async fn resolve_operation_auth(
     workspace_id: &crank_core::WorkspaceId,
     execution_config: &crank_core::ExecutionConfig,
 ) -> Result<Option<ResolvedAuth>, RuntimeError> {
-    resolve_runtime_auth_for_task(
+    if execution_config.auth_profile_ref.is_none() {
+        return Ok(None);
+    }
+    let span = Stage::AuthResolve.span();
+    let result = resolve_runtime_auth_for_task(
         &state.registry,
         &state.secret_crypto,
         workspace_id,
         execution_config,
     )
-    .await
+    .instrument(span.clone())
+    .await;
+    match &result {
+        Ok(_) => StageOutcome::Success.record(&span),
+        Err(_) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Configuration.record(&span);
+        }
+    }
+    result
 }
 
 async fn resolve_runtime_auth_for_task(
@@ -841,8 +1007,10 @@ async fn resolve_runtime_auth_for_task(
         return Ok(None);
     };
 
-    let auth_profile = registry
+    let auth_profile = observe_db_query(
-        .get_auth_profile(workspace_id, auth_profile_id)
+        DbOperation::AuthProfileRead,
+        registry.get_auth_profile(workspace_id, auth_profile_id),
+    )
     .await
     .map_err(|error| RuntimeError::SecretCrypto {
         operation: "load auth profile",
@@ -867,8 +1035,10 @@ async fn resolve_auth_profile(
     let used_at = OffsetDateTime::now_utc();
 
     for secret_id in auth_profile.config.secret_ids() {
-        let secret = registry
+        let secret = observe_db_query(
-            .get_secret(workspace_id, secret_id)
+            DbOperation::SecretRead,
+            registry.get_secret(workspace_id, secret_id),
+        )
         .await
         .map_err(|error| RuntimeError::SecretCrypto {
             operation: "load secret",
@@ -877,8 +1047,10 @@ async fn resolve_auth_profile(
         .ok_or_else(|| RuntimeError::MissingSecret {
             secret_id: secret_id.as_str().to_owned(),
         })?;
-        let version = registry
+        let version = observe_db_query(
-            .get_current_secret_version(workspace_id, secret_id)
+            DbOperation::SecretRead,
+            registry.get_current_secret_version(workspace_id, secret_id),
+        )
         .await
         .map_err(|error| RuntimeError::SecretCrypto {
             operation: "load current secret version",
@@ -892,8 +1064,10 @@ async fn resolve_auth_profile(
             &version.secret_version.key_version,
             &version.secret_version.ciphertext,
         )?;
-        registry
+        observe_db_query(
-            .touch_secret(workspace_id, secret_id, &used_at)
+            DbOperation::SecretTouch,
+            registry.touch_secret(workspace_id, secret_id, &used_at),
+        )
         .await
         .map_err(|error| RuntimeError::SecretCrypto {
             operation: "touch secret",
@@ -916,7 +1090,15 @@ async fn handle_base_tool_call(
     let tool = execution.tool;
     let arguments = execution.arguments;
     let operation = runtime_operation(&tool);
-    if let Some(response) = maybe_handle_approval_policy(
+    if tool
+        .operation
+        .execution_config
+        .approval_policy
+        .as_ref()
+        .is_some_and(|policy| policy.required)
+    {
+        let approval_span = Stage::ApprovalCheck.span();
+        let response = maybe_handle_approval_policy(
             &state,
             session,
             message,
@@ -925,9 +1107,22 @@ async fn handle_base_tool_call(
             &arguments,
             transport_request_id,
         )
-    .await
+        .instrument(approval_span.clone())
-    {
+        .await;
-        return response;
+        if let Some(result) = response {
+            return match result {
+                ApprovalPolicyResult::Required(response) => {
+                    StageOutcome::Required.record(&approval_span);
+                    response
+                }
+                ApprovalPolicyResult::Error(response) => {
+                    StageOutcome::Error.record(&approval_span);
+                    ErrorCategory::Approval.record(&approval_span);
+                    response
+                }
+            };
+        }
+        StageOutcome::Allowed.record(&approval_span);
     }
 
     let mut runtime_request_context = RuntimeRequestContext::from_request_id(transport_request_id)
@@ -964,7 +1159,7 @@ async fn handle_base_tool_call(
 
     match result {
         Ok(output) => {
-            if let Err(error) = persist_invocation(
+            persist_invocation(
                 &state,
                 &tool,
                 InvocationRecord {
@@ -980,15 +1175,12 @@ async fn handle_base_tool_call(
                     response_preview: output.clone(),
                 },
             )
-            .await
+            .await;
-            {
-                warn!(error = %error, "successful invocation log write failed");
-            }
 
             success_tool_response(message, response_mode, &session.protocol_version, output)
         }
         Err(error) => {
-            if let Err(log_error) = persist_invocation(
+            persist_invocation(
                 &state,
                 &tool,
                 InvocationRecord {
@@ -1004,10 +1196,7 @@ async fn handle_base_tool_call(
                     response_preview: Value::Null,
                 },
             )
-            .await
+            .await;
-            {
-                warn!(error = %log_error, "failed invocation log write failed");
-            }
 
             tool_error_response(
                 message,
@@ -1019,6 +1208,11 @@ async fn handle_base_tool_call(
     }
 }
 
+enum ApprovalPolicyResult {
+    Required(Response),
+    Error(Response),
+}
+
 async fn maybe_handle_approval_policy(
     state: &Arc<AppState>,
     session: &SessionState,
@@ -1027,7 +1221,7 @@ async fn maybe_handle_approval_policy(
     tool: &PublishedAgentTool,
     arguments: &Value,
     transport_request_id: &str,
-) -> Option<Response> {
+) -> Option<ApprovalPolicyResult> {
     let policy = tool.operation.execution_config.approval_policy.as_ref()?;
     if !policy.required {
         return None;
@@ -1066,35 +1260,12 @@ async fn maybe_create_custom_pending_approval(
     tool: &PublishedAgentTool,
     arguments: &Value,
     transport_request_id: &str,
-) -> Option<Response> {
+) -> Option<ApprovalPolicyResult> {
     let policy = tool.operation.execution_config.approval_policy.as_ref()?;
 
     let approval_id = ApprovalRequestId::new(format!("approval_{}", uuid::Uuid::now_v7().simple()));
     let now = OffsetDateTime::now_utc();
     let expires_at = now + time::Duration::seconds(i64::from(policy.ttl_seconds));
-    let approval_url = approval_url_for(tool, &approval_id);
-    let response_payload = json!({
-        "status": "approval_required",
-        "approval_id": approval_id.as_str(),
-        "approval_url": approval_url,
-        "approve": {
-            "method": "POST",
-            "url": format!("{approval_url}/approve"),
-            "body": { "approve": "yes" }
-        },
-        "deny": {
-            "method": "POST",
-            "url": format!("{approval_url}/deny"),
-            "body": { "approve": "no" }
-        },
-        "expires_at": expires_at,
-        "risk_level": policy.risk_level,
-        "payload_preview": if policy.show_payload_preview {
-            arguments.clone()
-        } else {
-            Value::Null
-        },
-    });
     let approval = ApprovalRequest {
         id: approval_id,
         workspace_id: tool.workspace_id.clone(),
@@ -1112,22 +1283,26 @@ async fn maybe_create_custom_pending_approval(
         decision_note: None,
     };
 
-    let persisted_approval = match state
+    let persisted_approval = match observe_db_query(
+        DbOperation::ApprovalWrite,
+        state
             .registry
             .create_approval_request(CreateApprovalRequest {
                 approval: &approval,
-        })
+            }),
+    )
     .await
     {
         Ok(approval) => approval,
-        Err(error) => return Some(internal_jsonrpc_error(message, error)),
+        Err(error) => {
+            return Some(ApprovalPolicyResult::Error(internal_jsonrpc_error(
+                message, error,
+            )));
+        }
     };
-    let response_payload = persisted_approval
+    let response_payload = approval_required_response(tool, &persisted_approval.approval, policy);
-        .approval
-        .response_payload
-        .unwrap_or(response_payload);
 
-    if let Err(error) = persist_invocation(
+    persist_invocation(
         state,
         tool,
         InvocationRecord {
@@ -1143,17 +1318,14 @@ async fn maybe_create_custom_pending_approval(
             response_preview: response_payload.clone(),
         },
     )
-    .await
+    .await;
-    {
-        warn!(error = %error, "pending approval invocation log write failed");
-    }
 
-    Some(success_tool_response(
+    Some(ApprovalPolicyResult::Required(success_tool_response(
         message,
         response_mode,
         &session.protocol_version,
         response_payload,
-    ))
+    )))
 }
 
 fn handle_elicitation_approval(
@@ -1164,9 +1336,9 @@ fn handle_elicitation_approval(
     arguments: &Value,
     elicitation_message: Option<&str>,
     transport_request_id: &str,
-) -> Response {
+) -> ApprovalPolicyResult {
     if !session.supports_elicitation {
-        return tool_error_response(
+        return ApprovalPolicyResult::Error(tool_error_response(
             message,
             response_mode,
             &session.protocol_version,
@@ -1179,7 +1351,7 @@ fn handle_elicitation_approval(
                     "Выберите Custom MCP Approval или подключите MCP-клиент с поддержкой elicitation.",
                 ),
             ),
-        );
+        ));
     }
 
     let payload_preview = tool
@@ -1190,7 +1362,7 @@ fn handle_elicitation_approval(
         .and_then(|policy| policy.show_payload_preview.then(|| arguments.clone()))
         .unwrap_or(Value::Null);
 
-    success_tool_response(
+    ApprovalPolicyResult::Required(success_tool_response(
         message,
         response_mode,
         &session.protocol_version,
@@ -1201,16 +1373,7 @@ fn handle_elicitation_approval(
             "payload_preview": payload_preview,
             "note": "This MCP client advertised elicitation support. Full elicitation/create continuation is handled by compatible client integrations.",
         }),
-    )
+    ))
-}
-
-fn approval_url_for(tool: &PublishedAgentTool, approval_id: &ApprovalRequestId) -> String {
-    format!(
-        "/v1/{}/{}/approvals/{}",
-        tool.workspace_slug,
-        tool.agent_slug,
-        approval_id.as_str()
-    )
 }
 
 async fn handle_initialize(
@@ -1394,51 +1557,6 @@ pub(super) fn build_request_preview(
     }
 }
 
-pub(super) struct InvocationRecord<'a> {
-    pub(super) request_id: Option<&'a str>,
-    pub(super) tool_name: &'a str,
-    pub(super) status: InvocationStatus,
-    pub(super) level: InvocationLevel,
-    pub(super) message: &'a str,
-    pub(super) status_code: Option<u16>,
-    pub(super) error_kind: Option<&'a str>,
-    pub(super) duration: Duration,
-    pub(super) request_preview: Value,
-    pub(super) response_preview: Value,
-}
-
-pub(super) async fn persist_invocation(
-    state: &Arc<AppState>,
-    tool: &PublishedAgentTool,
-    record: InvocationRecord<'_>,
-) -> Result<(), crank_registry::RegistryError> {
-    let created_at = OffsetDateTime::now_utc();
-    let duration_ms = u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX);
-    let log = InvocationLog {
-        id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
-        workspace_id: tool.workspace_id.clone(),
-        agent_id: Some(tool.agent_id.clone()),
-        operation_id: tool.operation.id.clone(),
-        source: InvocationSource::AgentToolCall,
-        level: record.level,
-        status: record.status,
-        tool_name: record.tool_name.to_owned(),
-        message: record.message.to_owned(),
-        request_id: record.request_id.map(ToOwned::to_owned),
-        status_code: record.status_code,
-        duration_ms,
-        error_kind: record.error_kind.map(ToOwned::to_owned),
-        request_preview: record.request_preview,
-        response_preview: record.response_preview,
-        created_at,
-    };
-
-    state
-        .registry
-        .create_invocation_log(CreateInvocationLogRequest { log: &log })
-        .await
-}
-
 fn success_tool_response(
     message: &Value,
     response_mode: ResponseMode,
@@ -1526,42 +1644,4 @@ pub(super) fn runtime_operation(tool: &PublishedAgentTool) -> RuntimeOperation {
 }
 
 #[cfg(test)]
-mod tests {
+mod tests;
-    use axum::body::to_bytes;
-    use serde_json::{Value, json};
-
-    use super::{ResponseMode, tool_error_response};
-    use crate::jsonrpc::CURRENT_PROTOCOL_VERSION;
-    use crate::tool_error::generic_tool_error_contract;
-
-    #[tokio::test]
-    async fn tool_error_response_includes_structured_context() {
-        let response = tool_error_response(
-            &json!({"jsonrpc": "2.0", "id": "req-1"}),
-            ResponseMode::Json,
-            CURRENT_PROTOCOL_VERSION,
-            generic_tool_error_contract(
-                "streaming_payload_error",
-                "request root must be an object",
-                "req-1",
-                false,
-                Some("Проверьте параметры вызова инструмента."),
-            ),
-        );
-
-        let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
-        let payload: Value = serde_json::from_slice(&body).unwrap();
-
-        assert_eq!(
-            payload["result"]["structuredContent"]["error"],
-            json!({
-                "code": "streaming_payload_error",
-                "error_code": "streaming_payload_error",
-                "message": "request root must be an object",
-                "recoverable": false,
-                "request_id": "req-1",
-                "suggested_action": "Проверьте параметры вызова инструмента."
-            })
-        );
-    }
-}

crates/crank-community-mcp/src/app/invocation_history.rs

@@ -0,0 +1,117 @@
+use std::{sync::Arc, time::Duration};
+
+use crank_core::{
+    InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus,
+};
+use crank_registry::{
+    CreateInvocationLogRequest, InvocationHistoryWriteOutcome, PublishedAgentTool,
+};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
+use serde_json::Value;
+use time::OffsetDateTime;
+use tracing::{Instrument, warn};
+
+use super::AppState;
+
+pub(crate) struct InvocationRecord<'a> {
+    pub(crate) request_id: Option<&'a str>,
+    pub(crate) tool_name: &'a str,
+    pub(crate) status: InvocationStatus,
+    pub(crate) level: InvocationLevel,
+    pub(crate) message: &'a str,
+    pub(crate) status_code: Option<u16>,
+    pub(crate) error_kind: Option<&'a str>,
+    pub(crate) duration: Duration,
+    pub(crate) request_preview: Value,
+    pub(crate) response_preview: Value,
+}
+
+pub(crate) async fn persist_invocation(
+    state: &Arc<AppState>,
+    tool: &PublishedAgentTool,
+    record: InvocationRecord<'_>,
+) -> InvocationHistoryWriteOutcome {
+    let log = InvocationLog {
+        id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
+        workspace_id: tool.workspace_id.clone(),
+        agent_id: Some(tool.agent_id.clone()),
+        operation_id: tool.operation.id.clone(),
+        source: InvocationSource::AgentToolCall,
+        level: record.level,
+        status: record.status,
+        tool_name: record.tool_name.to_owned(),
+        message: record.message.to_owned(),
+        request_id: record.request_id.map(ToOwned::to_owned),
+        status_code: record.status_code,
+        duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX),
+        error_kind: record.error_kind.map(ToOwned::to_owned),
+        request_preview: record.request_preview,
+        response_preview: record.response_preview,
+        created_at: OffsetDateTime::now_utc(),
+    };
+
+    let history_span = Stage::HistoryWrite.span();
+    let (outcome, db_span) = async {
+        let db_span = Stage::DbQuery
+            .db_span(DbOperation::InvocationHistoryWrite)
+            .expect("database stage");
+        let outcome = state
+            .registry
+            .create_invocation_log(CreateInvocationLogRequest { log: &log })
+            .instrument(db_span.clone())
+            .await;
+        (outcome, db_span)
+    }
+    .instrument(history_span.clone())
+    .await;
+    match outcome {
+        InvocationHistoryWriteOutcome::Recorded => {
+            StageOutcome::Success.record(&db_span);
+            StageOutcome::Success.record(&history_span);
+        }
+        InvocationHistoryWriteOutcome::Lost(_) => {
+            StageOutcome::Error.record(&db_span);
+            ErrorCategory::Database.record(&db_span);
+            StageOutcome::Error.record(&history_span);
+            ErrorCategory::History.record(&history_span);
+        }
+    }
+    drop(db_span);
+    drop(history_span);
+    observe_invocation_history_outcome(
+        outcome,
+        record.request_id,
+        record.status,
+        "agent_tool_call",
+    );
+    outcome
+}
+
+pub(super) fn observe_invocation_history_outcome(
+    outcome: InvocationHistoryWriteOutcome,
+    request_id: Option<&str>,
+    status: InvocationStatus,
+    source: &'static str,
+) {
+    let Some(loss) = outcome.loss() else {
+        return;
+    };
+    crank_observability::record_operational_incident(
+        crank_observability::OperationalIncident::InvocationHistoryLost,
+    );
+    warn!(
+        name: "mcp.invocation_history.lost",
+        request_id = request_id.unwrap_or_default(),
+        source,
+        invocation_status = invocation_status_label(status),
+        error_category = loss.category.as_str(),
+        "invocation history was not recorded"
+    );
+}
+
+fn invocation_status_label(status: InvocationStatus) -> &'static str {
+    match status {
+        InvocationStatus::Ok => "ok",
+        InvocationStatus::Error => "error",
+    }
+}

crates/crank-community-mcp/src/app/metrics.rs

@@ -0,0 +1,93 @@
+use std::sync::Arc;
+
+use axum::http::StatusCode;
+use serde_json::Value;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
+
+use crate::{
+    jsonrpc::{is_notification, is_response, method_name},
+    transport::ResponseMode,
+};
+
+pub(super) struct McpRequestMetrics {
+    method: &'static str,
+    response_mode: &'static str,
+    outcome: &'static str,
+}
+
+impl McpRequestMetrics {
+    pub(super) fn new(message: &Value) -> Self {
+        Self {
+            method: normalized_mcp_method(message),
+            response_mode: "unknown",
+            outcome: "rejected",
+        }
+    }
+
+    pub(super) fn set_response_mode(&mut self, mode: ResponseMode) -> ResponseMode {
+        self.response_mode = match mode {
+            ResponseMode::Json => "json",
+            ResponseMode::Sse => "sse",
+        };
+        mode
+    }
+
+    pub(super) fn complete(&mut self, status: StatusCode) {
+        self.outcome = match status.as_u16() {
+            200..=299 => "success",
+            400..=499 => "client_error",
+            500..=599 => "server_error",
+            _ => "other",
+        };
+    }
+}
+
+impl Drop for McpRequestMetrics {
+    fn drop(&mut self) {
+        ::metrics::counter!(
+            "crank_mcp_requests_total",
+            "method" => self.method,
+            "response_mode" => self.response_mode,
+            "outcome" => self.outcome
+        )
+        .increment(1);
+    }
+}
+
+pub(super) fn normalized_mcp_method(message: &Value) -> &'static str {
+    match method_name(message) {
+        Some("initialize") => "initialize",
+        Some("notifications/initialized") => "initialized",
+        Some("ping") => "ping",
+        Some("tools/list") => "tools_list",
+        Some("tools/call") => "tools_call",
+        Some(_) if is_notification(message) => "notification",
+        Some(_) => "unsupported",
+        None if is_response(message) => "response",
+        None => "invalid",
+    }
+}
+
+pub(super) struct ActiveSessionGuard {
+    _permit: OwnedSemaphorePermit,
+}
+
+impl ActiveSessionGuard {
+    pub(super) fn try_acquire(slots: &Arc<Semaphore>) -> Result<Self, ()> {
+        let permit = Arc::clone(slots).try_acquire_owned().map_err(|_| {
+            ::metrics::counter!(
+                "crank_runtime_limit_rejections_total",
+                "stage" => "mcp_session"
+            )
+            .increment(1);
+        })?;
+        ::metrics::gauge!("crank_mcp_active_sessions").increment(1.0);
+        Ok(Self { _permit: permit })
+    }
+}
+
+impl Drop for ActiveSessionGuard {
+    fn drop(&mut self) {
+        ::metrics::gauge!("crank_mcp_active_sessions").decrement(1.0);
+    }
+}

crates/crank-community-mcp/src/app/stages.rs

@@ -0,0 +1,86 @@
+use std::sync::Arc;
+
+use axum::http::{HeaderMap, StatusCode};
+use crank_core::PlatformApiKeyScope;
+use crank_registry::PlatformApiKeyRecord;
+use crank_runtime::RateLimitCheckError;
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
+use tracing::Instrument;
+
+use crate::{
+    access::{MachineAccessError, require_approval_access, require_machine_access},
+    app::{AgentRoutePath, AppState},
+    auth::VerifiedMachineCredential,
+    rate_limit::enforce_transport_rate_limit,
+};
+
+pub(super) async fn enforce_traced_rate_limit(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+) -> Result<(), RateLimitCheckError> {
+    let span = Stage::McpRateLimit.span();
+    let result = enforce_transport_rate_limit(state, path, headers)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(()) => StageOutcome::Allowed.record(&span),
+        Err(RateLimitCheckError::Rejected(_)) => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::RateLimit.record(&span);
+        }
+        Err(RateLimitCheckError::StoreUnavailable) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}
+
+pub(super) async fn require_traced_machine_access(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+    required_scope: PlatformApiKeyScope,
+) -> Result<VerifiedMachineCredential, MachineAccessError> {
+    let span = Stage::McpAccessCheck.span();
+    let result = require_machine_access(state, path, headers, required_scope)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(_) => StageOutcome::Allowed.record(&span),
+        Err(error) if error.is_denied() => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::Access.record(&span);
+        }
+        Err(_) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}
+
+pub(super) async fn require_traced_approval_access(
+    state: &Arc<AppState>,
+    path: &AgentRoutePath,
+    headers: &HeaderMap,
+    required_scope: PlatformApiKeyScope,
+) -> Result<PlatformApiKeyRecord, StatusCode> {
+    let span = Stage::McpAccessCheck.span();
+    let result = require_approval_access(state, path, headers, required_scope)
+        .instrument(span.clone())
+        .await;
+    match &result {
+        Ok(_) => StageOutcome::Allowed.record(&span),
+        Err(status) if *status == StatusCode::UNAUTHORIZED || *status == StatusCode::FORBIDDEN => {
+            StageOutcome::Denied.record(&span);
+            ErrorCategory::Access.record(&span);
+        }
+        Err(_) => {
+            StageOutcome::Error.record(&span);
+            ErrorCategory::Internal.record(&span);
+        }
+    }
+    result
+}

crates/crank-community-mcp/src/app/tests.rs

@@ -0,0 +1,149 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use axum::body::to_bytes;
+use crank_core::InvocationStatus;
+use crank_observability::{
+    ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
+    operational_incident_total,
+};
+use crank_registry::{
+    InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
+};
+use serde_json::{Value, json};
+use tracing_subscriber::fmt::MakeWriter;
+
+use super::{
+    ResponseMode, metrics::normalized_mcp_method, observe_invocation_history_outcome,
+    tool_error_response,
+};
+use crate::jsonrpc::CURRENT_PROTOCOL_VERSION;
+use crate::tool_error::generic_tool_error_contract;
+
+#[tokio::test]
+async fn tool_error_response_includes_structured_context() {
+    let response = tool_error_response(
+        &json!({"jsonrpc": "2.0", "id": "req-1"}),
+        ResponseMode::Json,
+        CURRENT_PROTOCOL_VERSION,
+        generic_tool_error_contract(
+            "streaming_payload_error",
+            "request root must be an object",
+            "req-1",
+            false,
+            Some("Проверьте параметры вызова инструмента."),
+        ),
+    );
+
+    let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
+    let payload: Value = serde_json::from_slice(&body).unwrap();
+
+    assert_eq!(
+        payload["result"]["structuredContent"]["error"],
+        json!({
+            "code": "streaming_payload_error",
+            "error_code": "streaming_payload_error",
+            "message": "request root must be an object",
+            "recoverable": false,
+            "request_id": "req-1",
+            "suggested_action": "Проверьте параметры вызова инструмента."
+        })
+    );
+}
+
+#[test]
+fn emits_bounded_history_loss_incident() {
+    let writer = SharedLogWriter::default();
+    let subscriber = crank_observability::build_subscriber(
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
+            "info",
+            RedactionLimits::default(),
+        ),
+        writer.clone(),
+    )
+    .unwrap();
+    let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+    let dispatch = tracing::Dispatch::new(subscriber);
+    let _guard = tracing::dispatcher::set_default(&dispatch);
+
+    observe_invocation_history_outcome(
+        InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
+            category: InvocationHistoryLossCategory::Unavailable,
+        }),
+        Some("req_mcp_dc08"),
+        InvocationStatus::Ok,
+        "agent_tool_call",
+    );
+
+    let output = writer.output();
+    assert!(!output.contains("dc08-canary-secret"));
+    let event: Value = output
+        .lines()
+        .map(|line| serde_json::from_str(line).unwrap())
+        .find(|event: &Value| event["event"] == "mcp.invocation_history.lost")
+        .unwrap();
+    assert_eq!(event["request_id"], "req_mcp_dc08");
+    assert_eq!(event["fields"]["source"], "agent_tool_call");
+    assert_eq!(event["fields"]["invocation_status"], "ok");
+    assert_eq!(event["fields"]["error_category"], "unavailable");
+    assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
+}
+
+#[test]
+fn mcp_metric_method_is_always_from_a_closed_set() {
+    assert_eq!(
+        normalized_mcp_method(&json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call"})),
+        "tools_call"
+    );
+    assert_eq!(
+        normalized_mcp_method(
+            &json!({"jsonrpc": "2.0", "id": 2, "method": "customer-controlled-method"})
+        ),
+        "unsupported"
+    );
+    assert_eq!(
+        normalized_mcp_method(
+            &json!({"jsonrpc": "2.0", "method": "customer-controlled-notification"})
+        ),
+        "notification"
+    );
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}

crates/crank-community-mcp/src/approval_execution.rs

@@ -5,11 +5,13 @@ use axum::{
     response::{IntoResponse, Response},
 };
 use crank_core::{ApprovalRequestStatus, InvocationLevel, InvocationSource, InvocationStatus};
+use crank_observability::RequestId;
 use crank_registry::{ApprovalRequestRecord, FinishApprovalRequest};
 use crank_runtime::{RuntimeExecutionRequest, RuntimeRequestContext};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
 use serde_json::json;
 use time::OffsetDateTime;
-use tracing::warn;
+use tracing::{Instrument, warn};
 
 use crate::{
     app::{
@@ -35,32 +37,81 @@ pub(super) fn spawn_approval_recovery(state: Arc<AppState>) {
 }
 
 async fn recover_approved_requests(state: &Arc<AppState>) {
+    fail_interrupted_requests(state).await;
+
     for _ in 0..32 {
         let now = OffsetDateTime::now_utc();
-        let approval = match state
+        let approval = match observe_db_query(
+            DbOperation::ApprovalWrite,
+            state
                 .registry
-            .claim_next_recoverable_approval_request(
+                .claim_next_recoverable_approval_request(now, now - RECOVERY_GRACE),
-                now,
-                now - RECOVERY_GRACE,
-                now - EXECUTION_LEASE,
         )
         .await
         {
             Ok(Some(approval)) => approval,
             Ok(None) => break,
-            Err(error) => {
+            Err(_) => {
-                warn!(error = %error, "approval recovery query failed");
+                warn!(
+                    name: "mcp.approval_recovery.query_failed",
+                    error_category = "registry",
+                    "approval recovery query failed"
+                );
                 break;
             }
         };
         let Some(path) = approval_agent_path(state, &approval).await else {
             continue;
         };
-        if execute_approved_request(state, &path, approval)
+        let recovery_span = Stage::ApprovalRecovery.span();
+        let result = execute_approved_request(state, &path, approval, None)
+            .instrument(recovery_span.clone())
+            .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&recovery_span),
+            Err(_) => {
+                StageOutcome::Error.record(&recovery_span);
+                ErrorCategory::Approval.record(&recovery_span);
+            }
+        }
+        drop(recovery_span);
+        if result.is_err() {
+            warn!(
+                name: "mcp.approval_recovery.execution_failed",
+                error_category = "runtime",
+                "recovered approval execution did not finish"
+            );
+        }
+    }
+}
+
+async fn fail_interrupted_requests(state: &Arc<AppState>) {
+    for _ in 0..32 {
+        let stale_before = OffsetDateTime::now_utc() - EXECUTION_LEASE;
+        match observe_db_query(
+            DbOperation::ApprovalWrite,
+            state
+                .registry
+                .fail_next_interrupted_approval_request(stale_before),
+        )
         .await
-            .is_err()
         {
-            warn!("recovered approval execution did not finish");
+            Ok(Some(approval)) => {
+                warn!(
+                    name: "mcp.approval_recovery.interrupted",
+                    approval_id = approval.approval.id.as_str(),
+                    "interrupted approval execution was not retried because its outcome is unknown"
+                );
+            }
+            Ok(None) => break,
+            Err(_) => {
+                warn!(
+                    name: "mcp.approval_recovery.interrupted_query_failed",
+                    error_category = "registry",
+                    "interrupted approval recovery query failed"
+                );
+                break;
+            }
         }
     }
 }
@@ -76,8 +127,12 @@ async fn approval_agent_path(
     {
         Ok(Some(workspace)) => workspace,
         Ok(None) => return None,
-        Err(error) => {
+        Err(_) => {
-            warn!(error = %error, "approval workspace lookup failed");
+            warn!(
+                name: "mcp.approval_recovery.workspace_lookup_failed",
+                error_category = "registry",
+                "approval workspace lookup failed"
+            );
             return None;
         }
     };
@@ -88,8 +143,12 @@ async fn approval_agent_path(
     {
         Ok(Some(agent)) => agent,
         Ok(None) => return None,
-        Err(error) => {
+        Err(_) => {
-            warn!(error = %error, "approval agent lookup failed");
+            warn!(
+                name: "mcp.approval_recovery.agent_lookup_failed",
+                error_category = "registry",
+                "approval agent lookup failed"
+            );
             return None;
         }
     };
@@ -103,7 +162,9 @@ pub(super) async fn execute_approved_request(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     approval: ApprovalRequestRecord,
+    request_id: Option<&str>,
 ) -> Result<ApprovalRequestRecord, Response> {
+    let request_id = RequestId::resolve(request_id).into_string();
     let tools = state
         .catalog
         .list_tools(&path.workspace_slug, &path.agent_slug)
@@ -123,8 +184,7 @@ pub(super) async fn execute_approved_request(
         &approval.approval.request_payload,
     );
     let started_at = Instant::now();
-    let runtime_request_context =
+    let runtime_request_context = RuntimeRequestContext::from_request_id(request_id.clone())
-        RuntimeRequestContext::from_request_id(approval.approval.id.as_str().to_owned())
         .with_response_cache_scope(
             tool.workspace_id.as_str().to_owned(),
             tool.agent_id.as_str().to_owned(),
@@ -176,11 +236,11 @@ pub(super) async fn execute_approved_request(
             ),
         };
 
-    if let Err(error) = persist_invocation(
+    persist_invocation(
         state,
         &tool,
         InvocationRecord {
-            request_id: Some(approval.approval.id.as_str()),
+            request_id: Some(&request_id),
             tool_name: &tool.tool_name,
             status: invocation_status,
             level: invocation_level,
@@ -192,11 +252,10 @@ pub(super) async fn execute_approved_request(
             response_preview: response_payload.clone(),
         },
     )
-    .await
+    .await;
-    {
-        warn!(error = %error, "approved invocation log write failed");
-    }
 
+    observe_db_query(
+        DbOperation::ApprovalWrite,
         state
             .registry
             .finish_approval_request(FinishApprovalRequest {
@@ -206,7 +265,8 @@ pub(super) async fn execute_approved_request(
                 status,
                 response_payload: Some(response_payload),
                 decision_note: None,
-        })
+            }),
+    )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
     .ok_or_else(|| StatusCode::CONFLICT.into_response())
@@ -216,6 +276,8 @@ async fn finish_unavailable_approval(
     state: &Arc<AppState>,
     approval: &ApprovalRequestRecord,
 ) -> Result<ApprovalRequestRecord, Response> {
+    observe_db_query(
+        DbOperation::ApprovalWrite,
         state
             .registry
             .finish_approval_request(FinishApprovalRequest {
@@ -230,7 +292,8 @@ async fn finish_unavailable_approval(
                     }
                 })),
                 decision_note: None,
-        })
+            }),
+    )
     .await
     .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
     .ok_or_else(|| StatusCode::CONFLICT.into_response())

crates/crank-community-mcp/src/approval_response.rs

@@ -0,0 +1,38 @@
+use crank_core::{ApprovalRequest, OperationApprovalPolicy};
+use crank_registry::PublishedAgentTool;
+use serde_json::{Value, json};
+
+pub(super) fn approval_required_response(
+    tool: &PublishedAgentTool,
+    approval: &ApprovalRequest,
+    policy: &OperationApprovalPolicy,
+) -> Value {
+    let approval_url = format!(
+        "/v1/{}/{}/approvals/{}",
+        tool.workspace_slug,
+        tool.agent_slug,
+        approval.id.as_str()
+    );
+    json!({
+        "status": "approval_required",
+        "approval_id": approval.id.as_str(),
+        "approval_url": approval_url,
+        "approve": {
+            "method": "POST",
+            "url": format!("{approval_url}/approve"),
+            "body": { "approve": "yes" }
+        },
+        "deny": {
+            "method": "POST",
+            "url": format!("{approval_url}/deny"),
+            "body": { "approve": "no" }
+        },
+        "expires_at": approval.expires_at,
+        "risk_level": approval.risk_level,
+        "payload_preview": if policy.show_payload_preview {
+            approval.request_payload.clone()
+        } else {
+            Value::Null
+        },
+    })
+}

crates/crank-community-mcp/src/catalog.rs

@@ -1,24 +1,27 @@
 use std::{
     collections::HashMap,
-    sync::Arc,
+    sync::{Arc, Weak},
     time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 
 use crank_core::{CacheScope, CoordinationStateStore, CoordinationStateValue};
 use crank_registry::{PostgresRegistry, PublishedAgentCatalog, PublishedAgentTool, RegistryError};
+use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
 use serde::{Deserialize, Serialize};
 use tokio::sync::{Mutex, RwLock};
-use tracing::{info, warn};
+use tracing::{Instrument, info, warn};
 
 use crate::manifest::analyze_published_tool_catalog;
 
+const MAX_LOCAL_CATALOGS: usize = 1_024;
+
 #[derive(Clone)]
 pub struct PublishedToolCatalog {
     registry: PostgresRegistry,
     refresh_interval: Duration,
     coordination_store: Arc<dyn CoordinationStateStore>,
     cached: Arc<RwLock<HashMap<CatalogKey, CachedCatalog>>>,
-    refresh_locks: Arc<Mutex<HashMap<CatalogKey, Arc<Mutex<()>>>>>,
+    refresh_locks: Arc<Mutex<HashMap<CatalogKey, Weak<Mutex<()>>>>>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -30,6 +33,14 @@ struct CatalogKey {
 struct CachedCatalog {
     loaded_at: Option<Instant>,
     catalog: PublishedAgentCatalog,
+    metrics: CatalogMetrics,
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct CatalogMetrics {
+    tool_count: usize,
+    estimated_context_tokens: usize,
+    warning_count: usize,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -66,6 +77,8 @@ impl PublishedToolCatalog {
         workspace_slug: &str,
         agent_slug: &str,
     ) -> Result<PublishedAgentCatalog, RegistryError> {
+        let span = Stage::McpCatalogLoad.span();
+        let result = async {
             self.refresh_if_stale(workspace_slug, agent_slug).await?;
             let guard = self.cached.read().await;
             guard
@@ -76,6 +89,18 @@ impl PublishedToolCatalog {
                     agent_slug: agent_slug.to_owned(),
                 })
         }
+        .instrument(span.clone())
+        .await;
+        match &result {
+            Ok(_) => StageOutcome::Success.record(&span),
+            Err(_) => {
+                StageOutcome::Error.record(&span);
+                ErrorCategory::Catalog.record(&span);
+            }
+        }
+        drop(span);
+        result
+    }
 
     async fn refresh_if_stale(
         &self,
@@ -98,11 +123,14 @@ impl PublishedToolCatalog {
 
         let refresh_lock = {
             let mut locks = self.refresh_locks.lock().await;
-            Arc::clone(
+            locks.retain(|_, lock| lock.strong_count() > 0);
-                locks
+            if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
-                    .entry(key.clone())
+                lock
-                    .or_insert_with(|| Arc::new(Mutex::new(()))),
+            } else {
-            )
+                let lock = Arc::new(Mutex::new(()));
+                locks.insert(key.clone(), Arc::downgrade(&lock));
+                lock
+            }
         };
         let _refresh_guard = refresh_lock.lock().await;
         let still_stale = {
@@ -117,50 +145,59 @@ impl PublishedToolCatalog {
         }
 
         if let Some((catalog, age)) = self.load_shared_snapshot(workspace_slug, agent_slug).await {
+            let metrics =
                 log_catalog_analysis(workspace_slug, agent_slug, "shared_cache", &catalog.tools);
-            let mut guard = self.cached.write().await;
+            self.store_local_catalog(
-            guard.insert(
                 key,
                 CachedCatalog {
                     loaded_at: Instant::now().checked_sub(age),
                     catalog,
+                    metrics,
                 },
-            );
+            )
+            .await;
             return Ok(());
         }
 
-        let catalog = match self
+        let db_span = Stage::DbQuery
+            .db_span(DbOperation::CatalogLoad)
+            .expect("database stage");
+        let catalog_result = self
             .registry
             .get_published_agent_catalog_by_slug(workspace_slug, agent_slug)
-            .await
+            .instrument(db_span.clone())
-        {
+            .await;
+        let catalog = match catalog_result {
             Ok(catalog) => catalog,
-            Err(error) => return Err(error),
+            Err(error) => {
+                StageOutcome::Error.record(&db_span);
+                ErrorCategory::Database.record(&db_span);
+                drop(db_span);
+                return Err(error);
+            }
         };
-        log_catalog_analysis(workspace_slug, agent_slug, "postgres", &catalog.tools);
+        StageOutcome::Success.record(&db_span);
+        drop(db_span);
+        let metrics = log_catalog_analysis(workspace_slug, agent_slug, "postgres", &catalog.tools);
         self.store_shared_snapshot(workspace_slug, agent_slug, &catalog)
             .await;
-        let mut guard = self.cached.write().await;
+        let published_tool_count = catalog.tools.len();
-        let previous_count = guard
+        let previous_count = self
-            .get(&key)
+            .store_local_catalog(
-            .map(|entry| entry.catalog.tools.len())
-            .unwrap_or_default();
-
-        guard.insert(
                 key,
                 CachedCatalog {
                     loaded_at: Some(Instant::now()),
                     catalog,
+                    metrics,
                 },
-        );
+            )
+            .await;
 
         info!(
+            name: "mcp.catalog.refreshed",
             workspace_slug,
             agent_slug,
-            published_tool_count = guard
+            published_tool_count,
-                .get(&CatalogKey::new(workspace_slug, agent_slug))
-                .map(|entry| entry.catalog.tools.len())
-                .unwrap_or_default(),
             previous_published_tool_count = previous_count,
             "published agent catalog refreshed"
         );
@@ -168,6 +205,27 @@ impl PublishedToolCatalog {
         Ok(())
     }
 
+    async fn store_local_catalog(&self, key: CatalogKey, entry: CachedCatalog) -> usize {
+        let mut guard = self.cached.write().await;
+        let previous_count = guard
+            .get(&key)
+            .map(|current| current.catalog.tools.len())
+            .unwrap_or_default();
+
+        if guard.len() >= MAX_LOCAL_CATALOGS && !guard.contains_key(&key) {
+            let oldest = guard
+                .iter()
+                .min_by_key(|(_, current)| current.loaded_at)
+                .map(|(candidate, _)| candidate.clone());
+            if let Some(oldest) = oldest {
+                guard.remove(&oldest);
+            }
+        }
+        guard.insert(key, entry);
+        record_catalog_metrics(guard.values().map(|entry| entry.metrics));
+        previous_count
+    }
+
     async fn load_shared_snapshot(
         &self,
         workspace_slug: &str,
@@ -226,12 +284,19 @@ fn log_catalog_analysis(
     agent_slug: &str,
     source: &str,
     tools: &[PublishedAgentTool],
-) {
+) -> CatalogMetrics {
     let analysis = match analyze_published_tool_catalog(tools) {
         Ok(analysis) => analysis,
-        Err(error) => {
+        Err(_) => {
-            warn!(workspace_slug, agent_slug, source, %error, "published catalog analysis failed");
+            warn!(
-            return;
+                name: "mcp.catalog.analysis_failed",
+                workspace_slug,
+                agent_slug,
+                source,
+                error_category = "catalog_validation",
+                "published catalog analysis failed"
+            );
+            return CatalogMetrics::default();
         }
     };
     let warning_count = analysis
@@ -242,6 +307,7 @@ fn log_catalog_analysis(
         .count();
 
     info!(
+        name: "mcp.catalog.analyzed",
         workspace_slug,
         agent_slug,
         source,
@@ -255,6 +321,30 @@ fn log_catalog_analysis(
         catalog_quality_warning_count = warning_count,
         "published agent catalog analyzed"
     );
+
+    CatalogMetrics {
+        tool_count: analysis.budget.tool_count,
+        estimated_context_tokens: analysis.budget.estimated_context_tokens,
+        warning_count,
+    }
+}
+
+fn record_catalog_metrics(metrics: impl Iterator<Item = CatalogMetrics>) {
+    let aggregate = metrics.fold(CatalogMetrics::default(), |mut aggregate, current| {
+        aggregate.tool_count = aggregate.tool_count.saturating_add(current.tool_count);
+        aggregate.estimated_context_tokens = aggregate
+            .estimated_context_tokens
+            .saturating_add(current.estimated_context_tokens);
+        aggregate.warning_count = aggregate
+            .warning_count
+            .saturating_add(current.warning_count);
+        aggregate
+    });
+
+    metrics::gauge!("crank_catalog_tools").set(aggregate.tool_count as f64);
+    metrics::gauge!("crank_catalog_estimated_context_tokens")
+        .set(aggregate.estimated_context_tokens as f64);
+    metrics::gauge!("crank_catalog_warnings").set(aggregate.warning_count as f64);
 }
 
 fn now_unix_ms() -> u64 {

crates/crank-community-mcp/src/lib.rs

@@ -1,14 +1,18 @@
 mod access;
 mod app;
 mod approval_execution;
+mod approval_response;
 pub mod auth;
 pub mod catalog;
 pub mod jsonrpc;
 pub mod manifest;
 mod rate_limit;
+mod request_context;
 pub mod session;
 pub mod tool_error;
 mod tool_search;
 mod transport;
 
-pub use app::{build_app, build_app_with_background_workers};
+pub use app::{
+    build_app, build_app_with_background_workers, build_app_with_background_workers_and_limits,
+};

crates/crank-community-mcp/src/rate_limit.rs

@@ -4,7 +4,7 @@ use axum::{
     http::{HeaderMap, HeaderValue, StatusCode, header::RETRY_AFTER},
     response::{IntoResponse, Response},
 };
-use crank_runtime::RateLimitRejection;
+use crank_runtime::RateLimitCheckError;
 use serde_json::{Value, json};
 
 use crate::{
@@ -14,19 +14,11 @@ use crate::{
     transport::{ResponseMode, session_id_from_headers, transport_response},
 };
 
-pub(super) async fn enforce_post_rate_limit(
-    state: &Arc<AppState>,
-    path: &AgentRoutePath,
-    headers: &HeaderMap,
-) -> Result<(), RateLimitRejection> {
-    enforce_transport_rate_limit(state, path, headers).await
-}
-
 pub(super) async fn enforce_transport_rate_limit(
     state: &Arc<AppState>,
     path: &AgentRoutePath,
     headers: &HeaderMap,
-) -> Result<(), RateLimitRejection> {
+) -> Result<(), RateLimitCheckError> {
     let key = rate_limit_key(path, headers);
     state.api_rate_limiter.check(&key).await
 }
@@ -35,8 +27,25 @@ pub(super) fn rate_limited_jsonrpc_response(
     message: &Value,
     response_mode: ResponseMode,
     protocol_version: &str,
-    rejection: RateLimitRejection,
+    error: RateLimitCheckError,
 ) -> Response {
+    let RateLimitCheckError::Rejected(rejection) = error else {
+        return transport_response(
+            StatusCode::SERVICE_UNAVAILABLE,
+            json!({
+                "jsonrpc": "2.0",
+                "id": request_id(message),
+                "error": {
+                    "code": -32603,
+                    "message": "rate limit service unavailable",
+                    "data": { "code": "rate_limit_unavailable" }
+                }
+            }),
+            response_mode,
+            None,
+            Some(protocol_version),
+        );
+    };
     let payload = json!({
         "jsonrpc": "2.0",
         "id": request_id(message),
@@ -61,11 +70,16 @@ pub(super) fn rate_limited_jsonrpc_response(
     response
 }
 
-pub(super) fn rate_limited_status_response(rejection: RateLimitRejection) -> Response {
+pub(super) fn rate_limited_status_response(error: RateLimitCheckError) -> Response {
+    match error {
+        RateLimitCheckError::Rejected(rejection) => {
             let mut response = StatusCode::TOO_MANY_REQUESTS.into_response();
             attach_retry_after_header(&mut response, rejection.retry_after_ms);
             response
         }
+        RateLimitCheckError::StoreUnavailable => StatusCode::SERVICE_UNAVAILABLE.into_response(),
+    }
+}
 
 fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String {
     if let Ok(Some(session_id)) = session_id_from_headers(headers) {

crates/crank-community-mcp/src/request_context.rs

@@ -0,0 +1,39 @@
+use axum::{extract::Request, http::HeaderValue, middleware::Next, response::Response};
+use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
+use tracing::{Instrument, info_span};
+
+use crate::transport::HEADER_X_REQUEST_ID;
+
+#[derive(Clone, Debug)]
+pub(super) struct RequestContext {
+    pub(super) request_id: String,
+}
+
+pub(super) async fn apply_request_context(mut request: Request, next: Next) -> Response {
+    let request_id = RequestId::resolve(
+        request
+            .headers()
+            .get(&HEADER_X_REQUEST_ID)
+            .and_then(|value| value.to_str().ok()),
+    )
+    .into_string();
+    let context = RequestContext {
+        request_id: request_id.clone(),
+    };
+    let span = info_span!(
+        target: "crank::trace",
+        "mcp.request",
+        request_id = %request_id,
+    );
+    set_remote_trace_parent(&span, request.headers());
+    request.extensions_mut().insert(context);
+
+    with_request_correlation(request_id.clone(), async move {
+        let mut response = next.run(request).instrument(span).await;
+        if let Ok(value) = HeaderValue::from_str(&request_id) {
+            response.headers_mut().insert(HEADER_X_REQUEST_ID, value);
+        }
+        response
+    })
+    .await
+}

crates/crank-community-mcp/src/session.rs

@@ -52,6 +52,8 @@ pub trait TransportSessionStore: Send + Sync {
     ) -> Result<bool, SessionStoreError>;
 
     async fn delete(&self, session_id: &str) -> Result<bool, SessionStoreError>;
+
+    async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError>;
 }
 
 pub type SharedSessionStore = Arc<dyn TransportSessionStore>;
@@ -62,6 +64,11 @@ pub struct PostgresTransportSessionStore {
 }
 
 impl PostgresTransportSessionStore {
+    pub async fn from_pool(pool: PgPool) -> Result<Self, SessionStoreError> {
+        apply_postgres_migrations(&pool).await?;
+        Ok(Self { pool })
+    }
+
     pub async fn connect_with_options_and_pool_config(
         connect_options: PgConnectOptions,
         pool_config: PostgresPoolConfig,
@@ -84,9 +91,7 @@ impl PostgresTransportSessionStore {
                 details: error.to_string(),
             })?;
 
-        apply_postgres_migrations(&pool).await?;
+        Self::from_pool(pool).await
-
-        Ok(Self { pool })
     }
 }
 
@@ -164,6 +169,13 @@ impl TransportSessionStore for InMemorySessionStore {
         let mut guard = self.inner.write().await;
         Ok(guard.remove(session_id).is_some())
     }
+
+    async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError> {
+        let mut guard = self.inner.write().await;
+        let before = guard.len();
+        guard.retain(|_, session| !is_expired(session, now));
+        Ok(u64::try_from(before.saturating_sub(guard.len())).unwrap_or(u64::MAX))
+    }
 }
 
 #[async_trait]
@@ -292,9 +304,68 @@ impl TransportSessionStore for PostgresTransportSessionStore {
 
         Ok(result.rows_affected() > 0)
     }
+
+    async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError> {
+        let result = query(
+            "delete from mcp_transport_sessions
+             where expires_at is not null and expires_at <= $1::timestamptz",
+        )
+        .bind(now)
+        .execute(&self.pool)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+
+        Ok(result.rows_affected())
+    }
 }
 
 async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreError> {
+    let mut transaction = pool.begin().await.map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+    query("select pg_advisory_xact_lock($1)")
+        .bind(0x4352_414E_4B4D_4350_i64)
+        .execute(&mut *transaction)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+    query(
+        "create table if not exists __crank_mcp_migrations (
+            version integer primary key,
+            checksum text not null,
+            applied_at timestamptz not null default now()
+        )",
+    )
+    .execute(&mut *transaction)
+    .await
+    .map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+    let applied = query("select checksum from __crank_mcp_migrations where version = 1")
+        .fetch_optional(&mut *transaction)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+    if let Some(row) = applied {
+        let checksum = row.get::<String, _>("checksum");
+        if checksum != "mcp-transport-sessions-v1" {
+            return Err(SessionStoreError {
+                details: format!("modified MCP migration version 1: {checksum}"),
+            });
+        }
+        transaction
+            .commit()
+            .await
+            .map_err(|error| SessionStoreError {
+                details: error.to_string(),
+            })?;
+        return Ok(());
+    }
+
     query(
         "create table if not exists mcp_transport_sessions (
             id text primary key,
@@ -308,14 +379,14 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
             expires_at timestamptz null
         )",
     )
-    .execute(pool)
+    .execute(&mut *transaction)
     .await
     .map_err(|error| SessionStoreError {
         details: error.to_string(),
     })?;
 
     query("alter table mcp_transport_sessions add column if not exists supports_elicitation boolean not null default false")
-        .execute(pool)
+        .execute(&mut *transaction)
         .await
         .map_err(|error| SessionStoreError {
             details: error.to_string(),
@@ -324,7 +395,7 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
     query(
         "alter table mcp_transport_sessions add column if not exists expires_at timestamptz null",
     )
-    .execute(pool)
+    .execute(&mut *transaction)
     .await
     .map_err(|error| SessionStoreError {
         details: error.to_string(),
@@ -334,7 +405,32 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
         "create index if not exists mcp_transport_sessions_workspace_agent_idx
          on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc)",
     )
-    .execute(pool)
+    .execute(&mut *transaction)
+    .await
+    .map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+
+    query(
+        "create index if not exists mcp_transport_sessions_expires_at_idx
+         on mcp_transport_sessions(expires_at)
+         where expires_at is not null",
+    )
+    .execute(&mut *transaction)
+    .await
+    .map_err(|error| SessionStoreError {
+        details: error.to_string(),
+    })?;
+
+    query("insert into __crank_mcp_migrations (version, checksum) values (1, $1)")
+        .bind("mcp-transport-sessions-v1")
+        .execute(&mut *transaction)
+        .await
+        .map_err(|error| SessionStoreError {
+            details: error.to_string(),
+        })?;
+    transaction
+        .commit()
         .await
         .map_err(|error| SessionStoreError {
             details: error.to_string(),

crates/crank-community-mcp/src/tool_error.rs

@@ -86,6 +86,10 @@ pub fn runtime_error_code(error: &RuntimeError) -> &'static str {
         RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
         RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
         RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
+        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
+        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
+        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
         RuntimeError::MissingAuthProfile { .. } => "auth_profile_not_found",
         RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
             "secret_not_found"
@@ -146,6 +150,19 @@ fn safe_runtime_error_message(error: &RuntimeError) -> String {
         RuntimeError::ConfirmationStoreUnavailable { .. } => {
             "Хранилище подтверждений временно недоступно.".to_owned()
         }
+        RuntimeError::IdempotencyStoreUnavailable { .. } => {
+            "Хранилище идемпотентности временно недоступно.".to_owned()
+        }
+        RuntimeError::IdempotencyInProgress { .. } => {
+            "Операция с этим ключом идемпотентности уже выполняется.".to_owned()
+        }
+        RuntimeError::IdempotencyConflict { .. } => {
+            "Ключ идемпотентности уже использован с другими параметрами.".to_owned()
+        }
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => {
+            "Результат предыдущего выполнения неизвестен; автоматический повтор заблокирован."
+                .to_owned()
+        }
         RuntimeError::MissingAuthProfile { .. } => "Профиль авторизации не найден.".to_owned(),
         RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
             "Секрет авторизации не найден.".to_owned()
@@ -169,6 +186,8 @@ fn is_recoverable_runtime_error(error: &RuntimeError) -> bool {
             | RuntimeError::ConcurrencyLimitExceeded { .. }
             | RuntimeError::SecretCrypto { .. }
             | RuntimeError::ConfirmationRequired { .. }
+            | RuntimeError::IdempotencyStoreUnavailable { .. }
+            | RuntimeError::IdempotencyInProgress { .. }
     )
 }
 
@@ -198,6 +217,14 @@ fn suggested_action(error: &RuntimeError) -> Option<&'static str> {
             Some("Запросите новый токен подтверждения.")
         }
         RuntimeError::ConfirmationStoreUnavailable { .. } => Some("Повторите запрос позже."),
+        RuntimeError::IdempotencyStoreUnavailable { .. }
+        | RuntimeError::IdempotencyInProgress { .. } => Some("Повторите запрос позже."),
+        RuntimeError::IdempotencyConflict { .. } => {
+            Some("Используйте новый ключ идемпотентности для изменённого запроса.")
+        }
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => {
+            Some("Проверьте результат во внешней системе перед ручным повтором.")
+        }
         RuntimeError::MissingAuthProfile { .. }
         | RuntimeError::MissingSecret { .. }
         | RuntimeError::MissingSecretVersion { .. }

crates/crank-community-mcp/src/tool_search.rs

@@ -3,6 +3,7 @@ use std::{collections::BTreeSet, sync::Arc};
 use axum::{http::StatusCode, response::Response};
 use crank_core::{ToolAccessMode, search_tool_catalog};
 use crank_registry::PublishedAgentCatalog;
+use crank_trace::{ErrorCategory, Stage, StageOutcome};
 use serde::Deserialize;
 use serde_json::{Value, json};
 
@@ -127,13 +128,25 @@ async fn execute_catalog_tool(
     mut arguments: Value,
     transport_request_id: &str,
 ) -> Response {
-    let Some(resolved) = resolve_generated_tool(&catalog.tools, tool_name) else {
+    let resolve_span = Stage::McpToolsResolve.span();
+    let resolved = resolve_span.in_scope(|| resolve_generated_tool(&catalog.tools, tool_name));
+    let resolved = match resolved {
+        Some(resolved) => {
+            StageOutcome::Success.record(&resolve_span);
+            drop(resolve_span);
+            resolved
+        }
+        None => {
+            StageOutcome::Error.record(&resolve_span);
+            ErrorCategory::Catalog.record(&resolve_span);
+            drop(resolve_span);
             return tool_not_found_response(
                 message,
                 response_mode,
                 &session.protocol_version,
                 tool_name,
             );
+        }
     };
     let confirmation_token = take_confirmation_token(&mut arguments);
     handle_tool_call(

crates/crank-community-mcp/src/transport.rs

@@ -22,7 +22,6 @@ use crate::jsonrpc::{
 pub(super) const HEADER_MCP_SESSION_ID: &str = "MCP-Session-Id";
 pub(super) const HEADER_MCP_PROTOCOL_VERSION: &str = "MCP-Protocol-Version";
 pub(super) const HEADER_X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id");
-const MAX_REQUEST_ID_LEN: usize = 128;
 
 #[derive(Clone, Copy)]
 pub(super) enum ResponseMode {
@@ -303,24 +302,6 @@ where
     response
 }
 
-pub(super) fn resolve_request_id(headers: &HeaderMap) -> String {
-    headers
-        .get(&HEADER_X_REQUEST_ID)
-        .and_then(|value| value.to_str().ok())
-        .map(str::trim)
-        .filter(|value| is_valid_request_id(value))
-        .map(ToOwned::to_owned)
-        .unwrap_or_else(|| uuid::Uuid::now_v7().to_string())
-}
-
-pub(super) fn is_valid_request_id(value: &str) -> bool {
-    !value.is_empty()
-        && value.len() <= MAX_REQUEST_ID_LEN
-        && value
-            .bytes()
-            .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
-}
-
 pub(super) fn extract_origin(url: &str) -> Option<String> {
     Some(parse_origin(url, false)?.origin().ascii_serialization())
 }

crates/crank-community-mcp/tests/integration/session.rs

@@ -97,3 +97,41 @@ async fn postgres_transport_sessions_evict_expired_rows_on_read() {
 
     assert_eq!(remaining, 0);
 }
+
+#[tokio::test]
+async fn postgres_transport_session_cleanup_removes_abandoned_expired_rows() {
+    let database_url = crank_test_support::postgres_schema_url("test_mcp_cleanup").await;
+    let store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
+        database_url.parse::<PgConnectOptions>().unwrap(),
+        PostgresPoolConfig::default(),
+    )
+    .await
+    .unwrap();
+    let now = OffsetDateTime::now_utc();
+
+    store
+        .create(
+            "2025-11-25",
+            "default",
+            "sales",
+            false,
+            now - time::Duration::hours(2),
+            Some(now - time::Duration::hours(1)),
+        )
+        .await
+        .unwrap();
+    let active = store
+        .create(
+            "2025-11-25",
+            "default",
+            "sales",
+            false,
+            now,
+            Some(now + time::Duration::hours(1)),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(store.cleanup_expired(now).await.unwrap(), 1);
+    assert!(store.get(&active).await.unwrap().is_some());
+}

crates/crank-community-mcp/tests/unit/session.rs

@@ -72,6 +72,31 @@ async fn drops_expired_in_memory_transport_sessions_on_read() {
     assert!(store.get(&session_id).await.unwrap().is_none());
 }
 
+#[tokio::test]
+async fn cleanup_removes_only_expired_sessions() {
+    let store = InMemorySessionStore::default();
+    let now = time::OffsetDateTime::now_utc();
+    let expired = store
+        .create("2025-11-25", "default", "sales", false, now, Some(now))
+        .await
+        .unwrap();
+    let active = store
+        .create(
+            "2025-11-25",
+            "default",
+            "sales",
+            false,
+            now,
+            Some(now + time::Duration::hours(1)),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(store.cleanup_expired(now).await.unwrap(), 1);
+    assert!(store.get(&expired).await.unwrap().is_none());
+    assert!(store.get(&active).await.unwrap().is_some());
+}
+
 #[test]
 fn formats_transport_session_store_error() {
     let error = SessionStoreError {

crates/crank-core/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-core"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-core/src/cache.rs

@@ -74,6 +74,12 @@ pub struct RateLimitBucketState {
     pub last_refill_unix_ms: i64,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RateLimitDecision {
+    Allowed,
+    Rejected { retry_after_ms: u64 },
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum ReplayGuardStatus {
@@ -86,6 +92,12 @@ pub struct CoordinationStateValue {
     pub payload: Value,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum CoordinationStateReservation {
+    Reserved,
+    Existing(CoordinationStateValue),
+}
+
 #[async_trait]
 pub trait ResponseCacheStore: Send + Sync {
     async fn get(&self, key: &str) -> Result<Option<CachedResponse>, CacheStoreError>;
@@ -108,6 +120,14 @@ pub trait RateLimitStateStore: Send + Sync {
         ttl: Duration,
     ) -> Result<(), CacheStoreError>;
     async fn delete_bucket(&self, key: &str) -> Result<(), CacheStoreError>;
+    async fn consume_token(
+        &self,
+        key: &str,
+        burst_tokens_micros: u64,
+        refill_per_second_micros: u64,
+        now_unix_ms: i64,
+        ttl: Duration,
+    ) -> Result<RateLimitDecision, CacheStoreError>;
 }
 
 #[async_trait]
@@ -135,6 +155,26 @@ pub trait CoordinationStateStore: Send + Sync {
         ttl: Duration,
     ) -> Result<(), CacheStoreError>;
     async fn delete_value(&self, scope: CacheScope, key: &str) -> Result<(), CacheStoreError>;
+    async fn take_value(
+        &self,
+        scope: CacheScope,
+        key: &str,
+    ) -> Result<Option<CoordinationStateValue>, CacheStoreError>;
+    async fn reserve_value(
+        &self,
+        scope: CacheScope,
+        key: &str,
+        value: CoordinationStateValue,
+        ttl: Duration,
+    ) -> Result<CoordinationStateReservation, CacheStoreError>;
+    async fn compare_and_set_value(
+        &self,
+        scope: CacheScope,
+        key: &str,
+        expected: &CoordinationStateValue,
+        value: CoordinationStateValue,
+        ttl: Duration,
+    ) -> Result<bool, CacheStoreError>;
 }
 
 #[derive(Debug, Error, PartialEq, Eq)]

crates/crank-core/src/lib.rs

@@ -66,8 +66,9 @@ pub mod domain {
 
 pub mod ports {
     pub use crate::cache::{
-        CacheStoreError, CoordinationStateStore, CoordinationStateValue, RateLimitStateStore,
+        CacheStoreError, CoordinationStateReservation, CoordinationStateStore,
-        ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
+        CoordinationStateValue, RateLimitDecision, RateLimitStateStore, ReplayGuardStatus,
+        ReplayGuardStore, ResponseCacheStore,
     };
     pub use crate::ext::access::{
         OwnerOnlyPolicyEngine, PolicyAction, PolicyDecision, PolicyEngine, PolicyScope,
@@ -108,8 +109,9 @@ pub use auth::{
 };
 pub use cache::{
     CacheBackend, CacheScope, CacheStoreError, CachedHeader, CachedResponse,
-    CoordinationStateStore, CoordinationStateValue, ParseCacheBackendError, RateLimitBucketState,
+    CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue,
-    RateLimitStateStore, ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
+    ParseCacheBackendError, RateLimitBucketState, RateLimitDecision, RateLimitStateStore,
+    ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
 };
 pub use edition::{
     EditionCapabilities, EditionLimits, MachineAccessMode, OperationSecurityLevel, ProductEdition,

crates/crank-import/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-import"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-mapping/Cargo.toml

@@ -3,6 +3,7 @@ name = "crank-mapping"
 edition.workspace = true
 license.workspace = true
 rust-version.workspace = true
+publish.workspace = true
 version.workspace = true
 
 [dependencies]

crates/crank-observability/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "crank-observability"
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+publish.workspace = true
+version.workspace = true
+
+[dependencies]
+axum.workspace = true
+metrics.workspace = true
+metrics-exporter-prometheus.workspace = true
+opentelemetry.workspace = true
+opentelemetry-otlp.workspace = true
+opentelemetry_sdk.workspace = true
+percent-encoding.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sentry.workspace = true
+sha2.workspace = true
+subtle.workspace = true
+thiserror.workspace = true
+time.workspace = true
+tokio = { workspace = true, features = ["net"] }
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true
+url.workspace = true
+uuid.workspace = true
+
+[dev-dependencies]
+opentelemetry-proto.workspace = true
+prost.workspace = true
+sentry = { workspace = true, features = ["test"] }
+tower.workspace = true

crates/crank-observability/src/config.rs

@@ -0,0 +1,170 @@
+use std::env;
+
+use thiserror::Error;
+
+use crate::RedactionLimits;
+
+const DEFAULT_ENVIRONMENT: &str = "development";
+const MAX_IDENTITY_LABEL_BYTES: usize = 64;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ServiceIdentity {
+    service: String,
+    version: String,
+    environment: String,
+}
+
+impl ServiceIdentity {
+    pub fn try_new(
+        service: impl Into<String>,
+        version: impl Into<String>,
+        environment: impl Into<String>,
+    ) -> Result<Self, ObservabilityConfigError> {
+        let identity = Self {
+            service: service.into(),
+            version: version.into(),
+            environment: environment.into(),
+        };
+        validate_label("service", &identity.service)?;
+        validate_label("version", &identity.version)?;
+        validate_label("environment", &identity.environment)?;
+        Ok(identity)
+    }
+
+    pub fn service(&self) -> &str {
+        &self.service
+    }
+
+    pub fn version(&self) -> &str {
+        &self.version
+    }
+
+    pub fn environment(&self) -> &str {
+        &self.environment
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct ObservabilityConfig {
+    identity: ServiceIdentity,
+    filter: String,
+    redaction_limits: RedactionLimits,
+}
+
+impl ObservabilityConfig {
+    pub fn new(
+        identity: ServiceIdentity,
+        filter: impl Into<String>,
+        redaction_limits: RedactionLimits,
+    ) -> Self {
+        Self {
+            identity,
+            filter: filter.into(),
+            redaction_limits,
+        }
+    }
+
+    pub fn from_env(
+        service: &'static str,
+        version: &'static str,
+        default_filter: &'static str,
+    ) -> Result<Self, ObservabilityConfigError> {
+        let environment = env_value_or_default(
+            "CRANK_ENVIRONMENT",
+            env::var("CRANK_ENVIRONMENT"),
+            DEFAULT_ENVIRONMENT,
+        )?;
+        let filter = env_value_or_default(
+            "CRANK_LOG_LEVEL",
+            env::var("CRANK_LOG_LEVEL"),
+            default_filter,
+        )?;
+        let identity = ServiceIdentity::try_new(service, version, environment)?;
+
+        Ok(Self::new(identity, filter, RedactionLimits::default()))
+    }
+
+    pub(crate) fn into_parts(self) -> (ServiceIdentity, String, RedactionLimits) {
+        (self.identity, self.filter, self.redaction_limits)
+    }
+
+    pub(crate) fn identity(&self) -> &ServiceIdentity {
+        &self.identity
+    }
+
+    pub(crate) fn redaction_limits(&self) -> RedactionLimits {
+        self.redaction_limits
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum ObservabilityConfigError {
+    #[error("invalid observability identity field: {field}")]
+    InvalidIdentity { field: &'static str },
+    #[error("observability environment variable is not valid UTF-8: {field}")]
+    InvalidEnvironmentEncoding { field: &'static str },
+}
+
+fn env_value_or_default(
+    field: &'static str,
+    value: Result<String, env::VarError>,
+    default: &'static str,
+) -> Result<String, ObservabilityConfigError> {
+    match value {
+        Ok(value) => Ok(value),
+        Err(env::VarError::NotPresent) => Ok(default.to_owned()),
+        Err(env::VarError::NotUnicode(_)) => {
+            Err(ObservabilityConfigError::InvalidEnvironmentEncoding { field })
+        }
+    }
+}
+
+fn validate_label(field: &'static str, value: &str) -> Result<(), ObservabilityConfigError> {
+    let valid = !value.is_empty()
+        && value.len() <= MAX_IDENTITY_LABEL_BYTES
+        && value
+            .bytes()
+            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+'));
+
+    if valid {
+        Ok(())
+    } else {
+        Err(ObservabilityConfigError::InvalidIdentity { field })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::ffi::OsString;
+
+    use super::{ObservabilityConfigError, ServiceIdentity, env_value_or_default};
+
+    #[test]
+    fn accepts_release_and_environment_labels() {
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1+build.7", "production")
+            .expect("identity must be valid");
+
+        assert_eq!(identity.service(), "admin-api");
+        assert_eq!(identity.version(), "0.3.1+build.7");
+        assert_eq!(identity.environment(), "production");
+    }
+
+    #[test]
+    fn rejects_non_utf8_environment_values() {
+        let error = env_value_or_default(
+            "CRANK_ENVIRONMENT",
+            Err(std::env::VarError::NotUnicode(OsString::from(
+                "invalid-environment",
+            ))),
+            "development",
+        )
+        .expect_err("non-UTF-8 values must not be replaced with defaults");
+
+        assert!(matches!(
+            error,
+            ObservabilityConfigError::InvalidEnvironmentEncoding {
+                field: "CRANK_ENVIRONMENT"
+            }
+        ));
+    }
+}

crates/crank-observability/src/correlation.rs

@@ -0,0 +1,39 @@
+use std::fmt;
+
+use uuid::Uuid;
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct RequestId(String);
+
+impl RequestId {
+    pub const MAX_LEN: usize = 128;
+
+    pub fn resolve(candidate: Option<&str>) -> Self {
+        candidate
+            .filter(|value| Self::is_valid(value))
+            .map(|value| Self(value.to_owned()))
+            .unwrap_or_else(|| Self(Uuid::now_v7().to_string()))
+    }
+
+    pub fn is_valid(value: &str) -> bool {
+        !value.is_empty()
+            && value.len() <= Self::MAX_LEN
+            && value
+                .bytes()
+                .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    pub fn into_string(self) -> String {
+        self.0
+    }
+}
+
+impl fmt::Display for RequestId {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str(self.as_str())
+    }
+}

crates/crank-observability/src/error_reporting.rs

@@ -0,0 +1,467 @@
+use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration};
+
+use sentry::{
+    ClientInitGuard, ClientOptions,
+    protocol::{Event, Level},
+    types::Dsn,
+};
+use thiserror::Error;
+
+use crate::{
+    RedactionLimits, ServiceIdentity, propagation::current_trace_id, redaction::truncate_string,
+};
+
+const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
+const CRITICAL_ERROR_MESSAGE: &str = "critical error";
+const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
+
+tokio::task_local! {
+    static REQUEST_ID: String;
+}
+
+pub struct SentryConfig {
+    dsn: Option<Dsn>,
+}
+
+impl SentryConfig {
+    pub fn parse(value: Option<&str>) -> Result<Self, SentryConfigError> {
+        let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
+            return Ok(Self { dsn: None });
+        };
+
+        let dsn = value
+            .parse::<Dsn>()
+            .map_err(|_| SentryConfigError::InvalidDsn)?;
+        Ok(Self { dsn: Some(dsn) })
+    }
+
+    pub fn from_env() -> Result<Self, SentryConfigError> {
+        match env::var(SENTRY_DSN_ENV) {
+            Ok(value) => Self::parse(Some(&value)),
+            Err(env::VarError::NotPresent) => Self::parse(None),
+            Err(env::VarError::NotUnicode(_)) => Err(SentryConfigError::InvalidEnvironmentEncoding),
+        }
+    }
+
+    pub fn enabled(&self) -> bool {
+        self.dsn.is_some()
+    }
+}
+
+impl fmt::Debug for SentryConfig {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("SentryConfig")
+            .field("enabled", &self.enabled())
+            .finish()
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum SentryConfigError {
+    #[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")]
+    InvalidDsn,
+    #[error("CRANK_SENTRY_DSN is not valid UTF-8")]
+    InvalidEnvironmentEncoding,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum CriticalErrorCategory {
+    Panic,
+    Startup,
+    Internal,
+    DataIntegrity,
+}
+
+impl CriticalErrorCategory {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Panic => "panic",
+            Self::Startup => "startup",
+            Self::Internal => "internal",
+            Self::DataIntegrity => "data_integrity",
+        }
+    }
+
+    fn parse(value: &str) -> Option<Self> {
+        match value {
+            "panic" => Some(Self::Panic),
+            "startup" => Some(Self::Startup),
+            "internal" => Some(Self::Internal),
+            "data_integrity" => Some(Self::DataIntegrity),
+            _ => None,
+        }
+    }
+}
+
+pub fn capture_critical_error(category: CriticalErrorCategory) {
+    let mut tags = correlation_tags();
+    tags.insert("category".to_owned(), category.as_str().to_owned());
+    sentry::capture_event(Event {
+        level: Level::Error,
+        message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
+        fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
+        tags,
+        ..Event::default()
+    });
+}
+
+pub async fn with_request_correlation<F>(request_id: String, future: F) -> F::Output
+where
+    F: Future,
+{
+    REQUEST_ID.scope(request_id, future).await
+}
+
+pub(crate) fn init_sentry(
+    identity: &ServiceIdentity,
+    limits: RedactionLimits,
+    config: SentryConfig,
+) -> Option<ClientInitGuard> {
+    let dsn = config.dsn?;
+    let identity = identity.clone();
+    let options = client_options(identity, limits);
+    Some(sentry::init((dsn, options)))
+}
+
+fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
+    let release = identity.version().to_owned();
+    let environment = identity.environment().to_owned();
+    let service = identity.service().to_owned();
+    let sanitizer_identity = identity.clone();
+
+    let mut options = ClientOptions::default();
+    options.release = Some(Cow::Owned(release));
+    options.environment = Some(Cow::Owned(environment));
+    options.server_name = Some(Cow::Owned(service));
+    options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled;
+    options.max_breadcrumbs = 0;
+    options.attach_stacktrace = false;
+    options.send_default_pii = false;
+    options.before_send = Some(std::sync::Arc::new(move |event| {
+        Some(sanitize_event(event, &sanitizer_identity, limits))
+    }));
+    options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT;
+    options.auto_session_tracking = false;
+    options.enable_logs = false;
+    options.enable_metrics = false;
+    options
+}
+
+fn sanitize_event(
+    event: Event<'static>,
+    identity: &ServiceIdentity,
+    limits: RedactionLimits,
+) -> Event<'static> {
+    let category = event
+        .tags
+        .get("category")
+        .and_then(|value| CriticalErrorCategory::parse(value))
+        .unwrap_or_else(|| {
+            if event.exception.is_empty() {
+                CriticalErrorCategory::Internal
+            } else {
+                CriticalErrorCategory::Panic
+            }
+        });
+    let mut tags = correlation_tags()
+        .into_iter()
+        .map(|(key, value)| (key, truncate_string(&value, limits.max_string_bytes)))
+        .collect::<BTreeMap<_, _>>();
+    for key in ["request_id", "trace_id"] {
+        if let Some(value) = event.tags.get(key) {
+            tags.entry(key.to_owned())
+                .or_insert_with(|| truncate_string(value, limits.max_string_bytes));
+        }
+    }
+    tags.insert("service".to_owned(), identity.service().to_owned());
+    tags.insert("category".to_owned(), category.as_str().to_owned());
+
+    enforce_event_budget(
+        Event {
+            event_id: event.event_id,
+            level: Level::Error,
+            fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
+            message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
+            timestamp: event.timestamp,
+            server_name: Some(Cow::Owned(identity.service().to_owned())),
+            release: Some(Cow::Owned(identity.version().to_owned())),
+            environment: Some(Cow::Owned(identity.environment().to_owned())),
+            tags,
+            ..Event::default()
+        },
+        limits.max_event_bytes,
+    )
+}
+
+fn correlation_tags() -> BTreeMap<String, String> {
+    let mut tags = BTreeMap::new();
+    if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) {
+        tags.insert("request_id".to_owned(), request_id);
+    }
+    if let Some(trace_id) = current_trace_id() {
+        tags.insert("trace_id".to_owned(), trace_id);
+    }
+    tags
+}
+
+fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
+    if serialized_event_len(&event) <= max_event_bytes {
+        return event;
+    }
+
+    event.tags.remove("request_id");
+    event.tags.remove("trace_id");
+    event
+}
+
+fn serialized_event_len(event: &Event<'_>) -> usize {
+    serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        collections::BTreeMap,
+        sync::{
+            Arc,
+            atomic::{AtomicUsize, Ordering},
+        },
+    };
+
+    use opentelemetry::trace::TracerProvider as _;
+    use opentelemetry_sdk::trace::SdkTracerProvider;
+    use sentry::{
+        Envelope, Hub,
+        protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values},
+    };
+
+    use super::{
+        CriticalErrorCategory, capture_critical_error, client_options, sanitize_event,
+        with_request_correlation,
+    };
+    use crate::{
+        ObservabilityConfig, RedactionLimits, ServiceIdentity,
+        logging::build_subscriber_with_tracer,
+    };
+
+    fn identity() -> ServiceIdentity {
+        ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity")
+    }
+
+    #[test]
+    fn client_disables_non_error_telemetry() {
+        let options = client_options(identity(), RedactionLimits::default());
+
+        assert_eq!(options.max_breadcrumbs, 0);
+        assert!(!options.attach_stacktrace);
+        assert!(!options.send_default_pii);
+        assert!(!options.auto_session_tracking);
+        assert!(!options.enable_logs);
+        assert!(!options.enable_metrics);
+        assert!(options.before_send.is_some());
+    }
+
+    #[test]
+    fn sanitizer_uses_a_strict_allowlist() {
+        let mut tags = BTreeMap::new();
+        tags.insert("category".to_owned(), "data_integrity".to_owned());
+        tags.insert("secret".to_owned(), "must-not-leak".to_owned());
+        let mut contexts = BTreeMap::new();
+        contexts.insert(
+            "secret".to_owned(),
+            Context::Other(BTreeMap::from([(
+                "token".to_owned(),
+                Value::String("must-not-leak".to_owned()),
+            )])),
+        );
+        let event = sentry::protocol::Event {
+            message: Some("password=must-not-leak".to_owned()),
+            request: Some(Request::default()),
+            user: Some(User::default()),
+            breadcrumbs: Values {
+                values: vec![Breadcrumb::default()],
+            },
+            exception: Values {
+                values: vec![Exception {
+                    value: Some("must-not-leak".to_owned()),
+                    ..Exception::default()
+                }],
+            },
+            contexts,
+            extra: BTreeMap::from([(
+                "payload".to_owned(),
+                Value::String("must-not-leak".to_owned()),
+            )]),
+            tags,
+            ..sentry::protocol::Event::default()
+        };
+
+        let cleaned = sanitize_event(event, &identity(), RedactionLimits::default());
+        let serialized = serde_json::to_string(&cleaned).expect("serialize event");
+
+        assert_eq!(cleaned.message.as_deref(), Some("critical error"));
+        assert_eq!(
+            cleaned.tags.get("category").map(String::as_str),
+            Some(CriticalErrorCategory::DataIntegrity.as_str())
+        );
+        assert!(cleaned.request.is_none());
+        assert!(cleaned.user.is_none());
+        assert!(cleaned.breadcrumbs.is_empty());
+        assert!(cleaned.exception.is_empty());
+        assert!(cleaned.contexts.is_empty());
+        assert!(cleaned.extra.is_empty());
+        assert!(!serialized.contains("must-not-leak"));
+        assert!(!serialized.contains("password"));
+    }
+
+    #[test]
+    fn sanitizer_honours_total_event_budget() {
+        let limits = RedactionLimits {
+            max_string_bytes: 8 * 1024,
+            max_event_bytes: 512,
+            ..RedactionLimits::default()
+        };
+        let event = sentry::protocol::Event {
+            tags: BTreeMap::from([
+                ("category".to_owned(), "internal".to_owned()),
+                ("request_id".to_owned(), "r".repeat(8 * 1024)),
+                ("trace_id".to_owned(), "t".repeat(8 * 1024)),
+            ]),
+            ..sentry::protocol::Event::default()
+        };
+
+        let cleaned = sanitize_event(event, &identity(), limits);
+        let serialized = serde_json::to_vec(&cleaned).expect("serialize event");
+
+        assert!(serialized.len() <= limits.max_event_bytes);
+        assert_eq!(
+            cleaned.tags.get("category").map(String::as_str),
+            Some("internal")
+        );
+        assert_eq!(
+            cleaned.tags.get("service").map(String::as_str),
+            Some("admin-api")
+        );
+        assert!(!cleaned.tags.contains_key("request_id"));
+        assert!(!cleaned.tags.contains_key("trace_id"));
+    }
+
+    #[test]
+    fn expected_application_errors_do_not_create_critical_events() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let events = sentry::test::with_captured_events_options(
+            || tracing::error!("ordinary product error"),
+            options,
+        );
+
+        assert!(events.is_empty());
+    }
+
+    #[test]
+    fn explicit_critical_error_is_correlated_and_sanitized() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .build()
+            .expect("runtime");
+        let provider = SdkTracerProvider::builder().build();
+        let tracer = provider.tracer("critical-error-test");
+        let subscriber = build_subscriber_with_tracer(
+            ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
+            std::io::sink,
+            Some(tracer),
+        )
+        .expect("subscriber");
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let events = sentry::test::with_captured_events_options(
+            || {
+                tracing::dispatcher::with_default(&dispatch, || {
+                    runtime.block_on(with_request_correlation("request-123".to_owned(), async {
+                        let span = tracing::info_span!(target: "crank::trace", "http.request");
+                        let _span_guard = span.enter();
+                        capture_critical_error(CriticalErrorCategory::DataIntegrity);
+                    }));
+                });
+            },
+            options,
+        );
+
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(event.message.as_deref(), Some("critical error"));
+        assert_eq!(
+            event.tags.get("category").map(String::as_str),
+            Some("data_integrity")
+        );
+        assert_eq!(
+            event.tags.get("request_id").map(String::as_str),
+            Some("request-123")
+        );
+        assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
+        assert_eq!(event.release.as_deref(), Some("1.2.3"));
+        assert_eq!(event.environment.as_deref(), Some("test"));
+        assert_eq!(event.server_name.as_deref(), Some("admin-api"));
+    }
+
+    #[test]
+    fn panic_creates_exactly_one_sanitized_critical_event() {
+        let options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        let events = sentry::test::with_captured_events_options(
+            || {
+                let result = std::panic::catch_unwind(|| {
+                    panic!("password=must-not-leak");
+                });
+                assert!(result.is_err());
+            },
+            options,
+        );
+
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(
+            event.tags.get("category").map(String::as_str),
+            Some("panic")
+        );
+        let serialized = serde_json::to_string(event).expect("serialize event");
+        assert!(!serialized.contains("must-not-leak"));
+        assert!(!serialized.contains("password"));
+    }
+
+    #[test]
+    fn receiver_failure_does_not_change_product_result_or_recurse() {
+        struct DroppingTransport {
+            attempts: AtomicUsize,
+        }
+
+        impl sentry::Transport for DroppingTransport {
+            fn send_envelope(&self, _envelope: Envelope) {
+                self.attempts.fetch_add(1, Ordering::Relaxed);
+            }
+        }
+
+        let transport = Arc::new(DroppingTransport {
+            attempts: AtomicUsize::new(0),
+        });
+        let mut options =
+            sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
+        options.dsn = Some(
+            "https://public@example.invalid/1"
+                .parse()
+                .expect("valid test DSN"),
+        );
+        options.transport = Some(Arc::new(transport.clone()));
+        let client = Arc::new(sentry::Client::from(options));
+        let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default())));
+
+        let product_result = Hub::run(hub, || {
+            capture_critical_error(CriticalErrorCategory::Internal);
+            42
+        });
+
+        assert_eq!(product_result, 42);
+        assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
+    }
+}

crates/crank-observability/src/incidents.rs

@@ -0,0 +1,36 @@
+use std::sync::atomic::{AtomicU64, Ordering};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum OperationalIncident {
+    InvocationHistoryLost,
+}
+
+static INVOCATION_HISTORY_LOST_TOTAL: AtomicU64 = AtomicU64::new(0);
+
+pub fn record_operational_incident(incident: OperationalIncident) {
+    let counter = counter(incident);
+    let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
+        value.checked_add(1)
+    });
+    match incident {
+        OperationalIncident::InvocationHistoryLost => {
+            metrics::counter!("crank_invocation_history_lost_total").increment(1);
+            metrics::counter!(
+                "crank_telemetry_export_failures_total",
+                "signal_type" => "invocation_history",
+                "exporter" => "postgres"
+            )
+            .increment(1);
+        }
+    }
+}
+
+pub fn operational_incident_total(incident: OperationalIncident) -> u64 {
+    counter(incident).load(Ordering::Relaxed)
+}
+
+fn counter(incident: OperationalIncident) -> &'static AtomicU64 {
+    match incident {
+        OperationalIncident::InvocationHistoryLost => &INVOCATION_HISTORY_LOST_TOTAL,
+    }
+}

crates/crank-observability/src/instrumentation.rs

@@ -0,0 +1,133 @@
+use std::time::Instant;
+
+use axum::{
+    extract::{MatchedPath, Request},
+    middleware::Next,
+    response::Response,
+};
+use metrics::{Gauge, Unit};
+
+use crate::{MetricKind, MetricUnit, metric_schema};
+
+pub async fn record_http_request(request: Request, next: Next) -> Response {
+    let route = request
+        .extensions()
+        .get::<MatchedPath>()
+        .map_or("unmatched", MatchedPath::as_str)
+        .to_owned();
+    let method = normalized_http_method(request.method().as_str());
+    let started_at = Instant::now();
+    let _inflight = GaugeGuard::increment("crank_http_inflight");
+
+    let response = next.run(request).await;
+    let status_class = status_class(response.status().as_u16());
+
+    metrics::counter!(
+        "crank_http_requests_total",
+        "route" => route.clone(),
+        "method" => method,
+        "status_class" => status_class
+    )
+    .increment(1);
+    metrics::histogram!(
+        "crank_http_request_duration_seconds",
+        "route" => route,
+        "method" => method
+    )
+    .record(started_at.elapsed().as_secs_f64());
+
+    response
+}
+
+pub fn record_db_pool_connections(total: u32, idle: usize) {
+    let idle = idle.min(total as usize) as f64;
+    metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(idle);
+    metrics::gauge!("crank_db_pool_connections", "state" => "used").set(f64::from(total) - idle);
+}
+
+pub(crate) fn register_metric_schema() {
+    for definition in metric_schema() {
+        let unit = match definition.unit {
+            MetricUnit::Count => Unit::Count,
+            MetricUnit::Seconds => Unit::Seconds,
+        };
+        match definition.kind {
+            MetricKind::Counter => {
+                metrics::describe_counter!(definition.name, unit, definition.description);
+            }
+            MetricKind::Gauge => {
+                metrics::describe_gauge!(definition.name, unit, definition.description);
+            }
+            MetricKind::Histogram => {
+                metrics::describe_histogram!(definition.name, unit, definition.description);
+            }
+        }
+    }
+
+    metrics::gauge!("crank_http_inflight").set(0.0);
+    metrics::gauge!("crank_mcp_active_sessions").set(0.0);
+    metrics::gauge!("crank_runtime_inflight").set(0.0);
+    metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(0.0);
+    metrics::gauge!("crank_db_pool_connections", "state" => "used").set(0.0);
+    metrics::gauge!("crank_catalog_tools").set(0.0);
+    metrics::gauge!("crank_catalog_estimated_context_tokens").set(0.0);
+    metrics::gauge!("crank_catalog_warnings").set(0.0);
+}
+
+fn normalized_http_method(method: &str) -> &'static str {
+    match method {
+        "GET" => "GET",
+        "POST" => "POST",
+        "PUT" => "PUT",
+        "PATCH" => "PATCH",
+        "DELETE" => "DELETE",
+        "OPTIONS" => "OPTIONS",
+        "HEAD" => "HEAD",
+        "CONNECT" => "CONNECT",
+        "TRACE" => "TRACE",
+        _ => "OTHER",
+    }
+}
+
+fn status_class(status: u16) -> &'static str {
+    match status {
+        100..=199 => "1xx",
+        200..=299 => "2xx",
+        300..=399 => "3xx",
+        400..=499 => "4xx",
+        500..=599 => "5xx",
+        _ => "other",
+    }
+}
+
+struct GaugeGuard {
+    gauge: Gauge,
+}
+
+impl GaugeGuard {
+    fn increment(name: &'static str) -> Self {
+        let gauge = metrics::gauge!(name);
+        gauge.increment(1.0);
+        Self { gauge }
+    }
+}
+
+impl Drop for GaugeGuard {
+    fn drop(&mut self) {
+        self.gauge.decrement(1.0);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{normalized_http_method, status_class};
+
+    #[test]
+    fn normalizes_unbounded_http_values() {
+        assert_eq!(normalized_http_method("GET"), "GET");
+        assert_eq!(normalized_http_method("CUSTOM-user-controlled"), "OTHER");
+        assert_eq!(status_class(204), "2xx");
+        assert_eq!(status_class(429), "4xx");
+        assert_eq!(status_class(999), "other");
+    }
+}

crates/crank-observability/src/lib.rs

@@ -0,0 +1,37 @@
+mod config;
+mod correlation;
+mod error_reporting;
+mod incidents;
+mod instrumentation;
+mod lifecycle;
+mod logging;
+mod metrics_schema;
+mod otlp;
+mod prometheus;
+mod propagation;
+mod redaction;
+mod schema;
+
+pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
+pub use correlation::RequestId;
+pub use error_reporting::{
+    CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
+    with_request_correlation,
+};
+pub use incidents::{OperationalIncident, operational_incident_total, record_operational_incident};
+pub use instrumentation::{record_db_pool_connections, record_http_request};
+pub use lifecycle::{ObservabilityInitError, ObservabilityLifecycle, init};
+pub use logging::build_subscriber;
+pub use metrics_schema::{
+    DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
+};
+pub use otlp::{
+    OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError, build_tracer_provider,
+};
+pub use prometheus::{
+    MetricsConfig, MetricsConfigError, MetricsServeError, MetricsSurface, MetricsSurfaceError,
+};
+pub use propagation::{inject_current_trace_context, set_remote_trace_parent};
+pub use redaction::{
+    REDACTED_MARKER, RedactionLimits, RedactionLimitsError, SafeJsonError, redact_value, safe_json,
+};

crates/crank-observability/src/lifecycle.rs

@@ -0,0 +1,90 @@
+use std::{fmt, io};
+
+use thiserror::Error;
+use tracing_subscriber::util::SubscriberInitExt;
+
+use crate::{
+    MetricsConfig, MetricsSurface, MetricsSurfaceError, ObservabilityConfig,
+    ObservabilityConfigError, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError,
+    RedactionLimitsError, SentryConfig, SentryConfigError, error_reporting::init_sentry,
+    instrumentation::register_metric_schema, logging::build_subscriber_with_tracer,
+    otlp::build_tracer_provider, prometheus::install_prometheus_recorder,
+    propagation::install_trace_context_propagator,
+};
+
+#[must_use = "observability resources must be retained until process shutdown"]
+pub struct ObservabilityLifecycle {
+    metrics_handle: metrics_exporter_prometheus::PrometheusHandle,
+    tracer_provider: Option<opentelemetry_sdk::trace::SdkTracerProvider>,
+    sentry_guard: Option<sentry::ClientInitGuard>,
+}
+
+impl ObservabilityLifecycle {
+    pub fn init(config: ObservabilityConfig) -> Result<Self, ObservabilityInitError> {
+        let identity = config.identity().clone();
+        let redaction_limits = config.redaction_limits();
+        let sentry_config = SentryConfig::from_env()?;
+        let trace_config = OtlpTraceConfig::from_env()?;
+        let tracing = build_tracer_provider(&identity, &trace_config)?;
+        let tracer = tracing.as_ref().map(|(_, tracer)| tracer.clone());
+        install_trace_context_propagator();
+        build_subscriber_with_tracer(config, io::stdout, tracer)?
+            .try_init()
+            .map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?;
+        let metrics_handle = install_prometheus_recorder(&identity)?;
+        register_metric_schema();
+        let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config);
+
+        Ok(Self {
+            metrics_handle,
+            tracer_provider: tracing.map(|(provider, _)| provider),
+            sentry_guard,
+        })
+    }
+
+    pub fn metrics_surface(&self, config: MetricsConfig) -> MetricsSurface {
+        MetricsSurface::new(config, self.metrics_handle.clone())
+    }
+
+    pub fn traces_enabled(&self) -> bool {
+        self.tracer_provider.is_some()
+    }
+
+    pub fn critical_errors_enabled(&self) -> bool {
+        self.sentry_guard.is_some()
+    }
+}
+
+impl fmt::Debug for ObservabilityLifecycle {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("ObservabilityLifecycle")
+            .field("traces_enabled", &self.traces_enabled())
+            .field("critical_errors_enabled", &self.critical_errors_enabled())
+            .finish_non_exhaustive()
+    }
+}
+
+pub fn init(config: ObservabilityConfig) -> Result<ObservabilityLifecycle, ObservabilityInitError> {
+    ObservabilityLifecycle::init(config)
+}
+
+#[derive(Debug, Error)]
+pub enum ObservabilityInitError {
+    #[error(transparent)]
+    InvalidConfig(#[from] ObservabilityConfigError),
+    #[error(transparent)]
+    InvalidRedactionLimits(#[from] RedactionLimitsError),
+    #[error("invalid log filter")]
+    InvalidFilter,
+    #[error("global tracing subscriber is already initialized")]
+    SubscriberAlreadyInitialized,
+    #[error(transparent)]
+    Metrics(#[from] MetricsSurfaceError),
+    #[error(transparent)]
+    OtlpConfig(#[from] OtlpTraceConfigError),
+    #[error(transparent)]
+    Otlp(#[from] OtlpTraceError),
+    #[error(transparent)]
+    SentryConfig(#[from] SentryConfigError),
+}

crates/crank-observability/src/logging.rs

@@ -0,0 +1,314 @@
+use std::fmt;
+
+use serde_json::{Map, Number, Value};
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tracing::{Event, Subscriber, field::Visit};
+use tracing_subscriber::{
+    EnvFilter, Layer,
+    filter::filter_fn,
+    fmt::{FmtContext, FormatEvent, FormatFields, MakeWriter, format::Writer},
+    layer::SubscriberExt,
+    registry::LookupSpan,
+};
+
+use crate::{
+    ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity,
+    propagation::current_trace_id,
+    redaction::{redact_value, truncate_string},
+    schema::LogEnvelope,
+};
+
+pub fn build_subscriber<W>(
+    config: ObservabilityConfig,
+    writer: W,
+) -> Result<impl Subscriber + Send + Sync, ObservabilityInitError>
+where
+    W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static,
+{
+    build_subscriber_with_tracer(config, writer, None)
+}
+
+pub(crate) fn build_subscriber_with_tracer<W>(
+    config: ObservabilityConfig,
+    writer: W,
+    tracer: Option<opentelemetry_sdk::trace::SdkTracer>,
+) -> Result<impl Subscriber + Send + Sync, ObservabilityInitError>
+where
+    W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static,
+{
+    let (identity, filter, limits) = config.into_parts();
+    limits.validate()?;
+    let filter = EnvFilter::try_new(filter).map_err(|_| ObservabilityInitError::InvalidFilter)?;
+    let formatter = JsonEventFormatter::new(identity, limits);
+    let fmt_layer = tracing_subscriber::fmt::layer()
+        .with_ansi(false)
+        .event_format(formatter)
+        .with_writer(writer)
+        .with_filter(filter);
+    let otel_layer = tracer.map(|tracer| {
+        tracing_opentelemetry::layer()
+            .with_tracer(tracer)
+            .with_filter(filter_fn(|metadata| {
+                metadata.is_span() && metadata.target() == "crank::trace"
+            }))
+    });
+
+    Ok(tracing_subscriber::registry()
+        .with(fmt_layer)
+        .with(otel_layer))
+}
+
+#[derive(Clone, Debug)]
+struct JsonEventFormatter {
+    identity: ServiceIdentity,
+    limits: RedactionLimits,
+}
+
+impl JsonEventFormatter {
+    fn new(identity: ServiceIdentity, limits: RedactionLimits) -> Self {
+        Self { identity, limits }
+    }
+
+    fn envelope(&self, event: &Event<'_>) -> Result<LogEnvelope, fmt::Error> {
+        let metadata = event.metadata();
+        let mut visitor = JsonFieldVisitor::default();
+        event.record(&mut visitor);
+        let mut raw_fields = visitor.fields;
+        let request_id = take_correlation_id(&mut raw_fields, "request_id")
+            .map(|value| truncate_string(&value, self.limits.max_string_bytes));
+        let trace_id = take_correlation_id(&mut raw_fields, "trace_id")
+            .or_else(current_trace_id)
+            .map(|value| truncate_string(&value, self.limits.max_string_bytes));
+        let cleaned = redact_value(&Value::Object(raw_fields), self.limits);
+        let fields = cleaned.as_object().cloned().unwrap_or_default();
+        let timestamp = OffsetDateTime::now_utc()
+            .format(&Rfc3339)
+            .map_err(|_| fmt::Error)?;
+
+        Ok(LogEnvelope {
+            timestamp,
+            level: metadata.level().as_str().to_owned(),
+            service: self.identity.service().to_owned(),
+            version: self.identity.version().to_owned(),
+            environment: self.identity.environment().to_owned(),
+            target: truncate_string(metadata.target(), self.limits.max_string_bytes),
+            event: truncate_string(metadata.name(), self.limits.max_string_bytes),
+            request_id,
+            trace_id,
+            fields,
+        })
+    }
+
+    fn serialize_bounded(&self, mut envelope: LogEnvelope) -> Result<String, fmt::Error> {
+        let line_budget = self.limits.max_event_bytes.saturating_sub(1);
+        let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
+        if serialized.len() <= line_budget {
+            return Ok(serialized);
+        }
+
+        envelope.fields = Map::from_iter([("truncated".to_owned(), Value::Bool(true))]);
+        let fallback_string_limit = self.limits.max_string_bytes.min(64);
+        envelope.target = truncate_string(&envelope.target, fallback_string_limit);
+        envelope.event = truncate_string(&envelope.event, fallback_string_limit);
+        envelope.request_id = envelope
+            .request_id
+            .map(|value| truncate_string(&value, fallback_string_limit));
+        envelope.trace_id = envelope
+            .trace_id
+            .map(|value| truncate_string(&value, fallback_string_limit));
+        let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
+        if serialized.len() <= line_budget {
+            return Ok(serialized);
+        }
+
+        envelope.request_id = None;
+        envelope.trace_id = None;
+        envelope.target = truncate_string(&envelope.target, 16);
+        envelope.event = truncate_string(&envelope.event, 16);
+        let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
+        (serialized.len() <= line_budget)
+            .then_some(serialized)
+            .ok_or(fmt::Error)
+    }
+}
+
+impl<S, N> FormatEvent<S, N> for JsonEventFormatter
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+    N: for<'writer> FormatFields<'writer> + 'static,
+{
+    fn format_event(
+        &self,
+        _ctx: &FmtContext<'_, S, N>,
+        mut writer: Writer<'_>,
+        event: &Event<'_>,
+    ) -> fmt::Result {
+        let serialized = self.serialize_bounded(self.envelope(event)?)?;
+        writer.write_str(&serialized)?;
+        writer.write_char('\n')
+    }
+}
+
+#[derive(Default)]
+struct JsonFieldVisitor {
+    fields: Map<String, Value>,
+}
+
+impl JsonFieldVisitor {
+    fn insert(&mut self, field: &tracing::field::Field, value: Value) {
+        self.fields.insert(field.name().to_owned(), value);
+    }
+}
+
+impl Visit for JsonFieldVisitor {
+    fn record_i64(&mut self, field: &tracing::field::Field, value: i64) {
+        self.insert(field, Value::Number(value.into()));
+    }
+
+    fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
+        self.insert(field, Value::Number(value.into()));
+    }
+
+    fn record_bool(&mut self, field: &tracing::field::Field, value: bool) {
+        self.insert(field, Value::Bool(value));
+    }
+
+    fn record_f64(&mut self, field: &tracing::field::Field, value: f64) {
+        let value = Number::from_f64(value)
+            .map(Value::Number)
+            .unwrap_or(Value::Null);
+        self.insert(field, value);
+    }
+
+    fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
+        self.insert(field, Value::String(value.to_owned()));
+    }
+
+    fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
+        let rendered = format!("{value:?}");
+        let value = if is_correlation_field(field.name()) {
+            Value::String(debug_scalar(&rendered))
+        } else {
+            match serde_json::from_str(&rendered) {
+                Ok(value @ (Value::Object(_) | Value::Array(_))) => value,
+                _ if field.name() == "message" || is_safe_display_scalar(&rendered) => {
+                    Value::String(rendered)
+                }
+                _ => Value::String(crate::REDACTED_MARKER.to_owned()),
+            }
+        };
+        self.insert(field, value);
+    }
+}
+
+fn take_correlation_id(fields: &mut Map<String, Value>, name: &str) -> Option<String> {
+    let value = fields.remove(name)?;
+    let value = match value {
+        Value::String(value) => value,
+        Value::Number(value) => value.to_string(),
+        Value::Bool(value) => value.to_string(),
+        Value::Null | Value::Array(_) | Value::Object(_) => return None,
+    };
+    (!value.is_empty()).then_some(value)
+}
+
+fn is_correlation_field(name: &str) -> bool {
+    matches!(name, "request_id" | "trace_id" | "correlation_id")
+}
+
+fn debug_scalar(rendered: &str) -> String {
+    serde_json::from_str::<String>(rendered).unwrap_or_else(|_| rendered.to_owned())
+}
+
+fn is_safe_display_scalar(rendered: &str) -> bool {
+    !rendered.is_empty()
+        && rendered.bytes().all(|byte| {
+            byte.is_ascii_alphanumeric()
+                || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'+' | b'@')
+        })
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        io,
+        sync::{Arc, Mutex},
+    };
+
+    use opentelemetry::{global, trace::TracerProvider as _};
+    use opentelemetry_sdk::{
+        error::OTelSdkResult,
+        propagation::TraceContextPropagator,
+        trace::{SdkTracerProvider, SpanData, SpanExporter},
+    };
+    use tracing::info;
+
+    use super::build_subscriber_with_tracer;
+    use crate::{
+        ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
+    };
+
+    #[test]
+    fn trace_spans_ignore_the_log_level_filter() {
+        global::set_text_map_propagator(TraceContextPropagator::new());
+        let provider = SdkTracerProvider::builder().build();
+        let tracer = provider.tracer("trace-filter-test");
+        let subscriber = build_subscriber_with_tracer(test_config("warn"), io::sink, Some(tracer))
+            .expect("subscriber must build");
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+        let span = tracing::info_span!(target: "crank::trace", "http.request");
+        let _span_guard = span.enter();
+        let mut headers = axum::http::HeaderMap::new();
+
+        assert!(inject_current_trace_context(&mut headers));
+        assert!(headers.contains_key("traceparent"));
+    }
+
+    #[test]
+    fn otel_layer_does_not_export_events() {
+        let exported = Arc::new(Mutex::new(Vec::new()));
+        let provider = SdkTracerProvider::builder()
+            .with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
+            .build();
+        let tracer = provider.tracer("event-filter-test");
+        let subscriber = build_subscriber_with_tracer(test_config("info"), io::sink, Some(tracer))
+            .expect("subscriber must build");
+        let dispatch = tracing::Dispatch::new(subscriber);
+
+        tracing::dispatcher::with_default(&dispatch, || {
+            let span = tracing::info_span!(target: "crank::trace", "http.request");
+            let _span_guard = span.enter();
+            info!(password = "canary-secret", "sensitive event");
+        });
+        provider.force_flush().expect("span must be exported");
+
+        let spans = exported.lock().expect("capture lock");
+        assert_eq!(spans.len(), 1);
+        assert!(spans[0].events.is_empty());
+        assert!(
+            !format!("{:?}", spans[0])
+                .as_bytes()
+                .windows(b"canary-secret".len())
+                .any(|window| window == b"canary-secret")
+        );
+    }
+
+    fn test_config(filter: &str) -> ObservabilityConfig {
+        ObservabilityConfig::new(
+            ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
+            filter,
+            RedactionLimits::default(),
+        )
+    }
+
+    #[derive(Clone, Debug)]
+    struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
+
+    impl SpanExporter for CapturingExporter {
+        async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
+            self.0.lock().expect("capture lock").extend(batch);
+            Ok(())
+        }
+    }
+}

crates/crank-observability/src/metrics_schema.rs

@@ -0,0 +1,159 @@
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum MetricKind {
+    Counter,
+    Gauge,
+    Histogram,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum MetricUnit {
+    Count,
+    Seconds,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct MetricDefinition {
+    pub name: &'static str,
+    pub kind: MetricKind,
+    pub unit: MetricUnit,
+    pub labels: &'static [&'static str],
+    pub description: &'static str,
+}
+
+pub const DURATION_BUCKETS_SECONDS: &[f64] = &[
+    0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
+];
+
+const METRIC_SCHEMA: &[MetricDefinition] = &[
+    counter(
+        "crank_http_requests_total",
+        &["route", "method", "status_class"],
+        "Total HTTP requests.",
+    ),
+    histogram(
+        "crank_http_request_duration_seconds",
+        &["route", "method"],
+        "HTTP request duration in seconds.",
+    ),
+    gauge(
+        "crank_http_inflight",
+        &[],
+        "HTTP requests currently being processed.",
+    ),
+    counter(
+        "crank_mcp_requests_total",
+        &["method", "response_mode", "outcome"],
+        "Total MCP JSON-RPC requests.",
+    ),
+    gauge(
+        "crank_mcp_active_sessions",
+        &[],
+        "Active MCP transport sessions.",
+    ),
+    counter(
+        "crank_tool_invocations_total",
+        &["source", "outcome", "error_kind"],
+        "Total tool invocations.",
+    ),
+    histogram(
+        "crank_tool_invocation_duration_seconds",
+        &["source", "outcome"],
+        "Tool invocation duration in seconds.",
+    ),
+    counter(
+        "crank_upstream_requests_total",
+        &["operation_kind", "outcome"],
+        "Total upstream requests.",
+    ),
+    histogram(
+        "crank_upstream_request_duration_seconds",
+        &["operation_kind", "outcome"],
+        "Upstream request duration in seconds.",
+    ),
+    gauge(
+        "crank_runtime_inflight",
+        &[],
+        "Runtime executions currently in progress.",
+    ),
+    counter(
+        "crank_runtime_limit_rejections_total",
+        &["stage"],
+        "Runtime executions rejected by a bounded limit.",
+    ),
+    gauge(
+        "crank_db_pool_connections",
+        &["state"],
+        "PostgreSQL pool connections by state.",
+    ),
+    gauge(
+        "crank_catalog_tools",
+        &[],
+        "Tools in the current published catalog.",
+    ),
+    gauge(
+        "crank_catalog_estimated_context_tokens",
+        &[],
+        "Estimated context tokens in the current published catalog.",
+    ),
+    gauge(
+        "crank_catalog_warnings",
+        &[],
+        "Warnings in the current published catalog.",
+    ),
+    counter(
+        "crank_invocation_history_lost_total",
+        &[],
+        "Invocation history records lost after an action completed.",
+    ),
+    counter(
+        "crank_telemetry_export_failures_total",
+        &["signal_type", "exporter"],
+        "Telemetry export failures.",
+    ),
+];
+
+pub const fn metric_schema() -> &'static [MetricDefinition] {
+    METRIC_SCHEMA
+}
+
+const fn counter(
+    name: &'static str,
+    labels: &'static [&'static str],
+    description: &'static str,
+) -> MetricDefinition {
+    MetricDefinition {
+        name,
+        kind: MetricKind::Counter,
+        unit: MetricUnit::Count,
+        labels,
+        description,
+    }
+}
+
+const fn gauge(
+    name: &'static str,
+    labels: &'static [&'static str],
+    description: &'static str,
+) -> MetricDefinition {
+    MetricDefinition {
+        name,
+        kind: MetricKind::Gauge,
+        unit: MetricUnit::Count,
+        labels,
+        description,
+    }
+}
+
+const fn histogram(
+    name: &'static str,
+    labels: &'static [&'static str],
+    description: &'static str,
+) -> MetricDefinition {
+    MetricDefinition {
+        name,
+        kind: MetricKind::Histogram,
+        unit: MetricUnit::Seconds,
+        labels,
+        description,
+    }
+}

crates/crank-observability/src/otlp.rs

@@ -0,0 +1,954 @@
+use std::{collections::HashMap, env, fmt, time::Duration};
+
+use axum::http::{HeaderName, HeaderValue};
+use opentelemetry::{
+    KeyValue, Value,
+    trace::{Status, TracerProvider as _},
+};
+use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig, WithHttpConfig};
+use opentelemetry_sdk::{
+    Resource,
+    error::OTelSdkResult,
+    trace::{
+        BatchConfigBuilder, BatchSpanProcessor, SdkTracer, SdkTracerProvider, SpanData,
+        SpanExporter as SpanExporterTrait,
+    },
+};
+use percent_encoding::percent_decode_str;
+use thiserror::Error;
+use url::Url;
+
+use crate::ServiceIdentity;
+
+const DEFAULT_EXPORT_TIMEOUT: Duration = Duration::from_secs(10);
+const DEFAULT_MAX_QUEUE_SIZE: usize = 2_048;
+const DEFAULT_MAX_EXPORT_BATCH_SIZE: usize = 512;
+const DEFAULT_SCHEDULE_DELAY: Duration = Duration::from_secs(5);
+const DEFAULT_BATCH_EXPORT_TIMEOUT: Duration = Duration::from_secs(30);
+const MAX_QUEUE_SIZE: usize = 65_536;
+const MAX_DURATION: Duration = Duration::from_secs(300);
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct OtlpBatchConfig {
+    max_queue_size: usize,
+    max_export_batch_size: usize,
+    scheduled_delay: Duration,
+    export_timeout: Duration,
+}
+
+impl OtlpBatchConfig {
+    pub fn try_new(
+        max_queue_size: usize,
+        max_export_batch_size: usize,
+        scheduled_delay: Duration,
+        export_timeout: Duration,
+    ) -> Result<Self, OtlpTraceConfigError> {
+        let valid = max_queue_size > 0
+            && max_queue_size <= MAX_QUEUE_SIZE
+            && max_export_batch_size > 0
+            && max_export_batch_size <= max_queue_size
+            && duration_is_bounded(scheduled_delay)
+            && duration_is_bounded(export_timeout);
+        if !valid {
+            return Err(OtlpTraceConfigError::InvalidBatchLimits);
+        }
+
+        Ok(Self {
+            max_queue_size,
+            max_export_batch_size,
+            scheduled_delay,
+            export_timeout,
+        })
+    }
+
+    pub fn max_queue_size(&self) -> usize {
+        self.max_queue_size
+    }
+
+    pub fn max_export_batch_size(&self) -> usize {
+        self.max_export_batch_size
+    }
+
+    pub fn scheduled_delay(&self) -> Duration {
+        self.scheduled_delay
+    }
+
+    pub fn export_timeout(&self) -> Duration {
+        self.export_timeout
+    }
+
+    fn sdk_config(&self) -> opentelemetry_sdk::trace::BatchConfig {
+        BatchConfigBuilder::default()
+            .with_max_queue_size(self.max_queue_size)
+            .with_max_export_batch_size(self.max_export_batch_size)
+            .with_scheduled_delay(self.scheduled_delay)
+            .build()
+    }
+}
+
+impl Default for OtlpBatchConfig {
+    fn default() -> Self {
+        Self {
+            max_queue_size: DEFAULT_MAX_QUEUE_SIZE,
+            max_export_batch_size: DEFAULT_MAX_EXPORT_BATCH_SIZE,
+            scheduled_delay: DEFAULT_SCHEDULE_DELAY,
+            export_timeout: DEFAULT_BATCH_EXPORT_TIMEOUT,
+        }
+    }
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct OtlpTraceConfig {
+    endpoint: Option<String>,
+    export_timeout: Duration,
+    batch: OtlpBatchConfig,
+    headers: HashMap<String, String>,
+}
+
+impl fmt::Debug for OtlpTraceConfig {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter
+            .debug_struct("OtlpTraceConfig")
+            .field("enabled", &self.is_enabled())
+            .field("export_timeout", &self.export_timeout)
+            .field("batch", &self.batch)
+            .field("header_count", &self.headers.len())
+            .finish()
+    }
+}
+
+impl OtlpTraceConfig {
+    pub fn from_env() -> Result<Self, OtlpTraceConfigError> {
+        OtlpEnvSettings::from_env()?.into_config()
+    }
+
+    fn from_settings(settings: OtlpEnvSettings) -> Result<Self, OtlpTraceConfigError> {
+        let endpoint = match settings.traces_endpoint {
+            Some(endpoint) => Some(validate_endpoint(endpoint, EndpointKind::Trace)?),
+            None => settings
+                .generic_endpoint
+                .map(|endpoint| validate_endpoint(endpoint, EndpointKind::Generic))
+                .transpose()?,
+        };
+        if endpoint.is_none() {
+            return Ok(Self {
+                endpoint: None,
+                export_timeout: DEFAULT_EXPORT_TIMEOUT,
+                batch: OtlpBatchConfig::default(),
+                headers: HashMap::new(),
+            });
+        }
+
+        let protocol = settings.traces_protocol.or(settings.generic_protocol);
+        let export_timeout = match settings.traces_timeout {
+            Some(timeout) => duration_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", Some(timeout))?,
+            None => duration_env("OTEL_EXPORTER_OTLP_TIMEOUT", settings.generic_timeout)?,
+        }
+        .unwrap_or(DEFAULT_EXPORT_TIMEOUT);
+        let batch = OtlpBatchConfig::try_new(
+            usize_env("OTEL_BSP_MAX_QUEUE_SIZE", settings.max_queue_size)?
+                .unwrap_or(DEFAULT_MAX_QUEUE_SIZE),
+            usize_env(
+                "OTEL_BSP_MAX_EXPORT_BATCH_SIZE",
+                settings.max_export_batch_size,
+            )?
+            .unwrap_or(DEFAULT_MAX_EXPORT_BATCH_SIZE),
+            duration_env("OTEL_BSP_SCHEDULE_DELAY", settings.scheduled_delay)?
+                .unwrap_or(DEFAULT_SCHEDULE_DELAY),
+            duration_env("OTEL_BSP_EXPORT_TIMEOUT", settings.batch_export_timeout)?
+                .unwrap_or(DEFAULT_BATCH_EXPORT_TIMEOUT),
+        )?;
+        let headers = settings
+            .traces_headers
+            .filter(|value| !value.is_empty())
+            .or(settings.generic_headers.filter(|value| !value.is_empty()))
+            .map(|value| parse_headers(&value))
+            .transpose()?
+            .unwrap_or_default();
+
+        Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, headers)
+    }
+
+    pub fn try_new(
+        endpoint: Option<String>,
+        protocol: Option<String>,
+        export_timeout: Duration,
+        batch: OtlpBatchConfig,
+    ) -> Result<Self, OtlpTraceConfigError> {
+        Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, HashMap::new())
+    }
+
+    fn try_new_with_headers(
+        endpoint: Option<String>,
+        protocol: Option<String>,
+        export_timeout: Duration,
+        batch: OtlpBatchConfig,
+        headers: HashMap<String, String>,
+    ) -> Result<Self, OtlpTraceConfigError> {
+        let endpoint = endpoint
+            .map(|endpoint| validate_endpoint(endpoint, EndpointKind::Trace))
+            .transpose()?;
+        if endpoint.is_some() && protocol.as_deref().unwrap_or("http/protobuf") != "http/protobuf" {
+            return Err(OtlpTraceConfigError::UnsupportedProtocol);
+        }
+        if !duration_is_bounded(export_timeout) {
+            return Err(OtlpTraceConfigError::InvalidDuration {
+                field: "OTEL_EXPORTER_OTLP_TIMEOUT",
+            });
+        }
+
+        Ok(Self {
+            endpoint,
+            export_timeout,
+            batch,
+            headers,
+        })
+    }
+
+    pub fn is_enabled(&self) -> bool {
+        self.endpoint.is_some()
+    }
+
+    pub fn export_timeout(&self) -> Duration {
+        self.export_timeout
+    }
+
+    pub fn batch(&self) -> &OtlpBatchConfig {
+        &self.batch
+    }
+
+    fn effective_export_timeout(&self) -> Duration {
+        self.export_timeout.min(self.batch.export_timeout)
+    }
+
+    #[cfg(test)]
+    fn endpoint(&self) -> Option<&str> {
+        self.endpoint.as_deref()
+    }
+
+    #[cfg(test)]
+    fn header(&self, name: &str) -> Option<&str> {
+        self.headers.get(name).map(String::as_str)
+    }
+}
+
+#[derive(Default)]
+struct OtlpEnvSettings {
+    traces_endpoint: Option<String>,
+    generic_endpoint: Option<String>,
+    traces_protocol: Option<String>,
+    generic_protocol: Option<String>,
+    traces_timeout: Option<String>,
+    generic_timeout: Option<String>,
+    traces_headers: Option<String>,
+    generic_headers: Option<String>,
+    max_queue_size: Option<String>,
+    max_export_batch_size: Option<String>,
+    scheduled_delay: Option<String>,
+    batch_export_timeout: Option<String>,
+}
+
+impl OtlpEnvSettings {
+    fn from_env() -> Result<Self, OtlpTraceConfigError> {
+        Ok(Self {
+            traces_endpoint: optional_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")?,
+            generic_endpoint: optional_env("OTEL_EXPORTER_OTLP_ENDPOINT")?,
+            traces_protocol: optional_env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")?,
+            generic_protocol: optional_env("OTEL_EXPORTER_OTLP_PROTOCOL")?,
+            traces_timeout: optional_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT")?,
+            generic_timeout: optional_env("OTEL_EXPORTER_OTLP_TIMEOUT")?,
+            traces_headers: optional_env("OTEL_EXPORTER_OTLP_TRACES_HEADERS")?,
+            generic_headers: optional_env("OTEL_EXPORTER_OTLP_HEADERS")?,
+            max_queue_size: optional_env("OTEL_BSP_MAX_QUEUE_SIZE")?,
+            max_export_batch_size: optional_env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE")?,
+            scheduled_delay: optional_env("OTEL_BSP_SCHEDULE_DELAY")?,
+            batch_export_timeout: optional_env("OTEL_BSP_EXPORT_TIMEOUT")?,
+        })
+    }
+
+    fn into_config(self) -> Result<OtlpTraceConfig, OtlpTraceConfigError> {
+        OtlpTraceConfig::from_settings(self)
+    }
+}
+
+#[derive(Debug, Error, Eq, PartialEq)]
+pub enum OtlpTraceConfigError {
+    #[error("OTLP environment variable is not valid UTF-8: {field}")]
+    InvalidEnvironmentEncoding { field: &'static str },
+    #[error("OTLP trace endpoint is invalid: {reason}")]
+    InvalidEndpoint { reason: &'static str },
+    #[error("OTLP trace protocol must be http/protobuf")]
+    UnsupportedProtocol,
+    #[error("OTLP numeric setting is invalid: {field}")]
+    InvalidNumber { field: &'static str },
+    #[error("OTLP duration setting is invalid: {field}")]
+    InvalidDuration { field: &'static str },
+    #[error("OTLP batch limits are invalid")]
+    InvalidBatchLimits,
+    #[error("OTLP trace headers are invalid")]
+    InvalidHeaders,
+}
+
+#[derive(Debug, Error)]
+pub enum OtlpTraceError {
+    #[error("failed to configure OTLP trace exporter")]
+    ExporterConfiguration,
+}
+
+pub fn build_tracer_provider(
+    identity: &ServiceIdentity,
+    config: &OtlpTraceConfig,
+) -> Result<Option<(SdkTracerProvider, SdkTracer)>, OtlpTraceError> {
+    let Some(endpoint) = config.endpoint.as_deref() else {
+        return Ok(None);
+    };
+    let exporter = SpanExporter::builder()
+        .with_http()
+        .with_protocol(Protocol::HttpBinary)
+        .with_endpoint(endpoint)
+        .with_timeout(config.effective_export_timeout())
+        .with_headers(config.headers.clone())
+        .build()
+        .map_err(|_| OtlpTraceError::ExporterConfiguration)?;
+    let processor = BatchSpanProcessor::builder(ObservedSpanExporter(exporter))
+        .with_batch_config(config.batch.sdk_config())
+        .build();
+    let resource = Resource::builder_empty()
+        .with_attributes([
+            KeyValue::new("service.name", identity.service().to_owned()),
+            KeyValue::new("service.version", identity.version().to_owned()),
+            KeyValue::new(
+                "deployment.environment.name",
+                identity.environment().to_owned(),
+            ),
+        ])
+        .build();
+    let provider = SdkTracerProvider::builder()
+        .with_span_processor(processor)
+        .with_resource(resource)
+        .build();
+    let tracer = provider.tracer("crank");
+
+    Ok(Some((provider, tracer)))
+}
+
+#[derive(Debug)]
+struct ObservedSpanExporter(SpanExporter);
+
+impl SpanExporterTrait for ObservedSpanExporter {
+    async fn export(&self, mut batch: Vec<SpanData>) -> OTelSdkResult {
+        sanitize_trace_batch(&mut batch);
+        let result = self.0.export(batch).await;
+        if result.is_err() {
+            metrics::counter!(
+                "crank_telemetry_export_failures_total",
+                "signal_type" => "trace",
+                "exporter" => "otlp"
+            )
+            .increment(1);
+        }
+        result
+    }
+
+    fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult {
+        self.0.shutdown_with_timeout(timeout)
+    }
+
+    fn force_flush(&self) -> OTelSdkResult {
+        self.0.force_flush()
+    }
+
+    fn set_resource(&mut self, resource: &Resource) {
+        self.0.set_resource(resource);
+    }
+}
+
+fn sanitize_trace_batch(batch: &mut Vec<SpanData>) {
+    batch.retain(|span| is_allowed_span_name(span.name.as_ref()));
+    for span in batch {
+        let original_attribute_count = span.attributes.len();
+        span.attributes.retain(is_allowed_span_attribute);
+        span.dropped_attributes_count = span
+            .dropped_attributes_count
+            .saturating_add((original_attribute_count - span.attributes.len()) as u32);
+        span.events = Default::default();
+        span.links = Default::default();
+        if matches!(span.status, Status::Error { .. }) {
+            span.status = Status::error("");
+        }
+    }
+}
+
+fn is_allowed_span_name(name: &str) -> bool {
+    matches!(
+        name,
+        "http.request"
+            | "mcp.request"
+            | "mcp.rate_limit"
+            | "mcp.access.check"
+            | "mcp.catalog.load"
+            | "mcp.tools.resolve"
+            | "approval.check"
+            | "runtime.execute"
+            | "runtime.arguments.map"
+            | "runtime.idempotency"
+            | "upstream.http"
+            | "runtime.response.transform"
+            | "auth.resolve"
+            | "approval.recovery"
+            | "history.write"
+            | "db.query"
+    )
+}
+
+fn is_allowed_span_attribute(attribute: &KeyValue) -> bool {
+    let Value::String(value) = &attribute.value else {
+        return false;
+    };
+    let value = value.as_str();
+    match attribute.key.as_str() {
+        "request_id" => crate::RequestId::is_valid(value),
+        "stage" => is_allowed_span_name(value),
+        "outcome" => matches!(
+            value,
+            "success"
+                | "error"
+                | "allowed"
+                | "denied"
+                | "required"
+                | "replay"
+                | "execute"
+                | "skipped"
+                | "cache_hit"
+        ),
+        "error.category" => matches!(
+            value,
+            "access"
+                | "rate_limit"
+                | "catalog"
+                | "approval"
+                | "idempotency"
+                | "schema"
+                | "mapping"
+                | "upstream"
+                | "transformation"
+                | "history"
+                | "database"
+                | "concurrency"
+                | "configuration"
+                | "internal"
+        ),
+        "db.system" => value == "postgresql",
+        "db.operation" => matches!(
+            value,
+            "machine_access.read"
+                | "machine_access.touch"
+                | "catalog.load"
+                | "approval.read"
+                | "approval.write"
+                | "auth_profile.read"
+                | "secret.read"
+                | "secret.touch"
+                | "invocation_history.write"
+        ),
+        _ => false,
+    }
+}
+
+#[derive(Clone, Copy)]
+enum EndpointKind {
+    Trace,
+    Generic,
+}
+
+fn validate_endpoint(endpoint: String, kind: EndpointKind) -> Result<String, OtlpTraceConfigError> {
+    let mut url = Url::parse(&endpoint).map_err(|_| OtlpTraceConfigError::InvalidEndpoint {
+        reason: "invalid URL",
+    })?;
+    if !matches!(url.scheme(), "http" | "https") {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "unsupported scheme",
+        });
+    }
+    if url.host_str().is_none() {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "host is required",
+        });
+    }
+    if !url.username().is_empty() || url.password().is_some() {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "credentials are forbidden",
+        });
+    }
+    if url.query().is_some() || url.fragment().is_some() {
+        return Err(OtlpTraceConfigError::InvalidEndpoint {
+            reason: "query and fragment are forbidden",
+        });
+    }
+    if matches!(kind, EndpointKind::Generic) {
+        let path = url.path().trim_end_matches('/');
+        url.set_path(&format!("{path}/v1/traces"));
+    }
+
+    Ok(url.into())
+}
+
+fn parse_headers(value: &str) -> Result<HashMap<String, String>, OtlpTraceConfigError> {
+    value
+        .split_terminator(',')
+        .map(str::trim)
+        .filter(|item| !item.is_empty())
+        .try_fold(HashMap::new(), |mut headers, item| {
+            let (name, encoded_value) = item
+                .split_once('=')
+                .ok_or(OtlpTraceConfigError::InvalidHeaders)?;
+            let name = HeaderName::from_bytes(name.trim().as_bytes())
+                .map_err(|_| OtlpTraceConfigError::InvalidHeaders)?;
+            let value = percent_decode_str(encoded_value.trim())
+                .decode_utf8()
+                .map_err(|_| OtlpTraceConfigError::InvalidHeaders)?
+                .into_owned();
+            if value.is_empty() || HeaderValue::from_str(&value).is_err() {
+                return Err(OtlpTraceConfigError::InvalidHeaders);
+            }
+            headers.insert(name.as_str().to_owned(), value);
+            Ok(headers)
+        })
+}
+
+fn optional_env(field: &'static str) -> Result<Option<String>, OtlpTraceConfigError> {
+    match env::var(field) {
+        Ok(value) if value.is_empty() => Ok(None),
+        Ok(value) => Ok(Some(value)),
+        Err(env::VarError::NotPresent) => Ok(None),
+        Err(env::VarError::NotUnicode(_)) => {
+            Err(OtlpTraceConfigError::InvalidEnvironmentEncoding { field })
+        }
+    }
+}
+
+fn usize_env(
+    field: &'static str,
+    value: Option<String>,
+) -> Result<Option<usize>, OtlpTraceConfigError> {
+    value
+        .map(|value| {
+            value
+                .parse()
+                .map_err(|_| OtlpTraceConfigError::InvalidNumber { field })
+        })
+        .transpose()
+}
+
+fn duration_env(
+    field: &'static str,
+    value: Option<String>,
+) -> Result<Option<Duration>, OtlpTraceConfigError> {
+    value
+        .map(|value| {
+            value
+                .parse::<u64>()
+                .map(Duration::from_millis)
+                .map_err(|_| OtlpTraceConfigError::InvalidDuration { field })
+        })
+        .transpose()
+}
+
+fn duration_is_bounded(duration: Duration) -> bool {
+    !duration.is_zero() && duration <= MAX_DURATION
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        io::{Read, Write},
+        net::TcpListener,
+        sync::mpsc,
+        thread,
+        time::{Duration, Instant},
+    };
+
+    use axum::{
+        Router,
+        body::{Body, to_bytes},
+        extract::Request,
+        middleware::Next,
+        response::Response,
+        routing::get,
+    };
+    use opentelemetry::{
+        KeyValue,
+        trace::{Span as _, Status, Tracer as _},
+    };
+    use opentelemetry_proto::tonic::{
+        collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value,
+    };
+    use prost::Message;
+    use tower::ServiceExt;
+    use tracing::{Instrument, info_span};
+    use tracing_subscriber::layer::SubscriberExt;
+
+    use super::{OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_tracer_provider};
+    use crate::ServiceIdentity;
+
+    #[test]
+    fn signal_specific_settings_override_generic_settings() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/custom".to_owned()),
+            generic_endpoint: Some("https://generic.example.test/otel".to_owned()),
+            traces_protocol: Some("http/protobuf".to_owned()),
+            generic_protocol: Some("grpc".to_owned()),
+            traces_timeout: Some("2500".to_owned()),
+            generic_timeout: Some("invalid-unused-fallback".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(
+            config.endpoint(),
+            Some("https://traces.example.test/custom")
+        );
+        assert_eq!(config.export_timeout(), Duration::from_millis(2500));
+    }
+
+    #[test]
+    fn disabled_export_ignores_inactive_settings() {
+        let config = OtlpEnvSettings {
+            traces_protocol: Some("grpc".to_owned()),
+            generic_protocol: Some("grpc".to_owned()),
+            traces_timeout: Some("invalid".to_owned()),
+            generic_timeout: Some("invalid".to_owned()),
+            max_queue_size: Some("invalid".to_owned()),
+            max_export_batch_size: Some("invalid".to_owned()),
+            scheduled_delay: Some("invalid".to_owned()),
+            batch_export_timeout: Some("invalid".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert!(!config.is_enabled());
+    }
+
+    #[test]
+    fn empty_signal_headers_use_generic_headers() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
+            traces_headers: Some(String::new()),
+            generic_headers: Some(
+                "authorization=Bearer%20canary-token,x-tenant=community".to_owned(),
+            ),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(config.header("authorization"), Some("Bearer canary-token"));
+        assert_eq!(config.header("x-tenant"), Some("community"));
+        assert!(!format!("{config:?}").contains("canary-token"));
+    }
+
+    #[test]
+    fn invalid_headers_return_a_safe_error() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
+            traces_headers: Some("authorization=canary-secret%0Ainjected".to_owned()),
+            ..OtlpEnvSettings::default()
+        };
+
+        let error = config.into_config().unwrap_err();
+        assert!(matches!(error, super::OtlpTraceConfigError::InvalidHeaders));
+        assert!(!error.to_string().contains("canary-secret"));
+    }
+
+    #[test]
+    fn stricter_batch_timeout_bounds_http_export() {
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
+            traces_protocol: Some("http/protobuf".to_owned()),
+            traces_timeout: Some("9000".to_owned()),
+            batch_export_timeout: Some("2500".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(
+            config.effective_export_timeout(),
+            Duration::from_millis(2500)
+        );
+    }
+
+    #[test]
+    fn generic_endpoint_receives_standard_trace_path() {
+        let config = OtlpEnvSettings {
+            generic_endpoint: Some("https://generic.example.test/otel/".to_owned()),
+            generic_protocol: Some("http/protobuf".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+
+        assert_eq!(
+            config.endpoint(),
+            Some("https://generic.example.test/otel/v1/traces")
+        );
+    }
+
+    #[test]
+    fn disabled_export_does_not_build_a_provider() {
+        let config = OtlpTraceConfig::try_new(
+            None,
+            None,
+            Duration::from_secs(1),
+            OtlpBatchConfig::default(),
+        )
+        .unwrap();
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap();
+
+        assert!(build_tracer_provider(&identity, &config).unwrap().is_none());
+    }
+
+    #[test]
+    fn real_http_protobuf_export_contains_resource_and_trace() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let (request_tx, request_rx) = mpsc::sync_channel(1);
+        let server = thread::spawn(move || {
+            let (mut stream, _) = listener.accept().unwrap();
+            let request = read_http_request(&mut stream);
+            stream
+                .write_all(
+                    b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
+                )
+                .unwrap();
+            request_tx.send(request).unwrap();
+        });
+        let config = OtlpEnvSettings {
+            traces_endpoint: Some(format!("http://{address}/v1/traces")),
+            traces_protocol: Some("http/protobuf".to_owned()),
+            traces_timeout: Some("2000".to_owned()),
+            traces_headers: Some(String::new()),
+            generic_headers: Some("authorization=Bearer%20canary-token".to_owned()),
+            max_queue_size: Some("16".to_owned()),
+            max_export_batch_size: Some("8".to_owned()),
+            scheduled_delay: Some("10".to_owned()),
+            batch_export_timeout: Some("2000".to_owned()),
+            ..OtlpEnvSettings::default()
+        }
+        .into_config()
+        .unwrap();
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "integration-test").unwrap();
+        let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
+        let mut span = tracer.start("http.request");
+        let trace_id = span.span_context().trace_id().to_bytes();
+        span.set_attribute(KeyValue::new("request_id", "req_otlp_contract"));
+        span.set_attribute(KeyValue::new("authorization", "Bearer canary-span-secret"));
+        span.add_event(
+            "canary-span-event",
+            vec![KeyValue::new("payload", "canary-span-secret")],
+        );
+        span.set_status(Status::error("canary-span-secret"));
+        span.end();
+
+        provider.force_flush().unwrap();
+        provider.shutdown().unwrap();
+        let request = request_rx.recv_timeout(Duration::from_secs(2)).unwrap();
+        server.join().unwrap();
+        let (headers, body) = split_http_request(&request);
+
+        assert!(headers.contains("POST /v1/traces HTTP/1.1"));
+        assert!(
+            headers
+                .to_ascii_lowercase()
+                .contains("content-type: application/x-protobuf")
+        );
+        assert!(
+            headers
+                .to_ascii_lowercase()
+                .contains("authorization: bearer canary-token")
+        );
+        let export = ExportTraceServiceRequest::decode(body).unwrap();
+        let resource_spans = export.resource_spans.first().unwrap();
+        let attributes = &resource_spans.resource.as_ref().unwrap().attributes;
+        assert_eq!(
+            string_attribute(attributes, "service.name"),
+            Some("admin-api")
+        );
+        assert_eq!(
+            string_attribute(attributes, "service.version"),
+            Some("0.3.1")
+        );
+        assert_eq!(
+            string_attribute(attributes, "deployment.environment.name"),
+            Some("integration-test")
+        );
+        assert_eq!(
+            resource_spans.scope_spans[0].spans[0].trace_id.as_slice(),
+            trace_id
+        );
+        let exported_span = &resource_spans.scope_spans[0].spans[0];
+        assert_eq!(exported_span.name, "http.request");
+        assert_eq!(
+            string_attribute(&exported_span.attributes, "request_id"),
+            Some("req_otlp_contract")
+        );
+        assert!(
+            exported_span
+                .attributes
+                .iter()
+                .all(|attribute| attribute.key != "authorization")
+        );
+        assert!(exported_span.events.is_empty());
+        assert_eq!(
+            exported_span
+                .status
+                .as_ref()
+                .map(|status| status.message.as_str()),
+            Some("")
+        );
+        assert!(
+            !body
+                .windows(b"canary-span-secret".len())
+                .any(|window| { window == b"canary-span-secret" })
+        );
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn unavailable_receiver_does_not_change_product_result() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = thread::spawn(move || {
+            let (stream, _) = listener.accept().unwrap();
+            drop(stream);
+        });
+        let config = OtlpTraceConfig::try_new(
+            Some(format!("http://{address}/v1/traces")),
+            Some("http/protobuf".to_owned()),
+            Duration::from_millis(250),
+            OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(250))
+                .unwrap(),
+        )
+        .unwrap();
+        let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "fault-test").unwrap();
+        let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
+        let subscriber =
+            tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+        let dispatch = tracing::Dispatch::new(subscriber);
+        let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
+        let app = Router::new()
+            .route("/product", get(|| async { "product-success" }))
+            .layer(axum::middleware::from_fn(trace_product_request));
+
+        let response = app
+            .oneshot(
+                Request::builder()
+                    .uri("/product")
+                    .body(Body::empty())
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        let status = response.status();
+        let body = to_bytes(response.into_body(), 64).await.unwrap();
+
+        assert_eq!(status, axum::http::StatusCode::OK);
+        assert_eq!(body.as_ref(), b"product-success");
+        assert!(provider.force_flush().is_err());
+        let _ = provider.shutdown();
+        server.join().unwrap();
+    }
+
+    async fn trace_product_request(request: Request, next: Next) -> Response {
+        next.run(request)
+            .instrument(info_span!(target: "crank::trace", "http.request"))
+            .await
+    }
+
+    #[test]
+    fn hanging_receiver_respects_the_stricter_export_timeout() {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = thread::spawn(move || {
+            let (stream, _) = listener.accept().unwrap();
+            thread::sleep(Duration::from_millis(750));
+            drop(stream);
+        });
+        let config = OtlpTraceConfig::try_new(
+            Some(format!("http://{address}/v1/traces")),
+            Some("http/protobuf".to_owned()),
+            Duration::from_secs(2),
+            OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(100))
+                .unwrap(),
+        )
+        .unwrap();
+        let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "timeout-test").unwrap();
+        let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
+        let mut span = tracer.start("http.request");
+        span.end();
+        let started_at = Instant::now();
+
+        assert!(provider.force_flush().is_err());
+        assert!(started_at.elapsed() < Duration::from_millis(500));
+        let _ = provider.shutdown();
+        server.join().unwrap();
+    }
+
+    fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
+        stream
+            .set_read_timeout(Some(Duration::from_secs(2)))
+            .unwrap();
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 4096];
+        loop {
+            let read = stream.read(&mut buffer).unwrap();
+            request.extend_from_slice(&buffer[..read]);
+            let Some(header_end) = find_bytes(&request, b"\r\n\r\n") else {
+                continue;
+            };
+            let headers = String::from_utf8_lossy(&request[..header_end]);
+            let content_length = headers
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("content-length")
+                        .then(|| value.trim().parse::<usize>().ok())
+                        .flatten()
+                })
+                .unwrap_or(0);
+            if request.len() >= header_end + 4 + content_length {
+                return request;
+            }
+        }
+    }
+
+    fn split_http_request(request: &[u8]) -> (&str, &[u8]) {
+        let header_end = find_bytes(request, b"\r\n\r\n").unwrap();
+        (
+            std::str::from_utf8(&request[..header_end]).unwrap(),
+            &request[header_end + 4..],
+        )
+    }
+
+    fn string_attribute<'a>(
+        attributes: &'a [opentelemetry_proto::tonic::common::v1::KeyValue],
+        key: &str,
+    ) -> Option<&'a str> {
+        attributes.iter().find_map(|attribute| {
+            let value = attribute.value.as_ref()?.value.as_ref()?;
+            (attribute.key == key)
+                .then_some(value)
+                .and_then(|value| match value {
+                    any_value::Value::StringValue(value) => Some(value.as_str()),
+                    _ => None,
+                })
+        })
+    }
+
+    fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+        haystack
+            .windows(needle.len())
+            .position(|candidate| candidate == needle)
+    }
+}

crates/crank-observability/src/prometheus.rs

@@ -0,0 +1,283 @@
+use std::{env, net::SocketAddr};
+
+use axum::{
+    Router,
+    extract::{Request, State},
+    http::{
+        HeaderMap, StatusCode,
+        header::{self, HeaderValue},
+    },
+    middleware::{self, Next},
+    response::{IntoResponse, Response},
+    routing::get,
+};
+use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle, PrometheusRecorder};
+use sha2::{Digest, Sha256};
+use subtle::ConstantTimeEq;
+use thiserror::Error;
+use tokio::net::TcpListener;
+
+use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
+
+const METRICS_ENABLED_ENV: &str = "CRANK_METRICS_ENABLED";
+const METRICS_TOKEN_ENV: &str = "CRANK_METRICS_BEARER_TOKEN";
+const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
+
+#[derive(Clone)]
+pub struct MetricsConfig {
+    enabled: bool,
+    bind_addr: SocketAddr,
+    token_digest: Option<[u8; 32]>,
+}
+
+impl std::fmt::Debug for MetricsConfig {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter
+            .debug_struct("MetricsConfig")
+            .field("enabled", &self.enabled)
+            .field("bind_addr", &self.bind_addr)
+            .field("authentication_configured", &self.token_digest.is_some())
+            .finish()
+    }
+}
+
+impl MetricsConfig {
+    pub fn new(
+        enabled: bool,
+        bind_addr: SocketAddr,
+        bearer_token: Option<String>,
+    ) -> Result<Self, MetricsConfigError> {
+        let token_digest = bearer_token
+            .filter(|token| !token.is_empty())
+            .map(|token| token_digest(token.as_bytes()));
+
+        if enabled && !bind_addr.ip().is_loopback() && token_digest.is_none() {
+            return Err(MetricsConfigError::MissingTokenForExternalBind);
+        }
+
+        Ok(Self {
+            enabled,
+            bind_addr,
+            token_digest,
+        })
+    }
+
+    pub fn from_env(
+        bind_env: &'static str,
+        default_bind: SocketAddr,
+    ) -> Result<Self, MetricsConfigError> {
+        let enabled = parse_enabled(env::var(METRICS_ENABLED_ENV))?;
+        let bind_addr = match env::var(bind_env) {
+            Ok(raw) => raw
+                .parse()
+                .map_err(|_| MetricsConfigError::InvalidBindAddress { field: bind_env })?,
+            Err(env::VarError::NotPresent) => default_bind,
+            Err(env::VarError::NotUnicode(_)) => {
+                return Err(MetricsConfigError::InvalidEnvironmentEncoding { field: bind_env });
+            }
+        };
+        let bearer_token = match env::var(METRICS_TOKEN_ENV) {
+            Ok(token) => Some(token),
+            Err(env::VarError::NotPresent) => None,
+            Err(env::VarError::NotUnicode(_)) => {
+                return Err(MetricsConfigError::InvalidEnvironmentEncoding {
+                    field: METRICS_TOKEN_ENV,
+                });
+            }
+        };
+
+        Self::new(enabled, bind_addr, bearer_token)
+    }
+
+    pub fn enabled(&self) -> bool {
+        self.enabled
+    }
+
+    pub fn bind_addr(&self) -> SocketAddr {
+        self.bind_addr
+    }
+
+    pub fn requires_authentication(&self) -> bool {
+        !self.bind_addr.ip().is_loopback()
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum MetricsConfigError {
+    #[error("metrics environment variable is not valid UTF-8: {field}")]
+    InvalidEnvironmentEncoding { field: &'static str },
+    #[error("metrics bind address is invalid: {field}")]
+    InvalidBindAddress { field: &'static str },
+    #[error("metrics enabled flag must be one of true, false, 1, 0")]
+    InvalidEnabledFlag,
+    #[error("external metrics bind requires a bearer token")]
+    MissingTokenForExternalBind,
+}
+
+#[derive(Clone)]
+struct MetricsState {
+    handle: PrometheusHandle,
+    token_digest: Option<[u8; 32]>,
+    requires_authentication: bool,
+}
+
+pub struct MetricsSurface {
+    config: MetricsConfig,
+    state: MetricsState,
+    _recorder: Option<PrometheusRecorder>,
+}
+
+impl MetricsSurface {
+    pub(crate) fn new(config: MetricsConfig, handle: PrometheusHandle) -> Self {
+        Self {
+            state: MetricsState {
+                handle,
+                token_digest: config.token_digest,
+                requires_authentication: config.requires_authentication(),
+            },
+            config,
+            _recorder: None,
+        }
+    }
+
+    pub fn for_test(
+        config: MetricsConfig,
+        identity: ServiceIdentity,
+    ) -> Result<Self, MetricsSurfaceError> {
+        let recorder = prometheus_builder(&identity)?.build_recorder();
+        let handle = recorder.handle();
+        let mut surface = Self::new(config, handle);
+        surface._recorder = Some(recorder);
+        Ok(surface)
+    }
+
+    pub fn router(&self) -> Router {
+        Router::new()
+            .route("/metrics", get(render_metrics))
+            .route("/health", get(metrics_health))
+            .layer(middleware::from_fn_with_state(
+                self.state.clone(),
+                authorize_metrics,
+            ))
+            .with_state(self.state.clone())
+    }
+
+    pub async fn bind(self) -> Result<MetricsServer, MetricsServeError> {
+        let listener = TcpListener::bind(self.config.bind_addr)
+            .await
+            .map_err(|_| MetricsServeError::Bind)?;
+        Ok(MetricsServer {
+            listener,
+            router: self.router(),
+        })
+    }
+}
+
+pub struct MetricsServer {
+    listener: TcpListener,
+    router: Router,
+}
+
+impl MetricsServer {
+    pub async fn serve(self) -> Result<(), MetricsServeError> {
+        axum::serve(self.listener, self.router)
+            .await
+            .map_err(|_| MetricsServeError::Serve)
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum MetricsSurfaceError {
+    #[error("failed to configure Prometheus recorder")]
+    RecorderConfiguration,
+}
+
+#[derive(Debug, Error)]
+pub enum MetricsServeError {
+    #[error("failed to bind metrics listener")]
+    Bind,
+    #[error("metrics listener stopped unexpectedly")]
+    Serve,
+}
+
+pub(crate) fn install_prometheus_recorder(
+    identity: &ServiceIdentity,
+) -> Result<PrometheusHandle, MetricsSurfaceError> {
+    prometheus_builder(identity)?
+        .install_recorder()
+        .map_err(|_| MetricsSurfaceError::RecorderConfiguration)
+}
+
+fn prometheus_builder(
+    identity: &ServiceIdentity,
+) -> Result<PrometheusBuilder, MetricsSurfaceError> {
+    PrometheusBuilder::new()
+        .set_buckets(DURATION_BUCKETS_SECONDS)
+        .map(|builder| {
+            builder
+                .add_global_label("service", identity.service())
+                .add_global_label("version", identity.version())
+                .add_global_label("environment", identity.environment())
+        })
+        .map_err(|_| MetricsSurfaceError::RecorderConfiguration)
+}
+
+async fn render_metrics(State(state): State<MetricsState>) -> Response {
+    let mut response = state.handle.render().into_response();
+    response.headers_mut().insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static(PROMETHEUS_CONTENT_TYPE),
+    );
+    response
+}
+
+async fn metrics_health() -> impl IntoResponse {
+    (StatusCode::OK, "ok\n")
+}
+
+async fn authorize_metrics(
+    State(state): State<MetricsState>,
+    request: Request,
+    next: Next,
+) -> Response {
+    if !state.requires_authentication {
+        return next.run(request).await;
+    }
+
+    let authorized = bearer_token(request.headers())
+        .map(token_digest)
+        .zip(state.token_digest)
+        .is_some_and(|(actual, expected)| bool::from(actual.ct_eq(&expected)));
+
+    if authorized {
+        next.run(request).await
+    } else {
+        StatusCode::UNAUTHORIZED.into_response()
+    }
+}
+
+fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
+    headers
+        .get(header::AUTHORIZATION)?
+        .as_bytes()
+        .strip_prefix(b"Bearer ")
+        .filter(|token| !token.is_empty())
+}
+
+fn token_digest(token: &[u8]) -> [u8; 32] {
+    Sha256::digest(token).into()
+}
+
+fn parse_enabled(value: Result<String, env::VarError>) -> Result<bool, MetricsConfigError> {
+    match value {
+        Ok(raw) => match raw.to_ascii_lowercase().as_str() {
+            "true" | "1" => Ok(true),
+            "false" | "0" => Ok(false),
+            _ => Err(MetricsConfigError::InvalidEnabledFlag),
+        },
+        Err(env::VarError::NotPresent) => Ok(true),
+        Err(env::VarError::NotUnicode(_)) => Err(MetricsConfigError::InvalidEnvironmentEncoding {
+            field: METRICS_ENABLED_ENV,
+        }),
+    }
+}

crates/crank-observability/src/propagation.rs

@@ -0,0 +1,70 @@
+use axum::http::{HeaderMap, HeaderName, HeaderValue};
+use opentelemetry::{
+    Context, global,
+    propagation::{Extractor, Injector},
+    trace::TraceContextExt,
+};
+use opentelemetry_sdk::propagation::TraceContextPropagator;
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+pub fn set_remote_trace_parent(span: &Span, headers: &HeaderMap) -> bool {
+    let context =
+        global::get_text_map_propagator(|propagator| propagator.extract(&HeaderExtractor(headers)));
+    let span_context = context.span().span_context().clone();
+    if !span_context.is_valid() || !span_context.is_remote() {
+        return false;
+    }
+
+    span.set_parent(context).is_ok()
+}
+
+pub fn inject_current_trace_context(headers: &mut HeaderMap) -> bool {
+    let context = Span::current().context();
+    if !context.span().span_context().is_valid() {
+        return false;
+    }
+
+    global::get_text_map_propagator(|propagator| {
+        propagator.inject_context(&context, &mut HeaderInjector(headers));
+    });
+    true
+}
+
+pub(crate) fn install_trace_context_propagator() {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+}
+
+struct HeaderExtractor<'a>(&'a HeaderMap);
+
+impl Extractor for HeaderExtractor<'_> {
+    fn get(&self, key: &str) -> Option<&str> {
+        self.0.get(key).and_then(|value| value.to_str().ok())
+    }
+
+    fn keys(&self) -> Vec<&str> {
+        self.0.keys().map(HeaderName::as_str).collect()
+    }
+}
+
+struct HeaderInjector<'a>(&'a mut HeaderMap);
+
+impl Injector for HeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(name) = HeaderName::try_from(key) else {
+            return;
+        };
+        let Ok(value) = HeaderValue::try_from(value) else {
+            return;
+        };
+        self.0.insert(name, value);
+    }
+}
+
+pub(crate) fn current_trace_id() -> Option<String> {
+    let context: Context = Span::current().context();
+    let span_context = context.span().span_context().clone();
+    span_context
+        .is_valid()
+        .then(|| span_context.trace_id().to_string())
+}

crates/crank-observability/src/redaction.rs

@@ -0,0 +1,292 @@
+use serde_json::{Map, Value};
+use thiserror::Error;
+
+pub const REDACTED_MARKER: &str = "[REDACTED]";
+const TRUNCATED_MARKER: &str = "[TRUNCATED]";
+const MIN_EVENT_BYTES: usize = 512;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct RedactionLimits {
+    pub max_string_bytes: usize,
+    pub max_array_items: usize,
+    pub max_object_fields: usize,
+    pub max_depth: usize,
+    pub max_event_bytes: usize,
+}
+
+impl Default for RedactionLimits {
+    fn default() -> Self {
+        Self {
+            max_string_bytes: 1024,
+            max_array_items: 32,
+            max_object_fields: 64,
+            max_depth: 8,
+            max_event_bytes: 16 * 1024,
+        }
+    }
+}
+
+impl RedactionLimits {
+    pub fn validate(self) -> Result<(), RedactionLimitsError> {
+        for (field, value, minimum) in [
+            (
+                "max_string_bytes",
+                self.max_string_bytes,
+                TRUNCATED_MARKER.len(),
+            ),
+            ("max_array_items", self.max_array_items, 1),
+            ("max_object_fields", self.max_object_fields, 1),
+            ("max_depth", self.max_depth, 1),
+            ("max_event_bytes", self.max_event_bytes, MIN_EVENT_BYTES),
+        ] {
+            if value < minimum {
+                return Err(RedactionLimitsError::TooSmall { field, minimum });
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum RedactionLimitsError {
+    #[error("invalid redaction limit {field}: minimum is {minimum}")]
+    TooSmall { field: &'static str, minimum: usize },
+}
+
+#[derive(Debug, Error)]
+pub enum SafeJsonError {
+    #[error(transparent)]
+    InvalidLimits(#[from] RedactionLimitsError),
+    #[error(transparent)]
+    Serialization(#[from] serde_json::Error),
+}
+
+pub fn redact_value(value: &Value, limits: RedactionLimits) -> Value {
+    redact_at_depth(value, limits, 0)
+}
+
+pub fn safe_json(value: &Value, limits: RedactionLimits) -> Result<String, SafeJsonError> {
+    limits.validate()?;
+    let serialized = serde_json::to_string(&redact_value(value, limits))?;
+    if serialized.len() <= limits.max_event_bytes {
+        return Ok(serialized);
+    }
+
+    Ok(serde_json::to_string(&serde_json::json!({
+        "truncated": true
+    }))?)
+}
+
+pub(crate) fn truncate_string(value: &str, max_bytes: usize) -> String {
+    if value.len() <= max_bytes {
+        return value.to_owned();
+    }
+    if max_bytes == 0 {
+        return String::new();
+    }
+
+    let marker = if max_bytes >= TRUNCATED_MARKER.len() {
+        TRUNCATED_MARKER
+    } else {
+        ""
+    };
+    let content_budget = max_bytes.saturating_sub(marker.len());
+    let mut boundary = content_budget.min(value.len());
+    while boundary > 0 && !value.is_char_boundary(boundary) {
+        boundary -= 1;
+    }
+
+    let mut truncated = String::with_capacity(max_bytes);
+    truncated.push_str(&value[..boundary]);
+    if marker.is_empty() {
+        let mut marker_boundary = max_bytes.min(TRUNCATED_MARKER.len());
+        while marker_boundary > 0 && !TRUNCATED_MARKER.is_char_boundary(marker_boundary) {
+            marker_boundary -= 1;
+        }
+        truncated.clear();
+        truncated.push_str(&TRUNCATED_MARKER[..marker_boundary]);
+    } else {
+        truncated.push_str(marker);
+    }
+    truncated
+}
+
+fn redact_at_depth(value: &Value, limits: RedactionLimits, depth: usize) -> Value {
+    if depth >= limits.max_depth {
+        return Value::String(TRUNCATED_MARKER.to_owned());
+    }
+
+    match value {
+        Value::Null | Value::Bool(_) | Value::Number(_) => value.clone(),
+        Value::String(value) => Value::String(truncate_string(value, limits.max_string_bytes)),
+        Value::Array(values) => redact_array(values, limits, depth),
+        Value::Object(values) => redact_object(values, limits, depth),
+    }
+}
+
+fn redact_array(values: &[Value], limits: RedactionLimits, depth: usize) -> Value {
+    if limits.max_array_items == 0 {
+        return Value::Array(Vec::new());
+    }
+
+    let truncated = values.len() > limits.max_array_items;
+    let value_limit = if truncated {
+        limits.max_array_items.saturating_sub(1)
+    } else {
+        limits.max_array_items
+    };
+    let mut output: Vec<_> = values
+        .iter()
+        .take(value_limit)
+        .map(|value| redact_at_depth(value, limits, depth + 1))
+        .collect();
+    if truncated {
+        output.push(Value::String(TRUNCATED_MARKER.to_owned()));
+    }
+    Value::Array(output)
+}
+
+fn redact_object(values: &Map<String, Value>, limits: RedactionLimits, depth: usize) -> Value {
+    if limits.max_object_fields == 0 {
+        return Value::Object(Map::new());
+    }
+
+    let truncated = values.len() > limits.max_object_fields;
+    let value_limit = if truncated {
+        limits.max_object_fields.saturating_sub(1)
+    } else {
+        limits.max_object_fields
+    };
+    let mut output = Map::new();
+
+    for (index, (key, value)) in values.iter().take(value_limit).enumerate() {
+        let cleaned = if is_sensitive_key(key) {
+            Value::String(REDACTED_MARKER.to_owned())
+        } else if is_url_key(key) {
+            value
+                .as_str()
+                .map(sanitize_url)
+                .map(|value| truncate_string(&value, limits.max_string_bytes))
+                .map(Value::String)
+                .unwrap_or_else(|| redact_at_depth(value, limits, depth + 1))
+        } else {
+            redact_at_depth(value, limits, depth + 1)
+        };
+        output.insert(
+            bounded_object_key(key, limits.max_string_bytes, index),
+            cleaned,
+        );
+    }
+    if truncated {
+        output.insert(
+            "_truncated".to_owned(),
+            Value::String(TRUNCATED_MARKER.to_owned()),
+        );
+    }
+
+    Value::Object(output)
+}
+
+fn normalized_key(key: &str) -> String {
+    key.bytes()
+        .filter(|byte| !matches!(byte, b'_' | b'-' | b'.'))
+        .map(|byte| byte.to_ascii_lowercase() as char)
+        .collect()
+}
+
+fn is_sensitive_key(key: &str) -> bool {
+    let key = normalized_key(key);
+    let exact_match = matches!(
+        key.as_str(),
+        "password"
+            | "passwd"
+            | "secret"
+            | "token"
+            | "apikey"
+            | "accesskey"
+            | "secretkey"
+            | "authorization"
+            | "proxyauthorization"
+            | "cookie"
+            | "setcookie"
+            | "query"
+            | "querystring"
+            | "rawquery"
+            | "urlquery"
+            | "payload"
+            | "body"
+            | "requestbody"
+            | "arguments"
+            | "result"
+            | "response"
+            | "context"
+            | "error"
+            | "errormessage"
+    );
+    let contains_high_risk_name = [
+        "password",
+        "passwd",
+        "secret",
+        "token",
+        "apikey",
+        "accesskey",
+        "authorization",
+        "cookie",
+    ]
+    .iter()
+    .any(|part| key.contains(part));
+
+    exact_match
+        || contains_high_risk_name
+        || key.ends_with("payload")
+        || key.ends_with("body")
+        || key.ends_with("arguments")
+        || key.ends_with("result")
+        || key.ends_with("response")
+        || key.ends_with("query")
+        || key.ends_with("context")
+        || key.starts_with("query")
+}
+
+fn is_url_key(key: &str) -> bool {
+    let key = normalized_key(key);
+    matches!(
+        key.as_str(),
+        "url" | "uri" | "endpoint" | "endpointurl" | "endpointuri" | "requesturl" | "targeturl"
+    ) || key.ends_with("url")
+        || key.ends_with("uri")
+        || key.ends_with("endpoint")
+}
+
+fn sanitize_url(value: &str) -> String {
+    let without_query = value
+        .find(['?', '#'])
+        .map(|index| &value[..index])
+        .unwrap_or(value);
+    let Some(scheme_end) = without_query.find("://") else {
+        return without_query.to_owned();
+    };
+    let authority_start = scheme_end + 3;
+    let authority_end = without_query[authority_start..]
+        .find('/')
+        .map(|index| authority_start + index)
+        .unwrap_or(without_query.len());
+    let authority = &without_query[authority_start..authority_end];
+    let Some(userinfo_end) = authority.rfind('@') else {
+        return without_query.to_owned();
+    };
+
+    format!(
+        "{}{}",
+        &without_query[..authority_start],
+        &without_query[authority_start + userinfo_end + 1..]
+    )
+}
+
+fn bounded_object_key(key: &str, max_bytes: usize, index: usize) -> String {
+    if key.len() <= max_bytes {
+        return key.to_owned();
+    }
+
+    truncate_string(&format!("_truncated_key_{index}"), max_bytes)
+}

crates/crank-observability/src/schema.rs

@@ -0,0 +1,18 @@
+use serde::Serialize;
+use serde_json::{Map, Value};
+
+#[derive(Debug, Serialize)]
+pub(crate) struct LogEnvelope {
+    pub timestamp: String,
+    pub level: String,
+    pub service: String,
+    pub version: String,
+    pub environment: String,
+    pub target: String,
+    pub event: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub request_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub trace_id: Option<String>,
+    pub fields: Map<String, Value>,
+}

crates/crank-observability/tests/correlation.rs

@@ -0,0 +1,42 @@
+use crank_observability::RequestId;
+use uuid::Version;
+
+#[test]
+fn preserves_valid_opaque_request_id() {
+    let request_id = RequestId::resolve(Some("req_test-123/abc"));
+
+    assert_eq!(request_id.as_str(), "req_test-123/abc");
+}
+
+#[test]
+fn replaces_missing_and_invalid_values_with_uuid_v7() {
+    for candidate in [
+        None,
+        Some(""),
+        Some("bad value"),
+        Some(" leading"),
+        Some("trailing "),
+        Some("bad,value"),
+        Some("bad;value"),
+        Some("я"),
+    ] {
+        let request_id = RequestId::resolve(candidate);
+        let parsed = uuid::Uuid::parse_str(request_id.as_str()).expect("generated UUID");
+
+        assert_eq!(parsed.get_version(), Some(Version::SortRand));
+    }
+}
+
+#[test]
+fn rejects_values_over_the_shared_limit() {
+    let oversized = "x".repeat(RequestId::MAX_LEN + 1);
+    let request_id = RequestId::resolve(Some(&oversized));
+
+    assert_ne!(request_id.as_str(), oversized);
+    assert_eq!(
+        uuid::Uuid::parse_str(request_id.as_str())
+            .expect("generated UUID")
+            .get_version(),
+        Some(Version::SortRand)
+    );
+}

crates/crank-observability/tests/critical_errors.rs

@@ -0,0 +1,33 @@
+use crank_observability::{CriticalErrorCategory, SentryConfig, SentryConfigError};
+
+#[test]
+fn missing_or_blank_dsn_disables_critical_error_channel() {
+    assert!(!SentryConfig::parse(None).expect("missing DSN").enabled());
+    assert!(!SentryConfig::parse(Some("")).expect("empty DSN").enabled());
+    assert!(
+        !SentryConfig::parse(Some("   "))
+            .expect("blank DSN")
+            .enabled()
+    );
+}
+
+#[test]
+fn invalid_explicit_dsn_is_rejected_without_echoing_the_value() {
+    let secret_value = "not-a-dsn?token=control-secret";
+    let error = SentryConfig::parse(Some(secret_value)).expect_err("invalid DSN must fail");
+
+    assert!(matches!(error, SentryConfigError::InvalidDsn));
+    assert!(!error.to_string().contains(secret_value));
+    assert!(!error.to_string().contains("control-secret"));
+}
+
+#[test]
+fn critical_error_categories_are_closed_and_stable() {
+    assert_eq!(CriticalErrorCategory::Panic.as_str(), "panic");
+    assert_eq!(CriticalErrorCategory::Startup.as_str(), "startup");
+    assert_eq!(CriticalErrorCategory::Internal.as_str(), "internal");
+    assert_eq!(
+        CriticalErrorCategory::DataIntegrity.as_str(),
+        "data_integrity"
+    );
+}

crates/crank-observability/tests/http_metrics.rs

@@ -0,0 +1,92 @@
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+use axum::{
+    Router,
+    body::{Body, to_bytes},
+    http::{Request, StatusCode},
+    middleware,
+    routing::get,
+};
+use crank_observability::{
+    MetricsConfig, ObservabilityConfig, RedactionLimits, ServiceIdentity, record_http_request,
+};
+use tower::ServiceExt;
+
+#[tokio::test]
+async fn http_metrics_use_matched_routes_and_closed_labels() {
+    let identity =
+        ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity");
+    let lifecycle = crank_observability::init(ObservabilityConfig::new(
+        identity,
+        "off",
+        RedactionLimits::default(),
+    ))
+    .expect("observability lifecycle");
+    let config = MetricsConfig::new(
+        true,
+        SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
+        None,
+    )
+    .expect("loopback metrics");
+    let metrics = lifecycle.metrics_surface(config).router();
+    let app = Router::new()
+        .route(
+            "/documents/{document_id}",
+            get(|| async { StatusCode::NO_CONTENT }),
+        )
+        .layer(middleware::from_fn(record_http_request));
+
+    let sensitive_path_segment = "customer-secret-document-id";
+    for index in 0..100 {
+        let response = app
+            .clone()
+            .oneshot(
+                Request::get(format!("/documents/{sensitive_path_segment}-{index}"))
+                    .body(Body::empty())
+                    .expect("request"),
+            )
+            .await
+            .expect("response");
+        assert_eq!(response.status(), StatusCode::NO_CONTENT);
+    }
+
+    let response = app
+        .oneshot(
+            Request::get("/unknown/customer-controlled-path")
+                .body(Body::empty())
+                .expect("request"),
+        )
+        .await
+        .expect("response");
+    assert_eq!(response.status(), StatusCode::NOT_FOUND);
+
+    let response = metrics
+        .oneshot(
+            Request::get("/metrics")
+                .body(Body::empty())
+                .expect("request"),
+        )
+        .await
+        .expect("metrics response");
+    let body = to_bytes(response.into_body(), 1024 * 1024)
+        .await
+        .expect("bounded metrics body");
+    let body = String::from_utf8(body.to_vec()).expect("utf-8 metrics");
+
+    assert!(body.contains("crank_http_requests_total"));
+    assert!(body.contains("route=\"/documents/{document_id}\""));
+    assert!(body.contains("method=\"GET\""));
+    assert!(body.contains("status_class=\"2xx\""));
+    assert!(body.contains("crank_http_request_duration_seconds_bucket"));
+    assert!(!body.contains(sensitive_path_segment));
+    assert_eq!(
+        body.lines()
+            .filter(|line| {
+                line.starts_with("crank_http_requests_total{")
+                    && line.contains("route=\"/documents/{document_id}\"")
+            })
+            .count(),
+        1,
+        "different entity ids must not create additional series"
+    );
+}

crates/crank-observability/tests/incidents.rs

@@ -0,0 +1,15 @@
+use crank_observability::{
+    OperationalIncident, operational_incident_total, record_operational_incident,
+};
+
+#[test]
+fn history_loss_counter_has_no_dynamic_dimensions() {
+    let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
+
+    record_operational_incident(OperationalIncident::InvocationHistoryLost);
+
+    assert!(
+        operational_incident_total(OperationalIncident::InvocationHistoryLost) > before,
+        "the closed incident counter must increase"
+    );
+}

crates/crank-observability/tests/json_logging.rs

@@ -0,0 +1,392 @@
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use crank_observability::{
+    ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber, safe_json,
+};
+use serde_json::Value;
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use tracing_subscriber::fmt::MakeWriter;
+
+#[derive(Clone, Default)]
+struct SharedWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().expect("test writer lock").clone())
+            .expect("log output must be UTF-8")
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedWriter {
+    type Writer = SharedWriterGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedWriterGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedWriterGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedWriterGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer
+            .lock()
+            .map_err(|_| io::Error::other("test writer lock poisoned"))?
+            .extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+fn capture(service: &'static str, emit: impl FnOnce()) -> Vec<Value> {
+    capture_with_limits(service, RedactionLimits::default(), emit)
+}
+
+fn capture_with_limits(
+    service: &'static str,
+    limits: RedactionLimits,
+    emit: impl FnOnce(),
+) -> Vec<Value> {
+    let writer = SharedWriter::default();
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new(service, "0.3.1", "test").expect("valid test identity"),
+        "info",
+        limits,
+    );
+    let subscriber =
+        build_subscriber(config, writer.clone()).expect("test subscriber must be built");
+
+    tracing::subscriber::with_default(subscriber, emit);
+
+    writer
+        .output()
+        .lines()
+        .map(|line| {
+            assert!(!line.contains('\u{1b}'), "ANSI is forbidden: {line}");
+            serde_json::from_str(line).expect("every line must be one JSON object")
+        })
+        .collect()
+}
+
+#[test]
+fn schema_contract_is_identical_for_both_services() {
+    let outputs = ["admin-api", "mcp-server"].map(|service| {
+        capture(service, || {
+            tracing::info!(
+                name: "service.started",
+                target: "crank::startup",
+                port = 3101_u64,
+                "service started"
+            );
+        })
+    });
+
+    for (service, events) in ["admin-api", "mcp-server"].into_iter().zip(outputs.iter()) {
+        assert_eq!(events.len(), 1);
+        let event = &events[0];
+        assert_eq!(event["service"], service);
+        assert_eq!(event["version"], "0.3.1");
+        assert_eq!(event["environment"], "test");
+        assert_eq!(event["level"], "INFO");
+        assert_eq!(event["target"], "crank::startup");
+        assert_eq!(event["event"], "service.started");
+        assert!(event["fields"].is_object());
+        assert_eq!(event["fields"]["port"], 3101);
+        assert_eq!(event["fields"]["message"], "service started");
+        let timestamp = event["timestamp"].as_str().expect("timestamp string");
+        let parsed =
+            OffsetDateTime::parse(timestamp, &Rfc3339).expect("timestamp must be RFC 3339");
+        assert_eq!(parsed.offset(), time::UtcOffset::UTC);
+    }
+
+    let first_keys: Vec<_> = outputs[0][0]
+        .as_object()
+        .expect("object")
+        .keys()
+        .cloned()
+        .collect();
+    let second_keys: Vec<_> = outputs[1][0]
+        .as_object()
+        .expect("object")
+        .keys()
+        .cloned()
+        .collect();
+    assert_eq!(first_keys, second_keys);
+}
+
+#[test]
+fn correlation_fields_are_distinct_and_only_present_when_recorded() {
+    let present = capture("admin-api", || {
+        tracing::info!(
+            name: "admin.request.completed",
+            request_id = "req-123",
+            trace_id = "trace-456"
+        );
+    });
+    assert_eq!(present[0]["request_id"], "req-123");
+    assert_eq!(present[0]["trace_id"], "trace-456");
+    assert!(present[0]["fields"].get("request_id").is_none());
+    assert!(present[0]["fields"].get("trace_id").is_none());
+
+    let absent = capture("mcp-server", || {
+        tracing::info!(name: "mcp.request.completed", status = 200_u64);
+    });
+    assert!(absent[0].get("request_id").is_none());
+    assert!(absent[0].get("trace_id").is_none());
+}
+
+#[test]
+fn correlation_fields_preserve_scalar_display_values_before_field_limits() {
+    let limits = RedactionLimits {
+        max_object_fields: 1,
+        ..RedactionLimits::default()
+    };
+    let request_id = "123";
+    let events = capture_with_limits("admin-api", limits, || {
+        tracing::info!(
+            name: "admin.request.completed",
+            alpha = "field that consumes the object budget",
+            request_id = %request_id,
+            trace_id = true,
+        );
+    });
+
+    assert_eq!(events[0]["request_id"], "123");
+    assert_eq!(events[0]["trace_id"], "true");
+}
+
+#[test]
+fn empty_correlation_fields_are_omitted() {
+    let events = capture("admin-api", || {
+        tracing::info!(
+            name: "admin.request.completed",
+            request_id = "",
+            trace_id = ""
+        );
+    });
+
+    assert!(events[0].get("request_id").is_none());
+    assert!(events[0].get("trace_id").is_none());
+}
+
+#[test]
+fn formatter_redacts_fields_before_serialization() {
+    let context = safe_json(
+        &serde_json::json!({
+            "nested": {
+                "access_token": "nested-canary-secret",
+                "endpoint": "https://example.test/private?key=nested-canary-secret"
+            }
+        }),
+        RedactionLimits::default(),
+    )
+    .expect("safe nested context");
+    let events = capture("admin-api", || {
+        tracing::warn!(
+            name: "admin.request.rejected",
+            password = "canary-secret",
+            url = "https://example.test/path?token=canary-secret",
+            safe_fields = %context,
+            unsafe_context = ?serde_json::json!({"password": "debug-canary-secret"}),
+            error_code = "invalid_request"
+        );
+    });
+    let serialized = serde_json::to_string(&events[0]).expect("event JSON");
+
+    assert_eq!(events[0]["fields"]["password"], "[REDACTED]");
+    assert_eq!(events[0]["fields"]["url"], "https://example.test/path");
+    assert_eq!(
+        events[0]["fields"]["safe_fields"]["nested"]["access_token"],
+        "[REDACTED]"
+    );
+    assert_eq!(
+        events[0]["fields"]["safe_fields"]["nested"]["endpoint"],
+        "https://example.test/private"
+    );
+    assert_eq!(events[0]["fields"]["unsafe_context"], "[REDACTED]");
+    assert_eq!(events[0]["fields"]["error_code"], "invalid_request");
+    assert!(!serialized.contains("canary-secret"));
+}
+
+#[test]
+fn arbitrary_debug_text_is_never_written_verbatim() {
+    struct Credentials;
+
+    impl std::fmt::Debug for Credentials {
+        fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            formatter.write_str("Credentials { password: \"debug-canary-secret\" }")
+        }
+    }
+
+    let events = capture("admin-api", || {
+        tracing::warn!(
+            name: "admin.debug.inspected",
+            details = ?Credentials
+        );
+    });
+    let serialized = serde_json::to_string(&events[0]).expect("event JSON");
+
+    assert_eq!(events[0]["fields"]["details"], "[REDACTED]");
+    assert!(!serialized.contains("debug-canary-secret"));
+}
+
+#[test]
+fn compound_sensitive_event_fields_are_redacted() {
+    let events = capture("admin-api", || {
+        tracing::warn!(
+            name: "admin.request.rejected",
+            client_api_key = "client-canary-secret",
+            authorization_header = "Bearer auth-canary-secret",
+            response_body = "response-canary-secret",
+            tool_arguments = "argument-canary-secret",
+            query_params = "query-canary-secret",
+        );
+    });
+    let serialized = serde_json::to_string(&events[0]).expect("event JSON");
+
+    for key in [
+        "client_api_key",
+        "authorization_header",
+        "response_body",
+        "tool_arguments",
+        "query_params",
+    ] {
+        assert_eq!(events[0]["fields"][key], "[REDACTED]");
+    }
+    assert!(!serialized.contains("canary-secret"));
+}
+
+#[test]
+fn oversized_event_falls_back_to_valid_bounded_json() {
+    let limits = RedactionLimits {
+        max_event_bytes: 512,
+        ..RedactionLimits::default()
+    };
+    let events = capture_with_limits("admin-api", limits, || {
+        tracing::info!(
+            name: "admin.payload.inspected",
+            description = %"x".repeat(1024)
+        );
+    });
+    let serialized = serde_json::to_vec(&events[0]).expect("bounded event JSON");
+
+    assert!(serialized.len() < limits.max_event_bytes);
+    assert_eq!(events[0]["fields"]["truncated"], true);
+}
+
+#[test]
+fn safe_json_honours_the_total_event_budget() {
+    let limits = RedactionLimits {
+        max_event_bytes: 512,
+        ..RedactionLimits::default()
+    };
+
+    let serialized = safe_json(
+        &serde_json::json!({"description": "x".repeat(4096)}),
+        limits,
+    )
+    .expect("safe JSON must remain serializable");
+
+    assert!(serialized.len() <= limits.max_event_bytes);
+    assert_eq!(
+        serde_json::from_str::<Value>(&serialized).expect("valid JSON")["truncated"],
+        true
+    );
+}
+
+#[test]
+fn subscriber_rejects_limits_that_cannot_hold_an_event() {
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
+        "info",
+        RedactionLimits {
+            max_event_bytes: 16,
+            ..RedactionLimits::default()
+        },
+    );
+
+    assert!(build_subscriber(config, SharedWriter::default()).is_err());
+}
+
+#[test]
+fn minimum_event_budget_handles_maximum_identity_labels() {
+    let writer = SharedWriter::default();
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new("s".repeat(64), "v".repeat(64), "e".repeat(64))
+            .expect("maximum identity labels are valid"),
+        "info",
+        RedactionLimits {
+            max_event_bytes: 512,
+            ..RedactionLimits::default()
+        },
+    );
+    let subscriber =
+        build_subscriber(config, writer.clone()).expect("minimum valid budget must be usable");
+
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::info!(
+            name: "event-name-that-is-intentionally-longer-than-the-fallback-limit",
+            description = %"x".repeat(4096),
+        );
+    });
+
+    let output = writer.output();
+    assert!(output.len() <= 512);
+    assert_eq!(output.lines().count(), 1);
+    serde_json::from_str::<Value>(output.trim_end()).expect("bounded line must remain valid JSON");
+}
+
+#[test]
+fn env_filter_is_applied_and_invalid_filter_is_safe() {
+    let writer = SharedWriter::default();
+    let config = ObservabilityConfig::new(
+        ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
+        "warn",
+        RedactionLimits::default(),
+    );
+    let subscriber =
+        build_subscriber(config, writer.clone()).expect("test subscriber must be built");
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::info!(name: "filtered.info", "filtered");
+        tracing::warn!(name: "visible.warning", "visible");
+    });
+    let output = writer.output();
+
+    assert!(!output.contains("filtered.info"));
+    assert!(output.contains("visible.warning"));
+
+    let invalid = ObservabilityConfig::new(
+        ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
+        "[not a valid filter",
+        RedactionLimits::default(),
+    );
+    let error = build_subscriber(invalid, SharedWriter::default())
+        .err()
+        .expect("invalid filter must fail");
+    assert_eq!(error.to_string(), "invalid log filter");
+    assert!(!error.to_string().contains("not a valid filter"));
+}
+
+#[test]
+fn service_identity_rejects_empty_or_unsafe_labels() {
+    for (service, version, environment) in [
+        ("", "0.3.1", "test"),
+        ("admin api", "0.3.1", "test"),
+        ("admin-api", "", "test"),
+        ("admin-api", "0.3.1", "prod\nsecret"),
+    ] {
+        assert!(ServiceIdentity::try_new(service, version, environment).is_err());
+    }
+}

crates/crank-observability/tests/lifecycle.rs

@@ -0,0 +1,22 @@
+use crank_observability::{
+    ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity, init,
+};
+
+fn config() -> ObservabilityConfig {
+    ObservabilityConfig::new(
+        ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"),
+        "info",
+        RedactionLimits::default(),
+    )
+}
+
+#[test]
+fn repeated_global_initialization_returns_typed_error() {
+    let _lifecycle = init(config()).expect("first initialization must succeed");
+    let error = init(config()).expect_err("second initialization must fail");
+
+    assert!(matches!(
+        error,
+        ObservabilityInitError::SubscriberAlreadyInitialized
+    ));
+}

crates/crank-observability/tests/otlp.rs

@@ -0,0 +1,56 @@
+use std::time::Duration;
+
+use crank_observability::{OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError};
+
+#[test]
+fn absent_endpoint_disables_export_without_background_resources() {
+    let config = OtlpTraceConfig::try_new(
+        None,
+        None,
+        Duration::from_secs(10),
+        OtlpBatchConfig::default(),
+    )
+    .expect("missing endpoint must be valid");
+
+    assert!(!config.is_enabled());
+}
+
+#[test]
+fn explicit_config_accepts_only_bounded_http_protobuf() {
+    let config = OtlpTraceConfig::try_new(
+        Some("https://collector.example.test/v1/traces".to_owned()),
+        Some("http/protobuf".to_owned()),
+        Duration::from_secs(3),
+        OtlpBatchConfig::try_new(256, 64, Duration::from_millis(500), Duration::from_secs(3))
+            .unwrap(),
+    )
+    .expect("bounded HTTP protobuf config must be valid");
+
+    assert!(config.is_enabled());
+    assert_eq!(config.export_timeout(), Duration::from_secs(3));
+    assert_eq!(config.batch().max_queue_size(), 256);
+    assert_eq!(config.batch().max_export_batch_size(), 64);
+}
+
+#[test]
+fn invalid_values_return_safe_typed_errors() {
+    let secret_endpoint = "https://user:canary-secret@collector.example.test/v1/traces";
+    let error = OtlpTraceConfig::try_new(
+        Some(secret_endpoint.to_owned()),
+        Some("grpc".to_owned()),
+        Duration::ZERO,
+        OtlpBatchConfig::default(),
+    )
+    .expect_err("credentials in endpoint must be rejected");
+
+    assert!(matches!(
+        error,
+        OtlpTraceConfigError::InvalidEndpoint { .. }
+    ));
+    assert!(!error.to_string().contains(secret_endpoint));
+    assert!(!error.to_string().contains("canary-secret"));
+
+    let error = OtlpBatchConfig::try_new(8, 9, Duration::from_millis(1), Duration::from_secs(1))
+        .expect_err("batch cannot exceed queue");
+    assert!(matches!(error, OtlpTraceConfigError::InvalidBatchLimits));
+}