bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

наблюдаемость: завершить базовый контур Community
CI / Rust Checks (push) Failing after 4m28s
Details
CI / UI Checks (push) Has been skipped
Details
CI / Frontend E2E (push) Has been skipped
Details
CI / Community Image Smoke (push) Has been skipped
Details
CI / Deploy (push) Has been skipped
Details

Browse Source 
Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.
This commit is contained in:
bsodfather
2026-07-31 01:01:14 +03:00
parent 99bd05c145
commit 0e8f1ca03a
160 changed files with 13506 additions and 1499 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/config.toml
+2
View File
@@ -0,0 +1,2 @@
[build]
jobs = 2
.env.example
+23
View File
@@ -29,7 +29,30 @@ CRANK_RUNTIME_MAX_CONCURRENT_JOBS=16
CRANK_OUTBOUND_ALLOWED_HOSTS=
CRANK_OUTBOUND_DENIED_HOSTS=
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
CRANK_ENVIRONMENT=development
CRANK_LOG_LEVEL=info
# Пустое значение отключает канал критических ошибок.
CRANK_SENTRY_DSN=
# Prometheus endpoints use separate listeners and stay on loopback by default.
CRANK_METRICS_ENABLED=true
CRANK_ADMIN_METRICS_BIND=127.0.0.1:9464
CRANK_MCP_METRICS_BIND=127.0.0.1:9465
# Required when either metrics listener uses a non-loopback address.
CRANK_METRICS_BEARER_TOKEN=
CRANK_INVOCATION_LOG_RETENTION_DAYS=30
# Пустой endpoint полностью отключает экспорт трасс.
OTEL_EXPORTER_OTLP_ENDPOINT=
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=
OTEL_EXPORTER_OTLP_TIMEOUT=10000
OTEL_EXPORTER_OTLP_TRACES_TIMEOUT=
OTEL_EXPORTER_OTLP_HEADERS=
OTEL_EXPORTER_OTLP_TRACES_HEADERS=
OTEL_BSP_MAX_QUEUE_SIZE=2048
OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512
OTEL_BSP_SCHEDULE_DELAY=5000
OTEL_BSP_EXPORT_TIMEOUT=30000
CRANK_MASTER_KEY=change-me-master-key
CRANK_SESSION_SECRET=change-me-session-secret
CRANK_PASSWORD_PEPPER=change-me-password-pepper
.gitea/workflows/ci.yml
+118 -48
View File
@@ -55,6 +55,9 @@ jobs:
docker --version
docker info
- name: Install dependency policy tool
run: cargo install cargo-deny --version 0.20.2 --locked
- name: Run tooling unit tests
run: python3 -m unittest discover -s tests/unit
@@ -67,6 +70,9 @@ jobs:
- name: Check Rust code health
run: scripts/check-rust-code-health.sh
- name: Check dependency licenses and advisories
run: cargo deny --locked check advisories bans licenses sources
- name: Check Rust boundaries
run: scripts/check-rust-boundaries.sh
@@ -95,6 +101,10 @@ jobs:
working-directory: apps/ui
run: npm ci
- name: Audit UI dependencies
working-directory: apps/ui
run: npm audit --audit-level=high
- name: Build UI bundle
working-directory: apps/ui
run: npm run build
@@ -179,9 +189,11 @@ jobs:
find .tmp/ui-e2e/logs -maxdepth 1 -type f -print -exec sed -n '1,220p' {} \; || true
deployment:
name: Deployment Manifests
name: Community Image Smoke
runs-on: ubuntu-latest
needs: ui
needs:
- rust
- ui
steps:
- name: Checkout
@@ -190,6 +202,58 @@ jobs:
- name: Validate Community deployment manifest
run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
- name: Build Community images
run: |
docker build -f apps/admin-api/Dockerfile -t crank/admin-api:ci .
docker build -f apps/mcp-server/Dockerfile -t crank/mcp-server:ci .
docker build -f apps/ui/Dockerfile -t crank/ui:ci .
- name: Start Community image stack
run: |
mkdir -p .tmp
cat > .tmp/community-smoke.env <<'EOF'
COMPOSE_PROJECT_NAME=crank-ci-smoke
POSTGRES_HOST=postgres
POSTGRES_PORT=5432
POSTGRES_DB=crank
POSTGRES_USER=crank
POSTGRES_PASSWORD=crank-ci-password
CRANK_ADMIN_API_IMAGE=crank/admin-api:ci
CRANK_MCP_SERVER_IMAGE=crank/mcp-server:ci
CRANK_UI_IMAGE=crank/ui:ci
CRANK_MASTER_KEY=0000000000000000000000000000000000000000000000000000000000000000
CRANK_SESSION_SECRET=ci-session-secret
CRANK_PASSWORD_PEPPER=ci-password-pepper
CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test
CRANK_BOOTSTRAP_ADMIN_PASSWORD=ci-admin-password
CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner
CRANK_BASE_URL=http://127.0.0.1:3000
CRANK_PUBLISH_BIND=127.0.0.1
CRANK_DEMO_SEED=true
EOF
docker compose -f deploy/community/docker-compose.images.yml \
--env-file .tmp/community-smoke.env --profile local-db up -d --wait
- name: Run authenticated Community image smoke
env:
CRANK_STAGING_ADMIN_EMAIL: owner@crank.test
CRANK_STAGING_ADMIN_PASSWORD: ci-admin-password
run: scripts/authenticated-product-smoke.sh http://127.0.0.1:3000
- name: Show Community image logs
if: failure()
run: |
docker compose -f deploy/community/docker-compose.images.yml \
--env-file .tmp/community-smoke.env --profile local-db ps || true
docker compose -f deploy/community/docker-compose.images.yml \
--env-file .tmp/community-smoke.env --profile local-db logs --no-color || true
- name: Stop Community image stack
if: always()
run: |
docker compose -f deploy/community/docker-compose.images.yml \
--env-file .tmp/community-smoke.env --profile local-db down -v --remove-orphans || true
deploy:
name: Deploy
runs-on: ubuntu-latest
@@ -245,6 +309,10 @@ jobs:
-t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \
-t '${{ env.UI_IMAGE }}:main' \
.
scripts/scan-images.sh \
'${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}'
docker push '${{ env.ADMIN_API_IMAGE }}:main'
docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}'
@@ -274,9 +342,14 @@ jobs:
run: |
. "$OPENBAO_ENV_FILE"
ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
"mkdir -p '$DEPLOY_PATH'"
"mkdir -p '$DEPLOY_PATH' && \
if [ -f '$DEPLOY_PATH/docker-compose.yml' ]; then \
cp '$DEPLOY_PATH/docker-compose.yml' '$DEPLOY_PATH/docker-compose.previous.yml'; \
fi"
rsync -az -e "ssh -p $DEPLOY_PORT" deploy/community/docker-compose.yml \
"$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/docker-compose.yml"
rsync -az -e "ssh -p $DEPLOY_PORT" scripts/deploy-community.sh \
"$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/deploy-community.sh"
- name: Write environment file
run: |
@@ -292,6 +365,11 @@ jobs:
append_if_set POSTGRES_USER "$POSTGRES_USER"
append_if_set POSTGRES_PASSWORD "$POSTGRES_PASSWORD"
append_if_set POSTGRES_HOST "$POSTGRES_HOST"
append_if_set POSTGRES_MAX_CONNECTIONS "${POSTGRES_MAX_CONNECTIONS:-}"
append_if_set POSTGRES_MIN_CONNECTIONS "${POSTGRES_MIN_CONNECTIONS:-}"
append_if_set POSTGRES_ACQUIRE_TIMEOUT_MS "${POSTGRES_ACQUIRE_TIMEOUT_MS:-}"
append_if_set POSTGRES_IDLE_TIMEOUT_MS "${POSTGRES_IDLE_TIMEOUT_MS:-}"
append_if_set POSTGRES_MAX_LIFETIME_MS "${POSTGRES_MAX_LIFETIME_MS:-}"
if [ -n "${POSTGRES_PORT:-}" ]; then
append_if_set POSTGRES_PORT "$POSTGRES_PORT"
elif [ -n "${PGBOUNCER_PORT:-}" ]; then
@@ -302,10 +380,35 @@ jobs:
append_if_set CRANK_ADMIN_BIND "$CRANK_ADMIN_BIND"
append_if_set CRANK_MCP_BIND "$CRANK_MCP_BIND"
append_if_set CRANK_MCP_REFRESH_MS "$CRANK_MCP_REFRESH_MS"
append_if_set CRANK_ADMIN_RATE_LIMIT_RPS "${CRANK_ADMIN_RATE_LIMIT_RPS:-}"
append_if_set CRANK_ADMIN_RATE_LIMIT_BURST "${CRANK_ADMIN_RATE_LIMIT_BURST:-}"
append_if_set CRANK_MCP_RATE_LIMIT_RPS "${CRANK_MCP_RATE_LIMIT_RPS:-}"
append_if_set CRANK_MCP_RATE_LIMIT_BURST "${CRANK_MCP_RATE_LIMIT_BURST:-}"
append_if_set CRANK_RUNTIME_MAX_CONCURRENT_UNARY "${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-}"
append_if_set CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS "${CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS:-}"
append_if_set CRANK_OUTBOUND_ALLOWED_HOSTS "${CRANK_OUTBOUND_ALLOWED_HOSTS:-}"
append_if_set CRANK_OUTBOUND_DENIED_HOSTS "${CRANK_OUTBOUND_DENIED_HOSTS:-}"
append_if_set CRANK_OUTBOUND_MAX_RESPONSE_BYTES "${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-}"
append_if_set CRANK_ENVIRONMENT "${CRANK_ENVIRONMENT:-production}"
append_if_set CRANK_LOG_LEVEL "$CRANK_LOG_LEVEL"
append_if_set CRANK_SENTRY_DSN "${CRANK_SENTRY_DSN:-}"
append_if_set CRANK_METRICS_ENABLED "${CRANK_METRICS_ENABLED:-}"
append_if_set CRANK_ADMIN_METRICS_BIND "${CRANK_ADMIN_METRICS_BIND:-}"
append_if_set CRANK_MCP_METRICS_BIND "${CRANK_MCP_METRICS_BIND:-}"
append_if_set CRANK_METRICS_BEARER_TOKEN "${CRANK_METRICS_BEARER_TOKEN:-}"
append_if_set CRANK_INVOCATION_LOG_RETENTION_DAYS "${CRANK_INVOCATION_LOG_RETENTION_DAYS:-}"
append_if_set OTEL_EXPORTER_OTLP_ENDPOINT "${OTEL_EXPORTER_OTLP_ENDPOINT:-}"
append_if_set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"
append_if_set OTEL_EXPORTER_OTLP_PROTOCOL "${OTEL_EXPORTER_OTLP_PROTOCOL:-}"
append_if_set OTEL_EXPORTER_OTLP_TRACES_PROTOCOL "${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-}"
append_if_set OTEL_EXPORTER_OTLP_TIMEOUT "${OTEL_EXPORTER_OTLP_TIMEOUT:-}"
append_if_set OTEL_EXPORTER_OTLP_TRACES_TIMEOUT "${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-}"
append_if_set OTEL_EXPORTER_OTLP_HEADERS "${OTEL_EXPORTER_OTLP_HEADERS:-}"
append_if_set OTEL_EXPORTER_OTLP_TRACES_HEADERS "${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-}"
append_if_set OTEL_BSP_MAX_QUEUE_SIZE "${OTEL_BSP_MAX_QUEUE_SIZE:-}"
append_if_set OTEL_BSP_MAX_EXPORT_BATCH_SIZE "${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-}"
append_if_set OTEL_BSP_SCHEDULE_DELAY "${OTEL_BSP_SCHEDULE_DELAY:-}"
append_if_set OTEL_BSP_EXPORT_TIMEOUT "${OTEL_BSP_EXPORT_TIMEOUT:-}"
append_if_set CRANK_MASTER_KEY "$CRANK_MASTER_KEY"
append_if_set CRANK_BASE_URL "$CRANK_BASE_URL"
append_if_set CRANK_CACHE_BACKEND "$CRANK_CACHE_BACKEND"
@@ -325,7 +428,10 @@ jobs:
printf 'CRANK_UI_IMAGE=%s:%s\n' '${{ env.UI_IMAGE }}' '${{ env.IMAGE_TAG }}'
} >> "$tmp_env"
cat "$tmp_env" | ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
"mkdir -p '$DEPLOY_PATH' && cat > '$DEPLOY_PATH/.env'"
"mkdir -p '$DEPLOY_PATH' && \
if [ -f '$DEPLOY_PATH/.env' ]; then \
cp '$DEPLOY_PATH/.env' '$DEPLOY_PATH/.env.previous'; \
fi && cat > '$DEPLOY_PATH/.env'"
rm -f "$tmp_env"
- name: Validate required environment variables
@@ -359,48 +465,12 @@ jobs:
- name: Deploy with Docker Compose
run: |
. "$OPENBAO_ENV_FILE"
ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" "
set -e
cd '$DEPLOY_PATH'
compose_profiles=''
cache_backend=\$(grep -E '^CRANK_CACHE_BACKEND=' .env | tail -n1 | cut -d= -f2- || true)
if [ \"\$cache_backend\" = 'valkey' ] || [ \"\$cache_backend\" = 'redis' ]; then
compose_profiles='--profile cache'
fi
echo '$DEPLOY_REGISTRY_TOKEN' | docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin
docker compose \$compose_profiles config -q
docker compose \$compose_profiles pull
docker compose \$compose_profiles down --remove-orphans
for container in \
crank-ui-1 \
crank-admin-api-1 \
crank-mcp-server-1 \
crank-postgres-1 \
crank-valkey-1 \
crank-community-ui-1 \
crank-community-admin-api-1 \
crank-community-mcp-server-1 \
crank-community-postgres-1 \
crank-community-valkey-1; do
if docker ps -a --format '{{.Names}}' | grep -Fx \"\$container\" >/dev/null; then
docker rm -f \"\$container\"
fi
done
echo 'Docker containers before freeing required ports:'
docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Ports}}'
for port in 3000 3001 3002; do
container_ids=\$(docker ps -aq --filter \"publish=\$port\")
if [ -n \"\$container_ids\" ]; then
echo \"Removing containers publishing port \$port\"
docker inspect --format '{{.Name}} {{json .NetworkSettings.Ports}}' \$container_ids || true
docker rm -f \$container_ids
fi
done
if command -v ss >/dev/null 2>&1; then
ss -ltnp '( sport = :3000 or sport = :3001 or sport = :3002 )' || true
fi
docker compose \$compose_profiles up -d --remove-orphans
"
printf '%s' "$DEPLOY_REGISTRY_TOKEN" | ssh -p "$DEPLOY_PORT" \
"$DEPLOY_USER@$DEPLOY_HOST" \
"docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin"
ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \
"chmod 700 '$DEPLOY_PATH/deploy-community.sh' && \
'$DEPLOY_PATH/deploy-community.sh' '$DEPLOY_PATH'"
- name: Verify health endpoints
run: |
@@ -410,8 +480,8 @@ jobs:
cd '$DEPLOY_PATH'
for attempt in \$(seq 1 30); do
if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \
&& curl --fail --silent http://127.0.0.1:3001/health >/dev/null \
&& curl --fail --silent http://127.0.0.1:3002/health >/dev/null; then
&& curl --fail --silent http://127.0.0.1:3001/ready >/dev/null \
&& curl --fail --silent http://127.0.0.1:3002/ready >/dev/null; then
exit 0
fi
sleep 2
.gitea/workflows/release.yml
+29
View File
@@ -49,6 +49,16 @@ jobs:
command -v bao
bao version
- name: Install dependency policy tool
run: cargo install cargo-deny --version 0.20.2 --locked
- name: Run release quality gates
run: |
cargo fmt --all --check
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test --workspace --all-targets
cargo deny --locked check advisories bans licenses sources
- name: Build release binaries
run: cargo build --release -p admin-api -p mcp-server
@@ -56,10 +66,25 @@ jobs:
working-directory: apps/ui
run: npm ci
- name: Audit UI dependencies
working-directory: apps/ui
run: npm audit --audit-level=high
- name: Build UI dist
working-directory: apps/ui
run: npm run build
- name: Install Playwright browser
working-directory: apps/ui
run: npx playwright install --with-deps chromium
- name: Run release end-to-end tests
working-directory: apps/ui
run: npm run e2e
- name: Validate deployment manifest
run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
- name: Package release artifacts
run: |
mkdir -p dist/release
@@ -105,6 +130,10 @@ jobs:
docker build -f apps/ui/Dockerfile \
-t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \
-t '${{ env.UI_IMAGE }}:latest' .
scripts/scan-images.sh \
'${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}'
docker push '${{ env.ADMIN_API_IMAGE }}:latest'
docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}'
Cargo.lock
Generated
+797 -34
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+28
View File
@@ -8,9 +8,11 @@ members = [
"crates/crank-import",
"crates/crank-schema",
"crates/crank-mapping",
"crates/crank-observability",
"crates/crank-registry",
"crates/crank-runtime",
"crates/crank-test-support",
"crates/crank-trace",
"crates/crank-adapter-rest",
]
resolver = "3"
@@ -20,6 +22,7 @@ edition = "2024"
license = "AGPL-3.0-only"
rust-version = "1.96"
version = "0.3.1"
publish = false
[workspace.dependencies]
aes-gcm = "0.10"
@@ -28,18 +31,43 @@ axum = "0.8"
axum-extra = { version = "0.12", features = ["cookie"] }
base64 = "0.22"
hkdf = "0.12"
metrics = "0.24.6"
metrics-exporter-prometheus = { version = "0.18.3", default-features = false }
opentelemetry = { version = "0.32.0", default-features = false, features = ["trace"] }
opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["http-proto", "reqwest-blocking-client", "reqwest-rustls", "trace"] }
opentelemetry-proto = { version = "0.32.0", default-features = false, features = ["gen-tonic-messages", "trace"] }
opentelemetry_sdk = { version = "0.32.1", default-features = false, features = ["trace"] }
percent-encoding = "2"
prost = "0.14"
rand = "0.10"
reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "rustls-tls"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"
sentry = { version = "0.49.0", default-features = false, features = ["backtrace", "panic", "rustls", "ureq"] }
sha2 = "0.10"
sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "macros", "json", "time", "uuid"] }
subtle = "2.6"
thiserror = "2"
time = { version = "0.3.53", features = ["formatting", "parsing", "serde"] }
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
tower = "0.5"
tracing = "0.1"
tracing-opentelemetry = { version = "0.33.0", default-features = false }
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
url = "2"
uuid = { version = "1", features = ["serde", "v7"] }
testcontainers = { version = "0.27", features = ["blocking"] }
testcontainers-modules = { version = "0.15", features = ["postgres", "blocking"] }
[profile.dev]
debug = "line-tables-only"
[profile.dev.package."*"]
debug = false
[profile.test]
debug = "line-tables-only"
[profile.test.package."*"]
debug = false
apps/admin-api/Cargo.toml
+8
View File
@@ -3,6 +3,7 @@ name = "admin-api"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[[bin]]
@@ -18,9 +19,12 @@ crank-community-auth = { path = "../../crates/crank-community-auth" }
crank-core = { path = "../../crates/crank-core" }
crank-import = { path = "../../crates/crank-import" }
crank-mapping = { path = "../../crates/crank-mapping" }
crank-observability = { path = "../../crates/crank-observability" }
crank-registry = { path = "../../crates/crank-registry" }
crank-runtime = { path = "../../crates/crank-runtime" }
crank-schema = { path = "../../crates/crank-schema" }
crank-trace = { path = "../../crates/crank-trace" }
metrics.workspace = true
rand.workspace = true
serde.workspace = true
serde_json.workspace = true
@@ -37,5 +41,9 @@ uuid.workspace = true
[dev-dependencies]
async-trait = "0.1"
crank-test-support = { path = "../../crates/crank-test-support" }
opentelemetry.workspace = true
opentelemetry_sdk.workspace = true
reqwest.workspace = true
serial_test = "3"
tower.workspace = true
tracing-opentelemetry.workspace = true
apps/admin-api/src/app.rs
+4
View File
@@ -166,6 +166,7 @@ pub fn build_app(state: AppState) -> Router {
Router::new()
.route("/health", get(crate::routes::health))
.route("/ready", get(crate::routes::readiness))
.nest(
"/api/auth",
Router::new()
@@ -178,6 +179,9 @@ pub fn build_app(state: AppState) -> Router {
apply_api_rate_limit,
))
.layer(middleware::from_fn(apply_request_context))
.layer(middleware::from_fn(
crank_observability::record_http_request,
))
.with_state(state)
}
apps/admin-api/src/dto.rs
+6 -1
View File
@@ -269,7 +269,12 @@ pub struct CreatedPlatformApiKeyResponse {
}
#[derive(Clone, Debug, Serialize)]
pub struct WorkspaceExportResponse {
pub struct WorkspaceCatalogSnapshotResponse {
pub kind: String,
pub format_version: String,
pub restorable: bool,
pub included: Vec<String>,
pub excluded: Vec<String>,
pub workspace: WorkspaceRecord,
pub operations: Vec<OperationSummaryView>,
pub agents: Vec<AgentSummaryView>,
apps/admin-api/src/error.rs
+30 -10
View File
@@ -137,16 +137,24 @@ impl ApiError {
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
match &self {
Self::Internal { message, .. } => {
error!(error_code = self.code(), error_message = %message)
Self::Internal { .. } => {
error!(
name: "admin.response.internal_error",
error_code = self.code(),
"internal API error response"
)
}
Self::Unauthorized { message, .. }
| Self::Forbidden { message, .. }
| Self::Validation { message, .. }
| Self::NotFound { message, .. }
| Self::Conflict { message, .. }
| Self::RateLimited { message, .. } => {
warn!(error_code = self.code(), error_message = %message)
Self::Unauthorized { .. }
| Self::Forbidden { .. }
| Self::Validation { .. }
| Self::NotFound { .. }
| Self::Conflict { .. }
| Self::RateLimited { .. } => {
warn!(
name: "admin.response.rejected",
error_code = self.code(),
"API request rejected"
)
}
}
@@ -351,6 +359,10 @@ impl From<RegistryError> for ApiError {
format!("import job {job_id} was not found"),
json!({ "job_id": job_id }),
),
RegistryError::ImportJobAlreadyApplied { job_id } => Self::conflict_with_context(
format!("import job {job_id} was already applied with different parameters"),
json!({ "job_id": job_id }),
),
RegistryError::Storage(_) | RegistryError::Serialization(_) => {
Self::internal(value.to_string())
}
@@ -407,6 +419,10 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
RuntimeError::ConfirmationRequired { .. } => "runtime_confirmation_required",
RuntimeError::InvalidConfirmationToken { .. } => "runtime_confirmation_error",
RuntimeError::ConfirmationStoreUnavailable { .. } => "runtime_confirmation_unavailable",
RuntimeError::IdempotencyStoreUnavailable { .. } => "runtime_idempotency_unavailable",
RuntimeError::IdempotencyInProgress { .. } => "runtime_idempotency_in_progress",
RuntimeError::IdempotencyConflict { .. } => "runtime_idempotency_conflict",
RuntimeError::IdempotencyOutcomeUnknown { .. } => "runtime_idempotency_outcome_unknown",
RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error",
RuntimeError::MissingAuthProfile { .. } => "runtime_auth_profile_error",
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
@@ -434,7 +450,11 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
"safety_class": safety_class,
})),
RuntimeError::InvalidConfirmationToken { operation_id }
| RuntimeError::ConfirmationStoreUnavailable { operation_id } => Some(json!({
| RuntimeError::ConfirmationStoreUnavailable { operation_id }
| RuntimeError::IdempotencyStoreUnavailable { operation_id }
| RuntimeError::IdempotencyInProgress { operation_id }
| RuntimeError::IdempotencyConflict { operation_id }
| RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({
"operation_id": operation_id,
})),
RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({
apps/admin-api/src/main.rs
+105 -9
View File
@@ -1,4 +1,4 @@
use std::{env, net::SocketAddr, path::PathBuf};
use std::{env, net::SocketAddr, path::PathBuf, time::Duration};
use admin_api::{
app::build_app,
@@ -7,22 +7,50 @@ use admin_api::{
state::AppState,
};
use crank_community_auth::PasswordIdentityProvider;
use crank_observability::{
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
capture_critical_error,
};
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use sqlx::postgres::PgConnectOptions;
use sqlx::{PgPool, postgres::PgConnectOptions};
use tokio::net::TcpListener;
use tracing::info;
use tracing::{info, warn};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt()
.with_env_filter(
env::var("CRANK_LOG_LEVEL").unwrap_or_else(|_| "admin_api=info,tower_http=info".into()),
)
.init();
let observability = crank_observability::init(ObservabilityConfig::from_env(
"admin-api",
env!("CARGO_PKG_VERSION"),
"admin_api=info,tower_http=info",
)?)?;
let mut startup_completed = false;
let result = run(&observability, &mut startup_completed).await;
if result.is_err() {
capture_critical_error(if startup_completed {
CriticalErrorCategory::Internal
} else {
CriticalErrorCategory::Startup
});
}
result
}
async fn run(
observability: &ObservabilityLifecycle,
startup_completed: &mut bool,
) -> Result<(), Box<dyn std::error::Error>> {
let metrics_config =
MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?;
let metrics_enabled = metrics_config.enabled();
let metrics_server = if metrics_config.enabled() {
Some(observability.metrics_surface(metrics_config).bind().await?)
} else {
None
};
let storage_root = PathBuf::from(
env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()),
@@ -36,6 +64,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
pool_config,
)
.await?;
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
}
let auth_settings = AuthSettings {
session_secret: env::var("CRANK_SESSION_SECRET")?,
password_pepper: env::var("CRANK_PASSWORD_PEPPER")?,
@@ -74,10 +105,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.with_outbound_http_policy(outbound_http_policy)
.with_identity_provider(std::sync::Arc::new(identity_provider))
.build();
let invocation_log_retention_days =
positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
service.bootstrap_admin_user().await?;
if env_flag("CRANK_DEMO_SEED") {
service.seed_demo_assets().await?;
}
spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days);
let state = AppState {
service,
api_rate_limiter: if cache_config.backend.is_external() {
@@ -92,6 +126,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
info!(
name: "admin.postgres_pool.configured",
runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
admin_rate_limit_rps = api_rate_limit.requests_per_second,
admin_rate_limit_burst = api_rate_limit.burst,
@@ -101,15 +136,76 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
acquire_timeout_ms = pool_config.acquire_timeout_ms,
idle_timeout_ms = pool_config.idle_timeout_ms,
max_lifetime_ms = pool_config.max_lifetime_ms,
invocation_log_retention_days,
"postgres pool configured"
);
info!("admin-api listening on {}", socket_addr);
info!(
name: "admin.server.listening",
bind_address = %socket_addr,
"admin-api listening"
);
*startup_completed = true;
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = axum::serve(listener, make_service) => result?,
result = metrics_server.serve() => result?,
}
} else {
axum::serve(listener, make_service).await?;
}
Ok(())
}
fn positive_i64_from_env(
name: &'static str,
default: i64,
) -> Result<i64, Box<dyn std::error::Error>> {
let value = match env::var(name) {
Ok(raw) => raw.parse::<i64>()?,
Err(env::VarError::NotPresent) => default,
Err(error) => return Err(error.into()),
};
if value <= 0 {
return Err(format!("{name} must be greater than zero").into());
}
Ok(value)
}
fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60 * 60));
loop {
interval.tick().await;
let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days);
match service.cleanup_invocation_logs_before(cutoff).await {
Ok(removed) if removed > 0 => info!(
name: "admin.invocation_log_cleanup.completed",
removed,
"expired invocation logs removed"
),
Ok(_) => {}
Err(_) => warn!(
name: "admin.invocation_log_cleanup.failed",
error_category = "registry_cleanup",
"failed to remove expired invocation logs"
),
}
}
});
}
fn spawn_postgres_pool_metrics(pool: PgPool) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
}
});
}
fn env_flag(name: &str) -> bool {
matches!(
env::var(name)
apps/admin-api/src/rate_limit.rs
+9 -4
View File
@@ -6,7 +6,7 @@ use axum::{
middleware::Next,
response::Response,
};
use crank_runtime::RateLimitRejection;
use crank_runtime::{RateLimitCheckError, RateLimitRejection};
use crate::{error::ApiError, state::AppState};
@@ -25,11 +25,16 @@ pub async fn apply_api_rate_limit(
peer_ip,
state.trust_forwarded_headers,
);
if let Err(rejection) = state.api_rate_limiter.check(&key).await {
return Err(ApiError::rate_limited_with_context(
if let Err(error) = state.api_rate_limiter.check(&key).await {
return match error {
RateLimitCheckError::Rejected(rejection) => Err(ApiError::rate_limited_with_context(
"request rate limit exceeded",
rejection_context(rejection),
));
)),
RateLimitCheckError::StoreUnavailable => {
Err(ApiError::internal("rate limit service unavailable"))
}
};
}
Ok(next.run(request).await)
apps/admin-api/src/request_context.rs
+22 -115
View File
@@ -4,11 +4,10 @@ use axum::{
middleware::Next,
response::Response,
};
use tracing::info;
use uuid::Uuid;
use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
use tracing::{Instrument, info, info_span};
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
const MAX_REQUEST_ID_LEN: usize = 128;
#[derive(Clone, Debug)]
pub struct RequestContext {
@@ -21,10 +20,18 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
};
let method = request.method().clone();
let path = request.uri().path().to_owned();
let span = info_span!(
target: "crank::trace",
"http.request",
request_id = %context.request_id,
);
set_remote_trace_parent(&span, request.headers());
request.extensions_mut().insert(context.clone());
let mut response = next.run(request).await;
with_request_correlation(context.request_id.clone(), async move {
let mut response = next.run(request).instrument(span).await;
info!(
name: "admin.request.completed",
request_id = %context.request_id,
method = %method,
path,
@@ -35,131 +42,31 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
response.headers_mut().insert(REQUEST_ID_HEADER, value);
}
response
})
.await
}
fn resolve_request_id(headers: &HeaderMap) -> String {
RequestId::resolve(
headers
.get(&REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| is_valid_request_id(value))
.map(ToOwned::to_owned)
.unwrap_or_else(|| Uuid::now_v7().to_string())
}
fn is_valid_request_id(value: &str) -> bool {
!value.is_empty()
&& value.len() <= MAX_REQUEST_ID_LEN
&& value
.bytes()
.all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
.and_then(|value| value.to_str().ok()),
)
.into_string()
}
#[cfg(test)]
mod tests {
use std::io;
use std::sync::{Arc, Mutex};
use axum::{Router, routing::get};
use reqwest::Client;
use tokio::net::TcpListener;
use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
use super::{REQUEST_ID_HEADER, apply_request_context, is_valid_request_id};
#[test]
fn accepts_visible_ascii_request_ids() {
assert!(is_valid_request_id("req_test_123"));
assert!(is_valid_request_id("trace-123/abc"));
assert!(crank_observability::RequestId::is_valid("req_test_123"));
assert!(crank_observability::RequestId::is_valid("trace-123/abc"));
}
#[test]
fn rejects_empty_or_control_request_ids() {
assert!(!is_valid_request_id(""));
assert!(!is_valid_request_id("bad value"));
assert!(!is_valid_request_id("bad\nvalue"));
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[tokio::test]
async fn logs_request_completion_with_request_id() {
let writer = SharedLogWriter::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(writer.clone())
.without_time()
.with_ansi(false)
.with_target(false)
.compact()
.with_filter(LevelFilter::INFO),
);
let dispatch = tracing::Dispatch::new(subscriber);
let app = Router::new()
.route("/probe", get(|| async { "ok" }))
.layer(axum::middleware::from_fn(apply_request_context));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let _guard = tracing::dispatcher::set_default(&dispatch);
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let response = Client::new()
.get(format!("http://{address}/probe"))
.header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(
response.headers()[REQUEST_ID_HEADER.as_str()]
.to_str()
.unwrap(),
"req_admin_trace_123"
);
let logs = writer.output();
assert!(logs.contains("admin request completed"));
assert!(logs.contains("req_admin_trace_123"));
assert!(logs.contains("GET"));
assert!(logs.contains("/probe"));
assert!(logs.contains("status=200"));
assert!(!crank_observability::RequestId::is_valid(""));
assert!(!crank_observability::RequestId::is_valid("bad value"));
assert!(!crank_observability::RequestId::is_valid("bad\nvalue"));
}
}
apps/admin-api/src/routes.rs
+25 -1
View File
@@ -10,12 +10,36 @@ pub mod secrets;
pub mod upstreams;
pub mod workspaces;
use axum::Json;
use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
use serde_json::json;
use crate::state::AppState;
pub async fn health() -> Json<serde_json::Value> {
Json(json!({
"service": "admin-api",
"status": "ok"
}))
}
pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
match state.service.readiness().await {
Ok(()) => (
StatusCode::OK,
Json(json!({
"service": "admin-api",
"status": "ready",
"checks": { "postgres": "ready" }
})),
),
Err(error) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "admin-api",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": error.to_string()
})),
),
}
}
apps/admin-api/src/routes/access.rs
+1 -1
View File
@@ -18,7 +18,7 @@ pub async fn export_workspace(
) -> Result<Json<Value>, ApiError> {
let exported = state
.service
.export_workspace(&path.workspace_id.as_str().into())
.export_workspace_catalog_snapshot(&path.workspace_id.as_str().into())
.await?;
Ok(Json(json!(exported)))
}
apps/admin-api/src/routes/auth.rs
+1 -1
View File
@@ -103,7 +103,7 @@ pub async fn change_password(
) -> Result<StatusCode, ApiError> {
state
.service
.change_password(&session.user.id, payload)
.change_password(&session.user.id, &session.session_id, payload)
.await?;
Ok(StatusCode::NO_CONTENT)
}
apps/admin-api/src/service.rs
+221 -20
View File
@@ -12,16 +12,18 @@ use crank_core::{
};
use crank_mapping::{MappingRule, MappingSet};
use crank_registry::{
AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary,
OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef,
OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
};
use crank_runtime::{
OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
};
use crank_schema::{Schema, SchemaKind};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tracing::Instrument;
use uuid::Uuid;
mod agents;
@@ -74,6 +76,11 @@ pub struct AdminServiceBuilder {
pub use crate::dto::*;
impl AdminService {
pub async fn readiness(&self) -> Result<(), ApiError> {
self.registry.ping().await?;
Ok(())
}
#[cfg(test)]
pub fn new(
registry: PostgresRegistry,
@@ -199,16 +206,33 @@ impl AdminServiceBuilder {
}
impl AdminService {
pub async fn export_workspace(
pub async fn export_workspace_catalog_snapshot(
&self,
workspace_id: &WorkspaceId,
) -> Result<WorkspaceExportResponse, ApiError> {
) -> Result<WorkspaceCatalogSnapshotResponse, ApiError> {
let workspace = self.get_workspace(workspace_id).await?;
let operations = self.list_operations(workspace_id).await?;
let agents = self.list_agents(workspace_id).await?;
let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?;
Ok(WorkspaceExportResponse {
Ok(WorkspaceCatalogSnapshotResponse {
kind: "workspace_catalog_snapshot".to_owned(),
format_version: "1".to_owned(),
restorable: false,
included: vec![
"workspace_settings".to_owned(),
"operation_summaries".to_owned(),
"agent_summaries".to_owned(),
"platform_api_key_metadata".to_owned(),
],
excluded: vec![
"operation_versions_and_samples".to_owned(),
"agent_versions_and_bindings".to_owned(),
"secret_metadata_and_values".to_owned(),
"secret_values".to_owned(),
"invocation_logs_and_usage".to_owned(),
"authentication_sessions".to_owned(),
],
workspace,
operations,
agents,
@@ -239,9 +263,13 @@ impl AdminService {
return Ok(None);
};
let auth_profile = self
.registry
.get_auth_profile(workspace_id, auth_profile_id)
let span = Stage::AuthResolve.span();
let result = async {
let auth_profile = observe_db_query(
DbOperation::AuthProfileRead,
self.registry
.get_auth_profile(workspace_id, auth_profile_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load auth profile",
@@ -255,6 +283,17 @@ impl AdminService {
.await
.map(Some)
}
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Configuration.record(&span);
}
}
result
}
async fn resolve_auth_profile(
&self,
@@ -265,9 +304,10 @@ impl AdminService {
let used_at = OffsetDateTime::now_utc();
for secret_id in auth_profile.config.secret_ids() {
let secret = self
.registry
.get_secret(workspace_id, secret_id)
let secret = observe_db_query(
DbOperation::SecretRead,
self.registry.get_secret(workspace_id, secret_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load secret",
@@ -276,9 +316,11 @@ impl AdminService {
.ok_or_else(|| RuntimeError::MissingSecret {
secret_id: secret_id.as_str().to_owned(),
})?;
let version = self
.registry
.get_current_secret_version(workspace_id, secret_id)
let version = observe_db_query(
DbOperation::SecretRead,
self.registry
.get_current_secret_version(workspace_id, secret_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load current secret version",
@@ -292,8 +334,11 @@ impl AdminService {
&version.secret_version.key_version,
&version.secret_version.ciphertext,
)?;
observe_db_query(
DbOperation::SecretTouch,
self.registry
.touch_secret(workspace_id, secret_id, &used_at)
.touch_secret(workspace_id, secret_id, &used_at),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "touch secret",
@@ -412,7 +457,7 @@ impl AdminService {
async fn record_invocation(
&self,
request: InvocationRecordRequest<'_>,
) -> Result<(), ApiError> {
) -> InvocationHistoryWriteOutcome {
let log = InvocationLog {
id: InvocationLogId::new(new_prefixed_id("log")),
workspace_id: request.workspace_id.clone(),
@@ -432,11 +477,70 @@ impl AdminService {
created_at: OffsetDateTime::now_utc(),
};
self.registry
let history_span = crank_trace::Stage::HistoryWrite.span();
let (outcome, db_span) = async {
let db_span = crank_trace::Stage::DbQuery
.db_span(crank_trace::DbOperation::InvocationHistoryWrite)
.expect("database stage");
let outcome = self
.registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.await?;
.instrument(db_span.clone())
.await;
(outcome, db_span)
}
.instrument(history_span.clone())
.await;
match outcome {
InvocationHistoryWriteOutcome::Recorded => {
crank_trace::StageOutcome::Success.record(&db_span);
crank_trace::StageOutcome::Success.record(&history_span);
}
InvocationHistoryWriteOutcome::Lost(_) => {
crank_trace::StageOutcome::Error.record(&db_span);
crank_trace::ErrorCategory::Database.record(&db_span);
crank_trace::StageOutcome::Error.record(&history_span);
crank_trace::ErrorCategory::History.record(&history_span);
}
}
drop(db_span);
drop(history_span);
observe_invocation_history_outcome(
outcome,
request.request_id,
request.status,
"admin_test_run",
);
outcome
}
}
Ok(())
fn observe_invocation_history_outcome(
outcome: InvocationHistoryWriteOutcome,
request_id: Option<&str>,
status: crank_core::InvocationStatus,
source: &'static str,
) {
let Some(loss) = outcome.loss() else {
return;
};
crank_observability::record_operational_incident(
crank_observability::OperationalIncident::InvocationHistoryLost,
);
tracing::warn!(
name: "admin.invocation_history.lost",
request_id = request_id.unwrap_or_default(),
source,
invocation_status = invocation_status_label(status),
error_category = loss.category.as_str(),
"invocation history was not recorded"
);
}
fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
match status {
crank_core::InvocationStatus::Ok => "ok",
crank_core::InvocationStatus::Error => "error",
}
}
@@ -543,6 +647,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
RuntimeError::RestAdapter(_) => "rest_error",
RuntimeError::ProtocolAdapter(_) => "adapter_error",
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
@@ -761,7 +869,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule {
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::{validate_profile_display_name, validate_profile_email};
use std::{
io,
sync::{Arc, Mutex},
};
use crank_core::InvocationStatus;
use crank_observability::{
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
operational_incident_total,
};
use crank_registry::{
InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
};
use serde_json::Value;
use tracing_subscriber::fmt::MakeWriter;
use super::{
observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email,
};
#[test]
fn validates_profile_identity_fields() {
@@ -782,6 +908,81 @@ mod tests {
assert!(validate_profile_display_name(&"x".repeat(81)).is_err());
assert!(validate_profile_email("owner <html>@crank.local").is_err());
}
#[test]
fn emits_bounded_history_loss_incident() {
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
observe_invocation_history_outcome(
InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
category: InvocationHistoryLossCategory::InvalidRecord,
}),
Some("req_admin_dc08"),
InvocationStatus::Error,
"admin_test_run",
);
let output = writer.output();
assert!(!output.contains("dc08-canary-secret"));
let event: Value = output
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.find(|event: &Value| event["event"] == "admin.invocation_history.lost")
.unwrap();
assert_eq!(event["request_id"], "req_admin_dc08");
assert_eq!(event["fields"]["source"], "admin_test_run");
assert_eq!(event["fields"]["invocation_status"], "error");
assert_eq!(event["fields"]["error_category"], "invalid_record");
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
}
fn enrich_operation_summary(
apps/admin-api/src/service/agents.rs
+23 -4
View File
@@ -296,7 +296,12 @@ impl AdminService {
bindings: &[],
})
.await?;
info!(agent_id = %agent_id.as_str(), version = 1, "agent created");
info!(
name: "admin.agent.created",
agent_id = %agent_id.as_str(),
version = 1,
"agent created"
);
Ok(CreatedAgentResponse {
agent_id: agent_id.as_str().to_owned(),
@@ -417,6 +422,7 @@ impl AdminService {
})
.await?;
info!(
name: "admin.agent.bindings_saved",
agent_id = %agent_id.as_str(),
version = current_version.version,
binding_count = bindings.len(),
@@ -531,7 +537,12 @@ impl AdminService {
published_by: None,
})
.await?;
info!(agent_id = %agent_id.as_str(), version, "agent published");
info!(
name: "admin.agent.published",
agent_id = %agent_id.as_str(),
version,
"agent published"
);
Ok(PublishAgentResponse {
agent_id: agent_id.as_str().to_owned(),
@@ -589,7 +600,11 @@ impl AdminService {
self.registry
.unpublish_agent(workspace_id, agent_id, &updated_at)
.await?;
info!(agent_id = %agent_id.as_str(), "agent moved to draft");
info!(
name: "admin.agent.unpublished",
agent_id = %agent_id.as_str(),
"agent moved to draft"
);
Ok(AgentMutationResult {
agent_id: agent_id.as_str().to_owned(),
@@ -609,7 +624,11 @@ impl AdminService {
self.registry
.archive_agent(workspace_id, agent_id, &updated_at)
.await?;
info!(agent_id = %agent_id.as_str(), "agent archived");
info!(
name: "admin.agent.archived",
agent_id = %agent_id.as_str(),
"agent archived"
);
Ok(AgentMutationResult {
agent_id: agent_id.as_str().to_owned(),
apps/admin-api/src/service/auth.rs
+7 -2
View File
@@ -22,7 +22,7 @@ impl AdminService {
)?;
let user_id = self
.registry
.upsert_bootstrap_user(
.ensure_bootstrap_user(
&self.auth_settings.bootstrap_admin.email,
&self.auth_settings.bootstrap_admin.display_name,
&password_hash,
@@ -227,6 +227,7 @@ impl AdminService {
pub async fn change_password(
&self,
user_id: &crank_core::UserId,
current_session_id: &UserSessionId,
payload: ChangePasswordPayload,
) -> Result<(), ApiError> {
if payload.new_password.len() < 12 {
@@ -257,7 +258,11 @@ impl AdminService {
let password_hash =
hash_password(&payload.new_password, &self.auth_settings.password_pepper)?;
self.registry
.update_user_password(user_id, &password_hash)
.update_user_password_and_revoke_other_sessions(
user_id,
current_session_id,
&password_hash,
)
.await?;
Ok(())
apps/admin-api/src/service/demo.rs
+2 -1
View File
@@ -131,6 +131,7 @@ impl AdminService {
Ok(()) => Ok(()),
Err(RegistryError::OperationHasPublishedAgentBindings { .. }) => {
tracing::warn!(
name: "admin.demo_operation.cleanup_skipped",
operation_id = %operation_id.as_str(),
"legacy demo operation is still bound to a published agent; leaving it in place"
);
@@ -335,7 +336,7 @@ impl AdminService {
}),
response_preview: demo_rest_response_sample(),
})
.await?;
.await;
Ok(())
}
}
apps/admin-api/src/service/import_export.rs
+2
View File
@@ -110,6 +110,7 @@ impl AdminService {
warnings,
};
info!(
name: "admin.operation.imported",
operation_id = %response.operation_id,
version = response.version,
"operation imported by upsert"
@@ -125,6 +126,7 @@ impl AdminService {
warnings,
};
info!(
name: "admin.operation.imported",
operation_id = %response.operation_id,
version = response.version,
"operation imported by upsert"
apps/admin-api/src/service/imports.rs
+85 -75
View File
@@ -8,9 +8,11 @@ use crank_import::rest::{
ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
};
use crank_registry::{
CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus,
ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind,
ImportJobStatus, ImportOperationDraft,
};
use serde_json::json;
use sha2::{Digest, Sha256};
use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
use tracing::{info, instrument};
@@ -50,7 +52,7 @@ impl AdminService {
kind: ImportJobKind::OpenApi,
source_format: &preview.source.format,
source_version: preview.source.version.as_deref(),
status: ImportJobStatus::Completed,
status: ImportJobStatus::Pending,
preview_payload: &preview_payload,
created_at: &now,
expires_at: &expires_at,
@@ -99,8 +101,12 @@ impl AdminService {
return Err(ApiError::validation("import job kind is not openapi"));
}
let preview: crank_import::rest::ImportPreview =
serde_json::from_value(job.preview_payload.clone())
let stored_preview = job
.preview_payload
.get("preview")
.cloned()
.unwrap_or_else(|| job.preview_payload.clone());
let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview)
.map_err(|error| ApiError::internal(error.to_string()))?;
let selected = payload
.selected_operation_keys
@@ -120,10 +126,8 @@ impl AdminService {
}
}
let mut created = Vec::new();
let mut skipped = Vec::new();
let mut findings = Vec::new();
let mut created_ids = Vec::new();
let mut operations = Vec::new();
for operation_key in selected {
let Some(candidate) = candidates.get(&operation_key) else {
@@ -137,44 +141,7 @@ impl AdminService {
let mut draft =
operation_draft_from_candidate(candidate, payload.server_url.as_deref());
attach_import_findings(&mut draft, candidate);
if let Some(existing_name) = self
.find_operation_by_name(workspace_id, &draft.name)
.await?
.map(|operation| operation.name)
{
if payload.conflict_mode == "skip" {
skipped.push(OpenApiImportSkippedOperation {
operation_key: candidate.key.clone(),
name: draft.name.clone(),
reason: "operation with this name already exists".to_owned(),
});
findings.push(ImportFinding {
code: "operation_name_conflict".to_owned(),
severity: ImportFindingSeverity::Warning,
message: format!(
"Операция {} уже существует и была пропущена.",
draft.name
),
operation_key: Some(candidate.key.clone()),
});
continue;
}
let renamed = self
.next_available_operation_name(workspace_id, &draft.name)
.await?;
findings.push(ImportFinding {
code: "operation_name_renamed".to_owned(),
severity: ImportFindingSeverity::Info,
message: format!(
"Операция {existing_name} уже существует, новый черновик создан как {renamed}."
),
operation_key: Some(candidate.key.clone()),
});
draft.name = renamed;
}
let payload = OperationPayload {
let operation = self.new_operation_snapshot(OperationPayload {
name: draft.name.clone(),
display_name: draft.display_name.clone(),
category: draft.category,
@@ -197,27 +164,74 @@ impl AdminService {
},
tool_description: draft.tool_description,
wizard_state: draft.wizard_state,
};
let result = self.create_operation(workspace_id, payload).await?;
created_ids.push(result.operation_id.clone());
created.push(OpenApiImportCreatedOperation {
operation_id: result.operation_id,
name: draft.name,
version: result.version,
})?;
operations.push(ImportOperationDraft {
operation_key: candidate.key.clone(),
operation,
});
}
let finished_at = OffsetDateTime::now_utc();
self.registry
.finish_import_job(FinishImportJobRequest {
let application_key = openapi_application_key(&payload)?;
let conflict_mode = if payload.conflict_mode == "skip" {
ImportConflictMode::Skip
} else {
ImportConflictMode::Rename
};
let applied = self
.registry
.apply_import_job(ApplyImportJobRequest {
id: job_id,
status: ImportJobStatus::Completed,
created_operation_ids: &json!(created_ids),
error_text: None,
workspace_id,
application_key: &application_key,
conflict_mode,
operations: &operations,
finished_at: &finished_at,
})
.await?;
let created = applied
.created
.iter()
.map(|operation| OpenApiImportCreatedOperation {
operation_id: operation.operation_id.as_str().to_owned(),
name: operation.name.clone(),
version: operation.version,
})
.collect::<Vec<_>>();
let mut findings = applied
.created
.iter()
.filter_map(|operation| {
operation.renamed_from.as_ref().map(|previous_name| ImportFinding {
code: "operation_name_renamed".to_owned(),
severity: ImportFindingSeverity::Info,
message: format!(
"Операция {previous_name} уже существует, новый черновик создан как {}.",
operation.name
),
operation_key: Some(operation.operation_key.clone()),
})
})
.collect::<Vec<_>>();
for operation in applied.skipped {
skipped.push(OpenApiImportSkippedOperation {
operation_key: operation.operation_key.clone(),
name: operation.name.clone(),
reason: "operation with this name already exists".to_owned(),
});
findings.push(ImportFinding {
code: operation.reason,
severity: ImportFindingSeverity::Warning,
message: format!(
"Операция {} уже существует и была пропущена.",
operation.name
),
operation_key: Some(operation.operation_key),
});
}
info!(
name: "admin.openapi_import.completed",
created = created.len(),
skipped = skipped.len(),
"openapi import created drafts"
@@ -229,25 +243,21 @@ impl AdminService {
findings,
})
}
}
async fn next_available_operation_name(
&self,
workspace_id: &WorkspaceId,
base_name: &str,
) -> Result<String, ApiError> {
for index in 2.. {
let candidate = format!("{base_name}_{index}");
if self
.find_operation_by_name(workspace_id, &candidate)
.await?
.is_none()
{
return Ok(candidate);
}
}
unreachable!()
}
fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
let selected_operation_keys = payload
.selected_operation_keys
.iter()
.cloned()
.collect::<BTreeSet<_>>();
let canonical = serde_json::to_vec(&json!({
"selected_operation_keys": selected_operation_keys,
"server_url": payload.server_url.as_deref(),
"conflict_mode": payload.conflict_mode.as_str(),
}))
.map_err(|error| ApiError::internal(error.to_string()))?;
Ok(format!("{:x}", Sha256::digest(canonical)))
}
fn attach_import_findings(
apps/admin-api/src/service/observability.rs
+10
View File
@@ -19,6 +19,16 @@ use crate::{
};
impl AdminService {
pub async fn cleanup_invocation_logs_before(
&self,
cutoff: OffsetDateTime,
) -> Result<u64, ApiError> {
self.registry
.delete_invocation_logs_before(cutoff)
.await
.map_err(ApiError::from)
}
#[instrument(skip(self))]
pub async fn list_logs(
&self,
apps/admin-api/src/service/operations.rs
+53 -24
View File
@@ -141,7 +141,6 @@ impl AdminService {
workspace_id: &WorkspaceId,
payload: OperationPayload,
) -> Result<CreatedOperationResponse, ApiError> {
self.validate_operation_payload(&payload)?;
self.ensure_workspace_exists(workspace_id).await?;
if self
@@ -155,10 +154,36 @@ impl AdminService {
));
}
let snapshot = self.new_operation_snapshot(payload)?;
let operation_id = snapshot.id.clone();
self.registry
.create_operation(workspace_id, &snapshot, None)
.await?;
info!(
name: "admin.operation.created",
operation_id = %operation_id.as_str(),
version = 1,
"operation created"
);
Ok(CreatedOperationResponse {
operation_id: operation_id.as_str().to_owned(),
workspace_id: workspace_id.as_str().to_owned(),
version: 1,
status: OperationStatus::Draft,
updated_at: format_timestamp(snapshot.updated_at),
})
}
pub(super) fn new_operation_snapshot(
&self,
payload: OperationPayload,
) -> Result<RegistryOperation, ApiError> {
self.validate_operation_payload(&payload)?;
let now = OffsetDateTime::now_utc();
let operation_id = OperationId::new(new_prefixed_id("op"));
let snapshot = RegistryOperation {
id: operation_id.clone(),
Ok(RegistryOperation {
id: OperationId::new(new_prefixed_id("op")),
name: payload.name,
display_name: payload.display_name,
category: payload.category,
@@ -183,19 +208,6 @@ impl AdminService {
created_at: now,
updated_at: now,
published_at: None,
};
self.registry
.create_operation(workspace_id, &snapshot, None)
.await?;
info!(operation_id = %operation_id.as_str(), version = 1, "operation created");
Ok(CreatedOperationResponse {
operation_id: operation_id.as_str().to_owned(),
workspace_id: workspace_id.as_str().to_owned(),
version: 1,
status: OperationStatus::Draft,
updated_at: format_timestamp(snapshot.updated_at),
})
}
@@ -279,7 +291,12 @@ impl AdminService {
created_by: None,
})
.await?;
info!(operation_id = %operation_id.as_str(), version, "operation version created");
info!(
name: "admin.operation.version_created",
operation_id = %operation_id.as_str(),
version,
"operation version created"
);
Ok(CreatedOperationResponse {
operation_id: operation_id.as_str().to_owned(),
@@ -364,7 +381,12 @@ impl AdminService {
published_by: None,
})
.await?;
info!(operation_id = %operation_id.as_str(), version, "operation published");
info!(
name: "admin.operation.published",
operation_id = %operation_id.as_str(),
version,
"operation published"
);
Ok(PublishResponse {
operation_id: operation_id.as_str().to_owned(),
@@ -431,10 +453,15 @@ impl AdminService {
.await?;
let runtime = RuntimeOperation::from(record.snapshot.clone());
let mode = ExecutionMode::Unary;
let request_preview =
match build_request_preview(&record.snapshot.input_mapping, &payload.input) {
let preview_span = crank_trace::Stage::RuntimeArgumentsMap.span();
let preview_result = preview_span
.in_scope(|| build_request_preview(&record.snapshot.input_mapping, &payload.input));
let request_preview = match preview_result {
Ok(preview) => preview,
Err(error) => {
crank_trace::StageOutcome::Error.record(&preview_span);
crank_trace::ErrorCategory::Mapping.record(&preview_span);
drop(preview_span);
self.record_invocation(InvocationRecordRequest {
workspace_id,
agent_id: None,
@@ -450,7 +477,7 @@ impl AdminService {
request_preview: Value::Null,
response_preview: Value::Null,
})
.await?;
.await;
return Ok(TestRunResult {
ok: false,
mode,
@@ -462,6 +489,8 @@ impl AdminService {
});
}
};
crank_trace::StageOutcome::Success.record(&preview_span);
drop(preview_span);
let resolved_auth = self
.resolve_operation_auth(workspace_id, &runtime.execution_config)
@@ -497,7 +526,7 @@ impl AdminService {
request_preview: request_preview.clone(),
response_preview: response_preview.clone(),
})
.await?;
.await;
Ok(TestRunResult {
ok: true,
mode,
@@ -524,7 +553,7 @@ impl AdminService {
request_preview: request_preview.clone(),
response_preview: Value::Null,
})
.await?;
.await;
Ok(TestRunResult {
ok: false,
mode,
apps/admin-api/src/service/samples.rs
+6 -1
View File
@@ -50,6 +50,7 @@ impl AdminService {
.save_sample_metadata(SaveSampleMetadataRequest { sample: &metadata })
.await?;
info!(
name: "admin.sample.saved",
operation_id = %operation_id.as_str(),
sample_id = %metadata.id.as_str(),
version,
@@ -119,7 +120,11 @@ impl AdminService {
input_mapping,
output_mapping,
};
info!(operation_id = %operation_id.as_str(), "draft generated from samples");
info!(
name: "admin.operation_draft.generated",
operation_id = %operation_id.as_str(),
"draft generated from samples"
);
Ok(result)
}
apps/admin-api/src/service/secrets.rs
+20 -4
View File
@@ -92,7 +92,11 @@ impl AdminService {
created_by,
})
.await?;
info!(secret_id = %secret.id.as_str(), "secret created");
info!(
name: "admin.secret.created",
secret_id = %secret.id.as_str(),
"secret created"
);
Ok(secret)
}
@@ -126,7 +130,11 @@ impl AdminService {
created_by,
})
.await?;
info!(secret_id = %secret_id.as_str(), "secret rotated");
info!(
name: "admin.secret.rotated",
secret_id = %secret_id.as_str(),
"secret rotated"
);
self.get_secret(workspace_id, secret_id).await
}
@@ -152,7 +160,11 @@ impl AdminService {
.into());
}
self.registry.delete_secret(workspace_id, secret_id).await?;
info!(secret_id = %secret_id.as_str(), "secret deleted");
info!(
name: "admin.secret.deleted",
secret_id = %secret_id.as_str(),
"secret deleted"
);
Ok(())
}
@@ -201,7 +213,11 @@ impl AdminService {
profile: &profile,
})
.await?;
info!(auth_profile_id = %profile.id.as_str(), "auth profile created");
info!(
name: "admin.auth_profile.created",
auth_profile_id = %profile.id.as_str(),
"auth profile created"
);
Ok(profile)
}
apps/admin-api/src/service/upstreams.rs
+5 -1
View File
@@ -75,7 +75,11 @@ impl AdminService {
upstream: &upstream,
})
.await?;
info!(upstream_id = %upstream.id.as_str(), "workspace upstream saved");
info!(
name: "admin.upstream.saved",
upstream_id = %upstream.id.as_str(),
"workspace upstream saved"
);
Ok(upstream)
}
apps/admin-api/tests/integration/community_access_usage.rs
+65 -1
View File
@@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind};
use serde_json::{Value, json};
use serial_test::serial;
use tokio::net::TcpListener;
use uuid::Version;
use admin_api::{
app::build_app,
@@ -40,6 +41,8 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
mod history_loss;
struct TestServer {
base_url: String,
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
@@ -385,6 +388,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() {
exported["workspace"]["workspace"]["id"],
DEFAULT_WORKSPACE_ID
);
assert_eq!(exported["kind"], "workspace_catalog_snapshot");
assert_eq!(exported["format_version"], "1");
assert_eq!(exported["restorable"], false);
assert_eq!(
exported["included"],
json!([
"workspace_settings",
"operation_summaries",
"agent_summaries",
"platform_api_key_metadata"
])
);
assert!(
exported["excluded"]
.as_array()
.unwrap()
.contains(&json!("secret_values"))
);
assert!(
exported["excluded"]
.as_array()
.unwrap()
.contains(&json!("invocation_logs_and_usage"))
);
assert!(exported.get("memberships").is_none());
assert!(exported.get("invitations").is_none());
@@ -574,6 +601,25 @@ async fn updates_profile_and_changes_password() {
.unwrap()
.to_owned();
let client = authorized_client(&base_url).await;
let second_client = reqwest::Client::builder()
.cookie_store(true)
.build()
.unwrap();
let second_login = second_client
.post(format!("{root_url}/api/auth/login"))
.json(&json!({
"email": TEST_AUTH_EMAIL,
"password": TEST_AUTH_PASSWORD,
}))
.send()
.await
.unwrap();
let second_login_status = second_login.status();
let second_login_body = second_login.text().await.unwrap();
assert!(
second_login_status.is_success(),
"second login failed with {second_login_status}: {second_login_body}"
);
let profile = assert_success_json(
client
@@ -615,6 +661,21 @@ async fn updates_profile_and_changes_password() {
.status();
assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);
let current_session_status = client
.get(format!("{root_url}/api/auth/profile"))
.send()
.await
.unwrap()
.status();
let other_session_status = second_client
.get(format!("{root_url}/api/auth/profile"))
.send()
.await
.unwrap()
.status();
assert_eq!(current_session_status, reqwest::StatusCode::OK);
assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
let relogin_client = reqwest::Client::builder()
.cookie_store(true)
.build()
@@ -855,7 +916,10 @@ async fn generates_request_id_for_test_run_invocations() {
.unwrap()
.to_owned();
assert!(!request_id.is_empty());
assert_eq!(
uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
Some(Version::SortRand)
);
response.error_for_status().unwrap();
let logs = client
apps/admin-api/tests/integration/community_access_usage/history_loss.rs
+141
View File
@@ -0,0 +1,141 @@
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use axum::{Json, Router, extract::State, routing::post};
use tokio::{net::TcpListener, sync::Notify};
use super::*;
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn preserves_external_success_when_invocation_history_is_lost() {
let registry = test_registry().await;
let registry_for_failure = registry.clone();
let storage_root = test_storage_root("observability_history_loss");
let upstream = spawn_blocking_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let created = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream.base_url,
"crm_history_loss",
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_id = created["operation_id"].as_str().unwrap().to_owned();
let request_client = client.clone();
let request_url = format!("{base_url}/operations/{operation_id}/test-runs");
let before = crank_observability::operational_incident_total(
crank_observability::OperationalIncident::InvocationHistoryLost,
);
let request = tokio::spawn(async move {
request_client
.post(request_url)
.header("x-request-id", "req_dc08_admin")
.json(&json!({
"version": 1,
"input": { "email": "dc08-canary-secret@example.com" }
}))
.send()
.await
.unwrap()
});
upstream.started.notified().await;
registry_for_failure
.delete_operation(
&WorkspaceId::new(DEFAULT_WORKSPACE_ID),
&OperationId::new(operation_id.clone()),
)
.await
.unwrap();
upstream.release.notify_one();
let response = request.await.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(
response.headers()["x-request-id"].to_str().unwrap(),
"req_dc08_admin"
);
let body = response.json::<Value>().await.unwrap();
assert_eq!(body["ok"], true);
assert_eq!(body["response_preview"]["id"], "lead_123");
assert_eq!(upstream.calls.load(Ordering::SeqCst), 1);
assert!(
crank_observability::operational_incident_total(
crank_observability::OperationalIncident::InvocationHistoryLost
) > before
);
let logs = client
.get(format!("{base_url}/logs?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert!(logs["items"].as_array().unwrap().is_empty());
}
struct BlockingUpstream {
base_url: String,
started: Arc<Notify>,
release: Arc<Notify>,
calls: Arc<AtomicUsize>,
}
#[derive(Clone)]
struct BlockingUpstreamState {
started: Arc<Notify>,
release: Arc<Notify>,
calls: Arc<AtomicUsize>,
}
async fn spawn_blocking_upstream_server() -> BlockingUpstream {
let state = BlockingUpstreamState {
started: Arc::new(Notify::new()),
release: Arc::new(Notify::new()),
calls: Arc::new(AtomicUsize::new(0)),
};
let app = Router::new()
.route("/crm/leads", post(blocking_create_lead))
.with_state(state.clone());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
BlockingUpstream {
base_url: format!("http://{address}"),
started: state.started,
release: state.release,
calls: state.calls,
}
}
async fn blocking_create_lead(
State(state): State<BlockingUpstreamState>,
Json(payload): Json<Value>,
) -> Json<Value> {
state.calls.fetch_add(1, Ordering::SeqCst);
state.started.notify_one();
state.release.notified().await;
Json(json!({
"id": "lead_123",
"status": "created",
"email": payload["email"]
}))
}
apps/admin-api/tests/integration/openapi_import.rs
+88 -3
View File
@@ -1,5 +1,6 @@
use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
use crank_core::WorkspaceId;
use crank_registry::ImportJobStatus;
use serial_test::serial;
use super::common::{
@@ -42,7 +43,7 @@ paths:
async fn previews_openapi_and_creates_draft_operations() {
let registry = test_registry().await;
let service = test_service(
registry,
registry.clone(),
test_storage_root("openapi_import"),
test_auth_settings(),
test_secret_crypto(),
@@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() {
preview.preview.groups[0].operations[0].suggested_name,
"latest_rates"
);
let preview_job = registry
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
.await
.unwrap()
.unwrap();
assert_eq!(preview_job.status, ImportJobStatus::Pending);
let created = service
.create_openapi_import(
@@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() {
.any(|finding| finding.code == "openapi_import.weak_tool_description")
);
let skip_preview = service
.preview_openapi_import(
&workspace_id,
OpenApiImportPreviewPayload {
document: OPENAPI3.to_owned(),
},
)
.await
.unwrap();
let skipped = service
.create_openapi_import(
&workspace_id,
&preview.job_id.as_str().into(),
&skip_preview.job_id.as_str().into(),
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() {
assert_eq!(skipped.skipped[0].name, "latest_rates");
assert_eq!(skipped.findings[0].code, "operation_name_conflict");
let rename_preview = service
.preview_openapi_import(
&workspace_id,
OpenApiImportPreviewPayload {
document: OPENAPI3.to_owned(),
},
)
.await
.unwrap();
let renamed = service
.create_openapi_import(
&workspace_id,
&preview.job_id.as_str().into(),
&rename_preview.job_id.as_str().into(),
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() {
assert_eq!(renamed.created[0].name, "latest_rates_2");
assert_eq!(renamed.findings[0].code, "operation_name_renamed");
}
#[tokio::test]
#[serial]
async fn concurrent_openapi_import_replays_the_same_atomic_result() {
let registry = test_registry().await;
let service = test_service(
registry,
test_storage_root("openapi_import_replay"),
test_auth_settings(),
test_secret_crypto(),
);
let workspace_id = WorkspaceId::new("ws_default");
let preview = service
.preview_openapi_import(
&workspace_id,
OpenApiImportPreviewPayload {
document: OPENAPI3.to_owned(),
},
)
.await
.unwrap();
let job_id = preview.job_id.as_str().into();
let payload = OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
conflict_mode: "rename".to_owned(),
};
let (first, second) = tokio::join!(
service.create_openapi_import(&workspace_id, &job_id, payload.clone()),
service.create_openapi_import(&workspace_id, &job_id, payload),
);
let first = first.unwrap();
let second = second.unwrap();
assert_eq!(first.created.len(), 1);
assert_eq!(second.created.len(), 1);
assert_eq!(
first.created[0].operation_id,
second.created[0].operation_id
);
assert_eq!(first.created[0].name, second.created[0].name);
assert_eq!(
service.list_operations(&workspace_id).await.unwrap().len(),
1
);
let conflicting_replay = service
.create_openapi_import(
&workspace_id,
&job_id,
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
conflict_mode: "skip".to_owned(),
},
)
.await;
assert!(conflicting_replay.is_err());
}
apps/admin-api/tests/integration/request_context.rs
+211
View File
@@ -0,0 +1,211 @@
use std::{
io,
sync::{Arc, Mutex},
};
use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
use axum::{
Router,
body::Body,
http::{HeaderMap, Request, StatusCode},
routing::get,
};
use crank_observability::{
ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
};
use opentelemetry::{global, trace::TracerProvider as _};
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
use uuid::Version;
#[tokio::test(flavor = "current_thread")]
async fn logs_request_completion_and_rejects_untrusted_values() {
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let dispatch = tracing::Dispatch::new(subscriber);
let app = probe_app();
let response = app
.clone()
.oneshot(
Request::builder()
.uri("/probe")
.header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers()[REQUEST_ID_HEADER.as_str()]
.to_str()
.unwrap(),
"req_admin_trace_123"
);
let event: serde_json::Value = writer
.output()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
.unwrap();
assert_eq!(event["request_id"], "req_admin_trace_123");
assert_eq!(event["fields"]["status"], 200);
let invalid_response = app
.oneshot(
Request::builder()
.uri("/probe")
.header(REQUEST_ID_HEADER.as_str(), "bad,value")
.header("traceparent", "canary-invalid-traceparent")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch)
.await
.unwrap();
let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
.to_str()
.unwrap();
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
assert!(!writer.output().contains("canary-invalid-traceparent"));
}
#[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent() {
global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("admin-request-context-test");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let app = trace_probe_app();
let valid = observed_trace_id(
app.clone()
.oneshot(
Request::builder()
.uri("/trace")
.header(
"traceparent",
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
)
.header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap(),
);
let invalid = observed_trace_id(
app.clone()
.oneshot(
Request::builder()
.uri("/trace")
.header("traceparent", "canary-invalid-traceparent")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap(),
);
let absent = observed_trace_id(
app.oneshot(
Request::builder()
.uri("/trace")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch)
.await
.unwrap(),
);
assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c");
assert_ne!(invalid, valid);
assert_ne!(absent, valid);
assert_ne!(invalid, absent);
provider.shutdown().unwrap();
}
fn probe_app() -> Router {
Router::new()
.route("/probe", get(|| async { "ok" }))
.layer(axum::middleware::from_fn(apply_request_context))
}
fn trace_probe_app() -> Router {
Router::new()
.route("/trace", get(observed_traceparent))
.layer(axum::middleware::from_fn(apply_request_context))
}
async fn observed_traceparent() -> HeaderMap {
let mut trace_headers = HeaderMap::new();
inject_current_trace_context(&mut trace_headers);
let mut response_headers = HeaderMap::new();
if let Some(traceparent) = trace_headers.remove("traceparent") {
response_headers.insert("x-observed-traceparent", traceparent);
}
response_headers
}
fn observed_trace_id(response: axum::response::Response) -> String {
let traceparent = response.headers()["x-observed-traceparent"]
.to_str()
.unwrap();
traceparent[3..35].to_owned()
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
apps/admin-api/tests/request_context.rs
+2
View File
@@ -0,0 +1,2 @@
#[path = "integration/request_context.rs"]
mod request_context;
apps/mcp-server/Cargo.toml
+9
View File
@@ -3,6 +3,7 @@ name = "mcp-server"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[[bin]]
@@ -15,10 +16,12 @@ axum.workspace = true
base64.workspace = true
crank-community-mcp = { path = "../../crates/crank-community-mcp" }
crank-core = { path = "../../crates/crank-core" }
crank-observability = { path = "../../crates/crank-observability" }
crank-registry = { path = "../../crates/crank-registry" }
crank-runtime = { path = "../../crates/crank-runtime" }
crank-schema = { path = "../../crates/crank-schema" }
futures-util = "0.3"
metrics.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
@@ -34,4 +37,10 @@ uuid.workspace = true
crank-mapping = { path = "../../crates/crank-mapping" }
crank-schema = { path = "../../crates/crank-schema" }
crank-test-support = { path = "../../crates/crank-test-support" }
opentelemetry.workspace = true
opentelemetry-proto.workspace = true
opentelemetry_sdk.workspace = true
prost.workspace = true
reqwest.workspace = true
tower.workspace = true
tracing-opentelemetry.workspace = true
apps/mcp-server/src/main.rs
+68 -19
View File
@@ -1,26 +1,53 @@
use std::{env, net::SocketAddr, time::Duration};
use crank_community_mcp::{
auth::CommunityMachineCredentialVerifier, build_app_with_background_workers,
auth::CommunityMachineCredentialVerifier, build_app_with_background_workers_and_limits,
session::PostgresTransportSessionStore,
};
use crank_observability::{
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
capture_critical_error,
};
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use sqlx::postgres::PgConnectOptions;
use sqlx::{PgPool, postgres::PgConnectOptions};
use tokio::net::TcpListener;
use tracing::info;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt()
.with_env_filter(
env::var("CRANK_LOG_LEVEL")
.unwrap_or_else(|_| "mcp_server=info,tower_http=info".into()),
)
.init();
let observability = crank_observability::init(ObservabilityConfig::from_env(
"mcp-server",
env!("CARGO_PKG_VERSION"),
"mcp_server=info,tower_http=info",
)?)?;
let mut startup_completed = false;
let result = run(&observability, &mut startup_completed).await;
if result.is_err() {
capture_critical_error(if startup_completed {
CriticalErrorCategory::Internal
} else {
CriticalErrorCategory::Startup
});
}
result
}
async fn run(
observability: &ObservabilityLifecycle,
startup_completed: &mut bool,
) -> Result<(), Box<dyn std::error::Error>> {
let metrics_config =
MetricsConfig::from_env("CRANK_MCP_METRICS_BIND", "127.0.0.1:9465".parse()?)?;
let metrics_enabled = metrics_config.enabled();
let metrics_server = if metrics_config.enabled() {
Some(observability.metrics_surface(metrics_config).bind().await?)
} else {
None
};
let bind_addr = env::var("CRANK_MCP_BIND").unwrap_or_else(|_| "0.0.0.0:3002".into());
let base_url = env::var("CRANK_BASE_URL").ok();
@@ -36,23 +63,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
let api_rate_limit = mcp_api_rate_limit_config_from_env()?;
let database_options = database_options_from_env()?;
let registry = PostgresRegistry::connect_with_options_and_pool_config(
database_options.clone(),
pool_config,
)
.await?;
let session_store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
database_options,
pool_config,
)
let registry =
PostgresRegistry::connect_with_options_and_pool_config(database_options, pool_config)
.await?;
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
}
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?;
let runtime = crank_runtime::community_from_env()?
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.build();
let app = build_app_with_background_workers(
let app = build_app_with_background_workers_and_limits(
registry,
refresh_interval,
base_url,
@@ -66,11 +90,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
cache_stores.coordination.clone(),
std::sync::Arc::new(session_store),
std::sync::Arc::new(CommunityMachineCredentialVerifier),
runtime_limits.max_concurrent_sessions,
);
let listener = TcpListener::bind(socket_addr).await?;
info!(
name: "mcp.postgres_pool.configured",
runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
runtime_max_concurrent_sessions = runtime_limits.max_concurrent_sessions,
mcp_rate_limit_rps = api_rate_limit.requests_per_second,
mcp_rate_limit_burst = api_rate_limit.burst,
cache_backend = %cache_config.backend,
@@ -81,9 +108,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
max_lifetime_ms = pool_config.max_lifetime_ms,
"postgres pool configured"
);
info!("mcp-server listening on {}", socket_addr);
info!(
name: "mcp.server.listening",
bind_address = %socket_addr,
"mcp-server listening"
);
*startup_completed = true;
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = axum::serve(listener, app) => result?,
result = metrics_server.serve() => result?,
}
} else {
axum::serve(listener, app).await?;
}
Ok(())
}
@@ -123,3 +162,13 @@ fn mcp_api_rate_limit_config_from_env() -> Result<RequestRateLimitConfig, Box<dy
Ok(RequestRateLimitConfig::new(requests_per_second, burst)?)
}
fn spawn_postgres_pool_metrics(pool: PgPool) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
}
});
}
apps/mcp-server/tests/execution_stages.rs
+4
View File
@@ -0,0 +1,4 @@
#[path = "integration/common.rs"]
mod common;
#[path = "integration/execution_stages.rs"]
mod execution_stages;
apps/mcp-server/tests/integration/catalog_access/approval_access.rs
+213 -7
View File
@@ -89,11 +89,16 @@ async fn approval_key_lists_and_decides_pending_requests() {
let approved = client
.post(&approve_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.header("x-request-id", "req_approval_execute_123")
.json(&json!({ "approve": "yes", "note": "confirmed by test" }))
.send()
.await
.unwrap();
assert_eq!(approved.status(), reqwest::StatusCode::OK);
assert_eq!(
approved.headers()["x-request-id"].to_str().unwrap(),
"req_approval_execute_123"
);
let approved_body = approved.json::<Value>().await.unwrap();
assert_eq!(
approved_body["approval"]["status"],
@@ -169,6 +174,10 @@ async fn approval_key_lists_and_decides_pending_requests() {
.await
.unwrap();
assert_eq!(logs.len(), 1);
assert_eq!(
logs[0].log.request_id.as_deref(),
Some("req_approval_execute_123")
);
}
#[tokio::test]
@@ -401,12 +410,7 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
let mcp_url = agent_mcp_url(&base_url, "sales-gated");
let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
let tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
json!({
let tool_call = json!({
"jsonrpc": "2.0",
"id": 9,
"method": "tools/call",
@@ -416,7 +420,13 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
"email": "ada@example.com"
}
}
}),
});
let tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
tool_call.clone(),
)
.await;
@@ -430,6 +440,19 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
.unwrap();
assert!(approval_id.starts_with("approval_"));
let repeated_tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
tool_call,
)
.await;
assert_eq!(
repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
"deduplicated tools/call must return the persisted approval id",
);
let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
let pending = client
.get(&approvals_url)
@@ -448,6 +471,189 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
);
}
#[tokio::test]
async fn approval_http_endpoints_enforce_request_rate_limit() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_approval_rate_limit");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-approval-rate-limit").await;
let approval_key = create_approval_platform_api_key(
&registry,
"sales-approval-rate-limit",
"approval-rate-limit",
)
.await;
let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
RequestRateLimitConfig::new(1, 1).unwrap(),
))
.await;
let approvals_url = format!(
"{}/approvals",
agent_mcp_url(&base_url, "sales-approval-rate-limit")
);
let client = reqwest::Client::new();
let allowed = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap();
assert_eq!(allowed.status(), reqwest::StatusCode::OK);
let limited = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap();
assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
assert!(limited.headers().contains_key(header::RETRY_AFTER));
}
#[tokio::test]
async fn recovery_does_not_repeat_interrupted_mutating_approval() {
let registry = test_registry().await;
let (upstream_base_url, upstream_calls) = spawn_counted_approval_upstream().await;
let operation = test_operation(&upstream_base_url, "crm_interrupted_approval");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-interrupted-approval").await;
let approval_key_name = "approval-interrupted";
create_approval_platform_api_key(&registry, "sales-interrupted-approval", approval_key_name)
.await;
let now = OffsetDateTime::now_utc();
let approval = ApprovalRequest {
id: ApprovalRequestId::new("approval_interrupted_mutation"),
workspace_id: test_workspace_id(),
agent_id: test_agent_id("sales-interrupted-approval"),
operation_id: operation.id.clone(),
operation_version: operation.version,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_payload: json!({"email": "interrupted@example.com"}),
response_payload: None,
created_at: now - time::Duration::minutes(10),
expires_at: now + time::Duration::minutes(5),
decided_at: None,
decided_by_key_id: None,
decision_note: None,
};
registry
.create_approval_request(CreateApprovalRequest {
approval: &approval,
})
.await
.unwrap();
let approval_key_id = PlatformApiKeyId::new(format!("pk_{approval_key_name}"));
registry
.decide_approval_request(crank_registry::DecideApprovalRequest {
workspace_id: &approval.workspace_id,
agent_id: &approval.agent_id,
approval_id: &approval.id,
status: ApprovalRequestStatus::Approved,
decided_at: now - time::Duration::minutes(10),
decided_by_key_id: &approval_key_id,
response_payload: Some(json!({"approve": "yes"})),
decision_note: None,
})
.await
.unwrap()
.unwrap();
registry
.claim_approval_request(
&approval.workspace_id,
&approval.agent_id,
&approval.id,
now - time::Duration::minutes(7),
)
.await
.unwrap()
.unwrap();
let _app = build_test_app_with_approval_recovery(registry.clone());
let failed = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let current = registry
.get_approval_request_for_agent(
&approval.workspace_id,
&approval.agent_id,
&approval.id,
)
.await
.unwrap()
.unwrap();
if current.approval.status == ApprovalRequestStatus::Failed {
break current;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("recovery must quarantine interrupted execution");
assert_eq!(
failed.approval.response_payload.unwrap()["error"]["code"],
"approval_execution_outcome_unknown"
);
assert_eq!(
upstream_calls.load(std::sync::atomic::Ordering::SeqCst),
0,
"recovery must not repeat a mutating upstream request"
);
}
fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router {
crank_community_mcp::build_app_with_background_workers(
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
SecretCrypto::new("test-master-key").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
Arc::new(InMemoryCoordinationStateStore::default()),
Arc::new(InMemorySessionStore::default()),
Arc::new(CommunityMachineCredentialVerifier),
)
}
async fn spawn_counted_approval_upstream() -> (String, Arc<std::sync::atomic::AtomicUsize>) {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let handler_calls = Arc::clone(&calls);
let app = Router::new().route(
"/crm/leads",
post(move |Json(payload): Json<Value>| {
let calls = Arc::clone(&handler_calls);
async move {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Json(json!({
"id": "lead_123",
"email": payload["email"]
}))
}
}),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(format!("http://{address}"), calls)
}
#[tokio::test]
async fn elicitation_approval_requires_client_capability() {
let registry = test_registry().await;
apps/mcp-server/tests/integration/execution_stages.rs
+376
View File
@@ -0,0 +1,376 @@
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use axum::{
Json, Router,
body::{Body, Bytes, to_bytes},
extract::State,
http::{HeaderMap, Request, StatusCode, header},
routing::post,
};
use crank_core::{InvocationSource, PlatformApiKeyScope};
use crank_observability::{
OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider,
};
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
use opentelemetry::global;
use opentelemetry_proto::tonic::{
collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value, trace::v1::Span,
};
use opentelemetry_sdk::propagation::TraceContextPropagator;
use prost::Message;
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::net::TcpListener;
use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::{Layer, filter::filter_fn, layer::SubscriberExt};
use super::common::{
build_test_app, create_platform_api_key, publish_agent_for_operation, test_operation,
test_registry, test_workspace_id,
};
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
const REQUEST_ID: &str = "req_stage_end_to_end";
const CANARY: &str = "dc-stage-canary-secret";
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exports_real_tool_stages_without_sensitive_data() {
global::set_text_map_propagator(TraceContextPropagator::new());
let (otlp_endpoint, request_rx, collector) = spawn_otlp_collector().await;
let trace_config = OtlpTraceConfig::try_new(
Some(otlp_endpoint),
Some("http/protobuf".to_owned()),
Duration::from_secs(2),
OtlpBatchConfig::try_new(128, 128, Duration::from_secs(300), Duration::from_secs(2))
.unwrap(),
)
.unwrap();
let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "integration-test").unwrap();
let (provider, tracer) = build_tracer_provider(&identity, &trace_config)
.unwrap()
.expect("enabled OTLP provider");
let subscriber = tracing_subscriber::registry().with(
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(filter_fn(|metadata| {
metadata.is_span() && metadata.target() == "crank::trace"
})),
);
let dispatch = tracing::Dispatch::new(subscriber);
let registry = test_registry().await;
let observed_traceparent = Arc::new(Mutex::new(None));
let upstream_base_url = spawn_upstream(Arc::clone(&observed_traceparent)).await;
let operation = test_operation(&upstream_base_url, "stage_end_to_end");
registry
.create_operation(&test_workspace_id(), &operation, Some("test"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("test"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "stage-agent").await;
let api_key = create_platform_api_key(
&registry,
"stage-agent",
"stage-key",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = build_test_app(registry.clone(), Duration::ZERO, None);
let call_result = async {
let initialized = send_jsonrpc(
app.clone(),
&api_key,
None,
None,
None,
json!({
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-11-25",
"capabilities": {}
}
}),
)
.await;
assert_eq!(initialized.status(), StatusCode::OK);
let session_id = initialized
.headers()
.get("MCP-Session-Id")
.unwrap()
.to_str()
.unwrap()
.to_owned();
let notification = send_jsonrpc(
app.clone(),
&api_key,
Some(&session_id),
None,
None,
json!({
"jsonrpc": "2.0",
"method": "notifications/initialized",
"params": {}
}),
)
.await;
assert_eq!(notification.status(), StatusCode::ACCEPTED);
send_jsonrpc(
app,
&api_key,
Some(&session_id),
Some(REQUEST_ID),
Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
json!({
"jsonrpc": "2.0",
"id": 2,
"method": "tools/call",
"params": {
"name": "stage_end_to_end",
"arguments": { "email": CANARY }
}
}),
)
.await
}
.with_subscriber(dispatch)
.await;
assert_eq!(call_result.status(), StatusCode::OK);
let body = to_bytes(call_result.into_body(), 1024 * 1024)
.await
.unwrap();
let body: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(body["result"]["isError"], false);
provider.force_flush().unwrap();
let request = tokio::time::timeout(Duration::from_secs(2), request_rx)
.await
.unwrap()
.unwrap();
collector.abort();
assert_eq!(
request
.headers
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok()),
Some("application/x-protobuf")
);
assert!(
!request
.body
.windows(CANARY.len())
.any(|window| window == CANARY.as_bytes())
);
let export = ExportTraceServiceRequest::decode(request.body).unwrap();
let spans = export
.resource_spans
.iter()
.flat_map(|resource| &resource.scope_spans)
.flat_map(|scope| &scope.spans)
.collect::<Vec<_>>();
let traceparent = observed_traceparent
.lock()
.unwrap()
.clone()
.expect("upstream traceparent");
assert_eq!(&traceparent[3..35], REMOTE_TRACE_ID);
let expected_trace_id = decode_trace_id(REMOTE_TRACE_ID);
let trace_spans = spans
.iter()
.copied()
.filter(|span| span.trace_id.as_slice() == expected_trace_id)
.collect::<Vec<_>>();
for expected in [
"mcp.request",
"mcp.rate_limit",
"mcp.access.check",
"mcp.catalog.load",
"mcp.tools.resolve",
"runtime.execute",
"runtime.arguments.map",
"upstream.http",
"runtime.response.transform",
"history.write",
"db.query",
] {
assert!(
trace_spans.iter().any(|span| span.name == expected),
"missing span {expected}"
);
}
assert!(!trace_spans.iter().any(|span| span.name == "approval.check"));
assert!(
!trace_spans
.iter()
.any(|span| span.name == "runtime.idempotency")
);
let root = trace_spans
.iter()
.find(|span| span.name == "mcp.request")
.expect("mcp root");
let runtime = trace_spans
.iter()
.find(|span| span.name == "runtime.execute")
.expect("runtime");
assert!(!runtime.parent_span_id.is_empty());
assert_eq!(runtime.parent_span_id, root.span_id);
let history = trace_spans
.iter()
.find(|span| span.name == "history.write")
.expect("history write");
let history_db = trace_spans
.iter()
.find(|span| {
span.name == "db.query"
&& string_attribute(span, "db.operation") == Some("invocation_history.write")
})
.expect("history PostgreSQL write");
assert_eq!(history_db.parent_span_id, history.span_id);
let logs = registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
limit: 10,
})
.await
.unwrap();
assert_eq!(logs.len(), 1);
assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID));
provider.shutdown().unwrap();
}
struct OtlpRequest {
headers: HeaderMap,
body: Bytes,
}
async fn spawn_otlp_collector() -> (
String,
tokio::sync::oneshot::Receiver<OtlpRequest>,
tokio::task::JoinHandle<()>,
) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let (request_tx, request_rx) = tokio::sync::oneshot::channel();
let sender = Arc::new(Mutex::new(Some(request_tx)));
let app = Router::new().route(
"/v1/traces",
post({
let sender = Arc::clone(&sender);
move |headers: HeaderMap, body: Bytes| {
let sender = Arc::clone(&sender);
async move {
if let Some(sender) = sender.lock().unwrap().take() {
let _ = sender.send(OtlpRequest { headers, body });
}
StatusCode::OK
}
}
}),
);
let collector = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(format!("http://{address}/v1/traces"), request_rx, collector)
}
fn decode_trace_id(value: &str) -> [u8; 16] {
let mut bytes = [0_u8; 16];
for (index, byte) in bytes.iter_mut().enumerate() {
*byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap();
}
bytes
}
fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> {
span.attributes.iter().find_map(|attribute| {
let value = attribute.value.as_ref()?.value.as_ref()?;
(attribute.key == key)
.then_some(value)
.and_then(|value| match value {
any_value::Value::StringValue(value) => Some(value.as_str()),
_ => None,
})
})
}
async fn send_jsonrpc(
app: Router,
api_key: &str,
session_id: Option<&str>,
request_id: Option<&str>,
traceparent: Option<&str>,
payload: Value,
) -> axum::response::Response {
let mut request = Request::builder()
.method("POST")
.uri("/v1/default/stage-agent")
.header(header::CONTENT_TYPE, "application/json")
.header(header::ACCEPT, "application/json, text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("MCP-Protocol-Version", "2025-11-25");
if let Some(session_id) = session_id {
request = request.header("MCP-Session-Id", session_id);
}
if let Some(request_id) = request_id {
request = request.header("x-request-id", request_id);
}
if let Some(traceparent) = traceparent {
request = request.header("traceparent", traceparent);
}
app.oneshot(request.body(Body::from(payload.to_string())).unwrap())
.await
.unwrap()
}
async fn spawn_upstream(observed: Arc<Mutex<Option<String>>>) -> String {
async fn create_lead(
State(observed): State<Arc<Mutex<Option<String>>>>,
headers: HeaderMap,
Json(_payload): Json<Value>,
) -> Json<Value> {
*observed.lock().unwrap() = headers
.get("traceparent")
.and_then(|value| value.to_str().ok())
.map(str::to_owned);
Json(json!({ "id": "lead_123" }))
}
let app = Router::new()
.route("/crm/leads", post(create_lead))
.with_state(observed);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{address}")
}
apps/mcp-server/tests/integration/request_context.rs
+130
View File
@@ -0,0 +1,130 @@
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use axum::{
body::Body,
http::{Request, StatusCode},
};
use opentelemetry::{
global,
trace::{TraceId, TracerProvider as _},
};
use opentelemetry_sdk::{
error::OTelSdkResult,
propagation::TraceContextPropagator,
trace::{SdkTracerProvider, SpanData, SpanExporter},
};
use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::layer::SubscriberExt;
use super::common::{build_test_app, test_registry};
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
#[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
global::set_text_map_propagator(TraceContextPropagator::new());
let exported = Arc::new(Mutex::new(Vec::new()));
let provider = SdkTracerProvider::builder()
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
.build();
let tracer = provider.tracer("mcp-request-context-test");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let app = build_test_app(test_registry().await, Duration::ZERO, None);
let (valid, invalid, absent) = async {
let valid = send_health(
app.clone(),
Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
Some("request-id-is-separate"),
)
.await;
let invalid = send_health(
app.clone(),
Some("canary-invalid-traceparent"),
Some("bad,value"),
)
.await;
let absent = send_health(app, None, None).await;
(valid, invalid, absent)
}
.with_subscriber(dispatch)
.await;
provider.force_flush().unwrap();
assert_eq!(valid.status, StatusCode::OK);
assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate"));
assert_eq!(invalid.status, StatusCode::OK);
assert_eq!(absent.status, StatusCode::OK);
assert!(valid.traceparent_response.is_none());
assert!(invalid.traceparent_response.is_none());
assert!(absent.traceparent_response.is_none());
let trace_ids: Vec<_> = exported
.lock()
.unwrap()
.iter()
.filter(|span| span.name.as_ref() == "mcp.request")
.map(|span| span.span_context.trace_id())
.collect();
assert_eq!(trace_ids.len(), 3);
assert_eq!(trace_ids[0].to_string(), REMOTE_TRACE_ID);
assert_ne!(trace_ids[1], trace_ids[0]);
assert_ne!(trace_ids[2], trace_ids[0]);
assert_ne!(trace_ids[1], trace_ids[2]);
assert!(!trace_ids.contains(&TraceId::INVALID));
provider.shutdown().unwrap();
}
async fn send_health(
app: axum::Router,
traceparent: Option<&str>,
request_id: Option<&str>,
) -> ProbeResponse {
let mut request = Request::builder().uri("/health");
if let Some(traceparent) = traceparent {
request = request.header("traceparent", traceparent);
}
if let Some(request_id) = request_id {
request = request.header("x-request-id", request_id);
}
let response = app
.oneshot(request.body(Body::empty()).unwrap())
.await
.unwrap();
ProbeResponse {
status: response.status(),
request_id: response
.headers()
.get("x-request-id")
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
traceparent_response: response
.headers()
.get("traceparent")
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
}
}
struct ProbeResponse {
status: StatusCode,
request_id: Option<String>,
traceparent_response: Option<String>,
}
#[derive(Clone, Debug)]
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
impl SpanExporter for CapturingExporter {
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
self.0.lock().unwrap().extend(batch);
Ok(())
}
}
apps/mcp-server/tests/integration/transport_protocol.rs
+51 -22
View File
@@ -38,7 +38,8 @@ use sha2::{Digest, Sha256};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::net::TcpListener;
use tokio::time::sleep;
use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
use tracing_subscriber::fmt::MakeWriter;
use uuid::Version;
use crank_community_mcp::{
auth::{CommunityMachineCredentialVerifier, SharedMachineCredentialVerifier},
@@ -46,6 +47,9 @@ use crank_community_mcp::{
catalog::PublishedToolCatalog,
session::{InMemorySessionStore, SharedSessionStore, TransportSessionStore},
};
use crank_observability::{
ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber,
};
fn test_workspace_id() -> WorkspaceId {
WorkspaceId::new("ws_default")
@@ -404,7 +408,10 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
.to_str()
.unwrap()
.to_owned();
assert!(!request_id.is_empty());
assert_eq!(
uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
Some(Version::SortRand)
);
let call_result = response.json::<Value>().await.unwrap();
assert_eq!(call_result["result"]["isError"], false);
@@ -427,7 +434,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str()));
}
#[tokio::test]
#[tokio::test(flavor = "current_thread")]
async fn emits_request_id_in_mcp_ingress_logs() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
@@ -456,6 +463,18 @@ async fn emits_request_id_in_mcp_ingress_logs() {
)
.await;
let writer = SharedLogWriter::default();
let subscriber = build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
let base_url = spawn_mcp_server(build_test_app(
registry,
Duration::from_millis(0),
@@ -464,18 +483,7 @@ async fn emits_request_id_in_mcp_ingress_logs() {
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-request-trace");
let writer = SharedLogWriter::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(writer.clone())
.without_time()
.with_ansi(false)
.with_target(false)
.compact()
.with_filter(LevelFilter::INFO),
);
let _ = tracing::subscriber::set_global_default(subscriber);
let response = post_jsonrpc_response(
&client,
&mcp_url,
@@ -500,14 +508,16 @@ async fn emits_request_id_in_mcp_ingress_logs() {
assert_eq!(response.status(), reqwest::StatusCode::OK);
let logs = writer.output();
assert!(
logs.contains("mcp request received"),
"captured logs did not include ingress marker: {logs}"
);
assert!(logs.contains("req_mcp_trace_123"));
assert!(logs.contains("sales-request-trace"));
assert!(logs.contains("default"));
assert!(logs.contains("initialize"));
let event = logs
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.find(|event| event["event"] == "mcp.request.received")
.unwrap();
assert_eq!(event["service"], "mcp-server");
assert_eq!(event["request_id"], "req_mcp_trace_123");
assert_eq!(event["fields"]["agent_slug"], "sales-request-trace");
assert_eq!(event["fields"]["workspace_slug"], "default");
assert_eq!(event["fields"]["jsonrpc_method"], "initialize");
}
#[tokio::test]
@@ -803,11 +813,30 @@ async fn get_requires_session_header() {
.get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
.header(header::ACCEPT, "text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("x-request-id", "req_early_mcp_error")
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
assert_eq!(
response.headers()["x-request-id"].to_str().unwrap(),
"req_early_mcp_error"
);
let invalid_response = client
.get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
.header(header::ACCEPT, "text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("x-request-id", "bad,value")
.send()
.await
.unwrap();
let generated = invalid_response.headers()["x-request-id"].to_str().unwrap();
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
}
#[tokio::test]
apps/mcp-server/tests/request_context.rs
+4
View File
@@ -0,0 +1,4 @@
#[path = "integration/common.rs"]
mod common;
#[path = "integration/request_context.rs"]
mod request_context;
apps/ui/html/workspace-setup.html
+2 -2
View File
@@ -97,8 +97,8 @@
</div>
<div class="danger-zone-action">
<div class="danger-zone-text">
<div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export all data</div>
<div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.</div>
<div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export workspace catalog</div>
<div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.</div>
</div>
<button class="btn-danger" id="export-workspace-btn" type="button" data-i18n="workspace_setup.export">Export</button>
</div>
apps/ui/js/i18n.js
+4 -4
View File
@@ -442,8 +442,8 @@ var TRANSLATIONS = {
'workspace_setup.create.subtitle': 'This Community installation uses one workspace for MCP operations and agents.',
'workspace_setup.create.footer': 'This Community installation uses one workspace.',
'workspace_setup.danger.title': 'Danger zone',
'workspace_setup.danger.export_title': 'Export all data',
'workspace_setup.danger.export_body': 'Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.',
'workspace_setup.danger.export_title': 'Export workspace catalog',
'workspace_setup.danger.export_body': 'Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.',
'workspace_setup.export': 'Export',
'workspace_setup.role.owner': 'Owner',
'workspace_setup.role.admin': 'Admin',
@@ -1334,8 +1334,8 @@ var TRANSLATIONS = {
'workspace_setup.create.subtitle': 'В Community используется один воркспейс для MCP-операций и агентов.',
'workspace_setup.create.footer': 'В Community используется один воркспейс.',
'workspace_setup.danger.title': 'Опасная зона',
'workspace_setup.danger.export_title': 'Экспортировать все данные',
'workspace_setup.danger.export_body': 'Скачать JSON-снимок настроек воркспейса, операций, агентов, секретов, данных использования и ключей доступа агентов.',
'workspace_setup.danger.export_title': 'Экспорт каталога рабочего пространства',
'workspace_setup.danger.export_body': 'Скачать невосстанавливаемый JSON-каталог настроек рабочего пространства, сводок операций и агентов, а также метаданных ключей API.',
'workspace_setup.export': 'Экспорт',
'workspace_setup.role.owner': 'Владелец',
'workspace_setup.role.admin': 'Администратор',
apps/ui/js/logs.js
+39 -3
View File
@@ -8,6 +8,8 @@ document.addEventListener('DOMContentLoaded', function () {
openId: null,
liveMode: true,
timer: null,
searchTimer: null,
refreshPromise: null,
workspaceId: null,
loading: false,
loadError: '',
@@ -457,7 +459,13 @@ document.addEventListener('DOMContentLoaded', function () {
}
async function refreshOperationalData() {
await Promise.all([loadLogs(), loadApprovals()]);
if (state.refreshPromise) {
return state.refreshPromise;
}
state.refreshPromise = Promise.all([loadLogs(), loadApprovals()]).finally(function () {
state.refreshPromise = null;
});
return state.refreshPromise;
}
async function loadLogDetail(logId) {
@@ -494,10 +502,14 @@ document.addEventListener('DOMContentLoaded', function () {
function startPolling() {
stopPolling();
if (!state.liveMode) {
if (!state.liveMode || document.hidden) {
return;
}
state.timer = setInterval(refreshOperationalData, 4000);
state.timer = setTimeout(async function poll() {
state.timer = null;
await refreshOperationalData();
startPolling();
}, 4000);
}
function toggleLive() {
@@ -526,7 +538,13 @@ document.addEventListener('DOMContentLoaded', function () {
if (logSearch) {
logSearch.addEventListener('input', function () {
state.search = this.value.trim();
if (state.searchTimer) {
clearTimeout(state.searchTimer);
}
state.searchTimer = setTimeout(function () {
state.searchTimer = null;
loadLogs();
}, 250);
});
}
@@ -572,6 +590,24 @@ document.addEventListener('DOMContentLoaded', function () {
refreshOperationalData();
});
document.addEventListener('visibilitychange', function () {
if (document.hidden) {
stopPolling();
return;
}
if (state.liveMode) {
refreshOperationalData().finally(startPolling);
}
});
window.addEventListener('pagehide', function () {
stopPolling();
if (state.searchTimer) {
clearTimeout(state.searchTimer);
state.searchTimer = null;
}
});
setLiveState();
startPolling();
apps/ui/js/workspace-setup.js
+1 -1
View File
@@ -196,7 +196,7 @@ async function exportWorkspaceSnapshot() {
var slug = workspaceFormState.workspaceRecord && workspaceFormState.workspaceRecord.workspace
? workspaceFormState.workspaceRecord.workspace.slug
: tKey('settings.nav.workspace');
downloadJsonFile(slug + '-snapshot.json', snapshot);
downloadJsonFile(slug + '-catalog-snapshot.json', snapshot);
} catch (error) {
if (window.CrankUi) {
window.CrankUi.error(error.message || tKey('workspace_setup.export_error'), tKey('workspace_setup.export_error_title'));
apps/ui/package-lock.json
Generated
+4 -4
View File
@@ -9,7 +9,7 @@
"@fontsource/inter": "5.2.8",
"@fontsource/jetbrains-mono": "5.2.8",
"alpinejs": "3.15.12",
"js-yaml": "5.2.1"
"js-yaml": "5.2.2"
},
"devDependencies": {
"@playwright/test": "1.61.1",
@@ -580,9 +580,9 @@
}
},
"node_modules/js-yaml": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz",
"integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==",
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz",
"integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==",
"funding": [
{
"type": "github",
apps/ui/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"@fontsource/inter": "5.2.8",
"@fontsource/jetbrains-mono": "5.2.8",
"alpinejs": "3.15.12",
"js-yaml": "5.2.1"
"js-yaml": "5.2.2"
},
"devDependencies": {
"@playwright/test": "1.61.1",
apps/ui/tests/e2e/wizard.spec.js
+3 -3
View File
@@ -161,15 +161,15 @@ test('wizard builds visual request mappings from JSON sample and path params', a
await page.goto('/wizard/');
await page.locator('[data-testid="wizard-protocol-rest"]').click();
await page.evaluate(() => window.CrankWizardShell.doGoToStep(2));
await page.evaluate(() => window.CrankWizardShell.goToStep(2));
await expect(page.locator('#step-panel-2')).toBeVisible();
await page.locator('#endpoint-path').fill('/rates/{date}');
await page.evaluate(() => window.CrankWizardShell.doGoToStep(3));
await page.evaluate(() => window.CrankWizardShell.goToStep(3));
await expect(page.locator('#step-panel-3-rest')).toBeVisible();
await page.locator('.method-card[data-method="GET"]').click();
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
await page.evaluate(() => window.CrankWizardShell.goToStep(5));
await expect(page.locator('#step-panel-5')).toBeVisible();
await page.locator('#wizard-input-sample').fill(JSON.stringify({
apps/ui/tests/e2e/workspace-settings.spec.js
+6
View File
@@ -11,6 +11,12 @@ test('workspace and settings pages show live session data', async ({ page }) =>
await expect(page.locator('#section-members')).toHaveCount(0);
await expect(page.locator('#section-invite')).toHaveCount(0);
await expect(page.locator('#delete-workspace-btn')).toHaveCount(0);
await expect(page.locator('[data-i18n="workspace_setup.danger.export_title"]')).toHaveText(
localized('Export workspace catalog', 'Экспорт каталога рабочего пространства'),
);
await expect(page.locator('[data-i18n="workspace_setup.danger.export_body"]')).not.toContainText(
localized('all data', 'все данные'),
);
await page.goto('/settings');
await expect(page.locator('.page-title')).toHaveText(localized('Account settings', 'Настройки аккаунта'));
crates/crank-adapter-rest/Cargo.toml
+8
View File
@@ -3,18 +3,26 @@ name = "crank-adapter-rest"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[dependencies]
async-trait = "0.1"
crank-core = { path = "../crank-core" }
crank-trace = { path = "../crank-trace" }
futures-util = "0.3"
metrics.workspace = true
opentelemetry.workspace = true
reqwest = { workspace = true, features = ["stream"] }
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tokio.workspace = true
tracing.workspace = true
tracing-opentelemetry.workspace = true
[dev-dependencies]
axum.workspace = true
opentelemetry_sdk.workspace = true
tokio.workspace = true
tracing-subscriber.workspace = true
crates/crank-adapter-rest/src/client.rs
+104 -1
View File
@@ -7,7 +7,9 @@ use std::{
};
use crank_core::{HttpMethod, RestTarget};
use crank_trace::{ErrorCategory, Stage, StageOutcome};
use futures_util::StreamExt;
use opentelemetry::{global, propagation::Injector, trace::TraceContextExt};
use reqwest::{
Client,
dns::{Addrs, Name, Resolve, Resolving},
@@ -15,6 +17,8 @@ use reqwest::{
redirect,
};
use serde_json::Value;
use tracing::{Instrument, Span};
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::{RestAdapterError, RestRequest, RestResponse};
@@ -66,10 +70,37 @@ impl RestAdapter {
&self,
target: &RestTarget,
request: &RestRequest,
) -> Result<RestResponse, RestAdapterError> {
let started_at = std::time::Instant::now();
let result = self.execute_inner(target, request).await;
let outcome = match &result {
Ok(_) => "success",
Err(error) => upstream_outcome(error),
};
metrics::counter!(
"crank_upstream_requests_total",
"operation_kind" => "rest",
"outcome" => outcome
)
.increment(1);
metrics::histogram!(
"crank_upstream_request_duration_seconds",
"operation_kind" => "rest",
"outcome" => outcome
)
.record(started_at.elapsed().as_secs_f64());
result
}
async fn execute_inner(
&self,
target: &RestTarget,
request: &RestRequest,
) -> Result<RestResponse, RestAdapterError> {
let url = build_url(target, request)?;
self.policy.validate_url(&url)?;
let headers = build_headers(target, request)?;
let mut headers = build_headers(target, request)?;
apply_current_trace_context(&mut headers);
let client =
self.client
.as_ref()
@@ -85,6 +116,8 @@ impl RestAdapter {
builder = builder.json(body);
}
let upstream_span = Stage::UpstreamHttp.span();
let result = async {
let response = builder.send().await?;
let status = response.status();
let headers = normalize_headers(response.headers());
@@ -103,6 +136,41 @@ impl RestAdapter {
body,
})
}
.instrument(upstream_span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&upstream_span),
Err(_) => {
StageOutcome::Error.record(&upstream_span);
ErrorCategory::Upstream.record(&upstream_span);
}
}
result
}
}
fn upstream_outcome(error: &RestAdapterError) -> &'static str {
match error {
RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
"client_error"
}
RestAdapterError::UnexpectedStatus { status, .. } if (500..600).contains(status) => {
"server_error"
}
RestAdapterError::UnexpectedStatus { .. } => "unexpected_status",
RestAdapterError::Transport(error) if error.is_timeout() => "timeout",
RestAdapterError::Transport(_) => "transport_error",
RestAdapterError::ResponseTooLarge { .. } => "response_too_large",
RestAdapterError::TargetNotAllowed { .. } => "rejected",
RestAdapterError::WindowExpired => "window_expired",
RestAdapterError::InvalidSseEvent => "invalid_response",
RestAdapterError::InvalidBaseUrl { .. }
| RestAdapterError::InvalidPathParameter { .. }
| RestAdapterError::InvalidQueryParameter { .. }
| RestAdapterError::InvalidHeaderName { .. }
| RestAdapterError::InvalidHeaderValue { .. } => "invalid_request",
RestAdapterError::InvalidConfiguration { .. } => "configuration",
}
}
impl Default for OutboundHttpPolicy {
@@ -407,6 +475,9 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
HeaderName::try_from(name).map_err(|_| RestAdapterError::InvalidHeaderName {
header: name.to_owned(),
})?;
if is_trace_propagation_header(&header_name) {
return Ok(());
}
let header_value =
HeaderValue::try_from(value).map_err(|_| RestAdapterError::InvalidHeaderValue {
header: name.to_owned(),
@@ -416,6 +487,38 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(),
Ok(())
}
fn is_trace_propagation_header(name: &HeaderName) -> bool {
matches!(name.as_str(), "traceparent" | "tracestate" | "baggage")
}
fn apply_current_trace_context(headers: &mut HeaderMap) {
for header in ["traceparent", "tracestate", "baggage"] {
headers.remove(header);
}
let context = Span::current().context();
if !context.span().span_context().is_valid() {
return;
}
global::get_text_map_propagator(|propagator| {
propagator.inject_context(&context, &mut ReqwestHeaderInjector(headers));
});
}
struct ReqwestHeaderInjector<'a>(&'a mut HeaderMap);
impl Injector for ReqwestHeaderInjector<'_> {
fn set(&mut self, key: &str, value: String) {
let Ok(name) = HeaderName::try_from(key) else {
return;
};
let Ok(value) = HeaderValue::try_from(value) else {
return;
};
self.0.insert(name, value);
}
}
async fn decode_body(
response: reqwest::Response,
max_response_bytes: usize,
crates/crank-adapter-rest/src/lib.rs
+4 -2
View File
@@ -26,13 +26,15 @@ impl ProtocolAdapter for RestAdapter {
&self,
target: &Target,
prepared: &PreparedRequest,
_context: &RuntimeRequestContext,
context: &RuntimeRequestContext,
) -> Result<AdapterResponse, ProtocolAdapterError> {
let target = rest_target(target)?;
let mut headers = prepared.headers.clone();
headers.extend(context.outbound_headers());
let request = RestRequest {
path_params: prepared.path_params.clone(),
query_params: prepared.query_params.clone(),
headers: prepared.headers.clone(),
headers,
body: prepared.body.clone(),
timeout_ms: prepared.timeout_ms,
};
crates/crank-adapter-rest/tests/integration/client.rs
+170 -3
View File
@@ -8,9 +8,19 @@ use axum::{
routing::{get, post},
};
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
use crank_core::{HttpMethod, RestTarget};
use crank_core::{
HttpMethod, PreparedRequest, ProtocolAdapter, RestTarget, RuntimeRequestContext, Target,
};
use opentelemetry::{
global,
trace::{TraceContextExt, TracerProvider as _},
};
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
use serde_json::{Value, json};
use tokio::net::TcpListener;
use tracing::Instrument;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use tracing_subscriber::layer::SubscriberExt;
#[tokio::test]
async fn executes_rest_request_and_normalizes_json_response() {
@@ -45,6 +55,120 @@ async fn executes_rest_request_and_normalizes_json_response() {
);
}
#[tokio::test]
async fn protocol_context_overrides_mapped_correlation_headers() {
let base_url = spawn_test_server().await;
let adapter = test_adapter();
let target = Target::Rest(RestTarget {
base_url,
method: HttpMethod::Post,
path_template: "/users/{user_id}".to_owned(),
static_headers: BTreeMap::from([
("x-request-id".to_owned(), "static-request".to_owned()),
(
"x-correlation-id".to_owned(),
"static-correlation".to_owned(),
),
]),
});
let prepared = PreparedRequest {
path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
headers: BTreeMap::from([
("x-request-id".to_owned(), "mapped-request".to_owned()),
(
"x-correlation-id".to_owned(),
"mapped-correlation".to_owned(),
),
]),
body: Some(json!({ "name": "Ada" })),
timeout_ms: 1_000,
..PreparedRequest::default()
};
let context = RuntimeRequestContext::new("req-runtime", "corr-runtime");
let response = adapter
.invoke_unary(&target, &prepared, &context)
.await
.unwrap();
assert_eq!(response.body["request_id"], "req-runtime");
assert_eq!(response.body["correlation_id"], "corr-runtime");
}
#[tokio::test(flavor = "current_thread")]
async fn current_trace_context_overrides_mapped_traceparent() {
global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("rest-propagation-test");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let span = tracing::info_span!("runtime.execute");
let context = span.context();
let expected_trace_id = context.span().span_context().trace_id().to_string();
let base_url = spawn_test_server().await;
let target = RestTarget {
base_url,
method: HttpMethod::Post,
path_template: "/users/{user_id}".to_owned(),
static_headers: BTreeMap::new(),
};
let request = RestRequest {
path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
query_params: BTreeMap::new(),
headers: BTreeMap::from([(
"traceparent".to_owned(),
"00-11111111111111111111111111111111-2222222222222222-01".to_owned(),
)]),
body: Some(json!({ "name": "Ada" })),
timeout_ms: 1_000,
};
let response = test_adapter()
.execute(&target, &request)
.instrument(span)
.await
.unwrap();
assert_eq!(
&response.body["traceparent"].as_str().unwrap()[3..35],
expected_trace_id
);
provider.shutdown().unwrap();
}
#[tokio::test]
async fn user_configured_propagation_headers_are_removed_without_trusted_context() {
let base_url = spawn_test_server().await;
let target = RestTarget {
base_url,
method: HttpMethod::Post,
path_template: "/users/{user_id}".to_owned(),
static_headers: BTreeMap::from([
(
"traceparent".to_owned(),
"untrusted\ninvalid-value".to_owned(),
),
("tracestate".to_owned(), "vendor=value".to_owned()),
("baggage".to_owned(), "secret=must-not-leave".to_owned()),
]),
};
let request = RestRequest {
path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]),
query_params: BTreeMap::new(),
headers: BTreeMap::new(),
body: Some(json!({ "name": "Ada" })),
timeout_ms: 1_000,
};
let response = test_adapter().execute(&target, &request).await.unwrap();
assert!(response.body.get("traceparent").is_none());
assert!(response.body.get("tracestate").is_none());
assert!(response.body.get("baggage").is_none());
}
#[tokio::test]
async fn returns_unexpected_status_with_normalized_body() {
let base_url = spawn_test_server().await;
@@ -186,14 +310,57 @@ async fn create_user(
.get("x-static")
.and_then(|value| value.to_str().ok())
.unwrap_or_default();
let request_id = headers
.get("x-request-id")
.and_then(|value| value.to_str().ok());
let correlation_id = headers
.get("x-correlation-id")
.and_then(|value| value.to_str().ok());
let traceparent = headers
.get("traceparent")
.and_then(|value| value.to_str().ok());
let tracestate = headers
.get("tracestate")
.and_then(|value| value.to_str().ok());
let baggage = headers.get("baggage").and_then(|value| value.to_str().ok());
Json(json!({
let mut response = json!({
"id": user_id,
"query": query.get("expand").cloned().unwrap_or_default(),
"trace": trace,
"static": static_header,
"payload": payload
}))
});
let response = response.as_object_mut().unwrap();
if let Some(request_id) = request_id {
response.insert(
"request_id".to_owned(),
Value::String(request_id.to_owned()),
);
}
if let Some(correlation_id) = correlation_id {
response.insert(
"correlation_id".to_owned(),
Value::String(correlation_id.to_owned()),
);
}
if let Some(traceparent) = traceparent {
response.insert(
"traceparent".to_owned(),
Value::String(traceparent.to_owned()),
);
}
if let Some(tracestate) = tracestate {
response.insert(
"tracestate".to_owned(),
Value::String(tracestate.to_owned()),
);
}
if let Some(baggage) = baggage {
response.insert("baggage".to_owned(), Value::String(baggage.to_owned()));
}
Json(Value::Object(response.clone()))
}
async fn fail() -> (axum::http::StatusCode, Json<Value>) {
crates/crank-community-auth/Cargo.toml
+1
View File
@@ -3,6 +3,7 @@ name = "crank-community-auth"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[dependencies]
crates/crank-community-auth/src/password_provider.rs
+5 -1
View File
@@ -52,7 +52,11 @@ impl IdentityProvider for PasswordIdentityProvider {
&self.password_pepper,
&user.password_hash,
) {
debug!(email = %payload.email, "password identity provider rejected credentials");
debug!(
name: "auth.password.rejected",
identity_provider = "password",
"password identity provider rejected credentials"
);
return Err(IdentityError::BadCredentials);
}
crates/crank-community-mcp/Cargo.toml
+8
View File
@@ -3,6 +3,7 @@ name = "crank-community-mcp"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[dependencies]
@@ -11,10 +12,13 @@ axum.workspace = true
base64.workspace = true
crank-adapter-rest = { path = "../crank-adapter-rest" }
crank-core = { path = "../crank-core" }
crank-observability = { path = "../crank-observability" }
crank-registry = { path = "../crank-registry" }
crank-runtime = { path = "../crank-runtime" }
crank-schema = { path = "../crank-schema" }
crank-trace = { path = "../crank-trace" }
futures-util = "0.3"
metrics.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
@@ -29,3 +33,7 @@ uuid.workspace = true
[dev-dependencies]
crank-mapping = { path = "../crank-mapping" }
crank-test-support = { path = "../crank-test-support" }
opentelemetry.workspace = true
opentelemetry_sdk.workspace = true
tracing-opentelemetry.workspace = true
tracing-subscriber.workspace = true
crates/crank-community-mcp/src/access.rs
+85 -17
View File
@@ -1,27 +1,54 @@
use std::sync::Arc;
use axum::http::{HeaderMap, StatusCode, header::AUTHORIZATION};
use axum::{
http::{HeaderMap, StatusCode, header::AUTHORIZATION},
response::{IntoResponse, Response},
};
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{OperationSecurityLevel, PlatformApiKeyScope};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
use sha2::{Digest, Sha256};
use time::OffsetDateTime;
use tracing::Instrument;
use crate::{
app::{AgentRoutePath, AppState},
auth::VerifiedMachineCredential,
};
#[derive(Clone, Copy, Debug)]
pub(super) enum MachineAccessError {
Denied(StatusCode),
Unavailable,
}
impl MachineAccessError {
pub(super) fn is_denied(self) -> bool {
matches!(self, Self::Denied(_))
}
}
impl IntoResponse for MachineAccessError {
fn into_response(self) -> Response {
match self {
Self::Denied(status) => status.into_response(),
Self::Unavailable => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
}
}
}
pub(super) async fn require_machine_access(
state: &Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
required_scope: PlatformApiKeyScope,
) -> Result<VerifiedMachineCredential, StatusCode> {
let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?;
) -> Result<VerifiedMachineCredential, MachineAccessError> {
let secret =
bearer_token(headers).ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))?;
let credential = resolve_machine_credential(state, path, secret).await?;
if !allows_scope(&credential.scopes, required_scope) {
return Err(StatusCode::FORBIDDEN);
return Err(MachineAccessError::Denied(StatusCode::FORBIDDEN));
}
Ok(credential)
@@ -35,12 +62,15 @@ pub(super) async fn require_approval_access(
) -> Result<crank_registry::PlatformApiKeyRecord, StatusCode> {
let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?;
let secret_hash = hash_access_secret(secret);
let Some(api_key) = state
let Some(api_key) = observe_db_query(
DbOperation::MachineAccessRead,
state
.registry
.get_approval_api_key_by_secret_for_agent_slug(
&path.workspace_slug,
&path.agent_slug,
&secret_hash,
),
)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
@@ -53,9 +83,14 @@ pub(super) async fn require_approval_access(
}
let used_at = OffsetDateTime::now_utc();
state
.registry
.touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
observe_db_query(
DbOperation::MachineAccessTouch,
state.registry.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&used_at,
),
)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
@@ -100,7 +135,7 @@ async fn resolve_machine_credential(
state: &Arc<AppState>,
path: &AgentRoutePath,
token: &str,
) -> Result<VerifiedMachineCredential, StatusCode> {
) -> Result<VerifiedMachineCredential, MachineAccessError> {
if let Some(credential) = verify_static_agent_key(state, path, token).await? {
return Ok(credential);
}
@@ -109,35 +144,68 @@ async fn resolve_machine_credential(
.credential_verifier
.verify_bearer_token(&path.workspace_slug, &path.agent_slug, token)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
.ok_or(StatusCode::UNAUTHORIZED)
.map_err(|_| MachineAccessError::Unavailable)?
.ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))
}
async fn verify_static_agent_key(
state: &Arc<AppState>,
path: &AgentRoutePath,
secret: &str,
) -> Result<Option<VerifiedMachineCredential>, StatusCode> {
) -> Result<Option<VerifiedMachineCredential>, MachineAccessError> {
let secret_hash = hash_access_secret(secret);
let Some(api_key) = state
let read_span = Stage::DbQuery
.db_span(DbOperation::MachineAccessRead)
.expect("database stage");
let api_key_result = state
.registry
.get_platform_api_key_by_secret_for_agent_slug(
&path.workspace_slug,
&path.agent_slug,
&secret_hash,
)
.instrument(read_span.clone())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
else {
.map_err(|_| MachineAccessError::Unavailable);
let api_key = match api_key_result {
Ok(api_key) => {
StageOutcome::Success.record(&read_span);
drop(read_span);
api_key
}
Err(status) => {
StageOutcome::Error.record(&read_span);
ErrorCategory::Database.record(&read_span);
drop(read_span);
return Err(status);
}
};
let Some(api_key) = api_key else {
return Ok(None);
};
let used_at = OffsetDateTime::now_utc();
state
let touch_span = Stage::DbQuery
.db_span(DbOperation::MachineAccessTouch)
.expect("database stage");
let touch_result = state
.registry
.touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
.instrument(touch_span.clone())
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
.map_err(|_| MachineAccessError::Unavailable);
match touch_result {
Ok(()) => {
StageOutcome::Success.record(&touch_span);
drop(touch_span);
}
Err(status) => {
StageOutcome::Error.record(&touch_span);
ErrorCategory::Database.record(&touch_span);
drop(touch_span);
return Err(status);
}
}
Ok(Some(VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
crates/crank-community-mcp/src/app.rs
+287 -207
View File
@@ -7,36 +7,38 @@ use std::{
use axum::{
Json, Router,
extract::{Path, State},
extract::{Extension, Path, State},
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response, sse::Event},
routing::{get, post},
};
use crank_core::{
ApprovalRequest, ApprovalRequestId, ApprovalRequestStatus, AuthProfile, CoordinationStateStore,
InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus,
OperationApprovalMode, PlatformApiKeyScope, SecretId,
InvocationLevel, InvocationSource, InvocationStatus, OperationApprovalMode,
PlatformApiKeyScope, SecretId,
};
use crank_registry::{
CreateApprovalRequest, CreateInvocationLogRequest, DecideApprovalRequest,
ExpireApprovalRequest, PostgresRegistry, PublishedAgentTool,
CreateApprovalRequest, DecideApprovalRequest, ExpireApprovalRequest, PostgresRegistry,
PublishedAgentTool,
};
use crank_runtime::{
RequestRateLimiter, ResolvedAuth, RuntimeError, RuntimeExecutionRequest, RuntimeExecutor,
RuntimeOperation, RuntimeRequestContext, SecretCrypto,
};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
use futures_util::stream;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use time::OffsetDateTime;
use tracing::{info, warn};
use tokio::sync::Semaphore;
use tracing::{Instrument, info, warn};
use crate::{
access::{
credential_allows_security_level, require_approval_access, require_machine_access,
serialize_machine_access_mode, serialize_security_level,
credential_allows_security_level, serialize_machine_access_mode, serialize_security_level,
},
approval_execution::{execute_approved_request, spawn_approval_recovery},
approval_response::approval_required_response,
auth::{SharedMachineCredentialVerifier, VerifiedMachineCredential},
catalog::PublishedToolCatalog,
jsonrpc::{
@@ -44,10 +46,8 @@ use crate::{
jsonrpc_result, method_name, negotiated_protocol_version, params, request_id,
},
manifest::catalog_tool_definitions,
rate_limit::{
enforce_post_rate_limit, enforce_transport_rate_limit, rate_limited_jsonrpc_response,
rate_limited_status_response,
},
rate_limit::{rate_limited_jsonrpc_response, rate_limited_status_response},
request_context::{RequestContext, apply_request_context},
session::{SessionState, SharedSessionStore},
tool_error::{
ToolErrorContract, generic_tool_error_contract, runtime_error_code,
@@ -56,13 +56,25 @@ use crate::{
tool_search::handle_catalog_tool_call,
transport::{
AllowedOrigins, HEADER_MCP_SESSION_ID, ResponseMode, json_response,
negotiate_post_response_mode, protocol_version_from_headers, resolve_request_id,
session_id_from_headers, sse_response, transport_response, validate_get_accept_header,
validate_origin, validate_session_protocol_version, with_request_id_header,
negotiate_post_response_mode, protocol_version_from_headers, session_id_from_headers,
sse_response, transport_response, validate_get_accept_header, validate_origin,
validate_session_protocol_version, with_request_id_header,
},
};
mod invocation_history;
mod metrics;
mod stages;
use self::metrics::{ActiveSessionGuard, McpRequestMetrics};
use self::stages::{
enforce_traced_rate_limit, require_traced_approval_access, require_traced_machine_access,
};
#[cfg(test)]
use invocation_history::observe_invocation_history_outcome;
pub(super) use invocation_history::{InvocationRecord, persist_invocation};
const TRANSPORT_SESSION_TTL_MS: u64 = 86_400_000;
const DEFAULT_MAX_CONCURRENT_SESSIONS: usize = 16;
const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60);
#[derive(Clone)]
pub(super) struct AppState {
@@ -72,6 +84,7 @@ pub(super) struct AppState {
pub(super) api_rate_limiter: RequestRateLimiter,
secret_crypto: SecretCrypto,
sessions: SharedSessionStore,
session_slots: Arc<Semaphore>,
pub(super) credential_verifier: SharedMachineCredentialVerifier,
allowed_origins: AllowedOrigins,
}
@@ -144,6 +157,7 @@ pub fn build_app(
coordination_store,
sessions,
credential_verifier,
DEFAULT_MAX_CONCURRENT_SESSIONS,
false,
)
}
@@ -159,6 +173,33 @@ pub fn build_app_with_background_workers(
coordination_store: Arc<dyn CoordinationStateStore>,
sessions: SharedSessionStore,
credential_verifier: SharedMachineCredentialVerifier,
) -> Router {
build_app_with_background_workers_and_limits(
registry,
refresh_interval,
public_base_url,
secret_crypto,
runtime,
api_rate_limiter,
coordination_store,
sessions,
credential_verifier,
DEFAULT_MAX_CONCURRENT_SESSIONS,
)
}
#[allow(clippy::too_many_arguments)]
pub fn build_app_with_background_workers_and_limits(
registry: PostgresRegistry,
refresh_interval: Duration,
public_base_url: Option<String>,
secret_crypto: SecretCrypto,
runtime: RuntimeExecutor,
api_rate_limiter: RequestRateLimiter,
coordination_store: Arc<dyn CoordinationStateStore>,
sessions: SharedSessionStore,
credential_verifier: SharedMachineCredentialVerifier,
max_concurrent_sessions: usize,
) -> Router {
build_app_inner(
registry,
@@ -170,6 +211,7 @@ pub fn build_app_with_background_workers(
coordination_store,
sessions,
credential_verifier,
max_concurrent_sessions,
true,
)
}
@@ -185,6 +227,7 @@ fn build_app_inner(
coordination_store: Arc<dyn CoordinationStateStore>,
sessions: SharedSessionStore,
credential_verifier: SharedMachineCredentialVerifier,
max_concurrent_sessions: usize,
start_background_workers: bool,
) -> Router {
let state = Arc::new(AppState {
@@ -194,15 +237,18 @@ fn build_app_inner(
api_rate_limiter,
secret_crypto,
sessions,
session_slots: Arc::new(Semaphore::new(max_concurrent_sessions)),
credential_verifier,
allowed_origins: AllowedOrigins::new(public_base_url),
});
if start_background_workers {
spawn_approval_recovery(Arc::clone(&state));
spawn_session_cleanup(Arc::clone(&state));
}
Router::new()
.route("/health", get(health))
.route("/ready", get(readiness))
.route(
"/v1/{workspace_slug}/{agent_slug}",
get(mcp_get).post(mcp_post).delete(mcp_delete),
@@ -224,6 +270,10 @@ fn build_app_inner(
post(deny_request),
)
.with_state(state)
.layer(axum::middleware::from_fn(apply_request_context))
.layer(axum::middleware::from_fn(
crank_observability::record_http_request,
))
}
async fn health() -> Json<Value> {
@@ -233,13 +283,64 @@ async fn health() -> Json<Value> {
}))
}
fn spawn_session_cleanup(state: Arc<AppState>) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(SESSION_CLEANUP_INTERVAL);
loop {
interval.tick().await;
match state
.sessions
.cleanup_expired(OffsetDateTime::now_utc())
.await
{
Ok(removed) if removed > 0 => {
info!(name: "mcp.session_cleanup.completed", removed);
}
Ok(_) => {}
Err(_) => {
warn!(name: "mcp.session_cleanup.failed", error_category = "session_store");
}
}
}
});
}
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
match state.registry.ping().await {
Ok(()) => Json(json!({
"service": "mcp-server",
"status": "ready",
"checks": { "postgres": "ready" }
}))
.into_response(),
Err(error) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": error.to_string()
})),
)
.into_response(),
}
}
async fn list_pending_approvals(
Path(path): Path<AgentRoutePath>,
State(state): State<Arc<AppState>>,
headers: HeaderMap,
) -> Response {
let key =
match require_approval_access(&state, &path, &headers, PlatformApiKeyScope::ReadPending)
if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await {
return rate_limited_status_response(rejection);
}
let key = match require_traced_approval_access(
&state,
&path,
&headers,
PlatformApiKeyScope::ReadPending,
)
.await
{
Ok(key) => key,
@@ -250,9 +351,12 @@ async fn list_pending_approvals(
return StatusCode::FORBIDDEN.into_response();
};
match state
match observe_db_query(
DbOperation::ApprovalRead,
state
.registry
.list_pending_approval_requests_for_agent(&key.api_key.workspace_id, agent_id)
.list_pending_approval_requests_for_agent(&key.api_key.workspace_id, agent_id),
)
.await
{
Ok(items) => Json(json!({ "items": items })).into_response(),
@@ -263,6 +367,7 @@ async fn list_pending_approvals(
async fn approve_request(
Path(path): Path<ApprovalRoutePath>,
State(state): State<Arc<AppState>>,
Extension(request_context): Extension<RequestContext>,
headers: HeaderMap,
Json(payload): Json<ApprovalDecisionPayload>,
) -> Response {
@@ -273,6 +378,7 @@ async fn approve_request(
payload,
PlatformApiKeyScope::Approve,
ApprovalRequestStatus::Approved,
Some(request_context.request_id),
)
.await
}
@@ -286,7 +392,11 @@ async fn get_approval_request(
workspace_slug: path.workspace_slug,
agent_slug: path.agent_slug,
};
let key = match require_approval_access(
if let Err(rejection) = enforce_traced_rate_limit(&state, &agent_path, &headers).await {
return rate_limited_status_response(rejection);
}
let key = match require_traced_approval_access(
&state,
&agent_path,
&headers,
@@ -318,6 +428,7 @@ async fn deny_request(
payload,
PlatformApiKeyScope::Deny,
ApprovalRequestStatus::Denied,
None,
)
.await
}
@@ -329,12 +440,18 @@ async fn decide_approval_request(
payload: ApprovalDecisionPayload,
required_scope: PlatformApiKeyScope,
status: ApprovalRequestStatus,
execution_request_id: Option<String>,
) -> Response {
let agent_path = AgentRoutePath {
workspace_slug: path.workspace_slug,
agent_slug: path.agent_slug,
};
let key = match require_approval_access(&state, &agent_path, &headers, required_scope).await {
if let Err(rejection) = enforce_traced_rate_limit(&state, &agent_path, &headers).await {
return rate_limited_status_response(rejection);
}
let key =
match require_traced_approval_access(&state, &agent_path, &headers, required_scope).await {
Ok(key) => key,
Err(status) => return status.into_response(),
};
@@ -357,7 +474,9 @@ async fn decide_approval_request(
};
let approval_id = ApprovalRequestId::new(path.approval_id);
match state
match observe_db_query(
DbOperation::ApprovalWrite,
state
.registry
.decide_approval_request(DecideApprovalRequest {
workspace_id: &key.api_key.workspace_id,
@@ -368,17 +487,19 @@ async fn decide_approval_request(
decided_by_key_id: &key.api_key.id,
response_payload: Some(json!({ "approve": payload.approve })),
decision_note: payload.note.as_deref(),
})
}),
)
.await
{
Ok(Some(record)) if status == ApprovalRequestStatus::Approved => {
let claimed = match state
.registry
.claim_approval_request(
let claimed = match observe_db_query(
DbOperation::ApprovalWrite,
state.registry.claim_approval_request(
&record.approval.workspace_id,
&record.approval.agent_id,
&record.approval.id,
OffsetDateTime::now_utc(),
),
)
.await
{
@@ -386,7 +507,14 @@ async fn decide_approval_request(
Ok(None) => return StatusCode::CONFLICT.into_response(),
Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(),
};
match execute_approved_request(&state, &agent_path, claimed).await {
match execute_approved_request(
&state,
&agent_path,
claimed,
execution_request_id.as_deref(),
)
.await
{
Ok(record) => Json(json!(record)).into_response(),
Err(response) => response,
}
@@ -406,9 +534,12 @@ async fn approval_record_response(
agent_id: &crank_core::AgentId,
approval_id: &ApprovalRequestId,
) -> Response {
match state
match observe_db_query(
DbOperation::ApprovalRead,
state
.registry
.get_approval_request_for_agent(workspace_id, agent_id, approval_id)
.get_approval_request_for_agent(workspace_id, agent_id, approval_id),
)
.await
{
Ok(Some(record))
@@ -429,9 +560,12 @@ async fn terminal_decision_response(
agent_id: &crank_core::AgentId,
approval_id: &ApprovalRequestId,
) -> Response {
match state
match observe_db_query(
DbOperation::ApprovalRead,
state
.registry
.get_approval_request_for_agent(workspace_id, agent_id, approval_id)
.get_approval_request_for_agent(workspace_id, agent_id, approval_id),
)
.await
{
Ok(Some(record))
@@ -463,14 +597,17 @@ async fn expire_approval_response(
agent_id: &crank_core::AgentId,
approval_id: &ApprovalRequestId,
) -> Response {
match state
match observe_db_query(
DbOperation::ApprovalWrite,
state
.registry
.expire_approval_request(ExpireApprovalRequest {
workspace_id,
agent_id,
approval_id,
expired_at: OffsetDateTime::now_utc(),
})
}),
)
.await
{
Ok(Some(record)) => Json(json!(record)).into_response(),
@@ -493,12 +630,12 @@ async fn mcp_get(
}
if let Err(status) =
require_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
require_traced_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
{
return status.into_response();
}
if let Err(rejection) = enforce_transport_rate_limit(&state, &path, &headers).await {
if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await {
return rate_limited_status_response(rejection);
}
@@ -527,9 +664,19 @@ async fn mcp_get(
return status.into_response();
}
let Ok(permit) = ActiveSessionGuard::try_acquire(&state.session_slots) else {
return StatusCode::TOO_MANY_REQUESTS.into_response();
};
let stream = stream::unfold(permit, |permit| async move {
tokio::time::sleep(Duration::from_millis(TRANSPORT_SESSION_TTL_MS)).await;
drop(permit);
None::<(Result<Event, Infallible>, _)>
});
sse_response(
StatusCode::OK,
stream::pending::<Result<Event, Infallible>>(),
stream,
Some(&session_id),
Some(&session.protocol_version),
)
@@ -541,12 +688,12 @@ async fn mcp_delete(
headers: HeaderMap,
) -> Response {
if let Err(status) =
require_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
require_traced_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await
{
return status.into_response();
}
if let Err(rejection) = enforce_transport_rate_limit(&state, &path, &headers).await {
if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await {
return rate_limited_status_response(rejection);
}
@@ -574,11 +721,14 @@ async fn mcp_delete(
async fn mcp_post(
Path(path): Path<AgentRoutePath>,
State(state): State<Arc<AppState>>,
Extension(request_context): Extension<RequestContext>,
headers: HeaderMap,
Json(message): Json<Value>,
) -> Response {
let transport_request_id = resolve_request_id(&headers);
let mut request_metrics = McpRequestMetrics::new(&message);
let transport_request_id = request_context.request_id;
info!(
name: "mcp.request.received",
request_id = %transport_request_id,
workspace_slug = %path.workspace_slug,
agent_slug = %path.agent_slug,
@@ -591,7 +741,7 @@ async fn mcp_post(
}
let response_mode = match negotiate_post_response_mode(&headers) {
Ok(mode) => mode,
Ok(mode) => request_metrics.set_response_mode(mode),
Err(status) => {
return with_request_id_header(status.into_response(), &transport_request_id);
}
@@ -608,13 +758,13 @@ async fn mcp_post(
}
};
if let Err(rejection) = enforce_post_rate_limit(&state, &path, &headers).await {
let rate_limit_result = enforce_traced_rate_limit(&state, &path, &headers).await;
if let Err(error) = rate_limit_result {
return with_request_id_header(
rate_limited_jsonrpc_response(&message, response_mode, &protocol_version, rejection),
rate_limited_jsonrpc_response(&message, response_mode, &protocol_version, error),
&transport_request_id,
);
}
if let Some(session_id) = headers.get(HEADER_MCP_SESSION_ID)
&& let Ok(session_id) = session_id.to_str()
{
@@ -639,10 +789,12 @@ async fn mcp_post(
Some("tools/call") => PlatformApiKeyScope::Write,
_ => PlatformApiKeyScope::Read,
};
let credential = match require_machine_access(&state, &path, &headers, required_scope).await {
let access_result =
require_traced_machine_access(&state, &path, &headers, required_scope).await;
let credential = match access_result {
Ok(credential) => credential,
Err(status) => {
return with_request_id_header(status.into_response(), &transport_request_id);
Err(error) => {
return with_request_id_header(error.into_response(), &transport_request_id);
}
};
@@ -768,6 +920,7 @@ async fn mcp_post(
),
};
request_metrics.complete(response.status());
with_request_id_header(response, &transport_request_id)
}
@@ -822,13 +975,26 @@ pub(super) async fn resolve_operation_auth(
workspace_id: &crank_core::WorkspaceId,
execution_config: &crank_core::ExecutionConfig,
) -> Result<Option<ResolvedAuth>, RuntimeError> {
resolve_runtime_auth_for_task(
if execution_config.auth_profile_ref.is_none() {
return Ok(None);
}
let span = Stage::AuthResolve.span();
let result = resolve_runtime_auth_for_task(
&state.registry,
&state.secret_crypto,
workspace_id,
execution_config,
)
.await
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Configuration.record(&span);
}
}
result
}
async fn resolve_runtime_auth_for_task(
@@ -841,8 +1007,10 @@ async fn resolve_runtime_auth_for_task(
return Ok(None);
};
let auth_profile = registry
.get_auth_profile(workspace_id, auth_profile_id)
let auth_profile = observe_db_query(
DbOperation::AuthProfileRead,
registry.get_auth_profile(workspace_id, auth_profile_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load auth profile",
@@ -867,8 +1035,10 @@ async fn resolve_auth_profile(
let used_at = OffsetDateTime::now_utc();
for secret_id in auth_profile.config.secret_ids() {
let secret = registry
.get_secret(workspace_id, secret_id)
let secret = observe_db_query(
DbOperation::SecretRead,
registry.get_secret(workspace_id, secret_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load secret",
@@ -877,8 +1047,10 @@ async fn resolve_auth_profile(
.ok_or_else(|| RuntimeError::MissingSecret {
secret_id: secret_id.as_str().to_owned(),
})?;
let version = registry
.get_current_secret_version(workspace_id, secret_id)
let version = observe_db_query(
DbOperation::SecretRead,
registry.get_current_secret_version(workspace_id, secret_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load current secret version",
@@ -892,8 +1064,10 @@ async fn resolve_auth_profile(
&version.secret_version.key_version,
&version.secret_version.ciphertext,
)?;
registry
.touch_secret(workspace_id, secret_id, &used_at)
observe_db_query(
DbOperation::SecretTouch,
registry.touch_secret(workspace_id, secret_id, &used_at),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "touch secret",
@@ -916,7 +1090,15 @@ async fn handle_base_tool_call(
let tool = execution.tool;
let arguments = execution.arguments;
let operation = runtime_operation(&tool);
if let Some(response) = maybe_handle_approval_policy(
if tool
.operation
.execution_config
.approval_policy
.as_ref()
.is_some_and(|policy| policy.required)
{
let approval_span = Stage::ApprovalCheck.span();
let response = maybe_handle_approval_policy(
&state,
session,
message,
@@ -925,9 +1107,22 @@ async fn handle_base_tool_call(
&arguments,
transport_request_id,
)
.await
{
return response;
.instrument(approval_span.clone())
.await;
if let Some(result) = response {
return match result {
ApprovalPolicyResult::Required(response) => {
StageOutcome::Required.record(&approval_span);
response
}
ApprovalPolicyResult::Error(response) => {
StageOutcome::Error.record(&approval_span);
ErrorCategory::Approval.record(&approval_span);
response
}
};
}
StageOutcome::Allowed.record(&approval_span);
}
let mut runtime_request_context = RuntimeRequestContext::from_request_id(transport_request_id)
@@ -964,7 +1159,7 @@ async fn handle_base_tool_call(
match result {
Ok(output) => {
if let Err(error) = persist_invocation(
persist_invocation(
&state,
&tool,
InvocationRecord {
@@ -980,15 +1175,12 @@ async fn handle_base_tool_call(
response_preview: output.clone(),
},
)
.await
{
warn!(error = %error, "successful invocation log write failed");
}
.await;
success_tool_response(message, response_mode, &session.protocol_version, output)
}
Err(error) => {
if let Err(log_error) = persist_invocation(
persist_invocation(
&state,
&tool,
InvocationRecord {
@@ -1004,10 +1196,7 @@ async fn handle_base_tool_call(
response_preview: Value::Null,
},
)
.await
{
warn!(error = %log_error, "failed invocation log write failed");
}
.await;
tool_error_response(
message,
@@ -1019,6 +1208,11 @@ async fn handle_base_tool_call(
}
}
enum ApprovalPolicyResult {
Required(Response),
Error(Response),
}
async fn maybe_handle_approval_policy(
state: &Arc<AppState>,
session: &SessionState,
@@ -1027,7 +1221,7 @@ async fn maybe_handle_approval_policy(
tool: &PublishedAgentTool,
arguments: &Value,
transport_request_id: &str,
) -> Option<Response> {
) -> Option<ApprovalPolicyResult> {
let policy = tool.operation.execution_config.approval_policy.as_ref()?;
if !policy.required {
return None;
@@ -1066,35 +1260,12 @@ async fn maybe_create_custom_pending_approval(
tool: &PublishedAgentTool,
arguments: &Value,
transport_request_id: &str,
) -> Option<Response> {
) -> Option<ApprovalPolicyResult> {
let policy = tool.operation.execution_config.approval_policy.as_ref()?;
let approval_id = ApprovalRequestId::new(format!("approval_{}", uuid::Uuid::now_v7().simple()));
let now = OffsetDateTime::now_utc();
let expires_at = now + time::Duration::seconds(i64::from(policy.ttl_seconds));
let approval_url = approval_url_for(tool, &approval_id);
let response_payload = json!({
"status": "approval_required",
"approval_id": approval_id.as_str(),
"approval_url": approval_url,
"approve": {
"method": "POST",
"url": format!("{approval_url}/approve"),
"body": { "approve": "yes" }
},
"deny": {
"method": "POST",
"url": format!("{approval_url}/deny"),
"body": { "approve": "no" }
},
"expires_at": expires_at,
"risk_level": policy.risk_level,
"payload_preview": if policy.show_payload_preview {
arguments.clone()
} else {
Value::Null
},
});
let approval = ApprovalRequest {
id: approval_id,
workspace_id: tool.workspace_id.clone(),
@@ -1112,22 +1283,26 @@ async fn maybe_create_custom_pending_approval(
decision_note: None,
};
let persisted_approval = match state
let persisted_approval = match observe_db_query(
DbOperation::ApprovalWrite,
state
.registry
.create_approval_request(CreateApprovalRequest {
approval: &approval,
})
}),
)
.await
{
Ok(approval) => approval,
Err(error) => return Some(internal_jsonrpc_error(message, error)),
Err(error) => {
return Some(ApprovalPolicyResult::Error(internal_jsonrpc_error(
message, error,
)));
}
};
let response_payload = persisted_approval
.approval
.response_payload
.unwrap_or(response_payload);
let response_payload = approval_required_response(tool, &persisted_approval.approval, policy);
if let Err(error) = persist_invocation(
persist_invocation(
state,
tool,
InvocationRecord {
@@ -1143,17 +1318,14 @@ async fn maybe_create_custom_pending_approval(
response_preview: response_payload.clone(),
},
)
.await
{
warn!(error = %error, "pending approval invocation log write failed");
}
.await;
Some(success_tool_response(
Some(ApprovalPolicyResult::Required(success_tool_response(
message,
response_mode,
&session.protocol_version,
response_payload,
))
)))
}
fn handle_elicitation_approval(
@@ -1164,9 +1336,9 @@ fn handle_elicitation_approval(
arguments: &Value,
elicitation_message: Option<&str>,
transport_request_id: &str,
) -> Response {
) -> ApprovalPolicyResult {
if !session.supports_elicitation {
return tool_error_response(
return ApprovalPolicyResult::Error(tool_error_response(
message,
response_mode,
&session.protocol_version,
@@ -1179,7 +1351,7 @@ fn handle_elicitation_approval(
"Выберите Custom MCP Approval или подключите MCP-клиент с поддержкой elicitation.",
),
),
);
));
}
let payload_preview = tool
@@ -1190,7 +1362,7 @@ fn handle_elicitation_approval(
.and_then(|policy| policy.show_payload_preview.then(|| arguments.clone()))
.unwrap_or(Value::Null);
success_tool_response(
ApprovalPolicyResult::Required(success_tool_response(
message,
response_mode,
&session.protocol_version,
@@ -1201,16 +1373,7 @@ fn handle_elicitation_approval(
"payload_preview": payload_preview,
"note": "This MCP client advertised elicitation support. Full elicitation/create continuation is handled by compatible client integrations.",
}),
)
}
fn approval_url_for(tool: &PublishedAgentTool, approval_id: &ApprovalRequestId) -> String {
format!(
"/v1/{}/{}/approvals/{}",
tool.workspace_slug,
tool.agent_slug,
approval_id.as_str()
)
))
}
async fn handle_initialize(
@@ -1394,51 +1557,6 @@ pub(super) fn build_request_preview(
}
}
pub(super) struct InvocationRecord<'a> {
pub(super) request_id: Option<&'a str>,
pub(super) tool_name: &'a str,
pub(super) status: InvocationStatus,
pub(super) level: InvocationLevel,
pub(super) message: &'a str,
pub(super) status_code: Option<u16>,
pub(super) error_kind: Option<&'a str>,
pub(super) duration: Duration,
pub(super) request_preview: Value,
pub(super) response_preview: Value,
}
pub(super) async fn persist_invocation(
state: &Arc<AppState>,
tool: &PublishedAgentTool,
record: InvocationRecord<'_>,
) -> Result<(), crank_registry::RegistryError> {
let created_at = OffsetDateTime::now_utc();
let duration_ms = u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX);
let log = InvocationLog {
id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
workspace_id: tool.workspace_id.clone(),
agent_id: Some(tool.agent_id.clone()),
operation_id: tool.operation.id.clone(),
source: InvocationSource::AgentToolCall,
level: record.level,
status: record.status,
tool_name: record.tool_name.to_owned(),
message: record.message.to_owned(),
request_id: record.request_id.map(ToOwned::to_owned),
status_code: record.status_code,
duration_ms,
error_kind: record.error_kind.map(ToOwned::to_owned),
request_preview: record.request_preview,
response_preview: record.response_preview,
created_at,
};
state
.registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.await
}
fn success_tool_response(
message: &Value,
response_mode: ResponseMode,
@@ -1526,42 +1644,4 @@ pub(super) fn runtime_operation(tool: &PublishedAgentTool) -> RuntimeOperation {
}
#[cfg(test)]
mod tests {
use axum::body::to_bytes;
use serde_json::{Value, json};
use super::{ResponseMode, tool_error_response};
use crate::jsonrpc::CURRENT_PROTOCOL_VERSION;
use crate::tool_error::generic_tool_error_contract;
#[tokio::test]
async fn tool_error_response_includes_structured_context() {
let response = tool_error_response(
&json!({"jsonrpc": "2.0", "id": "req-1"}),
ResponseMode::Json,
CURRENT_PROTOCOL_VERSION,
generic_tool_error_contract(
"streaming_payload_error",
"request root must be an object",
"req-1",
false,
Some("Проверьте параметры вызова инструмента."),
),
);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let payload: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(
payload["result"]["structuredContent"]["error"],
json!({
"code": "streaming_payload_error",
"error_code": "streaming_payload_error",
"message": "request root must be an object",
"recoverable": false,
"request_id": "req-1",
"suggested_action": "Проверьте параметры вызова инструмента."
})
);
}
}
mod tests;
crates/crank-community-mcp/src/app/invocation_history.rs
+117
View File
@@ -0,0 +1,117 @@
use std::{sync::Arc, time::Duration};
use crank_core::{
InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus,
};
use crank_registry::{
CreateInvocationLogRequest, InvocationHistoryWriteOutcome, PublishedAgentTool,
};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
use serde_json::Value;
use time::OffsetDateTime;
use tracing::{Instrument, warn};
use super::AppState;
pub(crate) struct InvocationRecord<'a> {
pub(crate) request_id: Option<&'a str>,
pub(crate) tool_name: &'a str,
pub(crate) status: InvocationStatus,
pub(crate) level: InvocationLevel,
pub(crate) message: &'a str,
pub(crate) status_code: Option<u16>,
pub(crate) error_kind: Option<&'a str>,
pub(crate) duration: Duration,
pub(crate) request_preview: Value,
pub(crate) response_preview: Value,
}
pub(crate) async fn persist_invocation(
state: &Arc<AppState>,
tool: &PublishedAgentTool,
record: InvocationRecord<'_>,
) -> InvocationHistoryWriteOutcome {
let log = InvocationLog {
id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
workspace_id: tool.workspace_id.clone(),
agent_id: Some(tool.agent_id.clone()),
operation_id: tool.operation.id.clone(),
source: InvocationSource::AgentToolCall,
level: record.level,
status: record.status,
tool_name: record.tool_name.to_owned(),
message: record.message.to_owned(),
request_id: record.request_id.map(ToOwned::to_owned),
status_code: record.status_code,
duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX),
error_kind: record.error_kind.map(ToOwned::to_owned),
request_preview: record.request_preview,
response_preview: record.response_preview,
created_at: OffsetDateTime::now_utc(),
};
let history_span = Stage::HistoryWrite.span();
let (outcome, db_span) = async {
let db_span = Stage::DbQuery
.db_span(DbOperation::InvocationHistoryWrite)
.expect("database stage");
let outcome = state
.registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.instrument(db_span.clone())
.await;
(outcome, db_span)
}
.instrument(history_span.clone())
.await;
match outcome {
InvocationHistoryWriteOutcome::Recorded => {
StageOutcome::Success.record(&db_span);
StageOutcome::Success.record(&history_span);
}
InvocationHistoryWriteOutcome::Lost(_) => {
StageOutcome::Error.record(&db_span);
ErrorCategory::Database.record(&db_span);
StageOutcome::Error.record(&history_span);
ErrorCategory::History.record(&history_span);
}
}
drop(db_span);
drop(history_span);
observe_invocation_history_outcome(
outcome,
record.request_id,
record.status,
"agent_tool_call",
);
outcome
}
pub(super) fn observe_invocation_history_outcome(
outcome: InvocationHistoryWriteOutcome,
request_id: Option<&str>,
status: InvocationStatus,
source: &'static str,
) {
let Some(loss) = outcome.loss() else {
return;
};
crank_observability::record_operational_incident(
crank_observability::OperationalIncident::InvocationHistoryLost,
);
warn!(
name: "mcp.invocation_history.lost",
request_id = request_id.unwrap_or_default(),
source,
invocation_status = invocation_status_label(status),
error_category = loss.category.as_str(),
"invocation history was not recorded"
);
}
fn invocation_status_label(status: InvocationStatus) -> &'static str {
match status {
InvocationStatus::Ok => "ok",
InvocationStatus::Error => "error",
}
}
crates/crank-community-mcp/src/app/metrics.rs
+93
View File
@@ -0,0 +1,93 @@
use std::sync::Arc;
use axum::http::StatusCode;
use serde_json::Value;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use crate::{
jsonrpc::{is_notification, is_response, method_name},
transport::ResponseMode,
};
pub(super) struct McpRequestMetrics {
method: &'static str,
response_mode: &'static str,
outcome: &'static str,
}
impl McpRequestMetrics {
pub(super) fn new(message: &Value) -> Self {
Self {
method: normalized_mcp_method(message),
response_mode: "unknown",
outcome: "rejected",
}
}
pub(super) fn set_response_mode(&mut self, mode: ResponseMode) -> ResponseMode {
self.response_mode = match mode {
ResponseMode::Json => "json",
ResponseMode::Sse => "sse",
};
mode
}
pub(super) fn complete(&mut self, status: StatusCode) {
self.outcome = match status.as_u16() {
200..=299 => "success",
400..=499 => "client_error",
500..=599 => "server_error",
_ => "other",
};
}
}
impl Drop for McpRequestMetrics {
fn drop(&mut self) {
::metrics::counter!(
"crank_mcp_requests_total",
"method" => self.method,
"response_mode" => self.response_mode,
"outcome" => self.outcome
)
.increment(1);
}
}
pub(super) fn normalized_mcp_method(message: &Value) -> &'static str {
match method_name(message) {
Some("initialize") => "initialize",
Some("notifications/initialized") => "initialized",
Some("ping") => "ping",
Some("tools/list") => "tools_list",
Some("tools/call") => "tools_call",
Some(_) if is_notification(message) => "notification",
Some(_) => "unsupported",
None if is_response(message) => "response",
None => "invalid",
}
}
pub(super) struct ActiveSessionGuard {
_permit: OwnedSemaphorePermit,
}
impl ActiveSessionGuard {
pub(super) fn try_acquire(slots: &Arc<Semaphore>) -> Result<Self, ()> {
let permit = Arc::clone(slots).try_acquire_owned().map_err(|_| {
::metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => "mcp_session"
)
.increment(1);
})?;
::metrics::gauge!("crank_mcp_active_sessions").increment(1.0);
Ok(Self { _permit: permit })
}
}
impl Drop for ActiveSessionGuard {
fn drop(&mut self) {
::metrics::gauge!("crank_mcp_active_sessions").decrement(1.0);
}
}
crates/crank-community-mcp/src/app/stages.rs
+86
View File
@@ -0,0 +1,86 @@
use std::sync::Arc;
use axum::http::{HeaderMap, StatusCode};
use crank_core::PlatformApiKeyScope;
use crank_registry::PlatformApiKeyRecord;
use crank_runtime::RateLimitCheckError;
use crank_trace::{ErrorCategory, Stage, StageOutcome};
use tracing::Instrument;
use crate::{
access::{MachineAccessError, require_approval_access, require_machine_access},
app::{AgentRoutePath, AppState},
auth::VerifiedMachineCredential,
rate_limit::enforce_transport_rate_limit,
};
pub(super) async fn enforce_traced_rate_limit(
state: &Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
) -> Result<(), RateLimitCheckError> {
let span = Stage::McpRateLimit.span();
let result = enforce_transport_rate_limit(state, path, headers)
.instrument(span.clone())
.await;
match &result {
Ok(()) => StageOutcome::Allowed.record(&span),
Err(RateLimitCheckError::Rejected(_)) => {
StageOutcome::Denied.record(&span);
ErrorCategory::RateLimit.record(&span);
}
Err(RateLimitCheckError::StoreUnavailable) => {
StageOutcome::Error.record(&span);
ErrorCategory::Internal.record(&span);
}
}
result
}
pub(super) async fn require_traced_machine_access(
state: &Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
required_scope: PlatformApiKeyScope,
) -> Result<VerifiedMachineCredential, MachineAccessError> {
let span = Stage::McpAccessCheck.span();
let result = require_machine_access(state, path, headers, required_scope)
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Allowed.record(&span),
Err(error) if error.is_denied() => {
StageOutcome::Denied.record(&span);
ErrorCategory::Access.record(&span);
}
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Internal.record(&span);
}
}
result
}
pub(super) async fn require_traced_approval_access(
state: &Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
required_scope: PlatformApiKeyScope,
) -> Result<PlatformApiKeyRecord, StatusCode> {
let span = Stage::McpAccessCheck.span();
let result = require_approval_access(state, path, headers, required_scope)
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Allowed.record(&span),
Err(status) if *status == StatusCode::UNAUTHORIZED || *status == StatusCode::FORBIDDEN => {
StageOutcome::Denied.record(&span);
ErrorCategory::Access.record(&span);
}
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Internal.record(&span);
}
}
result
}
crates/crank-community-mcp/src/app/tests.rs
+149
View File
@@ -0,0 +1,149 @@
use std::{
io,
sync::{Arc, Mutex},
};
use axum::body::to_bytes;
use crank_core::InvocationStatus;
use crank_observability::{
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
operational_incident_total,
};
use crank_registry::{
InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
};
use serde_json::{Value, json};
use tracing_subscriber::fmt::MakeWriter;
use super::{
ResponseMode, metrics::normalized_mcp_method, observe_invocation_history_outcome,
tool_error_response,
};
use crate::jsonrpc::CURRENT_PROTOCOL_VERSION;
use crate::tool_error::generic_tool_error_contract;
#[tokio::test]
async fn tool_error_response_includes_structured_context() {
let response = tool_error_response(
&json!({"jsonrpc": "2.0", "id": "req-1"}),
ResponseMode::Json,
CURRENT_PROTOCOL_VERSION,
generic_tool_error_contract(
"streaming_payload_error",
"request root must be an object",
"req-1",
false,
Some("Проверьте параметры вызова инструмента."),
),
);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let payload: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(
payload["result"]["structuredContent"]["error"],
json!({
"code": "streaming_payload_error",
"error_code": "streaming_payload_error",
"message": "request root must be an object",
"recoverable": false,
"request_id": "req-1",
"suggested_action": "Проверьте параметры вызова инструмента."
})
);
}
#[test]
fn emits_bounded_history_loss_incident() {
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
observe_invocation_history_outcome(
InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
category: InvocationHistoryLossCategory::Unavailable,
}),
Some("req_mcp_dc08"),
InvocationStatus::Ok,
"agent_tool_call",
);
let output = writer.output();
assert!(!output.contains("dc08-canary-secret"));
let event: Value = output
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.find(|event: &Value| event["event"] == "mcp.invocation_history.lost")
.unwrap();
assert_eq!(event["request_id"], "req_mcp_dc08");
assert_eq!(event["fields"]["source"], "agent_tool_call");
assert_eq!(event["fields"]["invocation_status"], "ok");
assert_eq!(event["fields"]["error_category"], "unavailable");
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
}
#[test]
fn mcp_metric_method_is_always_from_a_closed_set() {
assert_eq!(
normalized_mcp_method(&json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call"})),
"tools_call"
);
assert_eq!(
normalized_mcp_method(
&json!({"jsonrpc": "2.0", "id": 2, "method": "customer-controlled-method"})
),
"unsupported"
);
assert_eq!(
normalized_mcp_method(
&json!({"jsonrpc": "2.0", "method": "customer-controlled-notification"})
),
"notification"
);
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
crates/crank-community-mcp/src/approval_execution.rs
+88 -25
View File
@@ -5,11 +5,13 @@ use axum::{
response::{IntoResponse, Response},
};
use crank_core::{ApprovalRequestStatus, InvocationLevel, InvocationSource, InvocationStatus};
use crank_observability::RequestId;
use crank_registry::{ApprovalRequestRecord, FinishApprovalRequest};
use crank_runtime::{RuntimeExecutionRequest, RuntimeRequestContext};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
use serde_json::json;
use time::OffsetDateTime;
use tracing::warn;
use tracing::{Instrument, warn};
use crate::{
app::{
@@ -35,32 +37,81 @@ pub(super) fn spawn_approval_recovery(state: Arc<AppState>) {
}
async fn recover_approved_requests(state: &Arc<AppState>) {
fail_interrupted_requests(state).await;
for _ in 0..32 {
let now = OffsetDateTime::now_utc();
let approval = match state
let approval = match observe_db_query(
DbOperation::ApprovalWrite,
state
.registry
.claim_next_recoverable_approval_request(
now,
now - RECOVERY_GRACE,
now - EXECUTION_LEASE,
.claim_next_recoverable_approval_request(now, now - RECOVERY_GRACE),
)
.await
{
Ok(Some(approval)) => approval,
Ok(None) => break,
Err(error) => {
warn!(error = %error, "approval recovery query failed");
Err(_) => {
warn!(
name: "mcp.approval_recovery.query_failed",
error_category = "registry",
"approval recovery query failed"
);
break;
}
};
let Some(path) = approval_agent_path(state, &approval).await else {
continue;
};
if execute_approved_request(state, &path, approval)
let recovery_span = Stage::ApprovalRecovery.span();
let result = execute_approved_request(state, &path, approval, None)
.instrument(recovery_span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&recovery_span),
Err(_) => {
StageOutcome::Error.record(&recovery_span);
ErrorCategory::Approval.record(&recovery_span);
}
}
drop(recovery_span);
if result.is_err() {
warn!(
name: "mcp.approval_recovery.execution_failed",
error_category = "runtime",
"recovered approval execution did not finish"
);
}
}
}
async fn fail_interrupted_requests(state: &Arc<AppState>) {
for _ in 0..32 {
let stale_before = OffsetDateTime::now_utc() - EXECUTION_LEASE;
match observe_db_query(
DbOperation::ApprovalWrite,
state
.registry
.fail_next_interrupted_approval_request(stale_before),
)
.await
.is_err()
{
warn!("recovered approval execution did not finish");
Ok(Some(approval)) => {
warn!(
name: "mcp.approval_recovery.interrupted",
approval_id = approval.approval.id.as_str(),
"interrupted approval execution was not retried because its outcome is unknown"
);
}
Ok(None) => break,
Err(_) => {
warn!(
name: "mcp.approval_recovery.interrupted_query_failed",
error_category = "registry",
"interrupted approval recovery query failed"
);
break;
}
}
}
}
@@ -76,8 +127,12 @@ async fn approval_agent_path(
{
Ok(Some(workspace)) => workspace,
Ok(None) => return None,
Err(error) => {
warn!(error = %error, "approval workspace lookup failed");
Err(_) => {
warn!(
name: "mcp.approval_recovery.workspace_lookup_failed",
error_category = "registry",
"approval workspace lookup failed"
);
return None;
}
};
@@ -88,8 +143,12 @@ async fn approval_agent_path(
{
Ok(Some(agent)) => agent,
Ok(None) => return None,
Err(error) => {
warn!(error = %error, "approval agent lookup failed");
Err(_) => {
warn!(
name: "mcp.approval_recovery.agent_lookup_failed",
error_category = "registry",
"approval agent lookup failed"
);
return None;
}
};
@@ -103,7 +162,9 @@ pub(super) async fn execute_approved_request(
state: &Arc<AppState>,
path: &AgentRoutePath,
approval: ApprovalRequestRecord,
request_id: Option<&str>,
) -> Result<ApprovalRequestRecord, Response> {
let request_id = RequestId::resolve(request_id).into_string();
let tools = state
.catalog
.list_tools(&path.workspace_slug, &path.agent_slug)
@@ -123,8 +184,7 @@ pub(super) async fn execute_approved_request(
&approval.approval.request_payload,
);
let started_at = Instant::now();
let runtime_request_context =
RuntimeRequestContext::from_request_id(approval.approval.id.as_str().to_owned())
let runtime_request_context = RuntimeRequestContext::from_request_id(request_id.clone())
.with_response_cache_scope(
tool.workspace_id.as_str().to_owned(),
tool.agent_id.as_str().to_owned(),
@@ -176,11 +236,11 @@ pub(super) async fn execute_approved_request(
),
};
if let Err(error) = persist_invocation(
persist_invocation(
state,
&tool,
InvocationRecord {
request_id: Some(approval.approval.id.as_str()),
request_id: Some(&request_id),
tool_name: &tool.tool_name,
status: invocation_status,
level: invocation_level,
@@ -192,11 +252,10 @@ pub(super) async fn execute_approved_request(
response_preview: response_payload.clone(),
},
)
.await
{
warn!(error = %error, "approved invocation log write failed");
}
.await;
observe_db_query(
DbOperation::ApprovalWrite,
state
.registry
.finish_approval_request(FinishApprovalRequest {
@@ -206,7 +265,8 @@ pub(super) async fn execute_approved_request(
status,
response_payload: Some(response_payload),
decision_note: None,
})
}),
)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
.ok_or_else(|| StatusCode::CONFLICT.into_response())
@@ -216,6 +276,8 @@ async fn finish_unavailable_approval(
state: &Arc<AppState>,
approval: &ApprovalRequestRecord,
) -> Result<ApprovalRequestRecord, Response> {
observe_db_query(
DbOperation::ApprovalWrite,
state
.registry
.finish_approval_request(FinishApprovalRequest {
@@ -230,7 +292,8 @@ async fn finish_unavailable_approval(
}
})),
decision_note: None,
})
}),
)
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
.ok_or_else(|| StatusCode::CONFLICT.into_response())
crates/crank-community-mcp/src/approval_response.rs
+38
View File
@@ -0,0 +1,38 @@
use crank_core::{ApprovalRequest, OperationApprovalPolicy};
use crank_registry::PublishedAgentTool;
use serde_json::{Value, json};
pub(super) fn approval_required_response(
tool: &PublishedAgentTool,
approval: &ApprovalRequest,
policy: &OperationApprovalPolicy,
) -> Value {
let approval_url = format!(
"/v1/{}/{}/approvals/{}",
tool.workspace_slug,
tool.agent_slug,
approval.id.as_str()
);
json!({
"status": "approval_required",
"approval_id": approval.id.as_str(),
"approval_url": approval_url,
"approve": {
"method": "POST",
"url": format!("{approval_url}/approve"),
"body": { "approve": "yes" }
},
"deny": {
"method": "POST",
"url": format!("{approval_url}/deny"),
"body": { "approve": "no" }
},
"expires_at": approval.expires_at,
"risk_level": approval.risk_level,
"payload_preview": if policy.show_payload_preview {
approval.request_payload.clone()
} else {
Value::Null
},
})
}
crates/crank-community-mcp/src/catalog.rs
+122 -32
View File
@@ -1,24 +1,27 @@
use std::{
collections::HashMap,
sync::Arc,
sync::{Arc, Weak},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use crank_core::{CacheScope, CoordinationStateStore, CoordinationStateValue};
use crank_registry::{PostgresRegistry, PublishedAgentCatalog, PublishedAgentTool, RegistryError};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome};
use serde::{Deserialize, Serialize};
use tokio::sync::{Mutex, RwLock};
use tracing::{info, warn};
use tracing::{Instrument, info, warn};
use crate::manifest::analyze_published_tool_catalog;
const MAX_LOCAL_CATALOGS: usize = 1_024;
#[derive(Clone)]
pub struct PublishedToolCatalog {
registry: PostgresRegistry,
refresh_interval: Duration,
coordination_store: Arc<dyn CoordinationStateStore>,
cached: Arc<RwLock<HashMap<CatalogKey, CachedCatalog>>>,
refresh_locks: Arc<Mutex<HashMap<CatalogKey, Arc<Mutex<()>>>>>,
refresh_locks: Arc<Mutex<HashMap<CatalogKey, Weak<Mutex<()>>>>>,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -30,6 +33,14 @@ struct CatalogKey {
struct CachedCatalog {
loaded_at: Option<Instant>,
catalog: PublishedAgentCatalog,
metrics: CatalogMetrics,
}
#[derive(Clone, Copy, Debug, Default)]
struct CatalogMetrics {
tool_count: usize,
estimated_context_tokens: usize,
warning_count: usize,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -66,6 +77,8 @@ impl PublishedToolCatalog {
workspace_slug: &str,
agent_slug: &str,
) -> Result<PublishedAgentCatalog, RegistryError> {
let span = Stage::McpCatalogLoad.span();
let result = async {
self.refresh_if_stale(workspace_slug, agent_slug).await?;
let guard = self.cached.read().await;
guard
@@ -76,6 +89,18 @@ impl PublishedToolCatalog {
agent_slug: agent_slug.to_owned(),
})
}
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Catalog.record(&span);
}
}
drop(span);
result
}
async fn refresh_if_stale(
&self,
@@ -98,11 +123,14 @@ impl PublishedToolCatalog {
let refresh_lock = {
let mut locks = self.refresh_locks.lock().await;
Arc::clone(
locks
.entry(key.clone())
.or_insert_with(|| Arc::new(Mutex::new(()))),
)
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
lock
} else {
let lock = Arc::new(Mutex::new(()));
locks.insert(key.clone(), Arc::downgrade(&lock));
lock
}
};
let _refresh_guard = refresh_lock.lock().await;
let still_stale = {
@@ -117,50 +145,59 @@ impl PublishedToolCatalog {
}
if let Some((catalog, age)) = self.load_shared_snapshot(workspace_slug, agent_slug).await {
let metrics =
log_catalog_analysis(workspace_slug, agent_slug, "shared_cache", &catalog.tools);
let mut guard = self.cached.write().await;
guard.insert(
self.store_local_catalog(
key,
CachedCatalog {
loaded_at: Instant::now().checked_sub(age),
catalog,
metrics,
},
);
)
.await;
return Ok(());
}
let catalog = match self
let db_span = Stage::DbQuery
.db_span(DbOperation::CatalogLoad)
.expect("database stage");
let catalog_result = self
.registry
.get_published_agent_catalog_by_slug(workspace_slug, agent_slug)
.await
{
.instrument(db_span.clone())
.await;
let catalog = match catalog_result {
Ok(catalog) => catalog,
Err(error) => return Err(error),
Err(error) => {
StageOutcome::Error.record(&db_span);
ErrorCategory::Database.record(&db_span);
drop(db_span);
return Err(error);
}
};
log_catalog_analysis(workspace_slug, agent_slug, "postgres", &catalog.tools);
StageOutcome::Success.record(&db_span);
drop(db_span);
let metrics = log_catalog_analysis(workspace_slug, agent_slug, "postgres", &catalog.tools);
self.store_shared_snapshot(workspace_slug, agent_slug, &catalog)
.await;
let mut guard = self.cached.write().await;
let previous_count = guard
.get(&key)
.map(|entry| entry.catalog.tools.len())
.unwrap_or_default();
guard.insert(
let published_tool_count = catalog.tools.len();
let previous_count = self
.store_local_catalog(
key,
CachedCatalog {
loaded_at: Some(Instant::now()),
catalog,
metrics,
},
);
)
.await;
info!(
name: "mcp.catalog.refreshed",
workspace_slug,
agent_slug,
published_tool_count = guard
.get(&CatalogKey::new(workspace_slug, agent_slug))
.map(|entry| entry.catalog.tools.len())
.unwrap_or_default(),
published_tool_count,
previous_published_tool_count = previous_count,
"published agent catalog refreshed"
);
@@ -168,6 +205,27 @@ impl PublishedToolCatalog {
Ok(())
}
async fn store_local_catalog(&self, key: CatalogKey, entry: CachedCatalog) -> usize {
let mut guard = self.cached.write().await;
let previous_count = guard
.get(&key)
.map(|current| current.catalog.tools.len())
.unwrap_or_default();
if guard.len() >= MAX_LOCAL_CATALOGS && !guard.contains_key(&key) {
let oldest = guard
.iter()
.min_by_key(|(_, current)| current.loaded_at)
.map(|(candidate, _)| candidate.clone());
if let Some(oldest) = oldest {
guard.remove(&oldest);
}
}
guard.insert(key, entry);
record_catalog_metrics(guard.values().map(|entry| entry.metrics));
previous_count
}
async fn load_shared_snapshot(
&self,
workspace_slug: &str,
@@ -226,12 +284,19 @@ fn log_catalog_analysis(
agent_slug: &str,
source: &str,
tools: &[PublishedAgentTool],
) {
) -> CatalogMetrics {
let analysis = match analyze_published_tool_catalog(tools) {
Ok(analysis) => analysis,
Err(error) => {
warn!(workspace_slug, agent_slug, source, %error, "published catalog analysis failed");
return;
Err(_) => {
warn!(
name: "mcp.catalog.analysis_failed",
workspace_slug,
agent_slug,
source,
error_category = "catalog_validation",
"published catalog analysis failed"
);
return CatalogMetrics::default();
}
};
let warning_count = analysis
@@ -242,6 +307,7 @@ fn log_catalog_analysis(
.count();
info!(
name: "mcp.catalog.analyzed",
workspace_slug,
agent_slug,
source,
@@ -255,6 +321,30 @@ fn log_catalog_analysis(
catalog_quality_warning_count = warning_count,
"published agent catalog analyzed"
);
CatalogMetrics {
tool_count: analysis.budget.tool_count,
estimated_context_tokens: analysis.budget.estimated_context_tokens,
warning_count,
}
}
fn record_catalog_metrics(metrics: impl Iterator<Item = CatalogMetrics>) {
let aggregate = metrics.fold(CatalogMetrics::default(), |mut aggregate, current| {
aggregate.tool_count = aggregate.tool_count.saturating_add(current.tool_count);
aggregate.estimated_context_tokens = aggregate
.estimated_context_tokens
.saturating_add(current.estimated_context_tokens);
aggregate.warning_count = aggregate
.warning_count
.saturating_add(current.warning_count);
aggregate
});
metrics::gauge!("crank_catalog_tools").set(aggregate.tool_count as f64);
metrics::gauge!("crank_catalog_estimated_context_tokens")
.set(aggregate.estimated_context_tokens as f64);
metrics::gauge!("crank_catalog_warnings").set(aggregate.warning_count as f64);
}
fn now_unix_ms() -> u64 {
crates/crank-community-mcp/src/lib.rs
+5 -1
View File
@@ -1,14 +1,18 @@
mod access;
mod app;
mod approval_execution;
mod approval_response;
pub mod auth;
pub mod catalog;
pub mod jsonrpc;
pub mod manifest;
mod rate_limit;
mod request_context;
pub mod session;
pub mod tool_error;
mod tool_search;
mod transport;
pub use app::{build_app, build_app_with_background_workers};
pub use app::{
build_app, build_app_with_background_workers, build_app_with_background_workers_and_limits,
};
crates/crank-community-mcp/src/rate_limit.rs
+26 -12
View File
@@ -4,7 +4,7 @@ use axum::{
http::{HeaderMap, HeaderValue, StatusCode, header::RETRY_AFTER},
response::{IntoResponse, Response},
};
use crank_runtime::RateLimitRejection;
use crank_runtime::RateLimitCheckError;
use serde_json::{Value, json};
use crate::{
@@ -14,19 +14,11 @@ use crate::{
transport::{ResponseMode, session_id_from_headers, transport_response},
};
pub(super) async fn enforce_post_rate_limit(
state: &Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
) -> Result<(), RateLimitRejection> {
enforce_transport_rate_limit(state, path, headers).await
}
pub(super) async fn enforce_transport_rate_limit(
state: &Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
) -> Result<(), RateLimitRejection> {
) -> Result<(), RateLimitCheckError> {
let key = rate_limit_key(path, headers);
state.api_rate_limiter.check(&key).await
}
@@ -35,8 +27,25 @@ pub(super) fn rate_limited_jsonrpc_response(
message: &Value,
response_mode: ResponseMode,
protocol_version: &str,
rejection: RateLimitRejection,
error: RateLimitCheckError,
) -> Response {
let RateLimitCheckError::Rejected(rejection) = error else {
return transport_response(
StatusCode::SERVICE_UNAVAILABLE,
json!({
"jsonrpc": "2.0",
"id": request_id(message),
"error": {
"code": -32603,
"message": "rate limit service unavailable",
"data": { "code": "rate_limit_unavailable" }
}
}),
response_mode,
None,
Some(protocol_version),
);
};
let payload = json!({
"jsonrpc": "2.0",
"id": request_id(message),
@@ -61,10 +70,15 @@ pub(super) fn rate_limited_jsonrpc_response(
response
}
pub(super) fn rate_limited_status_response(rejection: RateLimitRejection) -> Response {
pub(super) fn rate_limited_status_response(error: RateLimitCheckError) -> Response {
match error {
RateLimitCheckError::Rejected(rejection) => {
let mut response = StatusCode::TOO_MANY_REQUESTS.into_response();
attach_retry_after_header(&mut response, rejection.retry_after_ms);
response
}
RateLimitCheckError::StoreUnavailable => StatusCode::SERVICE_UNAVAILABLE.into_response(),
}
}
fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String {
crates/crank-community-mcp/src/request_context.rs
+39
View File
@@ -0,0 +1,39 @@
use axum::{extract::Request, http::HeaderValue, middleware::Next, response::Response};
use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
use tracing::{Instrument, info_span};
use crate::transport::HEADER_X_REQUEST_ID;
#[derive(Clone, Debug)]
pub(super) struct RequestContext {
pub(super) request_id: String,
}
pub(super) async fn apply_request_context(mut request: Request, next: Next) -> Response {
let request_id = RequestId::resolve(
request
.headers()
.get(&HEADER_X_REQUEST_ID)
.and_then(|value| value.to_str().ok()),
)
.into_string();
let context = RequestContext {
request_id: request_id.clone(),
};
let span = info_span!(
target: "crank::trace",
"mcp.request",
request_id = %request_id,
);
set_remote_trace_parent(&span, request.headers());
request.extensions_mut().insert(context);
with_request_correlation(request_id.clone(), async move {
let mut response = next.run(request).instrument(span).await;
if let Ok(value) = HeaderValue::from_str(&request_id) {
response.headers_mut().insert(HEADER_X_REQUEST_ID, value);
}
response
})
.await
}
crates/crank-community-mcp/src/session.rs
+103 -7
View File
@@ -52,6 +52,8 @@ pub trait TransportSessionStore: Send + Sync {
) -> Result<bool, SessionStoreError>;
async fn delete(&self, session_id: &str) -> Result<bool, SessionStoreError>;
async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError>;
}
pub type SharedSessionStore = Arc<dyn TransportSessionStore>;
@@ -62,6 +64,11 @@ pub struct PostgresTransportSessionStore {
}
impl PostgresTransportSessionStore {
pub async fn from_pool(pool: PgPool) -> Result<Self, SessionStoreError> {
apply_postgres_migrations(&pool).await?;
Ok(Self { pool })
}
pub async fn connect_with_options_and_pool_config(
connect_options: PgConnectOptions,
pool_config: PostgresPoolConfig,
@@ -84,9 +91,7 @@ impl PostgresTransportSessionStore {
details: error.to_string(),
})?;
apply_postgres_migrations(&pool).await?;
Ok(Self { pool })
Self::from_pool(pool).await
}
}
@@ -164,6 +169,13 @@ impl TransportSessionStore for InMemorySessionStore {
let mut guard = self.inner.write().await;
Ok(guard.remove(session_id).is_some())
}
async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError> {
let mut guard = self.inner.write().await;
let before = guard.len();
guard.retain(|_, session| !is_expired(session, now));
Ok(u64::try_from(before.saturating_sub(guard.len())).unwrap_or(u64::MAX))
}
}
#[async_trait]
@@ -292,9 +304,68 @@ impl TransportSessionStore for PostgresTransportSessionStore {
Ok(result.rows_affected() > 0)
}
async fn cleanup_expired(&self, now: OffsetDateTime) -> Result<u64, SessionStoreError> {
let result = query(
"delete from mcp_transport_sessions
where expires_at is not null and expires_at <= $1::timestamptz",
)
.bind(now)
.execute(&self.pool)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
Ok(result.rows_affected())
}
}
async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreError> {
let mut transaction = pool.begin().await.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
query("select pg_advisory_xact_lock($1)")
.bind(0x4352_414E_4B4D_4350_i64)
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
query(
"create table if not exists __crank_mcp_migrations (
version integer primary key,
checksum text not null,
applied_at timestamptz not null default now()
)",
)
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
let applied = query("select checksum from __crank_mcp_migrations where version = 1")
.fetch_optional(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
if let Some(row) = applied {
let checksum = row.get::<String, _>("checksum");
if checksum != "mcp-transport-sessions-v1" {
return Err(SessionStoreError {
details: format!("modified MCP migration version 1: {checksum}"),
});
}
transaction
.commit()
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
return Ok(());
}
query(
"create table if not exists mcp_transport_sessions (
id text primary key,
@@ -308,14 +379,14 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
expires_at timestamptz null
)",
)
.execute(pool)
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
query("alter table mcp_transport_sessions add column if not exists supports_elicitation boolean not null default false")
.execute(pool)
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
@@ -324,7 +395,7 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
query(
"alter table mcp_transport_sessions add column if not exists expires_at timestamptz null",
)
.execute(pool)
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
@@ -334,7 +405,32 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro
"create index if not exists mcp_transport_sessions_workspace_agent_idx
on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc)",
)
.execute(pool)
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
query(
"create index if not exists mcp_transport_sessions_expires_at_idx
on mcp_transport_sessions(expires_at)
where expires_at is not null",
)
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
query("insert into __crank_mcp_migrations (version, checksum) values (1, $1)")
.bind("mcp-transport-sessions-v1")
.execute(&mut *transaction)
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
})?;
transaction
.commit()
.await
.map_err(|error| SessionStoreError {
details: error.to_string(),
crates/crank-community-mcp/src/tool_error.rs
+27
View File
@@ -86,6 +86,10 @@ pub fn runtime_error_code(error: &RuntimeError) -> &'static str {
RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
RuntimeError::MissingAuthProfile { .. } => "auth_profile_not_found",
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
"secret_not_found"
@@ -146,6 +150,19 @@ fn safe_runtime_error_message(error: &RuntimeError) -> String {
RuntimeError::ConfirmationStoreUnavailable { .. } => {
"Хранилище подтверждений временно недоступно.".to_owned()
}
RuntimeError::IdempotencyStoreUnavailable { .. } => {
"Хранилище идемпотентности временно недоступно.".to_owned()
}
RuntimeError::IdempotencyInProgress { .. } => {
"Операция с этим ключом идемпотентности уже выполняется.".to_owned()
}
RuntimeError::IdempotencyConflict { .. } => {
"Ключ идемпотентности уже использован с другими параметрами.".to_owned()
}
RuntimeError::IdempotencyOutcomeUnknown { .. } => {
"Результат предыдущего выполнения неизвестен; автоматический повтор заблокирован."
.to_owned()
}
RuntimeError::MissingAuthProfile { .. } => "Профиль авторизации не найден.".to_owned(),
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
"Секрет авторизации не найден.".to_owned()
@@ -169,6 +186,8 @@ fn is_recoverable_runtime_error(error: &RuntimeError) -> bool {
| RuntimeError::ConcurrencyLimitExceeded { .. }
| RuntimeError::SecretCrypto { .. }
| RuntimeError::ConfirmationRequired { .. }
| RuntimeError::IdempotencyStoreUnavailable { .. }
| RuntimeError::IdempotencyInProgress { .. }
)
}
@@ -198,6 +217,14 @@ fn suggested_action(error: &RuntimeError) -> Option<&'static str> {
Some("Запросите новый токен подтверждения.")
}
RuntimeError::ConfirmationStoreUnavailable { .. } => Some("Повторите запрос позже."),
RuntimeError::IdempotencyStoreUnavailable { .. }
| RuntimeError::IdempotencyInProgress { .. } => Some("Повторите запрос позже."),
RuntimeError::IdempotencyConflict { .. } => {
Some("Используйте новый ключ идемпотентности для изменённого запроса.")
}
RuntimeError::IdempotencyOutcomeUnknown { .. } => {
Some("Проверьте результат во внешней системе перед ручным повтором.")
}
RuntimeError::MissingAuthProfile { .. }
| RuntimeError::MissingSecret { .. }
| RuntimeError::MissingSecretVersion { .. }
crates/crank-community-mcp/src/tool_search.rs
+14 -1
View File
@@ -3,6 +3,7 @@ use std::{collections::BTreeSet, sync::Arc};
use axum::{http::StatusCode, response::Response};
use crank_core::{ToolAccessMode, search_tool_catalog};
use crank_registry::PublishedAgentCatalog;
use crank_trace::{ErrorCategory, Stage, StageOutcome};
use serde::Deserialize;
use serde_json::{Value, json};
@@ -127,13 +128,25 @@ async fn execute_catalog_tool(
mut arguments: Value,
transport_request_id: &str,
) -> Response {
let Some(resolved) = resolve_generated_tool(&catalog.tools, tool_name) else {
let resolve_span = Stage::McpToolsResolve.span();
let resolved = resolve_span.in_scope(|| resolve_generated_tool(&catalog.tools, tool_name));
let resolved = match resolved {
Some(resolved) => {
StageOutcome::Success.record(&resolve_span);
drop(resolve_span);
resolved
}
None => {
StageOutcome::Error.record(&resolve_span);
ErrorCategory::Catalog.record(&resolve_span);
drop(resolve_span);
return tool_not_found_response(
message,
response_mode,
&session.protocol_version,
tool_name,
);
}
};
let confirmation_token = take_confirmation_token(&mut arguments);
handle_tool_call(
crates/crank-community-mcp/src/transport.rs
-19
View File
@@ -22,7 +22,6 @@ use crate::jsonrpc::{
pub(super) const HEADER_MCP_SESSION_ID: &str = "MCP-Session-Id";
pub(super) const HEADER_MCP_PROTOCOL_VERSION: &str = "MCP-Protocol-Version";
pub(super) const HEADER_X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id");
const MAX_REQUEST_ID_LEN: usize = 128;
#[derive(Clone, Copy)]
pub(super) enum ResponseMode {
@@ -303,24 +302,6 @@ where
response
}
pub(super) fn resolve_request_id(headers: &HeaderMap) -> String {
headers
.get(&HEADER_X_REQUEST_ID)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| is_valid_request_id(value))
.map(ToOwned::to_owned)
.unwrap_or_else(|| uuid::Uuid::now_v7().to_string())
}
pub(super) fn is_valid_request_id(value: &str) -> bool {
!value.is_empty()
&& value.len() <= MAX_REQUEST_ID_LEN
&& value
.bytes()
.all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
}
pub(super) fn extract_origin(url: &str) -> Option<String> {
Some(parse_origin(url, false)?.origin().ascii_serialization())
}
crates/crank-community-mcp/tests/integration/session.rs
+38
View File
@@ -97,3 +97,41 @@ async fn postgres_transport_sessions_evict_expired_rows_on_read() {
assert_eq!(remaining, 0);
}
#[tokio::test]
async fn postgres_transport_session_cleanup_removes_abandoned_expired_rows() {
let database_url = crank_test_support::postgres_schema_url("test_mcp_cleanup").await;
let store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
database_url.parse::<PgConnectOptions>().unwrap(),
PostgresPoolConfig::default(),
)
.await
.unwrap();
let now = OffsetDateTime::now_utc();
store
.create(
"2025-11-25",
"default",
"sales",
false,
now - time::Duration::hours(2),
Some(now - time::Duration::hours(1)),
)
.await
.unwrap();
let active = store
.create(
"2025-11-25",
"default",
"sales",
false,
now,
Some(now + time::Duration::hours(1)),
)
.await
.unwrap();
assert_eq!(store.cleanup_expired(now).await.unwrap(), 1);
assert!(store.get(&active).await.unwrap().is_some());
}
crates/crank-community-mcp/tests/unit/session.rs
+25
View File
@@ -72,6 +72,31 @@ async fn drops_expired_in_memory_transport_sessions_on_read() {
assert!(store.get(&session_id).await.unwrap().is_none());
}
#[tokio::test]
async fn cleanup_removes_only_expired_sessions() {
let store = InMemorySessionStore::default();
let now = time::OffsetDateTime::now_utc();
let expired = store
.create("2025-11-25", "default", "sales", false, now, Some(now))
.await
.unwrap();
let active = store
.create(
"2025-11-25",
"default",
"sales",
false,
now,
Some(now + time::Duration::hours(1)),
)
.await
.unwrap();
assert_eq!(store.cleanup_expired(now).await.unwrap(), 1);
assert!(store.get(&expired).await.unwrap().is_none());
assert!(store.get(&active).await.unwrap().is_some());
}
#[test]
fn formats_transport_session_store_error() {
let error = SessionStoreError {
crates/crank-core/Cargo.toml
+1
View File
@@ -3,6 +3,7 @@ name = "crank-core"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[dependencies]
crates/crank-core/src/cache.rs
+40
View File
@@ -74,6 +74,12 @@ pub struct RateLimitBucketState {
pub last_refill_unix_ms: i64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RateLimitDecision {
Allowed,
Rejected { retry_after_ms: u64 },
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReplayGuardStatus {
@@ -86,6 +92,12 @@ pub struct CoordinationStateValue {
pub payload: Value,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum CoordinationStateReservation {
Reserved,
Existing(CoordinationStateValue),
}
#[async_trait]
pub trait ResponseCacheStore: Send + Sync {
async fn get(&self, key: &str) -> Result<Option<CachedResponse>, CacheStoreError>;
@@ -108,6 +120,14 @@ pub trait RateLimitStateStore: Send + Sync {
ttl: Duration,
) -> Result<(), CacheStoreError>;
async fn delete_bucket(&self, key: &str) -> Result<(), CacheStoreError>;
async fn consume_token(
&self,
key: &str,
burst_tokens_micros: u64,
refill_per_second_micros: u64,
now_unix_ms: i64,
ttl: Duration,
) -> Result<RateLimitDecision, CacheStoreError>;
}
#[async_trait]
@@ -135,6 +155,26 @@ pub trait CoordinationStateStore: Send + Sync {
ttl: Duration,
) -> Result<(), CacheStoreError>;
async fn delete_value(&self, scope: CacheScope, key: &str) -> Result<(), CacheStoreError>;
async fn take_value(
&self,
scope: CacheScope,
key: &str,
) -> Result<Option<CoordinationStateValue>, CacheStoreError>;
async fn reserve_value(
&self,
scope: CacheScope,
key: &str,
value: CoordinationStateValue,
ttl: Duration,
) -> Result<CoordinationStateReservation, CacheStoreError>;
async fn compare_and_set_value(
&self,
scope: CacheScope,
key: &str,
expected: &CoordinationStateValue,
value: CoordinationStateValue,
ttl: Duration,
) -> Result<bool, CacheStoreError>;
}
#[derive(Debug, Error, PartialEq, Eq)]
crates/crank-core/src/lib.rs
+6 -4
View File
@@ -66,8 +66,9 @@ pub mod domain {
pub mod ports {
pub use crate::cache::{
CacheStoreError, CoordinationStateStore, CoordinationStateValue, RateLimitStateStore,
ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
CacheStoreError, CoordinationStateReservation, CoordinationStateStore,
CoordinationStateValue, RateLimitDecision, RateLimitStateStore, ReplayGuardStatus,
ReplayGuardStore, ResponseCacheStore,
};
pub use crate::ext::access::{
OwnerOnlyPolicyEngine, PolicyAction, PolicyDecision, PolicyEngine, PolicyScope,
@@ -108,8 +109,9 @@ pub use auth::{
};
pub use cache::{
CacheBackend, CacheScope, CacheStoreError, CachedHeader, CachedResponse,
CoordinationStateStore, CoordinationStateValue, ParseCacheBackendError, RateLimitBucketState,
RateLimitStateStore, ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue,
ParseCacheBackendError, RateLimitBucketState, RateLimitDecision, RateLimitStateStore,
ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
};
pub use edition::{
EditionCapabilities, EditionLimits, MachineAccessMode, OperationSecurityLevel, ProductEdition,
crates/crank-import/Cargo.toml
+1
View File
@@ -3,6 +3,7 @@ name = "crank-import"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[dependencies]
crates/crank-mapping/Cargo.toml
+1
View File
@@ -3,6 +3,7 @@ name = "crank-mapping"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[dependencies]
crates/crank-observability/Cargo.toml
+35
View File
@@ -0,0 +1,35 @@
[package]
name = "crank-observability"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[dependencies]
axum.workspace = true
metrics.workspace = true
metrics-exporter-prometheus.workspace = true
opentelemetry.workspace = true
opentelemetry-otlp.workspace = true
opentelemetry_sdk.workspace = true
percent-encoding.workspace = true
serde.workspace = true
serde_json.workspace = true
sentry.workspace = true
sha2.workspace = true
subtle.workspace = true
thiserror.workspace = true
time.workspace = true
tokio = { workspace = true, features = ["net"] }
tracing.workspace = true
tracing-opentelemetry.workspace = true
tracing-subscriber.workspace = true
url.workspace = true
uuid.workspace = true
[dev-dependencies]
opentelemetry-proto.workspace = true
prost.workspace = true
sentry = { workspace = true, features = ["test"] }
tower.workspace = true
crates/crank-observability/src/config.rs
+170
View File
@@ -0,0 +1,170 @@
use std::env;
use thiserror::Error;
use crate::RedactionLimits;
const DEFAULT_ENVIRONMENT: &str = "development";
const MAX_IDENTITY_LABEL_BYTES: usize = 64;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ServiceIdentity {
service: String,
version: String,
environment: String,
}
impl ServiceIdentity {
pub fn try_new(
service: impl Into<String>,
version: impl Into<String>,
environment: impl Into<String>,
) -> Result<Self, ObservabilityConfigError> {
let identity = Self {
service: service.into(),
version: version.into(),
environment: environment.into(),
};
validate_label("service", &identity.service)?;
validate_label("version", &identity.version)?;
validate_label("environment", &identity.environment)?;
Ok(identity)
}
pub fn service(&self) -> &str {
&self.service
}
pub fn version(&self) -> &str {
&self.version
}
pub fn environment(&self) -> &str {
&self.environment
}
}
#[derive(Clone, Debug)]
pub struct ObservabilityConfig {
identity: ServiceIdentity,
filter: String,
redaction_limits: RedactionLimits,
}
impl ObservabilityConfig {
pub fn new(
identity: ServiceIdentity,
filter: impl Into<String>,
redaction_limits: RedactionLimits,
) -> Self {
Self {
identity,
filter: filter.into(),
redaction_limits,
}
}
pub fn from_env(
service: &'static str,
version: &'static str,
default_filter: &'static str,
) -> Result<Self, ObservabilityConfigError> {
let environment = env_value_or_default(
"CRANK_ENVIRONMENT",
env::var("CRANK_ENVIRONMENT"),
DEFAULT_ENVIRONMENT,
)?;
let filter = env_value_or_default(
"CRANK_LOG_LEVEL",
env::var("CRANK_LOG_LEVEL"),
default_filter,
)?;
let identity = ServiceIdentity::try_new(service, version, environment)?;
Ok(Self::new(identity, filter, RedactionLimits::default()))
}
pub(crate) fn into_parts(self) -> (ServiceIdentity, String, RedactionLimits) {
(self.identity, self.filter, self.redaction_limits)
}
pub(crate) fn identity(&self) -> &ServiceIdentity {
&self.identity
}
pub(crate) fn redaction_limits(&self) -> RedactionLimits {
self.redaction_limits
}
}
#[derive(Debug, Error)]
pub enum ObservabilityConfigError {
#[error("invalid observability identity field: {field}")]
InvalidIdentity { field: &'static str },
#[error("observability environment variable is not valid UTF-8: {field}")]
InvalidEnvironmentEncoding { field: &'static str },
}
fn env_value_or_default(
field: &'static str,
value: Result<String, env::VarError>,
default: &'static str,
) -> Result<String, ObservabilityConfigError> {
match value {
Ok(value) => Ok(value),
Err(env::VarError::NotPresent) => Ok(default.to_owned()),
Err(env::VarError::NotUnicode(_)) => {
Err(ObservabilityConfigError::InvalidEnvironmentEncoding { field })
}
}
}
fn validate_label(field: &'static str, value: &str) -> Result<(), ObservabilityConfigError> {
let valid = !value.is_empty()
&& value.len() <= MAX_IDENTITY_LABEL_BYTES
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+'));
if valid {
Ok(())
} else {
Err(ObservabilityConfigError::InvalidIdentity { field })
}
}
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use super::{ObservabilityConfigError, ServiceIdentity, env_value_or_default};
#[test]
fn accepts_release_and_environment_labels() {
let identity = ServiceIdentity::try_new("admin-api", "0.3.1+build.7", "production")
.expect("identity must be valid");
assert_eq!(identity.service(), "admin-api");
assert_eq!(identity.version(), "0.3.1+build.7");
assert_eq!(identity.environment(), "production");
}
#[test]
fn rejects_non_utf8_environment_values() {
let error = env_value_or_default(
"CRANK_ENVIRONMENT",
Err(std::env::VarError::NotUnicode(OsString::from(
"invalid-environment",
))),
"development",
)
.expect_err("non-UTF-8 values must not be replaced with defaults");
assert!(matches!(
error,
ObservabilityConfigError::InvalidEnvironmentEncoding {
field: "CRANK_ENVIRONMENT"
}
));
}
}
crates/crank-observability/src/correlation.rs
+39
View File
@@ -0,0 +1,39 @@
use std::fmt;
use uuid::Uuid;
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct RequestId(String);
impl RequestId {
pub const MAX_LEN: usize = 128;
pub fn resolve(candidate: Option<&str>) -> Self {
candidate
.filter(|value| Self::is_valid(value))
.map(|value| Self(value.to_owned()))
.unwrap_or_else(|| Self(Uuid::now_v7().to_string()))
}
pub fn is_valid(value: &str) -> bool {
!value.is_empty()
&& value.len() <= Self::MAX_LEN
&& value
.bytes()
.all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
}
pub fn as_str(&self) -> &str {
&self.0
}
pub fn into_string(self) -> String {
self.0
}
}
impl fmt::Display for RequestId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
crates/crank-observability/src/error_reporting.rs
+467
View File
@@ -0,0 +1,467 @@
use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration};
use sentry::{
ClientInitGuard, ClientOptions,
protocol::{Event, Level},
types::Dsn,
};
use thiserror::Error;
use crate::{
RedactionLimits, ServiceIdentity, propagation::current_trace_id, redaction::truncate_string,
};
const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN";
const CRITICAL_ERROR_MESSAGE: &str = "critical error";
const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
tokio::task_local! {
static REQUEST_ID: String;
}
pub struct SentryConfig {
dsn: Option<Dsn>,
}
impl SentryConfig {
pub fn parse(value: Option<&str>) -> Result<Self, SentryConfigError> {
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(Self { dsn: None });
};
let dsn = value
.parse::<Dsn>()
.map_err(|_| SentryConfigError::InvalidDsn)?;
Ok(Self { dsn: Some(dsn) })
}
pub fn from_env() -> Result<Self, SentryConfigError> {
match env::var(SENTRY_DSN_ENV) {
Ok(value) => Self::parse(Some(&value)),
Err(env::VarError::NotPresent) => Self::parse(None),
Err(env::VarError::NotUnicode(_)) => Err(SentryConfigError::InvalidEnvironmentEncoding),
}
}
pub fn enabled(&self) -> bool {
self.dsn.is_some()
}
}
impl fmt::Debug for SentryConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("SentryConfig")
.field("enabled", &self.enabled())
.finish()
}
}
#[derive(Debug, Error)]
pub enum SentryConfigError {
#[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")]
InvalidDsn,
#[error("CRANK_SENTRY_DSN is not valid UTF-8")]
InvalidEnvironmentEncoding,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CriticalErrorCategory {
Panic,
Startup,
Internal,
DataIntegrity,
}
impl CriticalErrorCategory {
pub const fn as_str(self) -> &'static str {
match self {
Self::Panic => "panic",
Self::Startup => "startup",
Self::Internal => "internal",
Self::DataIntegrity => "data_integrity",
}
}
fn parse(value: &str) -> Option<Self> {
match value {
"panic" => Some(Self::Panic),
"startup" => Some(Self::Startup),
"internal" => Some(Self::Internal),
"data_integrity" => Some(Self::DataIntegrity),
_ => None,
}
}
}
pub fn capture_critical_error(category: CriticalErrorCategory) {
let mut tags = correlation_tags();
tags.insert("category".to_owned(), category.as_str().to_owned());
sentry::capture_event(Event {
level: Level::Error,
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
tags,
..Event::default()
});
}
pub async fn with_request_correlation<F>(request_id: String, future: F) -> F::Output
where
F: Future,
{
REQUEST_ID.scope(request_id, future).await
}
pub(crate) fn init_sentry(
identity: &ServiceIdentity,
limits: RedactionLimits,
config: SentryConfig,
) -> Option<ClientInitGuard> {
let dsn = config.dsn?;
let identity = identity.clone();
let options = client_options(identity, limits);
Some(sentry::init((dsn, options)))
}
fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions {
let release = identity.version().to_owned();
let environment = identity.environment().to_owned();
let service = identity.service().to_owned();
let sanitizer_identity = identity.clone();
let mut options = ClientOptions::default();
options.release = Some(Cow::Owned(release));
options.environment = Some(Cow::Owned(environment));
options.server_name = Some(Cow::Owned(service));
options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled;
options.max_breadcrumbs = 0;
options.attach_stacktrace = false;
options.send_default_pii = false;
options.before_send = Some(std::sync::Arc::new(move |event| {
Some(sanitize_event(event, &sanitizer_identity, limits))
}));
options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT;
options.auto_session_tracking = false;
options.enable_logs = false;
options.enable_metrics = false;
options
}
fn sanitize_event(
event: Event<'static>,
identity: &ServiceIdentity,
limits: RedactionLimits,
) -> Event<'static> {
let category = event
.tags
.get("category")
.and_then(|value| CriticalErrorCategory::parse(value))
.unwrap_or_else(|| {
if event.exception.is_empty() {
CriticalErrorCategory::Internal
} else {
CriticalErrorCategory::Panic
}
});
let mut tags = correlation_tags()
.into_iter()
.map(|(key, value)| (key, truncate_string(&value, limits.max_string_bytes)))
.collect::<BTreeMap<_, _>>();
for key in ["request_id", "trace_id"] {
if let Some(value) = event.tags.get(key) {
tags.entry(key.to_owned())
.or_insert_with(|| truncate_string(value, limits.max_string_bytes));
}
}
tags.insert("service".to_owned(), identity.service().to_owned());
tags.insert("category".to_owned(), category.as_str().to_owned());
enforce_event_budget(
Event {
event_id: event.event_id,
level: Level::Error,
fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]),
message: Some(CRITICAL_ERROR_MESSAGE.to_owned()),
timestamp: event.timestamp,
server_name: Some(Cow::Owned(identity.service().to_owned())),
release: Some(Cow::Owned(identity.version().to_owned())),
environment: Some(Cow::Owned(identity.environment().to_owned())),
tags,
..Event::default()
},
limits.max_event_bytes,
)
}
fn correlation_tags() -> BTreeMap<String, String> {
let mut tags = BTreeMap::new();
if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) {
tags.insert("request_id".to_owned(), request_id);
}
if let Some(trace_id) = current_trace_id() {
tags.insert("trace_id".to_owned(), trace_id);
}
tags
}
fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Event<'static> {
if serialized_event_len(&event) <= max_event_bytes {
return event;
}
event.tags.remove("request_id");
event.tags.remove("trace_id");
event
}
fn serialized_event_len(event: &Event<'_>) -> usize {
serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len())
}
#[cfg(test)]
mod tests {
use std::{
collections::BTreeMap,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_sdk::trace::SdkTracerProvider;
use sentry::{
Envelope, Hub,
protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values},
};
use super::{
CriticalErrorCategory, capture_critical_error, client_options, sanitize_event,
with_request_correlation,
};
use crate::{
ObservabilityConfig, RedactionLimits, ServiceIdentity,
logging::build_subscriber_with_tracer,
};
fn identity() -> ServiceIdentity {
ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity")
}
#[test]
fn client_disables_non_error_telemetry() {
let options = client_options(identity(), RedactionLimits::default());
assert_eq!(options.max_breadcrumbs, 0);
assert!(!options.attach_stacktrace);
assert!(!options.send_default_pii);
assert!(!options.auto_session_tracking);
assert!(!options.enable_logs);
assert!(!options.enable_metrics);
assert!(options.before_send.is_some());
}
#[test]
fn sanitizer_uses_a_strict_allowlist() {
let mut tags = BTreeMap::new();
tags.insert("category".to_owned(), "data_integrity".to_owned());
tags.insert("secret".to_owned(), "must-not-leak".to_owned());
let mut contexts = BTreeMap::new();
contexts.insert(
"secret".to_owned(),
Context::Other(BTreeMap::from([(
"token".to_owned(),
Value::String("must-not-leak".to_owned()),
)])),
);
let event = sentry::protocol::Event {
message: Some("password=must-not-leak".to_owned()),
request: Some(Request::default()),
user: Some(User::default()),
breadcrumbs: Values {
values: vec![Breadcrumb::default()],
},
exception: Values {
values: vec![Exception {
value: Some("must-not-leak".to_owned()),
..Exception::default()
}],
},
contexts,
extra: BTreeMap::from([(
"payload".to_owned(),
Value::String("must-not-leak".to_owned()),
)]),
tags,
..sentry::protocol::Event::default()
};
let cleaned = sanitize_event(event, &identity(), RedactionLimits::default());
let serialized = serde_json::to_string(&cleaned).expect("serialize event");
assert_eq!(cleaned.message.as_deref(), Some("critical error"));
assert_eq!(
cleaned.tags.get("category").map(String::as_str),
Some(CriticalErrorCategory::DataIntegrity.as_str())
);
assert!(cleaned.request.is_none());
assert!(cleaned.user.is_none());
assert!(cleaned.breadcrumbs.is_empty());
assert!(cleaned.exception.is_empty());
assert!(cleaned.contexts.is_empty());
assert!(cleaned.extra.is_empty());
assert!(!serialized.contains("must-not-leak"));
assert!(!serialized.contains("password"));
}
#[test]
fn sanitizer_honours_total_event_budget() {
let limits = RedactionLimits {
max_string_bytes: 8 * 1024,
max_event_bytes: 512,
..RedactionLimits::default()
};
let event = sentry::protocol::Event {
tags: BTreeMap::from([
("category".to_owned(), "internal".to_owned()),
("request_id".to_owned(), "r".repeat(8 * 1024)),
("trace_id".to_owned(), "t".repeat(8 * 1024)),
]),
..sentry::protocol::Event::default()
};
let cleaned = sanitize_event(event, &identity(), limits);
let serialized = serde_json::to_vec(&cleaned).expect("serialize event");
assert!(serialized.len() <= limits.max_event_bytes);
assert_eq!(
cleaned.tags.get("category").map(String::as_str),
Some("internal")
);
assert_eq!(
cleaned.tags.get("service").map(String::as_str),
Some("admin-api")
);
assert!(!cleaned.tags.contains_key("request_id"));
assert!(!cleaned.tags.contains_key("trace_id"));
}
#[test]
fn expected_application_errors_do_not_create_critical_events() {
let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let events = sentry::test::with_captured_events_options(
|| tracing::error!("ordinary product error"),
options,
);
assert!(events.is_empty());
}
#[test]
fn explicit_critical_error_is_correlated_and_sanitized() {
let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("runtime");
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("critical-error-test");
let subscriber = build_subscriber_with_tracer(
ObservabilityConfig::new(identity(), "info", RedactionLimits::default()),
std::io::sink,
Some(tracer),
)
.expect("subscriber");
let dispatch = tracing::Dispatch::new(subscriber);
let events = sentry::test::with_captured_events_options(
|| {
tracing::dispatcher::with_default(&dispatch, || {
runtime.block_on(with_request_correlation("request-123".to_owned(), async {
let span = tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
capture_critical_error(CriticalErrorCategory::DataIntegrity);
}));
});
},
options,
);
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(event.message.as_deref(), Some("critical error"));
assert_eq!(
event.tags.get("category").map(String::as_str),
Some("data_integrity")
);
assert_eq!(
event.tags.get("request_id").map(String::as_str),
Some("request-123")
);
assert_eq!(event.tags.get("trace_id").map(String::len), Some(32));
assert_eq!(event.release.as_deref(), Some("1.2.3"));
assert_eq!(event.environment.as_deref(), Some("test"));
assert_eq!(event.server_name.as_deref(), Some("admin-api"));
}
#[test]
fn panic_creates_exactly_one_sanitized_critical_event() {
let options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
let events = sentry::test::with_captured_events_options(
|| {
let result = std::panic::catch_unwind(|| {
panic!("password=must-not-leak");
});
assert!(result.is_err());
},
options,
);
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(
event.tags.get("category").map(String::as_str),
Some("panic")
);
let serialized = serde_json::to_string(event).expect("serialize event");
assert!(!serialized.contains("must-not-leak"));
assert!(!serialized.contains("password"));
}
#[test]
fn receiver_failure_does_not_change_product_result_or_recurse() {
struct DroppingTransport {
attempts: AtomicUsize,
}
impl sentry::Transport for DroppingTransport {
fn send_envelope(&self, _envelope: Envelope) {
self.attempts.fetch_add(1, Ordering::Relaxed);
}
}
let transport = Arc::new(DroppingTransport {
attempts: AtomicUsize::new(0),
});
let mut options =
sentry::apply_defaults(client_options(identity(), RedactionLimits::default()));
options.dsn = Some(
"https://public@example.invalid/1"
.parse()
.expect("valid test DSN"),
);
options.transport = Some(Arc::new(transport.clone()));
let client = Arc::new(sentry::Client::from(options));
let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default())));
let product_result = Hub::run(hub, || {
capture_critical_error(CriticalErrorCategory::Internal);
42
});
assert_eq!(product_result, 42);
assert_eq!(transport.attempts.load(Ordering::Relaxed), 1);
}
}
crates/crank-observability/src/incidents.rs
+36
View File
@@ -0,0 +1,36 @@
use std::sync::atomic::{AtomicU64, Ordering};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum OperationalIncident {
InvocationHistoryLost,
}
static INVOCATION_HISTORY_LOST_TOTAL: AtomicU64 = AtomicU64::new(0);
pub fn record_operational_incident(incident: OperationalIncident) {
let counter = counter(incident);
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
value.checked_add(1)
});
match incident {
OperationalIncident::InvocationHistoryLost => {
metrics::counter!("crank_invocation_history_lost_total").increment(1);
metrics::counter!(
"crank_telemetry_export_failures_total",
"signal_type" => "invocation_history",
"exporter" => "postgres"
)
.increment(1);
}
}
}
pub fn operational_incident_total(incident: OperationalIncident) -> u64 {
counter(incident).load(Ordering::Relaxed)
}
fn counter(incident: OperationalIncident) -> &'static AtomicU64 {
match incident {
OperationalIncident::InvocationHistoryLost => &INVOCATION_HISTORY_LOST_TOTAL,
}
}
crates/crank-observability/src/instrumentation.rs
+133
View File
@@ -0,0 +1,133 @@
use std::time::Instant;
use axum::{
extract::{MatchedPath, Request},
middleware::Next,
response::Response,
};
use metrics::{Gauge, Unit};
use crate::{MetricKind, MetricUnit, metric_schema};
pub async fn record_http_request(request: Request, next: Next) -> Response {
let route = request
.extensions()
.get::<MatchedPath>()
.map_or("unmatched", MatchedPath::as_str)
.to_owned();
let method = normalized_http_method(request.method().as_str());
let started_at = Instant::now();
let _inflight = GaugeGuard::increment("crank_http_inflight");
let response = next.run(request).await;
let status_class = status_class(response.status().as_u16());
metrics::counter!(
"crank_http_requests_total",
"route" => route.clone(),
"method" => method,
"status_class" => status_class
)
.increment(1);
metrics::histogram!(
"crank_http_request_duration_seconds",
"route" => route,
"method" => method
)
.record(started_at.elapsed().as_secs_f64());
response
}
pub fn record_db_pool_connections(total: u32, idle: usize) {
let idle = idle.min(total as usize) as f64;
metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(idle);
metrics::gauge!("crank_db_pool_connections", "state" => "used").set(f64::from(total) - idle);
}
pub(crate) fn register_metric_schema() {
for definition in metric_schema() {
let unit = match definition.unit {
MetricUnit::Count => Unit::Count,
MetricUnit::Seconds => Unit::Seconds,
};
match definition.kind {
MetricKind::Counter => {
metrics::describe_counter!(definition.name, unit, definition.description);
}
MetricKind::Gauge => {
metrics::describe_gauge!(definition.name, unit, definition.description);
}
MetricKind::Histogram => {
metrics::describe_histogram!(definition.name, unit, definition.description);
}
}
}
metrics::gauge!("crank_http_inflight").set(0.0);
metrics::gauge!("crank_mcp_active_sessions").set(0.0);
metrics::gauge!("crank_runtime_inflight").set(0.0);
metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(0.0);
metrics::gauge!("crank_db_pool_connections", "state" => "used").set(0.0);
metrics::gauge!("crank_catalog_tools").set(0.0);
metrics::gauge!("crank_catalog_estimated_context_tokens").set(0.0);
metrics::gauge!("crank_catalog_warnings").set(0.0);
}
fn normalized_http_method(method: &str) -> &'static str {
match method {
"GET" => "GET",
"POST" => "POST",
"PUT" => "PUT",
"PATCH" => "PATCH",
"DELETE" => "DELETE",
"OPTIONS" => "OPTIONS",
"HEAD" => "HEAD",
"CONNECT" => "CONNECT",
"TRACE" => "TRACE",
_ => "OTHER",
}
}
fn status_class(status: u16) -> &'static str {
match status {
100..=199 => "1xx",
200..=299 => "2xx",
300..=399 => "3xx",
400..=499 => "4xx",
500..=599 => "5xx",
_ => "other",
}
}
struct GaugeGuard {
gauge: Gauge,
}
impl GaugeGuard {
fn increment(name: &'static str) -> Self {
let gauge = metrics::gauge!(name);
gauge.increment(1.0);
Self { gauge }
}
}
impl Drop for GaugeGuard {
fn drop(&mut self) {
self.gauge.decrement(1.0);
}
}
#[cfg(test)]
mod tests {
use super::{normalized_http_method, status_class};
#[test]
fn normalizes_unbounded_http_values() {
assert_eq!(normalized_http_method("GET"), "GET");
assert_eq!(normalized_http_method("CUSTOM-user-controlled"), "OTHER");
assert_eq!(status_class(204), "2xx");
assert_eq!(status_class(429), "4xx");
assert_eq!(status_class(999), "other");
}
}
crates/crank-observability/src/lib.rs
+37
View File
@@ -0,0 +1,37 @@
mod config;
mod correlation;
mod error_reporting;
mod incidents;
mod instrumentation;
mod lifecycle;
mod logging;
mod metrics_schema;
mod otlp;
mod prometheus;
mod propagation;
mod redaction;
mod schema;
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
pub use correlation::RequestId;
pub use error_reporting::{
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
with_request_correlation,
};
pub use incidents::{OperationalIncident, operational_incident_total, record_operational_incident};
pub use instrumentation::{record_db_pool_connections, record_http_request};
pub use lifecycle::{ObservabilityInitError, ObservabilityLifecycle, init};
pub use logging::build_subscriber;
pub use metrics_schema::{
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema,
};
pub use otlp::{
OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError, build_tracer_provider,
};
pub use prometheus::{
MetricsConfig, MetricsConfigError, MetricsServeError, MetricsSurface, MetricsSurfaceError,
};
pub use propagation::{inject_current_trace_context, set_remote_trace_parent};
pub use redaction::{
REDACTED_MARKER, RedactionLimits, RedactionLimitsError, SafeJsonError, redact_value, safe_json,
};
crates/crank-observability/src/lifecycle.rs
+90
View File
@@ -0,0 +1,90 @@
use std::{fmt, io};
use thiserror::Error;
use tracing_subscriber::util::SubscriberInitExt;
use crate::{
MetricsConfig, MetricsSurface, MetricsSurfaceError, ObservabilityConfig,
ObservabilityConfigError, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError,
RedactionLimitsError, SentryConfig, SentryConfigError, error_reporting::init_sentry,
instrumentation::register_metric_schema, logging::build_subscriber_with_tracer,
otlp::build_tracer_provider, prometheus::install_prometheus_recorder,
propagation::install_trace_context_propagator,
};
#[must_use = "observability resources must be retained until process shutdown"]
pub struct ObservabilityLifecycle {
metrics_handle: metrics_exporter_prometheus::PrometheusHandle,
tracer_provider: Option<opentelemetry_sdk::trace::SdkTracerProvider>,
sentry_guard: Option<sentry::ClientInitGuard>,
}
impl ObservabilityLifecycle {
pub fn init(config: ObservabilityConfig) -> Result<Self, ObservabilityInitError> {
let identity = config.identity().clone();
let redaction_limits = config.redaction_limits();
let sentry_config = SentryConfig::from_env()?;
let trace_config = OtlpTraceConfig::from_env()?;
let tracing = build_tracer_provider(&identity, &trace_config)?;
let tracer = tracing.as_ref().map(|(_, tracer)| tracer.clone());
install_trace_context_propagator();
build_subscriber_with_tracer(config, io::stdout, tracer)?
.try_init()
.map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?;
let metrics_handle = install_prometheus_recorder(&identity)?;
register_metric_schema();
let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config);
Ok(Self {
metrics_handle,
tracer_provider: tracing.map(|(provider, _)| provider),
sentry_guard,
})
}
pub fn metrics_surface(&self, config: MetricsConfig) -> MetricsSurface {
MetricsSurface::new(config, self.metrics_handle.clone())
}
pub fn traces_enabled(&self) -> bool {
self.tracer_provider.is_some()
}
pub fn critical_errors_enabled(&self) -> bool {
self.sentry_guard.is_some()
}
}
impl fmt::Debug for ObservabilityLifecycle {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ObservabilityLifecycle")
.field("traces_enabled", &self.traces_enabled())
.field("critical_errors_enabled", &self.critical_errors_enabled())
.finish_non_exhaustive()
}
}
pub fn init(config: ObservabilityConfig) -> Result<ObservabilityLifecycle, ObservabilityInitError> {
ObservabilityLifecycle::init(config)
}
#[derive(Debug, Error)]
pub enum ObservabilityInitError {
#[error(transparent)]
InvalidConfig(#[from] ObservabilityConfigError),
#[error(transparent)]
InvalidRedactionLimits(#[from] RedactionLimitsError),
#[error("invalid log filter")]
InvalidFilter,
#[error("global tracing subscriber is already initialized")]
SubscriberAlreadyInitialized,
#[error(transparent)]
Metrics(#[from] MetricsSurfaceError),
#[error(transparent)]
OtlpConfig(#[from] OtlpTraceConfigError),
#[error(transparent)]
Otlp(#[from] OtlpTraceError),
#[error(transparent)]
SentryConfig(#[from] SentryConfigError),
}
crates/crank-observability/src/logging.rs
+314
View File
@@ -0,0 +1,314 @@
use std::fmt;
use serde_json::{Map, Number, Value};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tracing::{Event, Subscriber, field::Visit};
use tracing_subscriber::{
EnvFilter, Layer,
filter::filter_fn,
fmt::{FmtContext, FormatEvent, FormatFields, MakeWriter, format::Writer},
layer::SubscriberExt,
registry::LookupSpan,
};
use crate::{
ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity,
propagation::current_trace_id,
redaction::{redact_value, truncate_string},
schema::LogEnvelope,
};
pub fn build_subscriber<W>(
config: ObservabilityConfig,
writer: W,
) -> Result<impl Subscriber + Send + Sync, ObservabilityInitError>
where
W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static,
{
build_subscriber_with_tracer(config, writer, None)
}
pub(crate) fn build_subscriber_with_tracer<W>(
config: ObservabilityConfig,
writer: W,
tracer: Option<opentelemetry_sdk::trace::SdkTracer>,
) -> Result<impl Subscriber + Send + Sync, ObservabilityInitError>
where
W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static,
{
let (identity, filter, limits) = config.into_parts();
limits.validate()?;
let filter = EnvFilter::try_new(filter).map_err(|_| ObservabilityInitError::InvalidFilter)?;
let formatter = JsonEventFormatter::new(identity, limits);
let fmt_layer = tracing_subscriber::fmt::layer()
.with_ansi(false)
.event_format(formatter)
.with_writer(writer)
.with_filter(filter);
let otel_layer = tracer.map(|tracer| {
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(filter_fn(|metadata| {
metadata.is_span() && metadata.target() == "crank::trace"
}))
});
Ok(tracing_subscriber::registry()
.with(fmt_layer)
.with(otel_layer))
}
#[derive(Clone, Debug)]
struct JsonEventFormatter {
identity: ServiceIdentity,
limits: RedactionLimits,
}
impl JsonEventFormatter {
fn new(identity: ServiceIdentity, limits: RedactionLimits) -> Self {
Self { identity, limits }
}
fn envelope(&self, event: &Event<'_>) -> Result<LogEnvelope, fmt::Error> {
let metadata = event.metadata();
let mut visitor = JsonFieldVisitor::default();
event.record(&mut visitor);
let mut raw_fields = visitor.fields;
let request_id = take_correlation_id(&mut raw_fields, "request_id")
.map(|value| truncate_string(&value, self.limits.max_string_bytes));
let trace_id = take_correlation_id(&mut raw_fields, "trace_id")
.or_else(current_trace_id)
.map(|value| truncate_string(&value, self.limits.max_string_bytes));
let cleaned = redact_value(&Value::Object(raw_fields), self.limits);
let fields = cleaned.as_object().cloned().unwrap_or_default();
let timestamp = OffsetDateTime::now_utc()
.format(&Rfc3339)
.map_err(|_| fmt::Error)?;
Ok(LogEnvelope {
timestamp,
level: metadata.level().as_str().to_owned(),
service: self.identity.service().to_owned(),
version: self.identity.version().to_owned(),
environment: self.identity.environment().to_owned(),
target: truncate_string(metadata.target(), self.limits.max_string_bytes),
event: truncate_string(metadata.name(), self.limits.max_string_bytes),
request_id,
trace_id,
fields,
})
}
fn serialize_bounded(&self, mut envelope: LogEnvelope) -> Result<String, fmt::Error> {
let line_budget = self.limits.max_event_bytes.saturating_sub(1);
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
if serialized.len() <= line_budget {
return Ok(serialized);
}
envelope.fields = Map::from_iter([("truncated".to_owned(), Value::Bool(true))]);
let fallback_string_limit = self.limits.max_string_bytes.min(64);
envelope.target = truncate_string(&envelope.target, fallback_string_limit);
envelope.event = truncate_string(&envelope.event, fallback_string_limit);
envelope.request_id = envelope
.request_id
.map(|value| truncate_string(&value, fallback_string_limit));
envelope.trace_id = envelope
.trace_id
.map(|value| truncate_string(&value, fallback_string_limit));
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
if serialized.len() <= line_budget {
return Ok(serialized);
}
envelope.request_id = None;
envelope.trace_id = None;
envelope.target = truncate_string(&envelope.target, 16);
envelope.event = truncate_string(&envelope.event, 16);
let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?;
(serialized.len() <= line_budget)
.then_some(serialized)
.ok_or(fmt::Error)
}
}
impl<S, N> FormatEvent<S, N> for JsonEventFormatter
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
N: for<'writer> FormatFields<'writer> + 'static,
{
fn format_event(
&self,
_ctx: &FmtContext<'_, S, N>,
mut writer: Writer<'_>,
event: &Event<'_>,
) -> fmt::Result {
let serialized = self.serialize_bounded(self.envelope(event)?)?;
writer.write_str(&serialized)?;
writer.write_char('\n')
}
}
#[derive(Default)]
struct JsonFieldVisitor {
fields: Map<String, Value>,
}
impl JsonFieldVisitor {
fn insert(&mut self, field: &tracing::field::Field, value: Value) {
self.fields.insert(field.name().to_owned(), value);
}
}
impl Visit for JsonFieldVisitor {
fn record_i64(&mut self, field: &tracing::field::Field, value: i64) {
self.insert(field, Value::Number(value.into()));
}
fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
self.insert(field, Value::Number(value.into()));
}
fn record_bool(&mut self, field: &tracing::field::Field, value: bool) {
self.insert(field, Value::Bool(value));
}
fn record_f64(&mut self, field: &tracing::field::Field, value: f64) {
let value = Number::from_f64(value)
.map(Value::Number)
.unwrap_or(Value::Null);
self.insert(field, value);
}
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.insert(field, Value::String(value.to_owned()));
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
let rendered = format!("{value:?}");
let value = if is_correlation_field(field.name()) {
Value::String(debug_scalar(&rendered))
} else {
match serde_json::from_str(&rendered) {
Ok(value @ (Value::Object(_) | Value::Array(_))) => value,
_ if field.name() == "message" || is_safe_display_scalar(&rendered) => {
Value::String(rendered)
}
_ => Value::String(crate::REDACTED_MARKER.to_owned()),
}
};
self.insert(field, value);
}
}
fn take_correlation_id(fields: &mut Map<String, Value>, name: &str) -> Option<String> {
let value = fields.remove(name)?;
let value = match value {
Value::String(value) => value,
Value::Number(value) => value.to_string(),
Value::Bool(value) => value.to_string(),
Value::Null | Value::Array(_) | Value::Object(_) => return None,
};
(!value.is_empty()).then_some(value)
}
fn is_correlation_field(name: &str) -> bool {
matches!(name, "request_id" | "trace_id" | "correlation_id")
}
fn debug_scalar(rendered: &str) -> String {
serde_json::from_str::<String>(rendered).unwrap_or_else(|_| rendered.to_owned())
}
fn is_safe_display_scalar(rendered: &str) -> bool {
!rendered.is_empty()
&& rendered.bytes().all(|byte| {
byte.is_ascii_alphanumeric()
|| matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'+' | b'@')
})
}
#[cfg(test)]
mod tests {
use std::{
io,
sync::{Arc, Mutex},
};
use opentelemetry::{global, trace::TracerProvider as _};
use opentelemetry_sdk::{
error::OTelSdkResult,
propagation::TraceContextPropagator,
trace::{SdkTracerProvider, SpanData, SpanExporter},
};
use tracing::info;
use super::build_subscriber_with_tracer;
use crate::{
ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
};
#[test]
fn trace_spans_ignore_the_log_level_filter() {
global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("trace-filter-test");
let subscriber = build_subscriber_with_tracer(test_config("warn"), io::sink, Some(tracer))
.expect("subscriber must build");
let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let span = tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
let mut headers = axum::http::HeaderMap::new();
assert!(inject_current_trace_context(&mut headers));
assert!(headers.contains_key("traceparent"));
}
#[test]
fn otel_layer_does_not_export_events() {
let exported = Arc::new(Mutex::new(Vec::new()));
let provider = SdkTracerProvider::builder()
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
.build();
let tracer = provider.tracer("event-filter-test");
let subscriber = build_subscriber_with_tracer(test_config("info"), io::sink, Some(tracer))
.expect("subscriber must build");
let dispatch = tracing::Dispatch::new(subscriber);
tracing::dispatcher::with_default(&dispatch, || {
let span = tracing::info_span!(target: "crank::trace", "http.request");
let _span_guard = span.enter();
info!(password = "canary-secret", "sensitive event");
});
provider.force_flush().expect("span must be exported");
let spans = exported.lock().expect("capture lock");
assert_eq!(spans.len(), 1);
assert!(spans[0].events.is_empty());
assert!(
!format!("{:?}", spans[0])
.as_bytes()
.windows(b"canary-secret".len())
.any(|window| window == b"canary-secret")
);
}
fn test_config(filter: &str) -> ObservabilityConfig {
ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
filter,
RedactionLimits::default(),
)
}
#[derive(Clone, Debug)]
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
impl SpanExporter for CapturingExporter {
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
self.0.lock().expect("capture lock").extend(batch);
Ok(())
}
}
}
crates/crank-observability/src/metrics_schema.rs
+159
View File
@@ -0,0 +1,159 @@
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MetricKind {
Counter,
Gauge,
Histogram,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MetricUnit {
Count,
Seconds,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MetricDefinition {
pub name: &'static str,
pub kind: MetricKind,
pub unit: MetricUnit,
pub labels: &'static [&'static str],
pub description: &'static str,
}
pub const DURATION_BUCKETS_SECONDS: &[f64] = &[
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
];
const METRIC_SCHEMA: &[MetricDefinition] = &[
counter(
"crank_http_requests_total",
&["route", "method", "status_class"],
"Total HTTP requests.",
),
histogram(
"crank_http_request_duration_seconds",
&["route", "method"],
"HTTP request duration in seconds.",
),
gauge(
"crank_http_inflight",
&[],
"HTTP requests currently being processed.",
),
counter(
"crank_mcp_requests_total",
&["method", "response_mode", "outcome"],
"Total MCP JSON-RPC requests.",
),
gauge(
"crank_mcp_active_sessions",
&[],
"Active MCP transport sessions.",
),
counter(
"crank_tool_invocations_total",
&["source", "outcome", "error_kind"],
"Total tool invocations.",
),
histogram(
"crank_tool_invocation_duration_seconds",
&["source", "outcome"],
"Tool invocation duration in seconds.",
),
counter(
"crank_upstream_requests_total",
&["operation_kind", "outcome"],
"Total upstream requests.",
),
histogram(
"crank_upstream_request_duration_seconds",
&["operation_kind", "outcome"],
"Upstream request duration in seconds.",
),
gauge(
"crank_runtime_inflight",
&[],
"Runtime executions currently in progress.",
),
counter(
"crank_runtime_limit_rejections_total",
&["stage"],
"Runtime executions rejected by a bounded limit.",
),
gauge(
"crank_db_pool_connections",
&["state"],
"PostgreSQL pool connections by state.",
),
gauge(
"crank_catalog_tools",
&[],
"Tools in the current published catalog.",
),
gauge(
"crank_catalog_estimated_context_tokens",
&[],
"Estimated context tokens in the current published catalog.",
),
gauge(
"crank_catalog_warnings",
&[],
"Warnings in the current published catalog.",
),
counter(
"crank_invocation_history_lost_total",
&[],
"Invocation history records lost after an action completed.",
),
counter(
"crank_telemetry_export_failures_total",
&["signal_type", "exporter"],
"Telemetry export failures.",
),
];
pub const fn metric_schema() -> &'static [MetricDefinition] {
METRIC_SCHEMA
}
const fn counter(
name: &'static str,
labels: &'static [&'static str],
description: &'static str,
) -> MetricDefinition {
MetricDefinition {
name,
kind: MetricKind::Counter,
unit: MetricUnit::Count,
labels,
description,
}
}
const fn gauge(
name: &'static str,
labels: &'static [&'static str],
description: &'static str,
) -> MetricDefinition {
MetricDefinition {
name,
kind: MetricKind::Gauge,
unit: MetricUnit::Count,
labels,
description,
}
}
const fn histogram(
name: &'static str,
labels: &'static [&'static str],
description: &'static str,
) -> MetricDefinition {
MetricDefinition {
name,
kind: MetricKind::Histogram,
unit: MetricUnit::Seconds,
labels,
description,
}
}
crates/crank-observability/src/otlp.rs
+954
View File
@@ -0,0 +1,954 @@
use std::{collections::HashMap, env, fmt, time::Duration};
use axum::http::{HeaderName, HeaderValue};
use opentelemetry::{
KeyValue, Value,
trace::{Status, TracerProvider as _},
};
use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig, WithHttpConfig};
use opentelemetry_sdk::{
Resource,
error::OTelSdkResult,
trace::{
BatchConfigBuilder, BatchSpanProcessor, SdkTracer, SdkTracerProvider, SpanData,
SpanExporter as SpanExporterTrait,
},
};
use percent_encoding::percent_decode_str;
use thiserror::Error;
use url::Url;
use crate::ServiceIdentity;
const DEFAULT_EXPORT_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_MAX_QUEUE_SIZE: usize = 2_048;
const DEFAULT_MAX_EXPORT_BATCH_SIZE: usize = 512;
const DEFAULT_SCHEDULE_DELAY: Duration = Duration::from_secs(5);
const DEFAULT_BATCH_EXPORT_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_QUEUE_SIZE: usize = 65_536;
const MAX_DURATION: Duration = Duration::from_secs(300);
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct OtlpBatchConfig {
max_queue_size: usize,
max_export_batch_size: usize,
scheduled_delay: Duration,
export_timeout: Duration,
}
impl OtlpBatchConfig {
pub fn try_new(
max_queue_size: usize,
max_export_batch_size: usize,
scheduled_delay: Duration,
export_timeout: Duration,
) -> Result<Self, OtlpTraceConfigError> {
let valid = max_queue_size > 0
&& max_queue_size <= MAX_QUEUE_SIZE
&& max_export_batch_size > 0
&& max_export_batch_size <= max_queue_size
&& duration_is_bounded(scheduled_delay)
&& duration_is_bounded(export_timeout);
if !valid {
return Err(OtlpTraceConfigError::InvalidBatchLimits);
}
Ok(Self {
max_queue_size,
max_export_batch_size,
scheduled_delay,
export_timeout,
})
}
pub fn max_queue_size(&self) -> usize {
self.max_queue_size
}
pub fn max_export_batch_size(&self) -> usize {
self.max_export_batch_size
}
pub fn scheduled_delay(&self) -> Duration {
self.scheduled_delay
}
pub fn export_timeout(&self) -> Duration {
self.export_timeout
}
fn sdk_config(&self) -> opentelemetry_sdk::trace::BatchConfig {
BatchConfigBuilder::default()
.with_max_queue_size(self.max_queue_size)
.with_max_export_batch_size(self.max_export_batch_size)
.with_scheduled_delay(self.scheduled_delay)
.build()
}
}
impl Default for OtlpBatchConfig {
fn default() -> Self {
Self {
max_queue_size: DEFAULT_MAX_QUEUE_SIZE,
max_export_batch_size: DEFAULT_MAX_EXPORT_BATCH_SIZE,
scheduled_delay: DEFAULT_SCHEDULE_DELAY,
export_timeout: DEFAULT_BATCH_EXPORT_TIMEOUT,
}
}
}
#[derive(Clone, Eq, PartialEq)]
pub struct OtlpTraceConfig {
endpoint: Option<String>,
export_timeout: Duration,
batch: OtlpBatchConfig,
headers: HashMap<String, String>,
}
impl fmt::Debug for OtlpTraceConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("OtlpTraceConfig")
.field("enabled", &self.is_enabled())
.field("export_timeout", &self.export_timeout)
.field("batch", &self.batch)
.field("header_count", &self.headers.len())
.finish()
}
}
impl OtlpTraceConfig {
pub fn from_env() -> Result<Self, OtlpTraceConfigError> {
OtlpEnvSettings::from_env()?.into_config()
}
fn from_settings(settings: OtlpEnvSettings) -> Result<Self, OtlpTraceConfigError> {
let endpoint = match settings.traces_endpoint {
Some(endpoint) => Some(validate_endpoint(endpoint, EndpointKind::Trace)?),
None => settings
.generic_endpoint
.map(|endpoint| validate_endpoint(endpoint, EndpointKind::Generic))
.transpose()?,
};
if endpoint.is_none() {
return Ok(Self {
endpoint: None,
export_timeout: DEFAULT_EXPORT_TIMEOUT,
batch: OtlpBatchConfig::default(),
headers: HashMap::new(),
});
}
let protocol = settings.traces_protocol.or(settings.generic_protocol);
let export_timeout = match settings.traces_timeout {
Some(timeout) => duration_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", Some(timeout))?,
None => duration_env("OTEL_EXPORTER_OTLP_TIMEOUT", settings.generic_timeout)?,
}
.unwrap_or(DEFAULT_EXPORT_TIMEOUT);
let batch = OtlpBatchConfig::try_new(
usize_env("OTEL_BSP_MAX_QUEUE_SIZE", settings.max_queue_size)?
.unwrap_or(DEFAULT_MAX_QUEUE_SIZE),
usize_env(
"OTEL_BSP_MAX_EXPORT_BATCH_SIZE",
settings.max_export_batch_size,
)?
.unwrap_or(DEFAULT_MAX_EXPORT_BATCH_SIZE),
duration_env("OTEL_BSP_SCHEDULE_DELAY", settings.scheduled_delay)?
.unwrap_or(DEFAULT_SCHEDULE_DELAY),
duration_env("OTEL_BSP_EXPORT_TIMEOUT", settings.batch_export_timeout)?
.unwrap_or(DEFAULT_BATCH_EXPORT_TIMEOUT),
)?;
let headers = settings
.traces_headers
.filter(|value| !value.is_empty())
.or(settings.generic_headers.filter(|value| !value.is_empty()))
.map(|value| parse_headers(&value))
.transpose()?
.unwrap_or_default();
Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, headers)
}
pub fn try_new(
endpoint: Option<String>,
protocol: Option<String>,
export_timeout: Duration,
batch: OtlpBatchConfig,
) -> Result<Self, OtlpTraceConfigError> {
Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, HashMap::new())
}
fn try_new_with_headers(
endpoint: Option<String>,
protocol: Option<String>,
export_timeout: Duration,
batch: OtlpBatchConfig,
headers: HashMap<String, String>,
) -> Result<Self, OtlpTraceConfigError> {
let endpoint = endpoint
.map(|endpoint| validate_endpoint(endpoint, EndpointKind::Trace))
.transpose()?;
if endpoint.is_some() && protocol.as_deref().unwrap_or("http/protobuf") != "http/protobuf" {
return Err(OtlpTraceConfigError::UnsupportedProtocol);
}
if !duration_is_bounded(export_timeout) {
return Err(OtlpTraceConfigError::InvalidDuration {
field: "OTEL_EXPORTER_OTLP_TIMEOUT",
});
}
Ok(Self {
endpoint,
export_timeout,
batch,
headers,
})
}
pub fn is_enabled(&self) -> bool {
self.endpoint.is_some()
}
pub fn export_timeout(&self) -> Duration {
self.export_timeout
}
pub fn batch(&self) -> &OtlpBatchConfig {
&self.batch
}
fn effective_export_timeout(&self) -> Duration {
self.export_timeout.min(self.batch.export_timeout)
}
#[cfg(test)]
fn endpoint(&self) -> Option<&str> {
self.endpoint.as_deref()
}
#[cfg(test)]
fn header(&self, name: &str) -> Option<&str> {
self.headers.get(name).map(String::as_str)
}
}
#[derive(Default)]
struct OtlpEnvSettings {
traces_endpoint: Option<String>,
generic_endpoint: Option<String>,
traces_protocol: Option<String>,
generic_protocol: Option<String>,
traces_timeout: Option<String>,
generic_timeout: Option<String>,
traces_headers: Option<String>,
generic_headers: Option<String>,
max_queue_size: Option<String>,
max_export_batch_size: Option<String>,
scheduled_delay: Option<String>,
batch_export_timeout: Option<String>,
}
impl OtlpEnvSettings {
fn from_env() -> Result<Self, OtlpTraceConfigError> {
Ok(Self {
traces_endpoint: optional_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")?,
generic_endpoint: optional_env("OTEL_EXPORTER_OTLP_ENDPOINT")?,
traces_protocol: optional_env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")?,
generic_protocol: optional_env("OTEL_EXPORTER_OTLP_PROTOCOL")?,
traces_timeout: optional_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT")?,
generic_timeout: optional_env("OTEL_EXPORTER_OTLP_TIMEOUT")?,
traces_headers: optional_env("OTEL_EXPORTER_OTLP_TRACES_HEADERS")?,
generic_headers: optional_env("OTEL_EXPORTER_OTLP_HEADERS")?,
max_queue_size: optional_env("OTEL_BSP_MAX_QUEUE_SIZE")?,
max_export_batch_size: optional_env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE")?,
scheduled_delay: optional_env("OTEL_BSP_SCHEDULE_DELAY")?,
batch_export_timeout: optional_env("OTEL_BSP_EXPORT_TIMEOUT")?,
})
}
fn into_config(self) -> Result<OtlpTraceConfig, OtlpTraceConfigError> {
OtlpTraceConfig::from_settings(self)
}
}
#[derive(Debug, Error, Eq, PartialEq)]
pub enum OtlpTraceConfigError {
#[error("OTLP environment variable is not valid UTF-8: {field}")]
InvalidEnvironmentEncoding { field: &'static str },
#[error("OTLP trace endpoint is invalid: {reason}")]
InvalidEndpoint { reason: &'static str },
#[error("OTLP trace protocol must be http/protobuf")]
UnsupportedProtocol,
#[error("OTLP numeric setting is invalid: {field}")]
InvalidNumber { field: &'static str },
#[error("OTLP duration setting is invalid: {field}")]
InvalidDuration { field: &'static str },
#[error("OTLP batch limits are invalid")]
InvalidBatchLimits,
#[error("OTLP trace headers are invalid")]
InvalidHeaders,
}
#[derive(Debug, Error)]
pub enum OtlpTraceError {
#[error("failed to configure OTLP trace exporter")]
ExporterConfiguration,
}
pub fn build_tracer_provider(
identity: &ServiceIdentity,
config: &OtlpTraceConfig,
) -> Result<Option<(SdkTracerProvider, SdkTracer)>, OtlpTraceError> {
let Some(endpoint) = config.endpoint.as_deref() else {
return Ok(None);
};
let exporter = SpanExporter::builder()
.with_http()
.with_protocol(Protocol::HttpBinary)
.with_endpoint(endpoint)
.with_timeout(config.effective_export_timeout())
.with_headers(config.headers.clone())
.build()
.map_err(|_| OtlpTraceError::ExporterConfiguration)?;
let processor = BatchSpanProcessor::builder(ObservedSpanExporter(exporter))
.with_batch_config(config.batch.sdk_config())
.build();
let resource = Resource::builder_empty()
.with_attributes([
KeyValue::new("service.name", identity.service().to_owned()),
KeyValue::new("service.version", identity.version().to_owned()),
KeyValue::new(
"deployment.environment.name",
identity.environment().to_owned(),
),
])
.build();
let provider = SdkTracerProvider::builder()
.with_span_processor(processor)
.with_resource(resource)
.build();
let tracer = provider.tracer("crank");
Ok(Some((provider, tracer)))
}
#[derive(Debug)]
struct ObservedSpanExporter(SpanExporter);
impl SpanExporterTrait for ObservedSpanExporter {
async fn export(&self, mut batch: Vec<SpanData>) -> OTelSdkResult {
sanitize_trace_batch(&mut batch);
let result = self.0.export(batch).await;
if result.is_err() {
metrics::counter!(
"crank_telemetry_export_failures_total",
"signal_type" => "trace",
"exporter" => "otlp"
)
.increment(1);
}
result
}
fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult {
self.0.shutdown_with_timeout(timeout)
}
fn force_flush(&self) -> OTelSdkResult {
self.0.force_flush()
}
fn set_resource(&mut self, resource: &Resource) {
self.0.set_resource(resource);
}
}
fn sanitize_trace_batch(batch: &mut Vec<SpanData>) {
batch.retain(|span| is_allowed_span_name(span.name.as_ref()));
for span in batch {
let original_attribute_count = span.attributes.len();
span.attributes.retain(is_allowed_span_attribute);
span.dropped_attributes_count = span
.dropped_attributes_count
.saturating_add((original_attribute_count - span.attributes.len()) as u32);
span.events = Default::default();
span.links = Default::default();
if matches!(span.status, Status::Error { .. }) {
span.status = Status::error("");
}
}
}
fn is_allowed_span_name(name: &str) -> bool {
matches!(
name,
"http.request"
| "mcp.request"
| "mcp.rate_limit"
| "mcp.access.check"
| "mcp.catalog.load"
| "mcp.tools.resolve"
| "approval.check"
| "runtime.execute"
| "runtime.arguments.map"
| "runtime.idempotency"
| "upstream.http"
| "runtime.response.transform"
| "auth.resolve"
| "approval.recovery"
| "history.write"
| "db.query"
)
}
fn is_allowed_span_attribute(attribute: &KeyValue) -> bool {
let Value::String(value) = &attribute.value else {
return false;
};
let value = value.as_str();
match attribute.key.as_str() {
"request_id" => crate::RequestId::is_valid(value),
"stage" => is_allowed_span_name(value),
"outcome" => matches!(
value,
"success"
| "error"
| "allowed"
| "denied"
| "required"
| "replay"
| "execute"
| "skipped"
| "cache_hit"
),
"error.category" => matches!(
value,
"access"
| "rate_limit"
| "catalog"
| "approval"
| "idempotency"
| "schema"
| "mapping"
| "upstream"
| "transformation"
| "history"
| "database"
| "concurrency"
| "configuration"
| "internal"
),
"db.system" => value == "postgresql",
"db.operation" => matches!(
value,
"machine_access.read"
| "machine_access.touch"
| "catalog.load"
| "approval.read"
| "approval.write"
| "auth_profile.read"
| "secret.read"
| "secret.touch"
| "invocation_history.write"
),
_ => false,
}
}
#[derive(Clone, Copy)]
enum EndpointKind {
Trace,
Generic,
}
fn validate_endpoint(endpoint: String, kind: EndpointKind) -> Result<String, OtlpTraceConfigError> {
let mut url = Url::parse(&endpoint).map_err(|_| OtlpTraceConfigError::InvalidEndpoint {
reason: "invalid URL",
})?;
if !matches!(url.scheme(), "http" | "https") {
return Err(OtlpTraceConfigError::InvalidEndpoint {
reason: "unsupported scheme",
});
}
if url.host_str().is_none() {
return Err(OtlpTraceConfigError::InvalidEndpoint {
reason: "host is required",
});
}
if !url.username().is_empty() || url.password().is_some() {
return Err(OtlpTraceConfigError::InvalidEndpoint {
reason: "credentials are forbidden",
});
}
if url.query().is_some() || url.fragment().is_some() {
return Err(OtlpTraceConfigError::InvalidEndpoint {
reason: "query and fragment are forbidden",
});
}
if matches!(kind, EndpointKind::Generic) {
let path = url.path().trim_end_matches('/');
url.set_path(&format!("{path}/v1/traces"));
}
Ok(url.into())
}
fn parse_headers(value: &str) -> Result<HashMap<String, String>, OtlpTraceConfigError> {
value
.split_terminator(',')
.map(str::trim)
.filter(|item| !item.is_empty())
.try_fold(HashMap::new(), |mut headers, item| {
let (name, encoded_value) = item
.split_once('=')
.ok_or(OtlpTraceConfigError::InvalidHeaders)?;
let name = HeaderName::from_bytes(name.trim().as_bytes())
.map_err(|_| OtlpTraceConfigError::InvalidHeaders)?;
let value = percent_decode_str(encoded_value.trim())
.decode_utf8()
.map_err(|_| OtlpTraceConfigError::InvalidHeaders)?
.into_owned();
if value.is_empty() || HeaderValue::from_str(&value).is_err() {
return Err(OtlpTraceConfigError::InvalidHeaders);
}
headers.insert(name.as_str().to_owned(), value);
Ok(headers)
})
}
fn optional_env(field: &'static str) -> Result<Option<String>, OtlpTraceConfigError> {
match env::var(field) {
Ok(value) if value.is_empty() => Ok(None),
Ok(value) => Ok(Some(value)),
Err(env::VarError::NotPresent) => Ok(None),
Err(env::VarError::NotUnicode(_)) => {
Err(OtlpTraceConfigError::InvalidEnvironmentEncoding { field })
}
}
}
fn usize_env(
field: &'static str,
value: Option<String>,
) -> Result<Option<usize>, OtlpTraceConfigError> {
value
.map(|value| {
value
.parse()
.map_err(|_| OtlpTraceConfigError::InvalidNumber { field })
})
.transpose()
}
fn duration_env(
field: &'static str,
value: Option<String>,
) -> Result<Option<Duration>, OtlpTraceConfigError> {
value
.map(|value| {
value
.parse::<u64>()
.map(Duration::from_millis)
.map_err(|_| OtlpTraceConfigError::InvalidDuration { field })
})
.transpose()
}
fn duration_is_bounded(duration: Duration) -> bool {
!duration.is_zero() && duration <= MAX_DURATION
}
#[cfg(test)]
mod tests {
use std::{
io::{Read, Write},
net::TcpListener,
sync::mpsc,
thread,
time::{Duration, Instant},
};
use axum::{
Router,
body::{Body, to_bytes},
extract::Request,
middleware::Next,
response::Response,
routing::get,
};
use opentelemetry::{
KeyValue,
trace::{Span as _, Status, Tracer as _},
};
use opentelemetry_proto::tonic::{
collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value,
};
use prost::Message;
use tower::ServiceExt;
use tracing::{Instrument, info_span};
use tracing_subscriber::layer::SubscriberExt;
use super::{OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_tracer_provider};
use crate::ServiceIdentity;
#[test]
fn signal_specific_settings_override_generic_settings() {
let config = OtlpEnvSettings {
traces_endpoint: Some("https://traces.example.test/custom".to_owned()),
generic_endpoint: Some("https://generic.example.test/otel".to_owned()),
traces_protocol: Some("http/protobuf".to_owned()),
generic_protocol: Some("grpc".to_owned()),
traces_timeout: Some("2500".to_owned()),
generic_timeout: Some("invalid-unused-fallback".to_owned()),
..OtlpEnvSettings::default()
}
.into_config()
.unwrap();
assert_eq!(
config.endpoint(),
Some("https://traces.example.test/custom")
);
assert_eq!(config.export_timeout(), Duration::from_millis(2500));
}
#[test]
fn disabled_export_ignores_inactive_settings() {
let config = OtlpEnvSettings {
traces_protocol: Some("grpc".to_owned()),
generic_protocol: Some("grpc".to_owned()),
traces_timeout: Some("invalid".to_owned()),
generic_timeout: Some("invalid".to_owned()),
max_queue_size: Some("invalid".to_owned()),
max_export_batch_size: Some("invalid".to_owned()),
scheduled_delay: Some("invalid".to_owned()),
batch_export_timeout: Some("invalid".to_owned()),
..OtlpEnvSettings::default()
}
.into_config()
.unwrap();
assert!(!config.is_enabled());
}
#[test]
fn empty_signal_headers_use_generic_headers() {
let config = OtlpEnvSettings {
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
traces_headers: Some(String::new()),
generic_headers: Some(
"authorization=Bearer%20canary-token,x-tenant=community".to_owned(),
),
..OtlpEnvSettings::default()
}
.into_config()
.unwrap();
assert_eq!(config.header("authorization"), Some("Bearer canary-token"));
assert_eq!(config.header("x-tenant"), Some("community"));
assert!(!format!("{config:?}").contains("canary-token"));
}
#[test]
fn invalid_headers_return_a_safe_error() {
let config = OtlpEnvSettings {
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
traces_headers: Some("authorization=canary-secret%0Ainjected".to_owned()),
..OtlpEnvSettings::default()
};
let error = config.into_config().unwrap_err();
assert!(matches!(error, super::OtlpTraceConfigError::InvalidHeaders));
assert!(!error.to_string().contains("canary-secret"));
}
#[test]
fn stricter_batch_timeout_bounds_http_export() {
let config = OtlpEnvSettings {
traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()),
traces_protocol: Some("http/protobuf".to_owned()),
traces_timeout: Some("9000".to_owned()),
batch_export_timeout: Some("2500".to_owned()),
..OtlpEnvSettings::default()
}
.into_config()
.unwrap();
assert_eq!(
config.effective_export_timeout(),
Duration::from_millis(2500)
);
}
#[test]
fn generic_endpoint_receives_standard_trace_path() {
let config = OtlpEnvSettings {
generic_endpoint: Some("https://generic.example.test/otel/".to_owned()),
generic_protocol: Some("http/protobuf".to_owned()),
..OtlpEnvSettings::default()
}
.into_config()
.unwrap();
assert_eq!(
config.endpoint(),
Some("https://generic.example.test/otel/v1/traces")
);
}
#[test]
fn disabled_export_does_not_build_a_provider() {
let config = OtlpTraceConfig::try_new(
None,
None,
Duration::from_secs(1),
OtlpBatchConfig::default(),
)
.unwrap();
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap();
assert!(build_tracer_provider(&identity, &config).unwrap().is_none());
}
#[test]
fn real_http_protobuf_export_contains_resource_and_trace() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let (request_tx, request_rx) = mpsc::sync_channel(1);
let server = thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let request = read_http_request(&mut stream);
stream
.write_all(
b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
)
.unwrap();
request_tx.send(request).unwrap();
});
let config = OtlpEnvSettings {
traces_endpoint: Some(format!("http://{address}/v1/traces")),
traces_protocol: Some("http/protobuf".to_owned()),
traces_timeout: Some("2000".to_owned()),
traces_headers: Some(String::new()),
generic_headers: Some("authorization=Bearer%20canary-token".to_owned()),
max_queue_size: Some("16".to_owned()),
max_export_batch_size: Some("8".to_owned()),
scheduled_delay: Some("10".to_owned()),
batch_export_timeout: Some("2000".to_owned()),
..OtlpEnvSettings::default()
}
.into_config()
.unwrap();
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "integration-test").unwrap();
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
let mut span = tracer.start("http.request");
let trace_id = span.span_context().trace_id().to_bytes();
span.set_attribute(KeyValue::new("request_id", "req_otlp_contract"));
span.set_attribute(KeyValue::new("authorization", "Bearer canary-span-secret"));
span.add_event(
"canary-span-event",
vec![KeyValue::new("payload", "canary-span-secret")],
);
span.set_status(Status::error("canary-span-secret"));
span.end();
provider.force_flush().unwrap();
provider.shutdown().unwrap();
let request = request_rx.recv_timeout(Duration::from_secs(2)).unwrap();
server.join().unwrap();
let (headers, body) = split_http_request(&request);
assert!(headers.contains("POST /v1/traces HTTP/1.1"));
assert!(
headers
.to_ascii_lowercase()
.contains("content-type: application/x-protobuf")
);
assert!(
headers
.to_ascii_lowercase()
.contains("authorization: bearer canary-token")
);
let export = ExportTraceServiceRequest::decode(body).unwrap();
let resource_spans = export.resource_spans.first().unwrap();
let attributes = &resource_spans.resource.as_ref().unwrap().attributes;
assert_eq!(
string_attribute(attributes, "service.name"),
Some("admin-api")
);
assert_eq!(
string_attribute(attributes, "service.version"),
Some("0.3.1")
);
assert_eq!(
string_attribute(attributes, "deployment.environment.name"),
Some("integration-test")
);
assert_eq!(
resource_spans.scope_spans[0].spans[0].trace_id.as_slice(),
trace_id
);
let exported_span = &resource_spans.scope_spans[0].spans[0];
assert_eq!(exported_span.name, "http.request");
assert_eq!(
string_attribute(&exported_span.attributes, "request_id"),
Some("req_otlp_contract")
);
assert!(
exported_span
.attributes
.iter()
.all(|attribute| attribute.key != "authorization")
);
assert!(exported_span.events.is_empty());
assert_eq!(
exported_span
.status
.as_ref()
.map(|status| status.message.as_str()),
Some("")
);
assert!(
!body
.windows(b"canary-span-secret".len())
.any(|window| { window == b"canary-span-secret" })
);
}
#[tokio::test(flavor = "current_thread")]
async fn unavailable_receiver_does_not_change_product_result() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = thread::spawn(move || {
let (stream, _) = listener.accept().unwrap();
drop(stream);
});
let config = OtlpTraceConfig::try_new(
Some(format!("http://{address}/v1/traces")),
Some("http/protobuf".to_owned()),
Duration::from_millis(250),
OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(250))
.unwrap(),
)
.unwrap();
let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "fault-test").unwrap();
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let app = Router::new()
.route("/product", get(|| async { "product-success" }))
.layer(axum::middleware::from_fn(trace_product_request));
let response = app
.oneshot(
Request::builder()
.uri("/product")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = to_bytes(response.into_body(), 64).await.unwrap();
assert_eq!(status, axum::http::StatusCode::OK);
assert_eq!(body.as_ref(), b"product-success");
assert!(provider.force_flush().is_err());
let _ = provider.shutdown();
server.join().unwrap();
}
async fn trace_product_request(request: Request, next: Next) -> Response {
next.run(request)
.instrument(info_span!(target: "crank::trace", "http.request"))
.await
}
#[test]
fn hanging_receiver_respects_the_stricter_export_timeout() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = thread::spawn(move || {
let (stream, _) = listener.accept().unwrap();
thread::sleep(Duration::from_millis(750));
drop(stream);
});
let config = OtlpTraceConfig::try_new(
Some(format!("http://{address}/v1/traces")),
Some("http/protobuf".to_owned()),
Duration::from_secs(2),
OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(100))
.unwrap(),
)
.unwrap();
let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "timeout-test").unwrap();
let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap();
let mut span = tracer.start("http.request");
span.end();
let started_at = Instant::now();
assert!(provider.force_flush().is_err());
assert!(started_at.elapsed() < Duration::from_millis(500));
let _ = provider.shutdown();
server.join().unwrap();
}
fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.unwrap();
let mut request = Vec::new();
let mut buffer = [0_u8; 4096];
loop {
let read = stream.read(&mut buffer).unwrap();
request.extend_from_slice(&buffer[..read]);
let Some(header_end) = find_bytes(&request, b"\r\n\r\n") else {
continue;
};
let headers = String::from_utf8_lossy(&request[..header_end]);
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().ok())
.flatten()
})
.unwrap_or(0);
if request.len() >= header_end + 4 + content_length {
return request;
}
}
}
fn split_http_request(request: &[u8]) -> (&str, &[u8]) {
let header_end = find_bytes(request, b"\r\n\r\n").unwrap();
(
std::str::from_utf8(&request[..header_end]).unwrap(),
&request[header_end + 4..],
)
}
fn string_attribute<'a>(
attributes: &'a [opentelemetry_proto::tonic::common::v1::KeyValue],
key: &str,
) -> Option<&'a str> {
attributes.iter().find_map(|attribute| {
let value = attribute.value.as_ref()?.value.as_ref()?;
(attribute.key == key)
.then_some(value)
.and_then(|value| match value {
any_value::Value::StringValue(value) => Some(value.as_str()),
_ => None,
})
})
}
fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack
.windows(needle.len())
.position(|candidate| candidate == needle)
}
}
crates/crank-observability/src/prometheus.rs
+283
View File
@@ -0,0 +1,283 @@
use std::{env, net::SocketAddr};
use axum::{
Router,
extract::{Request, State},
http::{
HeaderMap, StatusCode,
header::{self, HeaderValue},
},
middleware::{self, Next},
response::{IntoResponse, Response},
routing::get,
};
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle, PrometheusRecorder};
use sha2::{Digest, Sha256};
use subtle::ConstantTimeEq;
use thiserror::Error;
use tokio::net::TcpListener;
use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity};
const METRICS_ENABLED_ENV: &str = "CRANK_METRICS_ENABLED";
const METRICS_TOKEN_ENV: &str = "CRANK_METRICS_BEARER_TOKEN";
const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
#[derive(Clone)]
pub struct MetricsConfig {
enabled: bool,
bind_addr: SocketAddr,
token_digest: Option<[u8; 32]>,
}
impl std::fmt::Debug for MetricsConfig {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("MetricsConfig")
.field("enabled", &self.enabled)
.field("bind_addr", &self.bind_addr)
.field("authentication_configured", &self.token_digest.is_some())
.finish()
}
}
impl MetricsConfig {
pub fn new(
enabled: bool,
bind_addr: SocketAddr,
bearer_token: Option<String>,
) -> Result<Self, MetricsConfigError> {
let token_digest = bearer_token
.filter(|token| !token.is_empty())
.map(|token| token_digest(token.as_bytes()));
if enabled && !bind_addr.ip().is_loopback() && token_digest.is_none() {
return Err(MetricsConfigError::MissingTokenForExternalBind);
}
Ok(Self {
enabled,
bind_addr,
token_digest,
})
}
pub fn from_env(
bind_env: &'static str,
default_bind: SocketAddr,
) -> Result<Self, MetricsConfigError> {
let enabled = parse_enabled(env::var(METRICS_ENABLED_ENV))?;
let bind_addr = match env::var(bind_env) {
Ok(raw) => raw
.parse()
.map_err(|_| MetricsConfigError::InvalidBindAddress { field: bind_env })?,
Err(env::VarError::NotPresent) => default_bind,
Err(env::VarError::NotUnicode(_)) => {
return Err(MetricsConfigError::InvalidEnvironmentEncoding { field: bind_env });
}
};
let bearer_token = match env::var(METRICS_TOKEN_ENV) {
Ok(token) => Some(token),
Err(env::VarError::NotPresent) => None,
Err(env::VarError::NotUnicode(_)) => {
return Err(MetricsConfigError::InvalidEnvironmentEncoding {
field: METRICS_TOKEN_ENV,
});
}
};
Self::new(enabled, bind_addr, bearer_token)
}
pub fn enabled(&self) -> bool {
self.enabled
}
pub fn bind_addr(&self) -> SocketAddr {
self.bind_addr
}
pub fn requires_authentication(&self) -> bool {
!self.bind_addr.ip().is_loopback()
}
}
#[derive(Debug, Error)]
pub enum MetricsConfigError {
#[error("metrics environment variable is not valid UTF-8: {field}")]
InvalidEnvironmentEncoding { field: &'static str },
#[error("metrics bind address is invalid: {field}")]
InvalidBindAddress { field: &'static str },
#[error("metrics enabled flag must be one of true, false, 1, 0")]
InvalidEnabledFlag,
#[error("external metrics bind requires a bearer token")]
MissingTokenForExternalBind,
}
#[derive(Clone)]
struct MetricsState {
handle: PrometheusHandle,
token_digest: Option<[u8; 32]>,
requires_authentication: bool,
}
pub struct MetricsSurface {
config: MetricsConfig,
state: MetricsState,
_recorder: Option<PrometheusRecorder>,
}
impl MetricsSurface {
pub(crate) fn new(config: MetricsConfig, handle: PrometheusHandle) -> Self {
Self {
state: MetricsState {
handle,
token_digest: config.token_digest,
requires_authentication: config.requires_authentication(),
},
config,
_recorder: None,
}
}
pub fn for_test(
config: MetricsConfig,
identity: ServiceIdentity,
) -> Result<Self, MetricsSurfaceError> {
let recorder = prometheus_builder(&identity)?.build_recorder();
let handle = recorder.handle();
let mut surface = Self::new(config, handle);
surface._recorder = Some(recorder);
Ok(surface)
}
pub fn router(&self) -> Router {
Router::new()
.route("/metrics", get(render_metrics))
.route("/health", get(metrics_health))
.layer(middleware::from_fn_with_state(
self.state.clone(),
authorize_metrics,
))
.with_state(self.state.clone())
}
pub async fn bind(self) -> Result<MetricsServer, MetricsServeError> {
let listener = TcpListener::bind(self.config.bind_addr)
.await
.map_err(|_| MetricsServeError::Bind)?;
Ok(MetricsServer {
listener,
router: self.router(),
})
}
}
pub struct MetricsServer {
listener: TcpListener,
router: Router,
}
impl MetricsServer {
pub async fn serve(self) -> Result<(), MetricsServeError> {
axum::serve(self.listener, self.router)
.await
.map_err(|_| MetricsServeError::Serve)
}
}
#[derive(Debug, Error)]
pub enum MetricsSurfaceError {
#[error("failed to configure Prometheus recorder")]
RecorderConfiguration,
}
#[derive(Debug, Error)]
pub enum MetricsServeError {
#[error("failed to bind metrics listener")]
Bind,
#[error("metrics listener stopped unexpectedly")]
Serve,
}
pub(crate) fn install_prometheus_recorder(
identity: &ServiceIdentity,
) -> Result<PrometheusHandle, MetricsSurfaceError> {
prometheus_builder(identity)?
.install_recorder()
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
}
fn prometheus_builder(
identity: &ServiceIdentity,
) -> Result<PrometheusBuilder, MetricsSurfaceError> {
PrometheusBuilder::new()
.set_buckets(DURATION_BUCKETS_SECONDS)
.map(|builder| {
builder
.add_global_label("service", identity.service())
.add_global_label("version", identity.version())
.add_global_label("environment", identity.environment())
})
.map_err(|_| MetricsSurfaceError::RecorderConfiguration)
}
async fn render_metrics(State(state): State<MetricsState>) -> Response {
let mut response = state.handle.render().into_response();
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static(PROMETHEUS_CONTENT_TYPE),
);
response
}
async fn metrics_health() -> impl IntoResponse {
(StatusCode::OK, "ok\n")
}
async fn authorize_metrics(
State(state): State<MetricsState>,
request: Request,
next: Next,
) -> Response {
if !state.requires_authentication {
return next.run(request).await;
}
let authorized = bearer_token(request.headers())
.map(token_digest)
.zip(state.token_digest)
.is_some_and(|(actual, expected)| bool::from(actual.ct_eq(&expected)));
if authorized {
next.run(request).await
} else {
StatusCode::UNAUTHORIZED.into_response()
}
}
fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> {
headers
.get(header::AUTHORIZATION)?
.as_bytes()
.strip_prefix(b"Bearer ")
.filter(|token| !token.is_empty())
}
fn token_digest(token: &[u8]) -> [u8; 32] {
Sha256::digest(token).into()
}
fn parse_enabled(value: Result<String, env::VarError>) -> Result<bool, MetricsConfigError> {
match value {
Ok(raw) => match raw.to_ascii_lowercase().as_str() {
"true" | "1" => Ok(true),
"false" | "0" => Ok(false),
_ => Err(MetricsConfigError::InvalidEnabledFlag),
},
Err(env::VarError::NotPresent) => Ok(true),
Err(env::VarError::NotUnicode(_)) => Err(MetricsConfigError::InvalidEnvironmentEncoding {
field: METRICS_ENABLED_ENV,
}),
}
}
crates/crank-observability/src/propagation.rs
+70
View File
@@ -0,0 +1,70 @@
use axum::http::{HeaderMap, HeaderName, HeaderValue};
use opentelemetry::{
Context, global,
propagation::{Extractor, Injector},
trace::TraceContextExt,
};
use opentelemetry_sdk::propagation::TraceContextPropagator;
use tracing::Span;
use tracing_opentelemetry::OpenTelemetrySpanExt;
pub fn set_remote_trace_parent(span: &Span, headers: &HeaderMap) -> bool {
let context =
global::get_text_map_propagator(|propagator| propagator.extract(&HeaderExtractor(headers)));
let span_context = context.span().span_context().clone();
if !span_context.is_valid() || !span_context.is_remote() {
return false;
}
span.set_parent(context).is_ok()
}
pub fn inject_current_trace_context(headers: &mut HeaderMap) -> bool {
let context = Span::current().context();
if !context.span().span_context().is_valid() {
return false;
}
global::get_text_map_propagator(|propagator| {
propagator.inject_context(&context, &mut HeaderInjector(headers));
});
true
}
pub(crate) fn install_trace_context_propagator() {
global::set_text_map_propagator(TraceContextPropagator::new());
}
struct HeaderExtractor<'a>(&'a HeaderMap);
impl Extractor for HeaderExtractor<'_> {
fn get(&self, key: &str) -> Option<&str> {
self.0.get(key).and_then(|value| value.to_str().ok())
}
fn keys(&self) -> Vec<&str> {
self.0.keys().map(HeaderName::as_str).collect()
}
}
struct HeaderInjector<'a>(&'a mut HeaderMap);
impl Injector for HeaderInjector<'_> {
fn set(&mut self, key: &str, value: String) {
let Ok(name) = HeaderName::try_from(key) else {
return;
};
let Ok(value) = HeaderValue::try_from(value) else {
return;
};
self.0.insert(name, value);
}
}
pub(crate) fn current_trace_id() -> Option<String> {
let context: Context = Span::current().context();
let span_context = context.span().span_context().clone();
span_context
.is_valid()
.then(|| span_context.trace_id().to_string())
}
crates/crank-observability/src/redaction.rs
+292
View File
@@ -0,0 +1,292 @@
use serde_json::{Map, Value};
use thiserror::Error;
pub const REDACTED_MARKER: &str = "[REDACTED]";
const TRUNCATED_MARKER: &str = "[TRUNCATED]";
const MIN_EVENT_BYTES: usize = 512;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct RedactionLimits {
pub max_string_bytes: usize,
pub max_array_items: usize,
pub max_object_fields: usize,
pub max_depth: usize,
pub max_event_bytes: usize,
}
impl Default for RedactionLimits {
fn default() -> Self {
Self {
max_string_bytes: 1024,
max_array_items: 32,
max_object_fields: 64,
max_depth: 8,
max_event_bytes: 16 * 1024,
}
}
}
impl RedactionLimits {
pub fn validate(self) -> Result<(), RedactionLimitsError> {
for (field, value, minimum) in [
(
"max_string_bytes",
self.max_string_bytes,
TRUNCATED_MARKER.len(),
),
("max_array_items", self.max_array_items, 1),
("max_object_fields", self.max_object_fields, 1),
("max_depth", self.max_depth, 1),
("max_event_bytes", self.max_event_bytes, MIN_EVENT_BYTES),
] {
if value < minimum {
return Err(RedactionLimitsError::TooSmall { field, minimum });
}
}
Ok(())
}
}
#[derive(Debug, Error)]
pub enum RedactionLimitsError {
#[error("invalid redaction limit {field}: minimum is {minimum}")]
TooSmall { field: &'static str, minimum: usize },
}
#[derive(Debug, Error)]
pub enum SafeJsonError {
#[error(transparent)]
InvalidLimits(#[from] RedactionLimitsError),
#[error(transparent)]
Serialization(#[from] serde_json::Error),
}
pub fn redact_value(value: &Value, limits: RedactionLimits) -> Value {
redact_at_depth(value, limits, 0)
}
pub fn safe_json(value: &Value, limits: RedactionLimits) -> Result<String, SafeJsonError> {
limits.validate()?;
let serialized = serde_json::to_string(&redact_value(value, limits))?;
if serialized.len() <= limits.max_event_bytes {
return Ok(serialized);
}
Ok(serde_json::to_string(&serde_json::json!({
"truncated": true
}))?)
}
pub(crate) fn truncate_string(value: &str, max_bytes: usize) -> String {
if value.len() <= max_bytes {
return value.to_owned();
}
if max_bytes == 0 {
return String::new();
}
let marker = if max_bytes >= TRUNCATED_MARKER.len() {
TRUNCATED_MARKER
} else {
""
};
let content_budget = max_bytes.saturating_sub(marker.len());
let mut boundary = content_budget.min(value.len());
while boundary > 0 && !value.is_char_boundary(boundary) {
boundary -= 1;
}
let mut truncated = String::with_capacity(max_bytes);
truncated.push_str(&value[..boundary]);
if marker.is_empty() {
let mut marker_boundary = max_bytes.min(TRUNCATED_MARKER.len());
while marker_boundary > 0 && !TRUNCATED_MARKER.is_char_boundary(marker_boundary) {
marker_boundary -= 1;
}
truncated.clear();
truncated.push_str(&TRUNCATED_MARKER[..marker_boundary]);
} else {
truncated.push_str(marker);
}
truncated
}
fn redact_at_depth(value: &Value, limits: RedactionLimits, depth: usize) -> Value {
if depth >= limits.max_depth {
return Value::String(TRUNCATED_MARKER.to_owned());
}
match value {
Value::Null | Value::Bool(_) | Value::Number(_) => value.clone(),
Value::String(value) => Value::String(truncate_string(value, limits.max_string_bytes)),
Value::Array(values) => redact_array(values, limits, depth),
Value::Object(values) => redact_object(values, limits, depth),
}
}
fn redact_array(values: &[Value], limits: RedactionLimits, depth: usize) -> Value {
if limits.max_array_items == 0 {
return Value::Array(Vec::new());
}
let truncated = values.len() > limits.max_array_items;
let value_limit = if truncated {
limits.max_array_items.saturating_sub(1)
} else {
limits.max_array_items
};
let mut output: Vec<_> = values
.iter()
.take(value_limit)
.map(|value| redact_at_depth(value, limits, depth + 1))
.collect();
if truncated {
output.push(Value::String(TRUNCATED_MARKER.to_owned()));
}
Value::Array(output)
}
fn redact_object(values: &Map<String, Value>, limits: RedactionLimits, depth: usize) -> Value {
if limits.max_object_fields == 0 {
return Value::Object(Map::new());
}
let truncated = values.len() > limits.max_object_fields;
let value_limit = if truncated {
limits.max_object_fields.saturating_sub(1)
} else {
limits.max_object_fields
};
let mut output = Map::new();
for (index, (key, value)) in values.iter().take(value_limit).enumerate() {
let cleaned = if is_sensitive_key(key) {
Value::String(REDACTED_MARKER.to_owned())
} else if is_url_key(key) {
value
.as_str()
.map(sanitize_url)
.map(|value| truncate_string(&value, limits.max_string_bytes))
.map(Value::String)
.unwrap_or_else(|| redact_at_depth(value, limits, depth + 1))
} else {
redact_at_depth(value, limits, depth + 1)
};
output.insert(
bounded_object_key(key, limits.max_string_bytes, index),
cleaned,
);
}
if truncated {
output.insert(
"_truncated".to_owned(),
Value::String(TRUNCATED_MARKER.to_owned()),
);
}
Value::Object(output)
}
fn normalized_key(key: &str) -> String {
key.bytes()
.filter(|byte| !matches!(byte, b'_' | b'-' | b'.'))
.map(|byte| byte.to_ascii_lowercase() as char)
.collect()
}
fn is_sensitive_key(key: &str) -> bool {
let key = normalized_key(key);
let exact_match = matches!(
key.as_str(),
"password"
| "passwd"
| "secret"
| "token"
| "apikey"
| "accesskey"
| "secretkey"
| "authorization"
| "proxyauthorization"
| "cookie"
| "setcookie"
| "query"
| "querystring"
| "rawquery"
| "urlquery"
| "payload"
| "body"
| "requestbody"
| "arguments"
| "result"
| "response"
| "context"
| "error"
| "errormessage"
);
let contains_high_risk_name = [
"password",
"passwd",
"secret",
"token",
"apikey",
"accesskey",
"authorization",
"cookie",
]
.iter()
.any(|part| key.contains(part));
exact_match
|| contains_high_risk_name
|| key.ends_with("payload")
|| key.ends_with("body")
|| key.ends_with("arguments")
|| key.ends_with("result")
|| key.ends_with("response")
|| key.ends_with("query")
|| key.ends_with("context")
|| key.starts_with("query")
}
fn is_url_key(key: &str) -> bool {
let key = normalized_key(key);
matches!(
key.as_str(),
"url" | "uri" | "endpoint" | "endpointurl" | "endpointuri" | "requesturl" | "targeturl"
) || key.ends_with("url")
|| key.ends_with("uri")
|| key.ends_with("endpoint")
}
fn sanitize_url(value: &str) -> String {
let without_query = value
.find(['?', '#'])
.map(|index| &value[..index])
.unwrap_or(value);
let Some(scheme_end) = without_query.find("://") else {
return without_query.to_owned();
};
let authority_start = scheme_end + 3;
let authority_end = without_query[authority_start..]
.find('/')
.map(|index| authority_start + index)
.unwrap_or(without_query.len());
let authority = &without_query[authority_start..authority_end];
let Some(userinfo_end) = authority.rfind('@') else {
return without_query.to_owned();
};
format!(
"{}{}",
&without_query[..authority_start],
&without_query[authority_start + userinfo_end + 1..]
)
}
fn bounded_object_key(key: &str, max_bytes: usize, index: usize) -> String {
if key.len() <= max_bytes {
return key.to_owned();
}
truncate_string(&format!("_truncated_key_{index}"), max_bytes)
}
crates/crank-observability/src/schema.rs
+18
View File
@@ -0,0 +1,18 @@
use serde::Serialize;
use serde_json::{Map, Value};
#[derive(Debug, Serialize)]
pub(crate) struct LogEnvelope {
pub timestamp: String,
pub level: String,
pub service: String,
pub version: String,
pub environment: String,
pub target: String,
pub event: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub request_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub trace_id: Option<String>,
pub fields: Map<String, Value>,
}
crates/crank-observability/tests/correlation.rs
+42
View File
@@ -0,0 +1,42 @@
use crank_observability::RequestId;
use uuid::Version;
#[test]
fn preserves_valid_opaque_request_id() {
let request_id = RequestId::resolve(Some("req_test-123/abc"));
assert_eq!(request_id.as_str(), "req_test-123/abc");
}
#[test]
fn replaces_missing_and_invalid_values_with_uuid_v7() {
for candidate in [
None,
Some(""),
Some("bad value"),
Some(" leading"),
Some("trailing "),
Some("bad,value"),
Some("bad;value"),
Some("я"),
] {
let request_id = RequestId::resolve(candidate);
let parsed = uuid::Uuid::parse_str(request_id.as_str()).expect("generated UUID");
assert_eq!(parsed.get_version(), Some(Version::SortRand));
}
}
#[test]
fn rejects_values_over_the_shared_limit() {
let oversized = "x".repeat(RequestId::MAX_LEN + 1);
let request_id = RequestId::resolve(Some(&oversized));
assert_ne!(request_id.as_str(), oversized);
assert_eq!(
uuid::Uuid::parse_str(request_id.as_str())
.expect("generated UUID")
.get_version(),
Some(Version::SortRand)
);
}
crates/crank-observability/tests/critical_errors.rs
+33
View File
@@ -0,0 +1,33 @@
use crank_observability::{CriticalErrorCategory, SentryConfig, SentryConfigError};
#[test]
fn missing_or_blank_dsn_disables_critical_error_channel() {
assert!(!SentryConfig::parse(None).expect("missing DSN").enabled());
assert!(!SentryConfig::parse(Some("")).expect("empty DSN").enabled());
assert!(
!SentryConfig::parse(Some(" "))
.expect("blank DSN")
.enabled()
);
}
#[test]
fn invalid_explicit_dsn_is_rejected_without_echoing_the_value() {
let secret_value = "not-a-dsn?token=control-secret";
let error = SentryConfig::parse(Some(secret_value)).expect_err("invalid DSN must fail");
assert!(matches!(error, SentryConfigError::InvalidDsn));
assert!(!error.to_string().contains(secret_value));
assert!(!error.to_string().contains("control-secret"));
}
#[test]
fn critical_error_categories_are_closed_and_stable() {
assert_eq!(CriticalErrorCategory::Panic.as_str(), "panic");
assert_eq!(CriticalErrorCategory::Startup.as_str(), "startup");
assert_eq!(CriticalErrorCategory::Internal.as_str(), "internal");
assert_eq!(
CriticalErrorCategory::DataIntegrity.as_str(),
"data_integrity"
);
}
crates/crank-observability/tests/http_metrics.rs
+92
View File
@@ -0,0 +1,92 @@
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use axum::{
Router,
body::{Body, to_bytes},
http::{Request, StatusCode},
middleware,
routing::get,
};
use crank_observability::{
MetricsConfig, ObservabilityConfig, RedactionLimits, ServiceIdentity, record_http_request,
};
use tower::ServiceExt;
#[tokio::test]
async fn http_metrics_use_matched_routes_and_closed_labels() {
let identity =
ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity");
let lifecycle = crank_observability::init(ObservabilityConfig::new(
identity,
"off",
RedactionLimits::default(),
))
.expect("observability lifecycle");
let config = MetricsConfig::new(
true,
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464),
None,
)
.expect("loopback metrics");
let metrics = lifecycle.metrics_surface(config).router();
let app = Router::new()
.route(
"/documents/{document_id}",
get(|| async { StatusCode::NO_CONTENT }),
)
.layer(middleware::from_fn(record_http_request));
let sensitive_path_segment = "customer-secret-document-id";
for index in 0..100 {
let response = app
.clone()
.oneshot(
Request::get(format!("/documents/{sensitive_path_segment}-{index}"))
.body(Body::empty())
.expect("request"),
)
.await
.expect("response");
assert_eq!(response.status(), StatusCode::NO_CONTENT);
}
let response = app
.oneshot(
Request::get("/unknown/customer-controlled-path")
.body(Body::empty())
.expect("request"),
)
.await
.expect("response");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
let response = metrics
.oneshot(
Request::get("/metrics")
.body(Body::empty())
.expect("request"),
)
.await
.expect("metrics response");
let body = to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("bounded metrics body");
let body = String::from_utf8(body.to_vec()).expect("utf-8 metrics");
assert!(body.contains("crank_http_requests_total"));
assert!(body.contains("route=\"/documents/{document_id}\""));
assert!(body.contains("method=\"GET\""));
assert!(body.contains("status_class=\"2xx\""));
assert!(body.contains("crank_http_request_duration_seconds_bucket"));
assert!(!body.contains(sensitive_path_segment));
assert_eq!(
body.lines()
.filter(|line| {
line.starts_with("crank_http_requests_total{")
&& line.contains("route=\"/documents/{document_id}\"")
})
.count(),
1,
"different entity ids must not create additional series"
);
}
crates/crank-observability/tests/incidents.rs
+15
View File
@@ -0,0 +1,15 @@
use crank_observability::{
OperationalIncident, operational_incident_total, record_operational_incident,
};
#[test]
fn history_loss_counter_has_no_dynamic_dimensions() {
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
record_operational_incident(OperationalIncident::InvocationHistoryLost);
assert!(
operational_incident_total(OperationalIncident::InvocationHistoryLost) > before,
"the closed incident counter must increase"
);
}
crates/crank-observability/tests/json_logging.rs
+392
View File
@@ -0,0 +1,392 @@
use std::{
io,
sync::{Arc, Mutex},
};
use crank_observability::{
ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber, safe_json,
};
use serde_json::Value;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tracing_subscriber::fmt::MakeWriter;
#[derive(Clone, Default)]
struct SharedWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().expect("test writer lock").clone())
.expect("log output must be UTF-8")
}
}
impl<'a> MakeWriter<'a> for SharedWriter {
type Writer = SharedWriterGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedWriterGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedWriterGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedWriterGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer
.lock()
.map_err(|_| io::Error::other("test writer lock poisoned"))?
.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
fn capture(service: &'static str, emit: impl FnOnce()) -> Vec<Value> {
capture_with_limits(service, RedactionLimits::default(), emit)
}
fn capture_with_limits(
service: &'static str,
limits: RedactionLimits,
emit: impl FnOnce(),
) -> Vec<Value> {
let writer = SharedWriter::default();
let config = ObservabilityConfig::new(
ServiceIdentity::try_new(service, "0.3.1", "test").expect("valid test identity"),
"info",
limits,
);
let subscriber =
build_subscriber(config, writer.clone()).expect("test subscriber must be built");
tracing::subscriber::with_default(subscriber, emit);
writer
.output()
.lines()
.map(|line| {
assert!(!line.contains('\u{1b}'), "ANSI is forbidden: {line}");
serde_json::from_str(line).expect("every line must be one JSON object")
})
.collect()
}
#[test]
fn schema_contract_is_identical_for_both_services() {
let outputs = ["admin-api", "mcp-server"].map(|service| {
capture(service, || {
tracing::info!(
name: "service.started",
target: "crank::startup",
port = 3101_u64,
"service started"
);
})
});
for (service, events) in ["admin-api", "mcp-server"].into_iter().zip(outputs.iter()) {
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(event["service"], service);
assert_eq!(event["version"], "0.3.1");
assert_eq!(event["environment"], "test");
assert_eq!(event["level"], "INFO");
assert_eq!(event["target"], "crank::startup");
assert_eq!(event["event"], "service.started");
assert!(event["fields"].is_object());
assert_eq!(event["fields"]["port"], 3101);
assert_eq!(event["fields"]["message"], "service started");
let timestamp = event["timestamp"].as_str().expect("timestamp string");
let parsed =
OffsetDateTime::parse(timestamp, &Rfc3339).expect("timestamp must be RFC 3339");
assert_eq!(parsed.offset(), time::UtcOffset::UTC);
}
let first_keys: Vec<_> = outputs[0][0]
.as_object()
.expect("object")
.keys()
.cloned()
.collect();
let second_keys: Vec<_> = outputs[1][0]
.as_object()
.expect("object")
.keys()
.cloned()
.collect();
assert_eq!(first_keys, second_keys);
}
#[test]
fn correlation_fields_are_distinct_and_only_present_when_recorded() {
let present = capture("admin-api", || {
tracing::info!(
name: "admin.request.completed",
request_id = "req-123",
trace_id = "trace-456"
);
});
assert_eq!(present[0]["request_id"], "req-123");
assert_eq!(present[0]["trace_id"], "trace-456");
assert!(present[0]["fields"].get("request_id").is_none());
assert!(present[0]["fields"].get("trace_id").is_none());
let absent = capture("mcp-server", || {
tracing::info!(name: "mcp.request.completed", status = 200_u64);
});
assert!(absent[0].get("request_id").is_none());
assert!(absent[0].get("trace_id").is_none());
}
#[test]
fn correlation_fields_preserve_scalar_display_values_before_field_limits() {
let limits = RedactionLimits {
max_object_fields: 1,
..RedactionLimits::default()
};
let request_id = "123";
let events = capture_with_limits("admin-api", limits, || {
tracing::info!(
name: "admin.request.completed",
alpha = "field that consumes the object budget",
request_id = %request_id,
trace_id = true,
);
});
assert_eq!(events[0]["request_id"], "123");
assert_eq!(events[0]["trace_id"], "true");
}
#[test]
fn empty_correlation_fields_are_omitted() {
let events = capture("admin-api", || {
tracing::info!(
name: "admin.request.completed",
request_id = "",
trace_id = ""
);
});
assert!(events[0].get("request_id").is_none());
assert!(events[0].get("trace_id").is_none());
}
#[test]
fn formatter_redacts_fields_before_serialization() {
let context = safe_json(
&serde_json::json!({
"nested": {
"access_token": "nested-canary-secret",
"endpoint": "https://example.test/private?key=nested-canary-secret"
}
}),
RedactionLimits::default(),
)
.expect("safe nested context");
let events = capture("admin-api", || {
tracing::warn!(
name: "admin.request.rejected",
password = "canary-secret",
url = "https://example.test/path?token=canary-secret",
safe_fields = %context,
unsafe_context = ?serde_json::json!({"password": "debug-canary-secret"}),
error_code = "invalid_request"
);
});
let serialized = serde_json::to_string(&events[0]).expect("event JSON");
assert_eq!(events[0]["fields"]["password"], "[REDACTED]");
assert_eq!(events[0]["fields"]["url"], "https://example.test/path");
assert_eq!(
events[0]["fields"]["safe_fields"]["nested"]["access_token"],
"[REDACTED]"
);
assert_eq!(
events[0]["fields"]["safe_fields"]["nested"]["endpoint"],
"https://example.test/private"
);
assert_eq!(events[0]["fields"]["unsafe_context"], "[REDACTED]");
assert_eq!(events[0]["fields"]["error_code"], "invalid_request");
assert!(!serialized.contains("canary-secret"));
}
#[test]
fn arbitrary_debug_text_is_never_written_verbatim() {
struct Credentials;
impl std::fmt::Debug for Credentials {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("Credentials { password: \"debug-canary-secret\" }")
}
}
let events = capture("admin-api", || {
tracing::warn!(
name: "admin.debug.inspected",
details = ?Credentials
);
});
let serialized = serde_json::to_string(&events[0]).expect("event JSON");
assert_eq!(events[0]["fields"]["details"], "[REDACTED]");
assert!(!serialized.contains("debug-canary-secret"));
}
#[test]
fn compound_sensitive_event_fields_are_redacted() {
let events = capture("admin-api", || {
tracing::warn!(
name: "admin.request.rejected",
client_api_key = "client-canary-secret",
authorization_header = "Bearer auth-canary-secret",
response_body = "response-canary-secret",
tool_arguments = "argument-canary-secret",
query_params = "query-canary-secret",
);
});
let serialized = serde_json::to_string(&events[0]).expect("event JSON");
for key in [
"client_api_key",
"authorization_header",
"response_body",
"tool_arguments",
"query_params",
] {
assert_eq!(events[0]["fields"][key], "[REDACTED]");
}
assert!(!serialized.contains("canary-secret"));
}
#[test]
fn oversized_event_falls_back_to_valid_bounded_json() {
let limits = RedactionLimits {
max_event_bytes: 512,
..RedactionLimits::default()
};
let events = capture_with_limits("admin-api", limits, || {
tracing::info!(
name: "admin.payload.inspected",
description = %"x".repeat(1024)
);
});
let serialized = serde_json::to_vec(&events[0]).expect("bounded event JSON");
assert!(serialized.len() < limits.max_event_bytes);
assert_eq!(events[0]["fields"]["truncated"], true);
}
#[test]
fn safe_json_honours_the_total_event_budget() {
let limits = RedactionLimits {
max_event_bytes: 512,
..RedactionLimits::default()
};
let serialized = safe_json(
&serde_json::json!({"description": "x".repeat(4096)}),
limits,
)
.expect("safe JSON must remain serializable");
assert!(serialized.len() <= limits.max_event_bytes);
assert_eq!(
serde_json::from_str::<Value>(&serialized).expect("valid JSON")["truncated"],
true
);
}
#[test]
fn subscriber_rejects_limits_that_cannot_hold_an_event() {
let config = ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
"info",
RedactionLimits {
max_event_bytes: 16,
..RedactionLimits::default()
},
);
assert!(build_subscriber(config, SharedWriter::default()).is_err());
}
#[test]
fn minimum_event_budget_handles_maximum_identity_labels() {
let writer = SharedWriter::default();
let config = ObservabilityConfig::new(
ServiceIdentity::try_new("s".repeat(64), "v".repeat(64), "e".repeat(64))
.expect("maximum identity labels are valid"),
"info",
RedactionLimits {
max_event_bytes: 512,
..RedactionLimits::default()
},
);
let subscriber =
build_subscriber(config, writer.clone()).expect("minimum valid budget must be usable");
tracing::subscriber::with_default(subscriber, || {
tracing::info!(
name: "event-name-that-is-intentionally-longer-than-the-fallback-limit",
description = %"x".repeat(4096),
);
});
let output = writer.output();
assert!(output.len() <= 512);
assert_eq!(output.lines().count(), 1);
serde_json::from_str::<Value>(output.trim_end()).expect("bounded line must remain valid JSON");
}
#[test]
fn env_filter_is_applied_and_invalid_filter_is_safe() {
let writer = SharedWriter::default();
let config = ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
"warn",
RedactionLimits::default(),
);
let subscriber =
build_subscriber(config, writer.clone()).expect("test subscriber must be built");
tracing::subscriber::with_default(subscriber, || {
tracing::info!(name: "filtered.info", "filtered");
tracing::warn!(name: "visible.warning", "visible");
});
let output = writer.output();
assert!(!output.contains("filtered.info"));
assert!(output.contains("visible.warning"));
let invalid = ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"),
"[not a valid filter",
RedactionLimits::default(),
);
let error = build_subscriber(invalid, SharedWriter::default())
.err()
.expect("invalid filter must fail");
assert_eq!(error.to_string(), "invalid log filter");
assert!(!error.to_string().contains("not a valid filter"));
}
#[test]
fn service_identity_rejects_empty_or_unsafe_labels() {
for (service, version, environment) in [
("", "0.3.1", "test"),
("admin api", "0.3.1", "test"),
("admin-api", "", "test"),
("admin-api", "0.3.1", "prod\nsecret"),
] {
assert!(ServiceIdentity::try_new(service, version, environment).is_err());
}
}
crates/crank-observability/tests/lifecycle.rs
+22
View File
@@ -0,0 +1,22 @@
use crank_observability::{
ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity, init,
};
fn config() -> ObservabilityConfig {
ObservabilityConfig::new(
ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"),
"info",
RedactionLimits::default(),
)
}
#[test]
fn repeated_global_initialization_returns_typed_error() {
let _lifecycle = init(config()).expect("first initialization must succeed");
let error = init(config()).expect_err("second initialization must fail");
assert!(matches!(
error,
ObservabilityInitError::SubscriberAlreadyInitialized
));
}
crates/crank-observability/tests/otlp.rs
+56
View File
@@ -0,0 +1,56 @@
use std::time::Duration;
use crank_observability::{OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError};
#[test]
fn absent_endpoint_disables_export_without_background_resources() {
let config = OtlpTraceConfig::try_new(
None,
None,
Duration::from_secs(10),
OtlpBatchConfig::default(),
)
.expect("missing endpoint must be valid");
assert!(!config.is_enabled());
}
#[test]
fn explicit_config_accepts_only_bounded_http_protobuf() {
let config = OtlpTraceConfig::try_new(
Some("https://collector.example.test/v1/traces".to_owned()),
Some("http/protobuf".to_owned()),
Duration::from_secs(3),
OtlpBatchConfig::try_new(256, 64, Duration::from_millis(500), Duration::from_secs(3))
.unwrap(),
)
.expect("bounded HTTP protobuf config must be valid");
assert!(config.is_enabled());
assert_eq!(config.export_timeout(), Duration::from_secs(3));
assert_eq!(config.batch().max_queue_size(), 256);
assert_eq!(config.batch().max_export_batch_size(), 64);
}
#[test]
fn invalid_values_return_safe_typed_errors() {
let secret_endpoint = "https://user:canary-secret@collector.example.test/v1/traces";
let error = OtlpTraceConfig::try_new(
Some(secret_endpoint.to_owned()),
Some("grpc".to_owned()),
Duration::ZERO,
OtlpBatchConfig::default(),
)
.expect_err("credentials in endpoint must be rejected");
assert!(matches!(
error,
OtlpTraceConfigError::InvalidEndpoint { .. }
));
assert!(!error.to_string().contains(secret_endpoint));
assert!(!error.to_string().contains("canary-secret"));
let error = OtlpBatchConfig::try_new(8, 9, Duration::from_millis(1), Duration::from_secs(1))
.expect_err("batch cannot exceed queue");
assert!(matches!(error, OtlpTraceConfigError::InvalidBatchLimits));
}

Some files were not shown because too many files have changed in this diff Show More