diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..11700c7 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +jobs = 2 diff --git a/.env.example b/.env.example index 8a04937..a7650d5 100644 --- a/.env.example +++ b/.env.example @@ -29,7 +29,30 @@ CRANK_RUNTIME_MAX_CONCURRENT_JOBS=16 CRANK_OUTBOUND_ALLOWED_HOSTS= CRANK_OUTBOUND_DENIED_HOSTS= CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304 +CRANK_ENVIRONMENT=development CRANK_LOG_LEVEL=info +# Пустое значение отключает канал критических ошибок. +CRANK_SENTRY_DSN= +# Prometheus endpoints use separate listeners and stay on loopback by default. +CRANK_METRICS_ENABLED=true +CRANK_ADMIN_METRICS_BIND=127.0.0.1:9464 +CRANK_MCP_METRICS_BIND=127.0.0.1:9465 +# Required when either metrics listener uses a non-loopback address. +CRANK_METRICS_BEARER_TOKEN= +CRANK_INVOCATION_LOG_RETENTION_DAYS=30 +# Пустой endpoint полностью отключает экспорт трасс. +OTEL_EXPORTER_OTLP_ENDPOINT= +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_TRACES_PROTOCOL= +OTEL_EXPORTER_OTLP_TIMEOUT=10000 +OTEL_EXPORTER_OTLP_TRACES_TIMEOUT= +OTEL_EXPORTER_OTLP_HEADERS= +OTEL_EXPORTER_OTLP_TRACES_HEADERS= +OTEL_BSP_MAX_QUEUE_SIZE=2048 +OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512 +OTEL_BSP_SCHEDULE_DELAY=5000 +OTEL_BSP_EXPORT_TIMEOUT=30000 CRANK_MASTER_KEY=change-me-master-key CRANK_SESSION_SECRET=change-me-session-secret CRANK_PASSWORD_PEPPER=change-me-password-pepper diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 7cdd053..e20de36 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -55,6 +55,9 @@ jobs: docker --version docker info + - name: Install dependency policy tool + run: cargo install cargo-deny --version 0.20.2 --locked + - name: Run tooling unit tests run: python3 -m unittest discover -s tests/unit @@ -67,6 +70,9 @@ jobs: - name: Check Rust code health run: scripts/check-rust-code-health.sh + - name: Check dependency licenses and advisories + run: cargo deny --locked check advisories bans licenses sources + - name: Check Rust boundaries run: scripts/check-rust-boundaries.sh @@ -95,6 +101,10 @@ jobs: working-directory: apps/ui run: npm ci + - name: Audit UI dependencies + working-directory: apps/ui + run: npm audit --audit-level=high + - name: Build UI bundle working-directory: apps/ui run: npm run build @@ -179,9 +189,11 @@ jobs: find .tmp/ui-e2e/logs -maxdepth 1 -type f -print -exec sed -n '1,220p' {} \; || true deployment: - name: Deployment Manifests + name: Community Image Smoke runs-on: ubuntu-latest - needs: ui + needs: + - rust + - ui steps: - name: Checkout @@ -190,6 +202,58 @@ jobs: - name: Validate Community deployment manifest run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q + - name: Build Community images + run: | + docker build -f apps/admin-api/Dockerfile -t crank/admin-api:ci . + docker build -f apps/mcp-server/Dockerfile -t crank/mcp-server:ci . + docker build -f apps/ui/Dockerfile -t crank/ui:ci . + + - name: Start Community image stack + run: | + mkdir -p .tmp + cat > .tmp/community-smoke.env <<'EOF' + COMPOSE_PROJECT_NAME=crank-ci-smoke + POSTGRES_HOST=postgres + POSTGRES_PORT=5432 + POSTGRES_DB=crank + POSTGRES_USER=crank + POSTGRES_PASSWORD=crank-ci-password + CRANK_ADMIN_API_IMAGE=crank/admin-api:ci + CRANK_MCP_SERVER_IMAGE=crank/mcp-server:ci + CRANK_UI_IMAGE=crank/ui:ci + CRANK_MASTER_KEY=0000000000000000000000000000000000000000000000000000000000000000 + CRANK_SESSION_SECRET=ci-session-secret + CRANK_PASSWORD_PEPPER=ci-password-pepper + CRANK_BOOTSTRAP_ADMIN_EMAIL=owner@crank.test + CRANK_BOOTSTRAP_ADMIN_PASSWORD=ci-admin-password + CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME=CI Owner + CRANK_BASE_URL=http://127.0.0.1:3000 + CRANK_PUBLISH_BIND=127.0.0.1 + CRANK_DEMO_SEED=true + EOF + docker compose -f deploy/community/docker-compose.images.yml \ + --env-file .tmp/community-smoke.env --profile local-db up -d --wait + + - name: Run authenticated Community image smoke + env: + CRANK_STAGING_ADMIN_EMAIL: owner@crank.test + CRANK_STAGING_ADMIN_PASSWORD: ci-admin-password + run: scripts/authenticated-product-smoke.sh http://127.0.0.1:3000 + + - name: Show Community image logs + if: failure() + run: | + docker compose -f deploy/community/docker-compose.images.yml \ + --env-file .tmp/community-smoke.env --profile local-db ps || true + docker compose -f deploy/community/docker-compose.images.yml \ + --env-file .tmp/community-smoke.env --profile local-db logs --no-color || true + + - name: Stop Community image stack + if: always() + run: | + docker compose -f deploy/community/docker-compose.images.yml \ + --env-file .tmp/community-smoke.env --profile local-db down -v --remove-orphans || true + deploy: name: Deploy runs-on: ubuntu-latest @@ -245,6 +309,10 @@ jobs: -t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \ -t '${{ env.UI_IMAGE }}:main' \ . + scripts/scan-images.sh \ + '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \ + '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \ + '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' docker push '${{ env.ADMIN_API_IMAGE }}:main' docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' @@ -274,9 +342,14 @@ jobs: run: | . "$OPENBAO_ENV_FILE" ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \ - "mkdir -p '$DEPLOY_PATH'" + "mkdir -p '$DEPLOY_PATH' && \ + if [ -f '$DEPLOY_PATH/docker-compose.yml' ]; then \ + cp '$DEPLOY_PATH/docker-compose.yml' '$DEPLOY_PATH/docker-compose.previous.yml'; \ + fi" rsync -az -e "ssh -p $DEPLOY_PORT" deploy/community/docker-compose.yml \ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/docker-compose.yml" + rsync -az -e "ssh -p $DEPLOY_PORT" scripts/deploy-community.sh \ + "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/deploy-community.sh" - name: Write environment file run: | @@ -292,6 +365,11 @@ jobs: append_if_set POSTGRES_USER "$POSTGRES_USER" append_if_set POSTGRES_PASSWORD "$POSTGRES_PASSWORD" append_if_set POSTGRES_HOST "$POSTGRES_HOST" + append_if_set POSTGRES_MAX_CONNECTIONS "${POSTGRES_MAX_CONNECTIONS:-}" + append_if_set POSTGRES_MIN_CONNECTIONS "${POSTGRES_MIN_CONNECTIONS:-}" + append_if_set POSTGRES_ACQUIRE_TIMEOUT_MS "${POSTGRES_ACQUIRE_TIMEOUT_MS:-}" + append_if_set POSTGRES_IDLE_TIMEOUT_MS "${POSTGRES_IDLE_TIMEOUT_MS:-}" + append_if_set POSTGRES_MAX_LIFETIME_MS "${POSTGRES_MAX_LIFETIME_MS:-}" if [ -n "${POSTGRES_PORT:-}" ]; then append_if_set POSTGRES_PORT "$POSTGRES_PORT" elif [ -n "${PGBOUNCER_PORT:-}" ]; then @@ -302,10 +380,35 @@ jobs: append_if_set CRANK_ADMIN_BIND "$CRANK_ADMIN_BIND" append_if_set CRANK_MCP_BIND "$CRANK_MCP_BIND" append_if_set CRANK_MCP_REFRESH_MS "$CRANK_MCP_REFRESH_MS" + append_if_set CRANK_ADMIN_RATE_LIMIT_RPS "${CRANK_ADMIN_RATE_LIMIT_RPS:-}" + append_if_set CRANK_ADMIN_RATE_LIMIT_BURST "${CRANK_ADMIN_RATE_LIMIT_BURST:-}" + append_if_set CRANK_MCP_RATE_LIMIT_RPS "${CRANK_MCP_RATE_LIMIT_RPS:-}" + append_if_set CRANK_MCP_RATE_LIMIT_BURST "${CRANK_MCP_RATE_LIMIT_BURST:-}" + append_if_set CRANK_RUNTIME_MAX_CONCURRENT_UNARY "${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-}" + append_if_set CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS "${CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS:-}" append_if_set CRANK_OUTBOUND_ALLOWED_HOSTS "${CRANK_OUTBOUND_ALLOWED_HOSTS:-}" append_if_set CRANK_OUTBOUND_DENIED_HOSTS "${CRANK_OUTBOUND_DENIED_HOSTS:-}" append_if_set CRANK_OUTBOUND_MAX_RESPONSE_BYTES "${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-}" + append_if_set CRANK_ENVIRONMENT "${CRANK_ENVIRONMENT:-production}" append_if_set CRANK_LOG_LEVEL "$CRANK_LOG_LEVEL" + append_if_set CRANK_SENTRY_DSN "${CRANK_SENTRY_DSN:-}" + append_if_set CRANK_METRICS_ENABLED "${CRANK_METRICS_ENABLED:-}" + append_if_set CRANK_ADMIN_METRICS_BIND "${CRANK_ADMIN_METRICS_BIND:-}" + append_if_set CRANK_MCP_METRICS_BIND "${CRANK_MCP_METRICS_BIND:-}" + append_if_set CRANK_METRICS_BEARER_TOKEN "${CRANK_METRICS_BEARER_TOKEN:-}" + append_if_set CRANK_INVOCATION_LOG_RETENTION_DAYS "${CRANK_INVOCATION_LOG_RETENTION_DAYS:-}" + append_if_set OTEL_EXPORTER_OTLP_ENDPOINT "${OTEL_EXPORTER_OTLP_ENDPOINT:-}" + append_if_set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}" + append_if_set OTEL_EXPORTER_OTLP_PROTOCOL "${OTEL_EXPORTER_OTLP_PROTOCOL:-}" + append_if_set OTEL_EXPORTER_OTLP_TRACES_PROTOCOL "${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-}" + append_if_set OTEL_EXPORTER_OTLP_TIMEOUT "${OTEL_EXPORTER_OTLP_TIMEOUT:-}" + append_if_set OTEL_EXPORTER_OTLP_TRACES_TIMEOUT "${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-}" + append_if_set OTEL_EXPORTER_OTLP_HEADERS "${OTEL_EXPORTER_OTLP_HEADERS:-}" + append_if_set OTEL_EXPORTER_OTLP_TRACES_HEADERS "${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-}" + append_if_set OTEL_BSP_MAX_QUEUE_SIZE "${OTEL_BSP_MAX_QUEUE_SIZE:-}" + append_if_set OTEL_BSP_MAX_EXPORT_BATCH_SIZE "${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-}" + append_if_set OTEL_BSP_SCHEDULE_DELAY "${OTEL_BSP_SCHEDULE_DELAY:-}" + append_if_set OTEL_BSP_EXPORT_TIMEOUT "${OTEL_BSP_EXPORT_TIMEOUT:-}" append_if_set CRANK_MASTER_KEY "$CRANK_MASTER_KEY" append_if_set CRANK_BASE_URL "$CRANK_BASE_URL" append_if_set CRANK_CACHE_BACKEND "$CRANK_CACHE_BACKEND" @@ -325,7 +428,10 @@ jobs: printf 'CRANK_UI_IMAGE=%s:%s\n' '${{ env.UI_IMAGE }}' '${{ env.IMAGE_TAG }}' } >> "$tmp_env" cat "$tmp_env" | ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \ - "mkdir -p '$DEPLOY_PATH' && cat > '$DEPLOY_PATH/.env'" + "mkdir -p '$DEPLOY_PATH' && \ + if [ -f '$DEPLOY_PATH/.env' ]; then \ + cp '$DEPLOY_PATH/.env' '$DEPLOY_PATH/.env.previous'; \ + fi && cat > '$DEPLOY_PATH/.env'" rm -f "$tmp_env" - name: Validate required environment variables @@ -359,48 +465,12 @@ jobs: - name: Deploy with Docker Compose run: | . "$OPENBAO_ENV_FILE" - ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" " - set -e - cd '$DEPLOY_PATH' - compose_profiles='' - cache_backend=\$(grep -E '^CRANK_CACHE_BACKEND=' .env | tail -n1 | cut -d= -f2- || true) - if [ \"\$cache_backend\" = 'valkey' ] || [ \"\$cache_backend\" = 'redis' ]; then - compose_profiles='--profile cache' - fi - echo '$DEPLOY_REGISTRY_TOKEN' | docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin - docker compose \$compose_profiles config -q - docker compose \$compose_profiles pull - docker compose \$compose_profiles down --remove-orphans - for container in \ - crank-ui-1 \ - crank-admin-api-1 \ - crank-mcp-server-1 \ - crank-postgres-1 \ - crank-valkey-1 \ - crank-community-ui-1 \ - crank-community-admin-api-1 \ - crank-community-mcp-server-1 \ - crank-community-postgres-1 \ - crank-community-valkey-1; do - if docker ps -a --format '{{.Names}}' | grep -Fx \"\$container\" >/dev/null; then - docker rm -f \"\$container\" - fi - done - echo 'Docker containers before freeing required ports:' - docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Ports}}' - for port in 3000 3001 3002; do - container_ids=\$(docker ps -aq --filter \"publish=\$port\") - if [ -n \"\$container_ids\" ]; then - echo \"Removing containers publishing port \$port\" - docker inspect --format '{{.Name}} {{json .NetworkSettings.Ports}}' \$container_ids || true - docker rm -f \$container_ids - fi - done - if command -v ss >/dev/null 2>&1; then - ss -ltnp '( sport = :3000 or sport = :3001 or sport = :3002 )' || true - fi - docker compose \$compose_profiles up -d --remove-orphans - " + printf '%s' "$DEPLOY_REGISTRY_TOKEN" | ssh -p "$DEPLOY_PORT" \ + "$DEPLOY_USER@$DEPLOY_HOST" \ + "docker login '${{ env.REGISTRY }}' -u '$DEPLOY_REGISTRY_USER' --password-stdin" + ssh -p "$DEPLOY_PORT" "$DEPLOY_USER@$DEPLOY_HOST" \ + "chmod 700 '$DEPLOY_PATH/deploy-community.sh' && \ + '$DEPLOY_PATH/deploy-community.sh' '$DEPLOY_PATH'" - name: Verify health endpoints run: | @@ -410,8 +480,8 @@ jobs: cd '$DEPLOY_PATH' for attempt in \$(seq 1 30); do if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \ - && curl --fail --silent http://127.0.0.1:3001/health >/dev/null \ - && curl --fail --silent http://127.0.0.1:3002/health >/dev/null; then + && curl --fail --silent http://127.0.0.1:3001/ready >/dev/null \ + && curl --fail --silent http://127.0.0.1:3002/ready >/dev/null; then exit 0 fi sleep 2 diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index cad45c8..beaef31 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -49,6 +49,16 @@ jobs: command -v bao bao version + - name: Install dependency policy tool + run: cargo install cargo-deny --version 0.20.2 --locked + + - name: Run release quality gates + run: | + cargo fmt --all --check + cargo clippy --workspace --all-targets --all-features -- -D warnings + cargo test --workspace --all-targets + cargo deny --locked check advisories bans licenses sources + - name: Build release binaries run: cargo build --release -p admin-api -p mcp-server @@ -56,10 +66,25 @@ jobs: working-directory: apps/ui run: npm ci + - name: Audit UI dependencies + working-directory: apps/ui + run: npm audit --audit-level=high + - name: Build UI dist working-directory: apps/ui run: npm run build + - name: Install Playwright browser + working-directory: apps/ui + run: npx playwright install --with-deps chromium + + - name: Run release end-to-end tests + working-directory: apps/ui + run: npm run e2e + + - name: Validate deployment manifest + run: docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q + - name: Package release artifacts run: | mkdir -p dist/release @@ -105,6 +130,10 @@ jobs: docker build -f apps/ui/Dockerfile \ -t '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' \ -t '${{ env.UI_IMAGE }}:latest' . + scripts/scan-images.sh \ + '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \ + '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \ + '${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}' docker push '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' docker push '${{ env.ADMIN_API_IMAGE }}:latest' docker push '${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' diff --git a/Cargo.lock b/Cargo.lock index 46f1a0f..308b24b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "admin-api" version = "0.3.1" @@ -15,22 +30,29 @@ dependencies = [ "crank-core", "crank-import", "crank-mapping", + "crank-observability", "crank-registry", "crank-runtime", "crank-schema", "crank-test-support", + "crank-trace", + "metrics", + "opentelemetry", + "opentelemetry_sdk", "rand 0.10.2", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_yaml", "serial_test", "sha2 0.10.9", "sqlx", - "thiserror", + "thiserror 2.0.18", "time", "tokio", + "tower", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "uuid", ] @@ -191,6 +213,29 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "axum" version = "0.8.9" @@ -274,6 +319,21 @@ dependencies = [ "fastrand", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.22.1" @@ -356,7 +416,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_urlencoded", - "thiserror", + "thiserror 2.0.18", "time", "tokio", "tokio-stream", @@ -430,9 +490,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -478,6 +546,15 @@ dependencies = [ "inout", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -577,12 +654,19 @@ dependencies = [ "async-trait", "axum", "crank-core", + "crank-trace", "futures-util", - "reqwest", + "metrics", + "opentelemetry", + "opentelemetry_sdk", + "reqwest 0.12.28", "serde", "serde_json", - "thiserror", + "thiserror 2.0.18", "tokio", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", ] [[package]] @@ -597,7 +681,7 @@ dependencies = [ "crank-registry", "rand 0.10.2", "sha2 0.10.9", - "thiserror", + "thiserror 2.0.18", "time", "tracing", "uuid", @@ -613,20 +697,27 @@ dependencies = [ "crank-adapter-rest", "crank-core", "crank-mapping", + "crank-observability", "crank-registry", "crank-runtime", "crank-schema", "crank-test-support", + "crank-trace", "futures-util", - "reqwest", + "metrics", + "opentelemetry", + "opentelemetry_sdk", + "reqwest 0.12.28", "serde", "serde_json", "sha2 0.10.9", "sqlx", - "thiserror", + "thiserror 2.0.18", "time", "tokio", "tracing", + "tracing-opentelemetry", + "tracing-subscriber", "uuid", ] @@ -638,7 +729,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror", + "thiserror 2.0.18", "time", "tokio", ] @@ -653,7 +744,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -664,7 +755,36 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror", + "thiserror 2.0.18", +] + +[[package]] +name = "crank-observability" +version = "0.3.1" +dependencies = [ + "axum", + "metrics", + "metrics-exporter-prometheus", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-proto", + "opentelemetry_sdk", + "percent-encoding", + "prost", + "sentry", + "serde", + "serde_json", + "sha2 0.10.9", + "subtle", + "thiserror 2.0.18", + "time", + "tokio", + "tower", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", + "url", + "uuid", ] [[package]] @@ -679,7 +799,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sqlx", - "thiserror", + "thiserror 2.0.18", "time", "tokio", "uuid", @@ -697,13 +817,16 @@ dependencies = [ "crank-core", "crank-mapping", "crank-schema", + "crank-trace", "futures-util", "hkdf 0.12.4", + "metrics", "redis", "serde", "serde_json", "sha2 0.10.9", - "thiserror", + "testcontainers", + "thiserror 2.0.18", "time", "tokio", "tracing", @@ -719,7 +842,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -734,6 +857,14 @@ dependencies = [ "uuid", ] +[[package]] +name = "crank-trace" +version = "0.3.1" +dependencies = [ + "tracing", + "tracing-subscriber", +] + [[package]] name = "crc" version = "3.4.0" @@ -749,6 +880,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.13" @@ -836,6 +976,16 @@ dependencies = [ "syn", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid", +] + [[package]] name = "deranged" version = "0.5.8" @@ -904,6 +1054,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -956,6 +1112,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -1011,6 +1178,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -1110,6 +1283,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1169,6 +1357,12 @@ dependencies = [ "polyval", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "h2" version = "0.4.15" @@ -1188,6 +1382,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + [[package]] name = "hashbrown" version = "0.12.3" @@ -1621,6 +1821,60 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -1638,6 +1892,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "libc" version = "0.2.186" @@ -1693,6 +1958,19 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1724,20 +2002,28 @@ dependencies = [ "crank-community-mcp", "crank-core", "crank-mapping", + "crank-observability", "crank-registry", "crank-runtime", "crank-schema", "crank-test-support", "futures-util", - "reqwest", + "metrics", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest 0.12.28", "serde", "serde_json", "sha2 0.10.9", "sqlx", - "thiserror", + "thiserror 2.0.18", "time", "tokio", + "tower", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "uuid", ] @@ -1758,12 +2044,63 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64", + "evmap", + "indexmap 2.14.0", + "metrics", + "metrics-util", + "quanta", + "thiserror 2.0.18", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.4", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.2.1" @@ -1863,6 +2200,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1881,6 +2227,75 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.18", +] + +[[package]] +name = "opentelemetry-http" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest 0.13.4", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest 0.13.4", + "thiserror 2.0.18", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.4", + "thiserror 2.0.18", +] + [[package]] name = "parking" version = "2.2.1" @@ -2083,6 +2498,21 @@ dependencies = [ "psl-types", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.11" @@ -2097,7 +2527,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2 0.6.4", - "thiserror", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -2109,6 +2539,7 @@ version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", @@ -2119,7 +2550,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -2224,6 +2655,33 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "redis" version = "0.29.5" @@ -2349,6 +2807,45 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -2363,6 +2860,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2401,6 +2904,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -2432,12 +2936,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2455,6 +2987,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -2488,6 +3029,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2517,6 +3064,87 @@ dependencies = [ "libc", ] +[[package]] +name = "sentry" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e4790d8c2f43a6645ee2cb12aac2db79221fddd035b94c8166b88ea094408" +dependencies = [ + "cfg_aliases", + "httpdate", + "reqwest 0.13.4", + "rustls", + "sentry-backtrace", + "sentry-core", + "sentry-panic", + "sentry-tracing", + "ureq", +] + +[[package]] +name = "sentry-backtrace" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "326e106874a7ea90636f1ca42e2f7b912929d29307982cab37f81208c16cf043" +dependencies = [ + "backtrace", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-core" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f9efafefbb78d7e02cb06c10aec08b77e7500428389eabbf6d5a668325265e8" +dependencies = [ + "rand 0.9.4", + "sentry-types", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "sentry-panic" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fc536a3e1fc68d626ae8dd18b628cafd474d9d36fea74e4bbb4d038877f003" +dependencies = [ + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-tracing" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96be2253fe14b3fa10c1ba047af2d3e58ce85c8cb297bbd19d6fa81b604e7877" +dependencies = [ + "bitflags", + "sentry-backtrace", + "sentry-core", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sentry-types" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f967748632cd4c5405dbed45426b27b407c0e53ac59b7df1c163e7f5aa57a77c" +dependencies = [ + "debugid", + "hex", + "rand 0.9.4", + "serde", + "serde_json", + "thiserror 2.0.18", + "time", + "url", + "uuid", +] + [[package]] name = "serde" version = "1.0.228" @@ -2728,6 +3356,12 @@ dependencies = [ "libc", ] +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -2813,7 +3447,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", - "thiserror", + "thiserror 2.0.18", "time", "tokio", "tokio-stream", @@ -2857,7 +3491,7 @@ dependencies = [ "sqlx-postgres", "sqlx-sqlite", "syn", - "thiserror", + "thiserror 2.0.18", "tokio", "url", ] @@ -2884,7 +3518,7 @@ dependencies = [ "sha1", "sha2 0.11.0", "sqlx-core", - "thiserror", + "thiserror 2.0.18", "time", "tracing", "uuid", @@ -2920,7 +3554,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 2.0.18", "time", "tracing", "uuid", @@ -2946,7 +3580,7 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", - "thiserror", + "thiserror 2.0.18", "time", "tracing", "url", @@ -3060,7 +3694,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror", + "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-util", @@ -3076,13 +3710,33 @@ dependencies = [ "testcontainers", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -3354,6 +4008,20 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +dependencies = [ + "js-sys", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -3446,6 +4114,7 @@ dependencies = [ "rustls-pki-types", "ureq-proto", "utf8-zero", + "webpki-roots", ] [[package]] @@ -3515,6 +4184,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -3627,6 +4306,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.8" @@ -3658,6 +4346,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -3723,13 +4420,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3738,7 +4444,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3750,34 +4456,67 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3790,24 +4529,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 1b32b76..e135715 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,9 +8,11 @@ members = [ "crates/crank-import", "crates/crank-schema", "crates/crank-mapping", + "crates/crank-observability", "crates/crank-registry", "crates/crank-runtime", "crates/crank-test-support", + "crates/crank-trace", "crates/crank-adapter-rest", ] resolver = "3" @@ -20,6 +22,7 @@ edition = "2024" license = "AGPL-3.0-only" rust-version = "1.96" version = "0.3.1" +publish = false [workspace.dependencies] aes-gcm = "0.10" @@ -28,18 +31,43 @@ axum = "0.8" axum-extra = { version = "0.12", features = ["cookie"] } base64 = "0.22" hkdf = "0.12" +metrics = "0.24.6" +metrics-exporter-prometheus = { version = "0.18.3", default-features = false } +opentelemetry = { version = "0.32.0", default-features = false, features = ["trace"] } +opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["http-proto", "reqwest-blocking-client", "reqwest-rustls", "trace"] } +opentelemetry-proto = { version = "0.32.0", default-features = false, features = ["gen-tonic-messages", "trace"] } +opentelemetry_sdk = { version = "0.32.1", default-features = false, features = ["trace"] } +percent-encoding = "2" +prost = "0.14" rand = "0.10" reqwest = { version = "0.12", default-features = false, features = ["cookies", "json", "rustls-tls"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" +sentry = { version = "0.49.0", default-features = false, features = ["backtrace", "panic", "rustls", "ureq"] } sha2 = "0.10" sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "macros", "json", "time", "uuid"] } +subtle = "2.6" thiserror = "2" time = { version = "0.3.53", features = ["formatting", "parsing", "serde"] } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tower = "0.5" tracing = "0.1" +tracing-opentelemetry = { version = "0.33.0", default-features = false } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +url = "2" uuid = { version = "1", features = ["serde", "v7"] } testcontainers = { version = "0.27", features = ["blocking"] } testcontainers-modules = { version = "0.15", features = ["postgres", "blocking"] } + +[profile.dev] +debug = "line-tables-only" + +[profile.dev.package."*"] +debug = false + +[profile.test] +debug = "line-tables-only" + +[profile.test.package."*"] +debug = false diff --git a/apps/admin-api/Cargo.toml b/apps/admin-api/Cargo.toml index ed9d51f..da0e1db 100644 --- a/apps/admin-api/Cargo.toml +++ b/apps/admin-api/Cargo.toml @@ -3,6 +3,7 @@ name = "admin-api" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [[bin]] @@ -18,9 +19,12 @@ crank-community-auth = { path = "../../crates/crank-community-auth" } crank-core = { path = "../../crates/crank-core" } crank-import = { path = "../../crates/crank-import" } crank-mapping = { path = "../../crates/crank-mapping" } +crank-observability = { path = "../../crates/crank-observability" } crank-registry = { path = "../../crates/crank-registry" } crank-runtime = { path = "../../crates/crank-runtime" } crank-schema = { path = "../../crates/crank-schema" } +crank-trace = { path = "../../crates/crank-trace" } +metrics.workspace = true rand.workspace = true serde.workspace = true serde_json.workspace = true @@ -37,5 +41,9 @@ uuid.workspace = true [dev-dependencies] async-trait = "0.1" crank-test-support = { path = "../../crates/crank-test-support" } +opentelemetry.workspace = true +opentelemetry_sdk.workspace = true reqwest.workspace = true serial_test = "3" +tower.workspace = true +tracing-opentelemetry.workspace = true diff --git a/apps/admin-api/src/app.rs b/apps/admin-api/src/app.rs index 9ff95ae..51b2b22 100644 --- a/apps/admin-api/src/app.rs +++ b/apps/admin-api/src/app.rs @@ -166,6 +166,7 @@ pub fn build_app(state: AppState) -> Router { Router::new() .route("/health", get(crate::routes::health)) + .route("/ready", get(crate::routes::readiness)) .nest( "/api/auth", Router::new() @@ -178,6 +179,9 @@ pub fn build_app(state: AppState) -> Router { apply_api_rate_limit, )) .layer(middleware::from_fn(apply_request_context)) + .layer(middleware::from_fn( + crank_observability::record_http_request, + )) .with_state(state) } diff --git a/apps/admin-api/src/dto.rs b/apps/admin-api/src/dto.rs index 139b4ea..60fe641 100644 --- a/apps/admin-api/src/dto.rs +++ b/apps/admin-api/src/dto.rs @@ -269,7 +269,12 @@ pub struct CreatedPlatformApiKeyResponse { } #[derive(Clone, Debug, Serialize)] -pub struct WorkspaceExportResponse { +pub struct WorkspaceCatalogSnapshotResponse { + pub kind: String, + pub format_version: String, + pub restorable: bool, + pub included: Vec, + pub excluded: Vec, pub workspace: WorkspaceRecord, pub operations: Vec, pub agents: Vec, diff --git a/apps/admin-api/src/error.rs b/apps/admin-api/src/error.rs index fabf35f..cf2d0d4 100644 --- a/apps/admin-api/src/error.rs +++ b/apps/admin-api/src/error.rs @@ -137,16 +137,24 @@ impl ApiError { impl IntoResponse for ApiError { fn into_response(self) -> Response { match &self { - Self::Internal { message, .. } => { - error!(error_code = self.code(), error_message = %message) + Self::Internal { .. } => { + error!( + name: "admin.response.internal_error", + error_code = self.code(), + "internal API error response" + ) } - Self::Unauthorized { message, .. } - | Self::Forbidden { message, .. } - | Self::Validation { message, .. } - | Self::NotFound { message, .. } - | Self::Conflict { message, .. } - | Self::RateLimited { message, .. } => { - warn!(error_code = self.code(), error_message = %message) + Self::Unauthorized { .. } + | Self::Forbidden { .. } + | Self::Validation { .. } + | Self::NotFound { .. } + | Self::Conflict { .. } + | Self::RateLimited { .. } => { + warn!( + name: "admin.response.rejected", + error_code = self.code(), + "API request rejected" + ) } } @@ -351,6 +359,10 @@ impl From for ApiError { format!("import job {job_id} was not found"), json!({ "job_id": job_id }), ), + RegistryError::ImportJobAlreadyApplied { job_id } => Self::conflict_with_context( + format!("import job {job_id} was already applied with different parameters"), + json!({ "job_id": job_id }), + ), RegistryError::Storage(_) | RegistryError::Serialization(_) => { Self::internal(value.to_string()) } @@ -407,6 +419,10 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str { RuntimeError::ConfirmationRequired { .. } => "runtime_confirmation_required", RuntimeError::InvalidConfirmationToken { .. } => "runtime_confirmation_error", RuntimeError::ConfirmationStoreUnavailable { .. } => "runtime_confirmation_unavailable", + RuntimeError::IdempotencyStoreUnavailable { .. } => "runtime_idempotency_unavailable", + RuntimeError::IdempotencyInProgress { .. } => "runtime_idempotency_in_progress", + RuntimeError::IdempotencyConflict { .. } => "runtime_idempotency_conflict", + RuntimeError::IdempotencyOutcomeUnknown { .. } => "runtime_idempotency_outcome_unknown", RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error", RuntimeError::MissingAuthProfile { .. } => "runtime_auth_profile_error", RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => { @@ -434,7 +450,11 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option { "safety_class": safety_class, })), RuntimeError::InvalidConfirmationToken { operation_id } - | RuntimeError::ConfirmationStoreUnavailable { operation_id } => Some(json!({ + | RuntimeError::ConfirmationStoreUnavailable { operation_id } + | RuntimeError::IdempotencyStoreUnavailable { operation_id } + | RuntimeError::IdempotencyInProgress { operation_id } + | RuntimeError::IdempotencyConflict { operation_id } + | RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({ "operation_id": operation_id, })), RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({ diff --git a/apps/admin-api/src/main.rs b/apps/admin-api/src/main.rs index 37213fd..92fa1bb 100644 --- a/apps/admin-api/src/main.rs +++ b/apps/admin-api/src/main.rs @@ -1,4 +1,4 @@ -use std::{env, net::SocketAddr, path::PathBuf}; +use std::{env, net::SocketAddr, path::PathBuf, time::Duration}; use admin_api::{ app::build_app, @@ -7,22 +7,50 @@ use admin_api::{ state::AppState, }; use crank_community_auth::PasswordIdentityProvider; +use crank_observability::{ + CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle, + capture_critical_error, +}; use crank_registry::{PostgresPoolConfig, PostgresRegistry}; use crank_runtime::{ RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores, RuntimeLimits, SecretCrypto, }; -use sqlx::postgres::PgConnectOptions; +use sqlx::{PgPool, postgres::PgConnectOptions}; use tokio::net::TcpListener; -use tracing::info; +use tracing::{info, warn}; #[tokio::main] async fn main() -> Result<(), Box> { - tracing_subscriber::fmt() - .with_env_filter( - env::var("CRANK_LOG_LEVEL").unwrap_or_else(|_| "admin_api=info,tower_http=info".into()), - ) - .init(); + let observability = crank_observability::init(ObservabilityConfig::from_env( + "admin-api", + env!("CARGO_PKG_VERSION"), + "admin_api=info,tower_http=info", + )?)?; + let mut startup_completed = false; + let result = run(&observability, &mut startup_completed).await; + if result.is_err() { + capture_critical_error(if startup_completed { + CriticalErrorCategory::Internal + } else { + CriticalErrorCategory::Startup + }); + } + result +} + +async fn run( + observability: &ObservabilityLifecycle, + startup_completed: &mut bool, +) -> Result<(), Box> { + let metrics_config = + MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?; + let metrics_enabled = metrics_config.enabled(); + let metrics_server = if metrics_config.enabled() { + Some(observability.metrics_surface(metrics_config).bind().await?) + } else { + None + }; let storage_root = PathBuf::from( env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()), @@ -36,6 +64,9 @@ async fn main() -> Result<(), Box> { pool_config, ) .await?; + if metrics_enabled { + spawn_postgres_pool_metrics(registry.pool().clone()); + } let auth_settings = AuthSettings { session_secret: env::var("CRANK_SESSION_SECRET")?, password_pepper: env::var("CRANK_PASSWORD_PEPPER")?, @@ -74,10 +105,13 @@ async fn main() -> Result<(), Box> { .with_outbound_http_policy(outbound_http_policy) .with_identity_provider(std::sync::Arc::new(identity_provider)) .build(); + let invocation_log_retention_days = + positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?; service.bootstrap_admin_user().await?; if env_flag("CRANK_DEMO_SEED") { service.seed_demo_assets().await?; } + spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days); let state = AppState { service, api_rate_limiter: if cache_config.backend.is_external() { @@ -92,6 +126,7 @@ async fn main() -> Result<(), Box> { let make_service = app.into_make_service_with_connect_info::(); info!( + name: "admin.postgres_pool.configured", runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary, admin_rate_limit_rps = api_rate_limit.requests_per_second, admin_rate_limit_burst = api_rate_limit.burst, @@ -101,15 +136,76 @@ async fn main() -> Result<(), Box> { acquire_timeout_ms = pool_config.acquire_timeout_ms, idle_timeout_ms = pool_config.idle_timeout_ms, max_lifetime_ms = pool_config.max_lifetime_ms, + invocation_log_retention_days, "postgres pool configured" ); - info!("admin-api listening on {}", socket_addr); + info!( + name: "admin.server.listening", + bind_address = %socket_addr, + "admin-api listening" + ); + *startup_completed = true; - axum::serve(listener, make_service).await?; + if let Some(metrics_server) = metrics_server { + tokio::select! { + result = axum::serve(listener, make_service) => result?, + result = metrics_server.serve() => result?, + } + } else { + axum::serve(listener, make_service).await?; + } Ok(()) } +fn positive_i64_from_env( + name: &'static str, + default: i64, +) -> Result> { + let value = match env::var(name) { + Ok(raw) => raw.parse::()?, + Err(env::VarError::NotPresent) => default, + Err(error) => return Err(error.into()), + }; + if value <= 0 { + return Err(format!("{name} must be greater than zero").into()); + } + Ok(value) +} + +fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(60 * 60)); + loop { + interval.tick().await; + let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days); + match service.cleanup_invocation_logs_before(cutoff).await { + Ok(removed) if removed > 0 => info!( + name: "admin.invocation_log_cleanup.completed", + removed, + "expired invocation logs removed" + ), + Ok(_) => {} + Err(_) => warn!( + name: "admin.invocation_log_cleanup.failed", + error_category = "registry_cleanup", + "failed to remove expired invocation logs" + ), + } + } + }); +} + +fn spawn_postgres_pool_metrics(pool: PgPool) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + loop { + interval.tick().await; + crank_observability::record_db_pool_connections(pool.size(), pool.num_idle()); + } + }); +} + fn env_flag(name: &str) -> bool { matches!( env::var(name) diff --git a/apps/admin-api/src/rate_limit.rs b/apps/admin-api/src/rate_limit.rs index b41fec8..a23630a 100644 --- a/apps/admin-api/src/rate_limit.rs +++ b/apps/admin-api/src/rate_limit.rs @@ -6,7 +6,7 @@ use axum::{ middleware::Next, response::Response, }; -use crank_runtime::RateLimitRejection; +use crank_runtime::{RateLimitCheckError, RateLimitRejection}; use crate::{error::ApiError, state::AppState}; @@ -25,11 +25,16 @@ pub async fn apply_api_rate_limit( peer_ip, state.trust_forwarded_headers, ); - if let Err(rejection) = state.api_rate_limiter.check(&key).await { - return Err(ApiError::rate_limited_with_context( - "request rate limit exceeded", - rejection_context(rejection), - )); + if let Err(error) = state.api_rate_limiter.check(&key).await { + return match error { + RateLimitCheckError::Rejected(rejection) => Err(ApiError::rate_limited_with_context( + "request rate limit exceeded", + rejection_context(rejection), + )), + RateLimitCheckError::StoreUnavailable => { + Err(ApiError::internal("rate limit service unavailable")) + } + }; } Ok(next.run(request).await) diff --git a/apps/admin-api/src/request_context.rs b/apps/admin-api/src/request_context.rs index cda184e..8af3ab1 100644 --- a/apps/admin-api/src/request_context.rs +++ b/apps/admin-api/src/request_context.rs @@ -4,11 +4,10 @@ use axum::{ middleware::Next, response::Response, }; -use tracing::info; -use uuid::Uuid; +use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation}; +use tracing::{Instrument, info, info_span}; pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id"); -const MAX_REQUEST_ID_LEN: usize = 128; #[derive(Clone, Debug)] pub struct RequestContext { @@ -21,145 +20,53 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response }; let method = request.method().clone(); let path = request.uri().path().to_owned(); + let span = info_span!( + target: "crank::trace", + "http.request", + request_id = %context.request_id, + ); + set_remote_trace_parent(&span, request.headers()); request.extensions_mut().insert(context.clone()); - let mut response = next.run(request).await; - info!( - request_id = %context.request_id, - method = %method, - path, - status = response.status().as_u16(), - "admin request completed" - ); - if let Ok(value) = HeaderValue::from_str(&context.request_id) { - response.headers_mut().insert(REQUEST_ID_HEADER, value); - } - response + with_request_correlation(context.request_id.clone(), async move { + let mut response = next.run(request).instrument(span).await; + info!( + name: "admin.request.completed", + request_id = %context.request_id, + method = %method, + path, + status = response.status().as_u16(), + "admin request completed" + ); + if let Ok(value) = HeaderValue::from_str(&context.request_id) { + response.headers_mut().insert(REQUEST_ID_HEADER, value); + } + response + }) + .await } fn resolve_request_id(headers: &HeaderMap) -> String { - headers - .get(&REQUEST_ID_HEADER) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| is_valid_request_id(value)) - .map(ToOwned::to_owned) - .unwrap_or_else(|| Uuid::now_v7().to_string()) -} - -fn is_valid_request_id(value: &str) -> bool { - !value.is_empty() - && value.len() <= MAX_REQUEST_ID_LEN - && value - .bytes() - .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';') + RequestId::resolve( + headers + .get(&REQUEST_ID_HEADER) + .and_then(|value| value.to_str().ok()), + ) + .into_string() } #[cfg(test)] mod tests { - use std::io; - use std::sync::{Arc, Mutex}; - - use axum::{Router, routing::get}; - use reqwest::Client; - use tokio::net::TcpListener; - use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*}; - - use super::{REQUEST_ID_HEADER, apply_request_context, is_valid_request_id}; - #[test] fn accepts_visible_ascii_request_ids() { - assert!(is_valid_request_id("req_test_123")); - assert!(is_valid_request_id("trace-123/abc")); + assert!(crank_observability::RequestId::is_valid("req_test_123")); + assert!(crank_observability::RequestId::is_valid("trace-123/abc")); } #[test] fn rejects_empty_or_control_request_ids() { - assert!(!is_valid_request_id("")); - assert!(!is_valid_request_id("bad value")); - assert!(!is_valid_request_id("bad\nvalue")); - } - - #[derive(Clone, Default)] - struct SharedLogWriter { - buffer: Arc>>, - } - - impl SharedLogWriter { - fn output(&self) -> String { - String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap() - } - } - - impl<'a> MakeWriter<'a> for SharedLogWriter { - type Writer = SharedLogGuard; - - fn make_writer(&'a self) -> Self::Writer { - SharedLogGuard { - buffer: Arc::clone(&self.buffer), - } - } - } - - struct SharedLogGuard { - buffer: Arc>>, - } - - impl io::Write for SharedLogGuard { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.buffer.lock().unwrap().extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - - #[tokio::test] - async fn logs_request_completion_with_request_id() { - let writer = SharedLogWriter::default(); - let subscriber = tracing_subscriber::registry().with( - tracing_subscriber::fmt::layer() - .with_writer(writer.clone()) - .without_time() - .with_ansi(false) - .with_target(false) - .compact() - .with_filter(LevelFilter::INFO), - ); - let dispatch = tracing::Dispatch::new(subscriber); - let app = Router::new() - .route("/probe", get(|| async { "ok" })) - .layer(axum::middleware::from_fn(apply_request_context)); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - - let _guard = tracing::dispatcher::set_default(&dispatch); - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - - let response = Client::new() - .get(format!("http://{address}/probe")) - .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123") - .send() - .await - .unwrap(); - - assert_eq!(response.status(), reqwest::StatusCode::OK); - assert_eq!( - response.headers()[REQUEST_ID_HEADER.as_str()] - .to_str() - .unwrap(), - "req_admin_trace_123" - ); - - let logs = writer.output(); - assert!(logs.contains("admin request completed")); - assert!(logs.contains("req_admin_trace_123")); - assert!(logs.contains("GET")); - assert!(logs.contains("/probe")); - assert!(logs.contains("status=200")); + assert!(!crank_observability::RequestId::is_valid("")); + assert!(!crank_observability::RequestId::is_valid("bad value")); + assert!(!crank_observability::RequestId::is_valid("bad\nvalue")); } } diff --git a/apps/admin-api/src/routes.rs b/apps/admin-api/src/routes.rs index 25a1dee..f12cd64 100644 --- a/apps/admin-api/src/routes.rs +++ b/apps/admin-api/src/routes.rs @@ -10,12 +10,36 @@ pub mod secrets; pub mod upstreams; pub mod workspaces; -use axum::Json; +use axum::{Json, extract::State, http::StatusCode, response::IntoResponse}; use serde_json::json; +use crate::state::AppState; + pub async fn health() -> Json { Json(json!({ "service": "admin-api", "status": "ok" })) } + +pub async fn readiness(State(state): State) -> impl IntoResponse { + match state.service.readiness().await { + Ok(()) => ( + StatusCode::OK, + Json(json!({ + "service": "admin-api", + "status": "ready", + "checks": { "postgres": "ready" } + })), + ), + Err(error) => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "service": "admin-api", + "status": "not_ready", + "checks": { "postgres": "not_ready" }, + "error": error.to_string() + })), + ), + } +} diff --git a/apps/admin-api/src/routes/access.rs b/apps/admin-api/src/routes/access.rs index fceb8c6..caecca9 100644 --- a/apps/admin-api/src/routes/access.rs +++ b/apps/admin-api/src/routes/access.rs @@ -18,7 +18,7 @@ pub async fn export_workspace( ) -> Result, ApiError> { let exported = state .service - .export_workspace(&path.workspace_id.as_str().into()) + .export_workspace_catalog_snapshot(&path.workspace_id.as_str().into()) .await?; Ok(Json(json!(exported))) } diff --git a/apps/admin-api/src/routes/auth.rs b/apps/admin-api/src/routes/auth.rs index 7cfb6a7..df96f10 100644 --- a/apps/admin-api/src/routes/auth.rs +++ b/apps/admin-api/src/routes/auth.rs @@ -103,7 +103,7 @@ pub async fn change_password( ) -> Result { state .service - .change_password(&session.user.id, payload) + .change_password(&session.user.id, &session.session_id, payload) .await?; Ok(StatusCode::NO_CONTENT) } diff --git a/apps/admin-api/src/service.rs b/apps/admin-api/src/service.rs index fd7f3e5..aabdfac 100644 --- a/apps/admin-api/src/service.rs +++ b/apps/admin-api/src/service.rs @@ -12,16 +12,18 @@ use crank_core::{ }; use crank_mapping::{MappingRule, MappingSet}; use crank_registry::{ - AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary, - OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket, + AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef, + OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket, }; use crank_runtime::{ OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto, }; use crank_schema::{Schema, SchemaKind}; +use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tracing::Instrument; use uuid::Uuid; mod agents; @@ -74,6 +76,11 @@ pub struct AdminServiceBuilder { pub use crate::dto::*; impl AdminService { + pub async fn readiness(&self) -> Result<(), ApiError> { + self.registry.ping().await?; + Ok(()) + } + #[cfg(test)] pub fn new( registry: PostgresRegistry, @@ -199,16 +206,33 @@ impl AdminServiceBuilder { } impl AdminService { - pub async fn export_workspace( + pub async fn export_workspace_catalog_snapshot( &self, workspace_id: &WorkspaceId, - ) -> Result { + ) -> Result { let workspace = self.get_workspace(workspace_id).await?; let operations = self.list_operations(workspace_id).await?; let agents = self.list_agents(workspace_id).await?; let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?; - Ok(WorkspaceExportResponse { + Ok(WorkspaceCatalogSnapshotResponse { + kind: "workspace_catalog_snapshot".to_owned(), + format_version: "1".to_owned(), + restorable: false, + included: vec![ + "workspace_settings".to_owned(), + "operation_summaries".to_owned(), + "agent_summaries".to_owned(), + "platform_api_key_metadata".to_owned(), + ], + excluded: vec![ + "operation_versions_and_samples".to_owned(), + "agent_versions_and_bindings".to_owned(), + "secret_metadata_and_values".to_owned(), + "secret_values".to_owned(), + "invocation_logs_and_usage".to_owned(), + "authentication_sessions".to_owned(), + ], workspace, operations, agents, @@ -239,9 +263,13 @@ impl AdminService { return Ok(None); }; - let auth_profile = self - .registry - .get_auth_profile(workspace_id, auth_profile_id) + let span = Stage::AuthResolve.span(); + let result = async { + let auth_profile = observe_db_query( + DbOperation::AuthProfileRead, + self.registry + .get_auth_profile(workspace_id, auth_profile_id), + ) .await .map_err(|error| RuntimeError::SecretCrypto { operation: "load auth profile", @@ -251,9 +279,20 @@ impl AdminService { auth_profile_id: auth_profile_id.as_str().to_owned(), })?; - self.resolve_auth_profile(workspace_id, &auth_profile) - .await - .map(Some) + self.resolve_auth_profile(workspace_id, &auth_profile) + .await + .map(Some) + } + .instrument(span.clone()) + .await; + match &result { + Ok(_) => StageOutcome::Success.record(&span), + Err(_) => { + StageOutcome::Error.record(&span); + ErrorCategory::Configuration.record(&span); + } + } + result } async fn resolve_auth_profile( @@ -265,40 +304,46 @@ impl AdminService { let used_at = OffsetDateTime::now_utc(); for secret_id in auth_profile.config.secret_ids() { - let secret = self - .registry - .get_secret(workspace_id, secret_id) - .await - .map_err(|error| RuntimeError::SecretCrypto { - operation: "load secret", - details: error.to_string(), - })? - .ok_or_else(|| RuntimeError::MissingSecret { - secret_id: secret_id.as_str().to_owned(), - })?; - let version = self - .registry - .get_current_secret_version(workspace_id, secret_id) - .await - .map_err(|error| RuntimeError::SecretCrypto { - operation: "load current secret version", - details: error.to_string(), - })? - .ok_or_else(|| RuntimeError::MissingSecretVersion { - secret_id: secret_id.as_str().to_owned(), - version: secret.secret.current_version, - })?; + let secret = observe_db_query( + DbOperation::SecretRead, + self.registry.get_secret(workspace_id, secret_id), + ) + .await + .map_err(|error| RuntimeError::SecretCrypto { + operation: "load secret", + details: error.to_string(), + })? + .ok_or_else(|| RuntimeError::MissingSecret { + secret_id: secret_id.as_str().to_owned(), + })?; + let version = observe_db_query( + DbOperation::SecretRead, + self.registry + .get_current_secret_version(workspace_id, secret_id), + ) + .await + .map_err(|error| RuntimeError::SecretCrypto { + operation: "load current secret version", + details: error.to_string(), + })? + .ok_or_else(|| RuntimeError::MissingSecretVersion { + secret_id: secret_id.as_str().to_owned(), + version: secret.secret.current_version, + })?; let plaintext = self.secret_crypto.decrypt( &version.secret_version.key_version, &version.secret_version.ciphertext, )?; - self.registry - .touch_secret(workspace_id, secret_id, &used_at) - .await - .map_err(|error| RuntimeError::SecretCrypto { - operation: "touch secret", - details: error.to_string(), - })?; + observe_db_query( + DbOperation::SecretTouch, + self.registry + .touch_secret(workspace_id, secret_id, &used_at), + ) + .await + .map_err(|error| RuntimeError::SecretCrypto { + operation: "touch secret", + details: error.to_string(), + })?; secrets.insert(secret_id.clone(), plaintext); } @@ -412,7 +457,7 @@ impl AdminService { async fn record_invocation( &self, request: InvocationRecordRequest<'_>, - ) -> Result<(), ApiError> { + ) -> InvocationHistoryWriteOutcome { let log = InvocationLog { id: InvocationLogId::new(new_prefixed_id("log")), workspace_id: request.workspace_id.clone(), @@ -432,11 +477,70 @@ impl AdminService { created_at: OffsetDateTime::now_utc(), }; - self.registry - .create_invocation_log(CreateInvocationLogRequest { log: &log }) - .await?; + let history_span = crank_trace::Stage::HistoryWrite.span(); + let (outcome, db_span) = async { + let db_span = crank_trace::Stage::DbQuery + .db_span(crank_trace::DbOperation::InvocationHistoryWrite) + .expect("database stage"); + let outcome = self + .registry + .create_invocation_log(CreateInvocationLogRequest { log: &log }) + .instrument(db_span.clone()) + .await; + (outcome, db_span) + } + .instrument(history_span.clone()) + .await; + match outcome { + InvocationHistoryWriteOutcome::Recorded => { + crank_trace::StageOutcome::Success.record(&db_span); + crank_trace::StageOutcome::Success.record(&history_span); + } + InvocationHistoryWriteOutcome::Lost(_) => { + crank_trace::StageOutcome::Error.record(&db_span); + crank_trace::ErrorCategory::Database.record(&db_span); + crank_trace::StageOutcome::Error.record(&history_span); + crank_trace::ErrorCategory::History.record(&history_span); + } + } + drop(db_span); + drop(history_span); + observe_invocation_history_outcome( + outcome, + request.request_id, + request.status, + "admin_test_run", + ); + outcome + } +} - Ok(()) +fn observe_invocation_history_outcome( + outcome: InvocationHistoryWriteOutcome, + request_id: Option<&str>, + status: crank_core::InvocationStatus, + source: &'static str, +) { + let Some(loss) = outcome.loss() else { + return; + }; + crank_observability::record_operational_incident( + crank_observability::OperationalIncident::InvocationHistoryLost, + ); + tracing::warn!( + name: "admin.invocation_history.lost", + request_id = request_id.unwrap_or_default(), + source, + invocation_status = invocation_status_label(status), + error_category = loss.category.as_str(), + "invocation history was not recorded" + ); +} + +fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str { + match status { + crank_core::InvocationStatus::Ok => "ok", + crank_core::InvocationStatus::Error => "error", } } @@ -543,6 +647,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str { RuntimeError::ConfirmationRequired { .. } => "confirmation_required", RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token", RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable", + RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable", + RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress", + RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict", + RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown", RuntimeError::RestAdapter(_) => "rest_error", RuntimeError::ProtocolAdapter(_) => "adapter_error", RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol", @@ -761,7 +869,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule { #[cfg(test)] #[allow(clippy::items_after_test_module)] mod tests { - use super::{validate_profile_display_name, validate_profile_email}; + use std::{ + io, + sync::{Arc, Mutex}, + }; + + use crank_core::InvocationStatus; + use crank_observability::{ + ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity, + operational_incident_total, + }; + use crank_registry::{ + InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome, + }; + use serde_json::Value; + use tracing_subscriber::fmt::MakeWriter; + + use super::{ + observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email, + }; #[test] fn validates_profile_identity_fields() { @@ -782,6 +908,81 @@ mod tests { assert!(validate_profile_display_name(&"x".repeat(81)).is_err()); assert!(validate_profile_email("owner @crank.local").is_err()); } + + #[test] + fn emits_bounded_history_loss_incident() { + let writer = SharedLogWriter::default(); + let subscriber = crank_observability::build_subscriber( + ObservabilityConfig::new( + ServiceIdentity::try_new("admin-api", "test", "test").unwrap(), + "info", + RedactionLimits::default(), + ), + writer.clone(), + ) + .unwrap(); + let before = operational_incident_total(OperationalIncident::InvocationHistoryLost); + let dispatch = tracing::Dispatch::new(subscriber); + let _guard = tracing::dispatcher::set_default(&dispatch); + + observe_invocation_history_outcome( + InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss { + category: InvocationHistoryLossCategory::InvalidRecord, + }), + Some("req_admin_dc08"), + InvocationStatus::Error, + "admin_test_run", + ); + + let output = writer.output(); + assert!(!output.contains("dc08-canary-secret")); + let event: Value = output + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .find(|event: &Value| event["event"] == "admin.invocation_history.lost") + .unwrap(); + assert_eq!(event["request_id"], "req_admin_dc08"); + assert_eq!(event["fields"]["source"], "admin_test_run"); + assert_eq!(event["fields"]["invocation_status"], "error"); + assert_eq!(event["fields"]["error_category"], "invalid_record"); + assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before); + } + + #[derive(Clone, Default)] + struct SharedLogWriter { + buffer: Arc>>, + } + + impl SharedLogWriter { + fn output(&self) -> String { + String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap() + } + } + + impl<'a> MakeWriter<'a> for SharedLogWriter { + type Writer = SharedLogGuard; + + fn make_writer(&'a self) -> Self::Writer { + SharedLogGuard { + buffer: Arc::clone(&self.buffer), + } + } + } + + struct SharedLogGuard { + buffer: Arc>>, + } + + impl io::Write for SharedLogGuard { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.buffer.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } } fn enrich_operation_summary( diff --git a/apps/admin-api/src/service/agents.rs b/apps/admin-api/src/service/agents.rs index e056ab4..bf72666 100644 --- a/apps/admin-api/src/service/agents.rs +++ b/apps/admin-api/src/service/agents.rs @@ -296,7 +296,12 @@ impl AdminService { bindings: &[], }) .await?; - info!(agent_id = %agent_id.as_str(), version = 1, "agent created"); + info!( + name: "admin.agent.created", + agent_id = %agent_id.as_str(), + version = 1, + "agent created" + ); Ok(CreatedAgentResponse { agent_id: agent_id.as_str().to_owned(), @@ -417,6 +422,7 @@ impl AdminService { }) .await?; info!( + name: "admin.agent.bindings_saved", agent_id = %agent_id.as_str(), version = current_version.version, binding_count = bindings.len(), @@ -531,7 +537,12 @@ impl AdminService { published_by: None, }) .await?; - info!(agent_id = %agent_id.as_str(), version, "agent published"); + info!( + name: "admin.agent.published", + agent_id = %agent_id.as_str(), + version, + "agent published" + ); Ok(PublishAgentResponse { agent_id: agent_id.as_str().to_owned(), @@ -589,7 +600,11 @@ impl AdminService { self.registry .unpublish_agent(workspace_id, agent_id, &updated_at) .await?; - info!(agent_id = %agent_id.as_str(), "agent moved to draft"); + info!( + name: "admin.agent.unpublished", + agent_id = %agent_id.as_str(), + "agent moved to draft" + ); Ok(AgentMutationResult { agent_id: agent_id.as_str().to_owned(), @@ -609,7 +624,11 @@ impl AdminService { self.registry .archive_agent(workspace_id, agent_id, &updated_at) .await?; - info!(agent_id = %agent_id.as_str(), "agent archived"); + info!( + name: "admin.agent.archived", + agent_id = %agent_id.as_str(), + "agent archived" + ); Ok(AgentMutationResult { agent_id: agent_id.as_str().to_owned(), diff --git a/apps/admin-api/src/service/auth.rs b/apps/admin-api/src/service/auth.rs index e0afade..a49f14b 100644 --- a/apps/admin-api/src/service/auth.rs +++ b/apps/admin-api/src/service/auth.rs @@ -22,7 +22,7 @@ impl AdminService { )?; let user_id = self .registry - .upsert_bootstrap_user( + .ensure_bootstrap_user( &self.auth_settings.bootstrap_admin.email, &self.auth_settings.bootstrap_admin.display_name, &password_hash, @@ -227,6 +227,7 @@ impl AdminService { pub async fn change_password( &self, user_id: &crank_core::UserId, + current_session_id: &UserSessionId, payload: ChangePasswordPayload, ) -> Result<(), ApiError> { if payload.new_password.len() < 12 { @@ -257,7 +258,11 @@ impl AdminService { let password_hash = hash_password(&payload.new_password, &self.auth_settings.password_pepper)?; self.registry - .update_user_password(user_id, &password_hash) + .update_user_password_and_revoke_other_sessions( + user_id, + current_session_id, + &password_hash, + ) .await?; Ok(()) diff --git a/apps/admin-api/src/service/demo.rs b/apps/admin-api/src/service/demo.rs index 04a8e7f..9f151ce 100644 --- a/apps/admin-api/src/service/demo.rs +++ b/apps/admin-api/src/service/demo.rs @@ -131,6 +131,7 @@ impl AdminService { Ok(()) => Ok(()), Err(RegistryError::OperationHasPublishedAgentBindings { .. }) => { tracing::warn!( + name: "admin.demo_operation.cleanup_skipped", operation_id = %operation_id.as_str(), "legacy demo operation is still bound to a published agent; leaving it in place" ); @@ -335,7 +336,7 @@ impl AdminService { }), response_preview: demo_rest_response_sample(), }) - .await?; + .await; Ok(()) } } diff --git a/apps/admin-api/src/service/import_export.rs b/apps/admin-api/src/service/import_export.rs index 4de3e8e..33e2271 100644 --- a/apps/admin-api/src/service/import_export.rs +++ b/apps/admin-api/src/service/import_export.rs @@ -110,6 +110,7 @@ impl AdminService { warnings, }; info!( + name: "admin.operation.imported", operation_id = %response.operation_id, version = response.version, "operation imported by upsert" @@ -125,6 +126,7 @@ impl AdminService { warnings, }; info!( + name: "admin.operation.imported", operation_id = %response.operation_id, version = response.version, "operation imported by upsert" diff --git a/apps/admin-api/src/service/imports.rs b/apps/admin-api/src/service/imports.rs index 8253bf8..c171657 100644 --- a/apps/admin-api/src/service/imports.rs +++ b/apps/admin-api/src/service/imports.rs @@ -8,9 +8,11 @@ use crank_import::rest::{ ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate, }; use crank_registry::{ - CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus, + ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind, + ImportJobStatus, ImportOperationDraft, }; use serde_json::json; +use sha2::{Digest, Sha256}; use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; use tracing::{info, instrument}; @@ -50,7 +52,7 @@ impl AdminService { kind: ImportJobKind::OpenApi, source_format: &preview.source.format, source_version: preview.source.version.as_deref(), - status: ImportJobStatus::Completed, + status: ImportJobStatus::Pending, preview_payload: &preview_payload, created_at: &now, expires_at: &expires_at, @@ -99,9 +101,13 @@ impl AdminService { return Err(ApiError::validation("import job kind is not openapi")); } - let preview: crank_import::rest::ImportPreview = - serde_json::from_value(job.preview_payload.clone()) - .map_err(|error| ApiError::internal(error.to_string()))?; + let stored_preview = job + .preview_payload + .get("preview") + .cloned() + .unwrap_or_else(|| job.preview_payload.clone()); + let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview) + .map_err(|error| ApiError::internal(error.to_string()))?; let selected = payload .selected_operation_keys .iter() @@ -120,10 +126,8 @@ impl AdminService { } } - let mut created = Vec::new(); let mut skipped = Vec::new(); - let mut findings = Vec::new(); - let mut created_ids = Vec::new(); + let mut operations = Vec::new(); for operation_key in selected { let Some(candidate) = candidates.get(&operation_key) else { @@ -137,44 +141,7 @@ impl AdminService { let mut draft = operation_draft_from_candidate(candidate, payload.server_url.as_deref()); attach_import_findings(&mut draft, candidate); - if let Some(existing_name) = self - .find_operation_by_name(workspace_id, &draft.name) - .await? - .map(|operation| operation.name) - { - if payload.conflict_mode == "skip" { - skipped.push(OpenApiImportSkippedOperation { - operation_key: candidate.key.clone(), - name: draft.name.clone(), - reason: "operation with this name already exists".to_owned(), - }); - findings.push(ImportFinding { - code: "operation_name_conflict".to_owned(), - severity: ImportFindingSeverity::Warning, - message: format!( - "Операция {} уже существует и была пропущена.", - draft.name - ), - operation_key: Some(candidate.key.clone()), - }); - continue; - } - - let renamed = self - .next_available_operation_name(workspace_id, &draft.name) - .await?; - findings.push(ImportFinding { - code: "operation_name_renamed".to_owned(), - severity: ImportFindingSeverity::Info, - message: format!( - "Операция {existing_name} уже существует, новый черновик создан как {renamed}." - ), - operation_key: Some(candidate.key.clone()), - }); - draft.name = renamed; - } - - let payload = OperationPayload { + let operation = self.new_operation_snapshot(OperationPayload { name: draft.name.clone(), display_name: draft.display_name.clone(), category: draft.category, @@ -197,27 +164,74 @@ impl AdminService { }, tool_description: draft.tool_description, wizard_state: draft.wizard_state, - }; - let result = self.create_operation(workspace_id, payload).await?; - created_ids.push(result.operation_id.clone()); - created.push(OpenApiImportCreatedOperation { - operation_id: result.operation_id, - name: draft.name, - version: result.version, + })?; + operations.push(ImportOperationDraft { + operation_key: candidate.key.clone(), + operation, }); } let finished_at = OffsetDateTime::now_utc(); - self.registry - .finish_import_job(FinishImportJobRequest { + let application_key = openapi_application_key(&payload)?; + let conflict_mode = if payload.conflict_mode == "skip" { + ImportConflictMode::Skip + } else { + ImportConflictMode::Rename + }; + let applied = self + .registry + .apply_import_job(ApplyImportJobRequest { id: job_id, - status: ImportJobStatus::Completed, - created_operation_ids: &json!(created_ids), - error_text: None, + workspace_id, + application_key: &application_key, + conflict_mode, + operations: &operations, finished_at: &finished_at, }) .await?; + + let created = applied + .created + .iter() + .map(|operation| OpenApiImportCreatedOperation { + operation_id: operation.operation_id.as_str().to_owned(), + name: operation.name.clone(), + version: operation.version, + }) + .collect::>(); + let mut findings = applied + .created + .iter() + .filter_map(|operation| { + operation.renamed_from.as_ref().map(|previous_name| ImportFinding { + code: "operation_name_renamed".to_owned(), + severity: ImportFindingSeverity::Info, + message: format!( + "Операция {previous_name} уже существует, новый черновик создан как {}.", + operation.name + ), + operation_key: Some(operation.operation_key.clone()), + }) + }) + .collect::>(); + for operation in applied.skipped { + skipped.push(OpenApiImportSkippedOperation { + operation_key: operation.operation_key.clone(), + name: operation.name.clone(), + reason: "operation with this name already exists".to_owned(), + }); + findings.push(ImportFinding { + code: operation.reason, + severity: ImportFindingSeverity::Warning, + message: format!( + "Операция {} уже существует и была пропущена.", + operation.name + ), + operation_key: Some(operation.operation_key), + }); + } info!( + name: "admin.openapi_import.completed", created = created.len(), skipped = skipped.len(), "openapi import created drafts" @@ -229,25 +243,21 @@ impl AdminService { findings, }) } +} - async fn next_available_operation_name( - &self, - workspace_id: &WorkspaceId, - base_name: &str, - ) -> Result { - for index in 2.. { - let candidate = format!("{base_name}_{index}"); - if self - .find_operation_by_name(workspace_id, &candidate) - .await? - .is_none() - { - return Ok(candidate); - } - } - - unreachable!() - } +fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result { + let selected_operation_keys = payload + .selected_operation_keys + .iter() + .cloned() + .collect::>(); + let canonical = serde_json::to_vec(&json!({ + "selected_operation_keys": selected_operation_keys, + "server_url": payload.server_url.as_deref(), + "conflict_mode": payload.conflict_mode.as_str(), + })) + .map_err(|error| ApiError::internal(error.to_string()))?; + Ok(format!("{:x}", Sha256::digest(canonical))) } fn attach_import_findings( diff --git a/apps/admin-api/src/service/observability.rs b/apps/admin-api/src/service/observability.rs index 6d72501..8837fb2 100644 --- a/apps/admin-api/src/service/observability.rs +++ b/apps/admin-api/src/service/observability.rs @@ -19,6 +19,16 @@ use crate::{ }; impl AdminService { + pub async fn cleanup_invocation_logs_before( + &self, + cutoff: OffsetDateTime, + ) -> Result { + self.registry + .delete_invocation_logs_before(cutoff) + .await + .map_err(ApiError::from) + } + #[instrument(skip(self))] pub async fn list_logs( &self, diff --git a/apps/admin-api/src/service/operations.rs b/apps/admin-api/src/service/operations.rs index f667ec0..c7f5f5a 100644 --- a/apps/admin-api/src/service/operations.rs +++ b/apps/admin-api/src/service/operations.rs @@ -141,7 +141,6 @@ impl AdminService { workspace_id: &WorkspaceId, payload: OperationPayload, ) -> Result { - self.validate_operation_payload(&payload)?; self.ensure_workspace_exists(workspace_id).await?; if self @@ -155,10 +154,36 @@ impl AdminService { )); } + let snapshot = self.new_operation_snapshot(payload)?; + let operation_id = snapshot.id.clone(); + + self.registry + .create_operation(workspace_id, &snapshot, None) + .await?; + info!( + name: "admin.operation.created", + operation_id = %operation_id.as_str(), + version = 1, + "operation created" + ); + + Ok(CreatedOperationResponse { + operation_id: operation_id.as_str().to_owned(), + workspace_id: workspace_id.as_str().to_owned(), + version: 1, + status: OperationStatus::Draft, + updated_at: format_timestamp(snapshot.updated_at), + }) + } + + pub(super) fn new_operation_snapshot( + &self, + payload: OperationPayload, + ) -> Result { + self.validate_operation_payload(&payload)?; let now = OffsetDateTime::now_utc(); - let operation_id = OperationId::new(new_prefixed_id("op")); - let snapshot = RegistryOperation { - id: operation_id.clone(), + Ok(RegistryOperation { + id: OperationId::new(new_prefixed_id("op")), name: payload.name, display_name: payload.display_name, category: payload.category, @@ -183,19 +208,6 @@ impl AdminService { created_at: now, updated_at: now, published_at: None, - }; - - self.registry - .create_operation(workspace_id, &snapshot, None) - .await?; - info!(operation_id = %operation_id.as_str(), version = 1, "operation created"); - - Ok(CreatedOperationResponse { - operation_id: operation_id.as_str().to_owned(), - workspace_id: workspace_id.as_str().to_owned(), - version: 1, - status: OperationStatus::Draft, - updated_at: format_timestamp(snapshot.updated_at), }) } @@ -279,7 +291,12 @@ impl AdminService { created_by: None, }) .await?; - info!(operation_id = %operation_id.as_str(), version, "operation version created"); + info!( + name: "admin.operation.version_created", + operation_id = %operation_id.as_str(), + version, + "operation version created" + ); Ok(CreatedOperationResponse { operation_id: operation_id.as_str().to_owned(), @@ -364,7 +381,12 @@ impl AdminService { published_by: None, }) .await?; - info!(operation_id = %operation_id.as_str(), version, "operation published"); + info!( + name: "admin.operation.published", + operation_id = %operation_id.as_str(), + version, + "operation published" + ); Ok(PublishResponse { operation_id: operation_id.as_str().to_owned(), @@ -431,37 +453,44 @@ impl AdminService { .await?; let runtime = RuntimeOperation::from(record.snapshot.clone()); let mode = ExecutionMode::Unary; - let request_preview = - match build_request_preview(&record.snapshot.input_mapping, &payload.input) { - Ok(preview) => preview, - Err(error) => { - self.record_invocation(InvocationRecordRequest { - workspace_id, - agent_id: None, - operation: &record.snapshot, - request_id: Some(request_id), - source: InvocationSource::AdminTestRun, - level: InvocationLevel::Error, - status: InvocationStatus::Error, - message: "mapping preview failed".to_owned(), - status_code: None, - error_kind: Some("mapping".to_owned()), - duration_ms: 0, - request_preview: Value::Null, - response_preview: Value::Null, - }) - .await?; - return Ok(TestRunResult { - ok: false, - mode, - request_preview: Value::Null, - response_preview: Value::Null, - errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping( - error, - ))], - }); - } - }; + let preview_span = crank_trace::Stage::RuntimeArgumentsMap.span(); + let preview_result = preview_span + .in_scope(|| build_request_preview(&record.snapshot.input_mapping, &payload.input)); + let request_preview = match preview_result { + Ok(preview) => preview, + Err(error) => { + crank_trace::StageOutcome::Error.record(&preview_span); + crank_trace::ErrorCategory::Mapping.record(&preview_span); + drop(preview_span); + self.record_invocation(InvocationRecordRequest { + workspace_id, + agent_id: None, + operation: &record.snapshot, + request_id: Some(request_id), + source: InvocationSource::AdminTestRun, + level: InvocationLevel::Error, + status: InvocationStatus::Error, + message: "mapping preview failed".to_owned(), + status_code: None, + error_kind: Some("mapping".to_owned()), + duration_ms: 0, + request_preview: Value::Null, + response_preview: Value::Null, + }) + .await; + return Ok(TestRunResult { + ok: false, + mode, + request_preview: Value::Null, + response_preview: Value::Null, + errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping( + error, + ))], + }); + } + }; + crank_trace::StageOutcome::Success.record(&preview_span); + drop(preview_span); let resolved_auth = self .resolve_operation_auth(workspace_id, &runtime.execution_config) @@ -497,7 +526,7 @@ impl AdminService { request_preview: request_preview.clone(), response_preview: response_preview.clone(), }) - .await?; + .await; Ok(TestRunResult { ok: true, mode, @@ -524,7 +553,7 @@ impl AdminService { request_preview: request_preview.clone(), response_preview: Value::Null, }) - .await?; + .await; Ok(TestRunResult { ok: false, mode, diff --git a/apps/admin-api/src/service/samples.rs b/apps/admin-api/src/service/samples.rs index af6341d..ab23415 100644 --- a/apps/admin-api/src/service/samples.rs +++ b/apps/admin-api/src/service/samples.rs @@ -50,6 +50,7 @@ impl AdminService { .save_sample_metadata(SaveSampleMetadataRequest { sample: &metadata }) .await?; info!( + name: "admin.sample.saved", operation_id = %operation_id.as_str(), sample_id = %metadata.id.as_str(), version, @@ -119,7 +120,11 @@ impl AdminService { input_mapping, output_mapping, }; - info!(operation_id = %operation_id.as_str(), "draft generated from samples"); + info!( + name: "admin.operation_draft.generated", + operation_id = %operation_id.as_str(), + "draft generated from samples" + ); Ok(result) } diff --git a/apps/admin-api/src/service/secrets.rs b/apps/admin-api/src/service/secrets.rs index 009abe7..33079ec 100644 --- a/apps/admin-api/src/service/secrets.rs +++ b/apps/admin-api/src/service/secrets.rs @@ -92,7 +92,11 @@ impl AdminService { created_by, }) .await?; - info!(secret_id = %secret.id.as_str(), "secret created"); + info!( + name: "admin.secret.created", + secret_id = %secret.id.as_str(), + "secret created" + ); Ok(secret) } @@ -126,7 +130,11 @@ impl AdminService { created_by, }) .await?; - info!(secret_id = %secret_id.as_str(), "secret rotated"); + info!( + name: "admin.secret.rotated", + secret_id = %secret_id.as_str(), + "secret rotated" + ); self.get_secret(workspace_id, secret_id).await } @@ -152,7 +160,11 @@ impl AdminService { .into()); } self.registry.delete_secret(workspace_id, secret_id).await?; - info!(secret_id = %secret_id.as_str(), "secret deleted"); + info!( + name: "admin.secret.deleted", + secret_id = %secret_id.as_str(), + "secret deleted" + ); Ok(()) } @@ -201,7 +213,11 @@ impl AdminService { profile: &profile, }) .await?; - info!(auth_profile_id = %profile.id.as_str(), "auth profile created"); + info!( + name: "admin.auth_profile.created", + auth_profile_id = %profile.id.as_str(), + "auth profile created" + ); Ok(profile) } diff --git a/apps/admin-api/src/service/upstreams.rs b/apps/admin-api/src/service/upstreams.rs index 4cbf7be..8113f51 100644 --- a/apps/admin-api/src/service/upstreams.rs +++ b/apps/admin-api/src/service/upstreams.rs @@ -75,7 +75,11 @@ impl AdminService { upstream: &upstream, }) .await?; - info!(upstream_id = %upstream.id.as_str(), "workspace upstream saved"); + info!( + name: "admin.upstream.saved", + upstream_id = %upstream.id.as_str(), + "workspace upstream saved" + ); Ok(upstream) } diff --git a/apps/admin-api/tests/integration/community_access_usage.rs b/apps/admin-api/tests/integration/community_access_usage.rs index c01b50a..313423d 100644 --- a/apps/admin-api/tests/integration/community_access_usage.rs +++ b/apps/admin-api/tests/integration/community_access_usage.rs @@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind}; use serde_json::{Value, json}; use serial_test::serial; use tokio::net::TcpListener; +use uuid::Version; use admin_api::{ app::build_app, @@ -40,6 +41,8 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper"; const TEST_SESSION_SECRET: &str = "test-session-secret"; const TEST_MASTER_KEY: &str = "test-master-key"; +mod history_loss; + struct TestServer { base_url: String, shutdown: Option>, @@ -385,6 +388,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() { exported["workspace"]["workspace"]["id"], DEFAULT_WORKSPACE_ID ); + assert_eq!(exported["kind"], "workspace_catalog_snapshot"); + assert_eq!(exported["format_version"], "1"); + assert_eq!(exported["restorable"], false); + assert_eq!( + exported["included"], + json!([ + "workspace_settings", + "operation_summaries", + "agent_summaries", + "platform_api_key_metadata" + ]) + ); + assert!( + exported["excluded"] + .as_array() + .unwrap() + .contains(&json!("secret_values")) + ); + assert!( + exported["excluded"] + .as_array() + .unwrap() + .contains(&json!("invocation_logs_and_usage")) + ); assert!(exported.get("memberships").is_none()); assert!(exported.get("invitations").is_none()); @@ -574,6 +601,25 @@ async fn updates_profile_and_changes_password() { .unwrap() .to_owned(); let client = authorized_client(&base_url).await; + let second_client = reqwest::Client::builder() + .cookie_store(true) + .build() + .unwrap(); + let second_login = second_client + .post(format!("{root_url}/api/auth/login")) + .json(&json!({ + "email": TEST_AUTH_EMAIL, + "password": TEST_AUTH_PASSWORD, + })) + .send() + .await + .unwrap(); + let second_login_status = second_login.status(); + let second_login_body = second_login.text().await.unwrap(); + assert!( + second_login_status.is_success(), + "second login failed with {second_login_status}: {second_login_body}" + ); let profile = assert_success_json( client @@ -615,6 +661,21 @@ async fn updates_profile_and_changes_password() { .status(); assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT); + let current_session_status = client + .get(format!("{root_url}/api/auth/profile")) + .send() + .await + .unwrap() + .status(); + let other_session_status = second_client + .get(format!("{root_url}/api/auth/profile")) + .send() + .await + .unwrap() + .status(); + assert_eq!(current_session_status, reqwest::StatusCode::OK); + assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED); + let relogin_client = reqwest::Client::builder() .cookie_store(true) .build() @@ -855,7 +916,10 @@ async fn generates_request_id_for_test_run_invocations() { .unwrap() .to_owned(); - assert!(!request_id.is_empty()); + assert_eq!( + uuid::Uuid::parse_str(&request_id).unwrap().get_version(), + Some(Version::SortRand) + ); response.error_for_status().unwrap(); let logs = client diff --git a/apps/admin-api/tests/integration/community_access_usage/history_loss.rs b/apps/admin-api/tests/integration/community_access_usage/history_loss.rs new file mode 100644 index 0000000..93da145 --- /dev/null +++ b/apps/admin-api/tests/integration/community_access_usage/history_loss.rs @@ -0,0 +1,141 @@ +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use axum::{Json, Router, extract::State, routing::post}; +use tokio::{net::TcpListener, sync::Notify}; + +use super::*; + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn preserves_external_success_when_invocation_history_is_lost() { + let registry = test_registry().await; + let registry_for_failure = registry.clone(); + let storage_root = test_storage_root("observability_history_loss"); + let upstream = spawn_blocking_upstream_server().await; + let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await; + let client = authorized_client(&base_url).await; + + let created = client + .post(format!("{base_url}/operations")) + .json(&test_operation_payload( + &upstream.base_url, + "crm_history_loss", + )) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + let operation_id = created["operation_id"].as_str().unwrap().to_owned(); + let request_client = client.clone(); + let request_url = format!("{base_url}/operations/{operation_id}/test-runs"); + let before = crank_observability::operational_incident_total( + crank_observability::OperationalIncident::InvocationHistoryLost, + ); + + let request = tokio::spawn(async move { + request_client + .post(request_url) + .header("x-request-id", "req_dc08_admin") + .json(&json!({ + "version": 1, + "input": { "email": "dc08-canary-secret@example.com" } + })) + .send() + .await + .unwrap() + }); + + upstream.started.notified().await; + registry_for_failure + .delete_operation( + &WorkspaceId::new(DEFAULT_WORKSPACE_ID), + &OperationId::new(operation_id.clone()), + ) + .await + .unwrap(); + upstream.release.notify_one(); + + let response = request.await.unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!( + response.headers()["x-request-id"].to_str().unwrap(), + "req_dc08_admin" + ); + let body = response.json::().await.unwrap(); + assert_eq!(body["ok"], true); + assert_eq!(body["response_preview"]["id"], "lead_123"); + assert_eq!(upstream.calls.load(Ordering::SeqCst), 1); + assert!( + crank_observability::operational_incident_total( + crank_observability::OperationalIncident::InvocationHistoryLost + ) > before + ); + + let logs = client + .get(format!("{base_url}/logs?period=7d")) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + assert!(logs["items"].as_array().unwrap().is_empty()); +} + +struct BlockingUpstream { + base_url: String, + started: Arc, + release: Arc, + calls: Arc, +} + +#[derive(Clone)] +struct BlockingUpstreamState { + started: Arc, + release: Arc, + calls: Arc, +} + +async fn spawn_blocking_upstream_server() -> BlockingUpstream { + let state = BlockingUpstreamState { + started: Arc::new(Notify::new()), + release: Arc::new(Notify::new()), + calls: Arc::new(AtomicUsize::new(0)), + }; + let app = Router::new() + .route("/crm/leads", post(blocking_create_lead)) + .with_state(state.clone()); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + BlockingUpstream { + base_url: format!("http://{address}"), + started: state.started, + release: state.release, + calls: state.calls, + } +} + +async fn blocking_create_lead( + State(state): State, + Json(payload): Json, +) -> Json { + state.calls.fetch_add(1, Ordering::SeqCst); + state.started.notify_one(); + state.release.notified().await; + + Json(json!({ + "id": "lead_123", + "status": "created", + "email": payload["email"] + })) +} diff --git a/apps/admin-api/tests/integration/openapi_import.rs b/apps/admin-api/tests/integration/openapi_import.rs index 55e146e..24a6085 100644 --- a/apps/admin-api/tests/integration/openapi_import.rs +++ b/apps/admin-api/tests/integration/openapi_import.rs @@ -1,5 +1,6 @@ use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload}; use crank_core::WorkspaceId; +use crank_registry::ImportJobStatus; use serial_test::serial; use super::common::{ @@ -42,7 +43,7 @@ paths: async fn previews_openapi_and_creates_draft_operations() { let registry = test_registry().await; let service = test_service( - registry, + registry.clone(), test_storage_root("openapi_import"), test_auth_settings(), test_secret_crypto(), @@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() { preview.preview.groups[0].operations[0].suggested_name, "latest_rates" ); + let preview_job = registry + .get_import_job(&workspace_id, &preview.job_id.as_str().into()) + .await + .unwrap() + .unwrap(); + assert_eq!(preview_job.status, ImportJobStatus::Pending); let created = service .create_openapi_import( @@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() { .any(|finding| finding.code == "openapi_import.weak_tool_description") ); + let skip_preview = service + .preview_openapi_import( + &workspace_id, + OpenApiImportPreviewPayload { + document: OPENAPI3.to_owned(), + }, + ) + .await + .unwrap(); let skipped = service .create_openapi_import( &workspace_id, - &preview.job_id.as_str().into(), + &skip_preview.job_id.as_str().into(), OpenApiImportCreatePayload { selected_operation_keys: vec!["GET /v2/latest".to_owned()], server_url: Some("https://api.frankfurter.dev".to_owned()), @@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() { assert_eq!(skipped.skipped[0].name, "latest_rates"); assert_eq!(skipped.findings[0].code, "operation_name_conflict"); + let rename_preview = service + .preview_openapi_import( + &workspace_id, + OpenApiImportPreviewPayload { + document: OPENAPI3.to_owned(), + }, + ) + .await + .unwrap(); let renamed = service .create_openapi_import( &workspace_id, - &preview.job_id.as_str().into(), + &rename_preview.job_id.as_str().into(), OpenApiImportCreatePayload { selected_operation_keys: vec!["GET /v2/latest".to_owned()], server_url: Some("https://api.frankfurter.dev".to_owned()), @@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() { assert_eq!(renamed.created[0].name, "latest_rates_2"); assert_eq!(renamed.findings[0].code, "operation_name_renamed"); } + +#[tokio::test] +#[serial] +async fn concurrent_openapi_import_replays_the_same_atomic_result() { + let registry = test_registry().await; + let service = test_service( + registry, + test_storage_root("openapi_import_replay"), + test_auth_settings(), + test_secret_crypto(), + ); + let workspace_id = WorkspaceId::new("ws_default"); + let preview = service + .preview_openapi_import( + &workspace_id, + OpenApiImportPreviewPayload { + document: OPENAPI3.to_owned(), + }, + ) + .await + .unwrap(); + let job_id = preview.job_id.as_str().into(); + let payload = OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /v2/latest".to_owned()], + server_url: Some("https://api.frankfurter.dev".to_owned()), + conflict_mode: "rename".to_owned(), + }; + + let (first, second) = tokio::join!( + service.create_openapi_import(&workspace_id, &job_id, payload.clone()), + service.create_openapi_import(&workspace_id, &job_id, payload), + ); + let first = first.unwrap(); + let second = second.unwrap(); + + assert_eq!(first.created.len(), 1); + assert_eq!(second.created.len(), 1); + assert_eq!( + first.created[0].operation_id, + second.created[0].operation_id + ); + assert_eq!(first.created[0].name, second.created[0].name); + assert_eq!( + service.list_operations(&workspace_id).await.unwrap().len(), + 1 + ); + + let conflicting_replay = service + .create_openapi_import( + &workspace_id, + &job_id, + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /v2/latest".to_owned()], + server_url: Some("https://api.frankfurter.dev".to_owned()), + conflict_mode: "skip".to_owned(), + }, + ) + .await; + assert!(conflicting_replay.is_err()); +} diff --git a/apps/admin-api/tests/integration/request_context.rs b/apps/admin-api/tests/integration/request_context.rs new file mode 100644 index 0000000..dd3d9e2 --- /dev/null +++ b/apps/admin-api/tests/integration/request_context.rs @@ -0,0 +1,211 @@ +use std::{ + io, + sync::{Arc, Mutex}, +}; + +use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context}; +use axum::{ + Router, + body::Body, + http::{HeaderMap, Request, StatusCode}, + routing::get, +}; +use crank_observability::{ + ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context, +}; +use opentelemetry::{global, trace::TracerProvider as _}; +use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider}; +use tower::ServiceExt; +use tracing::instrument::WithSubscriber; +use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt}; +use uuid::Version; + +#[tokio::test(flavor = "current_thread")] +async fn logs_request_completion_and_rejects_untrusted_values() { + let writer = SharedLogWriter::default(); + let subscriber = crank_observability::build_subscriber( + ObservabilityConfig::new( + ServiceIdentity::try_new("admin-api", "test", "test").unwrap(), + "info", + RedactionLimits::default(), + ), + writer.clone(), + ) + .unwrap(); + let dispatch = tracing::Dispatch::new(subscriber); + let app = probe_app(); + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/probe") + .header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123") + .body(Body::empty()) + .unwrap(), + ) + .with_subscriber(dispatch.clone()) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[REQUEST_ID_HEADER.as_str()] + .to_str() + .unwrap(), + "req_admin_trace_123" + ); + let event: serde_json::Value = writer + .output() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .find(|event: &serde_json::Value| event["event"] == "admin.request.completed") + .unwrap(); + assert_eq!(event["request_id"], "req_admin_trace_123"); + assert_eq!(event["fields"]["status"], 200); + + let invalid_response = app + .oneshot( + Request::builder() + .uri("/probe") + .header(REQUEST_ID_HEADER.as_str(), "bad,value") + .header("traceparent", "canary-invalid-traceparent") + .body(Body::empty()) + .unwrap(), + ) + .with_subscriber(dispatch) + .await + .unwrap(); + let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()] + .to_str() + .unwrap(); + assert_eq!( + uuid::Uuid::parse_str(generated).unwrap().get_version(), + Some(Version::SortRand) + ); + assert!(!writer.output().contains("canary-invalid-traceparent")); +} + +#[tokio::test(flavor = "current_thread")] +async fn covers_valid_invalid_and_absent_traceparent() { + global::set_text_map_propagator(TraceContextPropagator::new()); + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("admin-request-context-test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + let dispatch = tracing::Dispatch::new(subscriber); + let app = trace_probe_app(); + + let valid = observed_trace_id( + app.clone() + .oneshot( + Request::builder() + .uri("/trace") + .header( + "traceparent", + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + ) + .header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate") + .body(Body::empty()) + .unwrap(), + ) + .with_subscriber(dispatch.clone()) + .await + .unwrap(), + ); + let invalid = observed_trace_id( + app.clone() + .oneshot( + Request::builder() + .uri("/trace") + .header("traceparent", "canary-invalid-traceparent") + .body(Body::empty()) + .unwrap(), + ) + .with_subscriber(dispatch.clone()) + .await + .unwrap(), + ); + let absent = observed_trace_id( + app.oneshot( + Request::builder() + .uri("/trace") + .body(Body::empty()) + .unwrap(), + ) + .with_subscriber(dispatch) + .await + .unwrap(), + ); + + assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c"); + assert_ne!(invalid, valid); + assert_ne!(absent, valid); + assert_ne!(invalid, absent); + provider.shutdown().unwrap(); +} + +fn probe_app() -> Router { + Router::new() + .route("/probe", get(|| async { "ok" })) + .layer(axum::middleware::from_fn(apply_request_context)) +} + +fn trace_probe_app() -> Router { + Router::new() + .route("/trace", get(observed_traceparent)) + .layer(axum::middleware::from_fn(apply_request_context)) +} + +async fn observed_traceparent() -> HeaderMap { + let mut trace_headers = HeaderMap::new(); + inject_current_trace_context(&mut trace_headers); + let mut response_headers = HeaderMap::new(); + if let Some(traceparent) = trace_headers.remove("traceparent") { + response_headers.insert("x-observed-traceparent", traceparent); + } + response_headers +} + +fn observed_trace_id(response: axum::response::Response) -> String { + let traceparent = response.headers()["x-observed-traceparent"] + .to_str() + .unwrap(); + traceparent[3..35].to_owned() +} + +#[derive(Clone, Default)] +struct SharedLogWriter { + buffer: Arc>>, +} + +impl SharedLogWriter { + fn output(&self) -> String { + String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap() + } +} + +impl<'a> MakeWriter<'a> for SharedLogWriter { + type Writer = SharedLogGuard; + + fn make_writer(&'a self) -> Self::Writer { + SharedLogGuard { + buffer: Arc::clone(&self.buffer), + } + } +} + +struct SharedLogGuard { + buffer: Arc>>, +} + +impl io::Write for SharedLogGuard { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.buffer.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/apps/admin-api/tests/request_context.rs b/apps/admin-api/tests/request_context.rs new file mode 100644 index 0000000..71b9dde --- /dev/null +++ b/apps/admin-api/tests/request_context.rs @@ -0,0 +1,2 @@ +#[path = "integration/request_context.rs"] +mod request_context; diff --git a/apps/mcp-server/Cargo.toml b/apps/mcp-server/Cargo.toml index 2e10602..048e8cf 100644 --- a/apps/mcp-server/Cargo.toml +++ b/apps/mcp-server/Cargo.toml @@ -3,6 +3,7 @@ name = "mcp-server" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [[bin]] @@ -15,10 +16,12 @@ axum.workspace = true base64.workspace = true crank-community-mcp = { path = "../../crates/crank-community-mcp" } crank-core = { path = "../../crates/crank-core" } +crank-observability = { path = "../../crates/crank-observability" } crank-registry = { path = "../../crates/crank-registry" } crank-runtime = { path = "../../crates/crank-runtime" } crank-schema = { path = "../../crates/crank-schema" } futures-util = "0.3" +metrics.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true @@ -34,4 +37,10 @@ uuid.workspace = true crank-mapping = { path = "../../crates/crank-mapping" } crank-schema = { path = "../../crates/crank-schema" } crank-test-support = { path = "../../crates/crank-test-support" } +opentelemetry.workspace = true +opentelemetry-proto.workspace = true +opentelemetry_sdk.workspace = true +prost.workspace = true reqwest.workspace = true +tower.workspace = true +tracing-opentelemetry.workspace = true diff --git a/apps/mcp-server/src/main.rs b/apps/mcp-server/src/main.rs index bcd797b..6ad3e48 100644 --- a/apps/mcp-server/src/main.rs +++ b/apps/mcp-server/src/main.rs @@ -1,26 +1,53 @@ use std::{env, net::SocketAddr, time::Duration}; use crank_community_mcp::{ - auth::CommunityMachineCredentialVerifier, build_app_with_background_workers, + auth::CommunityMachineCredentialVerifier, build_app_with_background_workers_and_limits, session::PostgresTransportSessionStore, }; +use crank_observability::{ + CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle, + capture_critical_error, +}; use crank_registry::{PostgresPoolConfig, PostgresRegistry}; use crank_runtime::{ RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores, RuntimeLimits, SecretCrypto, }; -use sqlx::postgres::PgConnectOptions; +use sqlx::{PgPool, postgres::PgConnectOptions}; use tokio::net::TcpListener; use tracing::info; #[tokio::main] async fn main() -> Result<(), Box> { - tracing_subscriber::fmt() - .with_env_filter( - env::var("CRANK_LOG_LEVEL") - .unwrap_or_else(|_| "mcp_server=info,tower_http=info".into()), - ) - .init(); + let observability = crank_observability::init(ObservabilityConfig::from_env( + "mcp-server", + env!("CARGO_PKG_VERSION"), + "mcp_server=info,tower_http=info", + )?)?; + let mut startup_completed = false; + let result = run(&observability, &mut startup_completed).await; + if result.is_err() { + capture_critical_error(if startup_completed { + CriticalErrorCategory::Internal + } else { + CriticalErrorCategory::Startup + }); + } + result +} + +async fn run( + observability: &ObservabilityLifecycle, + startup_completed: &mut bool, +) -> Result<(), Box> { + let metrics_config = + MetricsConfig::from_env("CRANK_MCP_METRICS_BIND", "127.0.0.1:9465".parse()?)?; + let metrics_enabled = metrics_config.enabled(); + let metrics_server = if metrics_config.enabled() { + Some(observability.metrics_surface(metrics_config).bind().await?) + } else { + None + }; let bind_addr = env::var("CRANK_MCP_BIND").unwrap_or_else(|_| "0.0.0.0:3002".into()); let base_url = env::var("CRANK_BASE_URL").ok(); @@ -36,23 +63,20 @@ async fn main() -> Result<(), Box> { let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?; let api_rate_limit = mcp_api_rate_limit_config_from_env()?; let database_options = database_options_from_env()?; - let registry = PostgresRegistry::connect_with_options_and_pool_config( - database_options.clone(), - pool_config, - ) - .await?; - let session_store = PostgresTransportSessionStore::connect_with_options_and_pool_config( - database_options, - pool_config, - ) - .await?; + let registry = + PostgresRegistry::connect_with_options_and_pool_config(database_options, pool_config) + .await?; + if metrics_enabled { + spawn_postgres_pool_metrics(registry.pool().clone()); + } + let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?; let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?; let runtime = crank_runtime::community_from_env()? .with_limits(runtime_limits) .with_response_cache(cache_stores.response.clone()) .with_coordination_store(cache_stores.coordination.clone()) .build(); - let app = build_app_with_background_workers( + let app = build_app_with_background_workers_and_limits( registry, refresh_interval, base_url, @@ -66,11 +90,14 @@ async fn main() -> Result<(), Box> { cache_stores.coordination.clone(), std::sync::Arc::new(session_store), std::sync::Arc::new(CommunityMachineCredentialVerifier), + runtime_limits.max_concurrent_sessions, ); let listener = TcpListener::bind(socket_addr).await?; info!( + name: "mcp.postgres_pool.configured", runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary, + runtime_max_concurrent_sessions = runtime_limits.max_concurrent_sessions, mcp_rate_limit_rps = api_rate_limit.requests_per_second, mcp_rate_limit_burst = api_rate_limit.burst, cache_backend = %cache_config.backend, @@ -81,9 +108,21 @@ async fn main() -> Result<(), Box> { max_lifetime_ms = pool_config.max_lifetime_ms, "postgres pool configured" ); - info!("mcp-server listening on {}", socket_addr); + info!( + name: "mcp.server.listening", + bind_address = %socket_addr, + "mcp-server listening" + ); + *startup_completed = true; - axum::serve(listener, app).await?; + if let Some(metrics_server) = metrics_server { + tokio::select! { + result = axum::serve(listener, app) => result?, + result = metrics_server.serve() => result?, + } + } else { + axum::serve(listener, app).await?; + } Ok(()) } @@ -123,3 +162,13 @@ fn mcp_api_rate_limit_config_from_env() -> Result().await.unwrap(); assert_eq!( approved_body["approval"]["status"], @@ -169,6 +174,10 @@ async fn approval_key_lists_and_decides_pending_requests() { .await .unwrap(); assert_eq!(logs.len(), 1); + assert_eq!( + logs[0].log.request_id.as_deref(), + Some("req_approval_execute_123") + ); } #[tokio::test] @@ -401,22 +410,23 @@ async fn tool_call_with_approval_policy_creates_pending_request() { let mcp_url = agent_mcp_url(&base_url, "sales-gated"); let initialized_session = initialize_session(&client, &mcp_url, &api_key).await; + let tool_call = json!({ + "jsonrpc": "2.0", + "id": 9, + "method": "tools/call", + "params": { + "name": "crm_requires_human_approval", + "arguments": { + "email": "ada@example.com" + } + } + }); let tool_result = post_jsonrpc( &client, &mcp_url, &api_key, Some(&initialized_session), - json!({ - "jsonrpc": "2.0", - "id": 9, - "method": "tools/call", - "params": { - "name": "crm_requires_human_approval", - "arguments": { - "email": "ada@example.com" - } - } - }), + tool_call.clone(), ) .await; @@ -430,6 +440,19 @@ async fn tool_call_with_approval_policy_creates_pending_request() { .unwrap(); assert!(approval_id.starts_with("approval_")); + let repeated_tool_result = post_jsonrpc( + &client, + &mcp_url, + &api_key, + Some(&initialized_session), + tool_call, + ) + .await; + assert_eq!( + repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id, + "deduplicated tools/call must return the persisted approval id", + ); + let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated")); let pending = client .get(&approvals_url) @@ -448,6 +471,189 @@ async fn tool_call_with_approval_policy_creates_pending_request() { ); } +#[tokio::test] +async fn approval_http_endpoints_enforce_request_rate_limit() { + let registry = test_registry().await; + let upstream_base_url = spawn_upstream_server().await; + let operation = test_operation(&upstream_base_url, "crm_approval_rate_limit"); + registry + .create_operation(&test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish_agent_for_operation(®istry, &operation, "sales-approval-rate-limit").await; + let approval_key = create_approval_platform_api_key( + ®istry, + "sales-approval-rate-limit", + "approval-rate-limit", + ) + .await; + let base_url = spawn_mcp_server(build_test_app_with_rate_limit( + registry, + Duration::from_millis(0), + Some("https://crank.example.com".to_owned()), + RequestRateLimitConfig::new(1, 1).unwrap(), + )) + .await; + let approvals_url = format!( + "{}/approvals", + agent_mcp_url(&base_url, "sales-approval-rate-limit") + ); + let client = reqwest::Client::new(); + + let allowed = client + .get(&approvals_url) + .header(header::AUTHORIZATION, format!("Bearer {approval_key}")) + .send() + .await + .unwrap(); + assert_eq!(allowed.status(), reqwest::StatusCode::OK); + + let limited = client + .get(&approvals_url) + .header(header::AUTHORIZATION, format!("Bearer {approval_key}")) + .send() + .await + .unwrap(); + assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS); + assert!(limited.headers().contains_key(header::RETRY_AFTER)); +} + +#[tokio::test] +async fn recovery_does_not_repeat_interrupted_mutating_approval() { + let registry = test_registry().await; + let (upstream_base_url, upstream_calls) = spawn_counted_approval_upstream().await; + let operation = test_operation(&upstream_base_url, "crm_interrupted_approval"); + registry + .create_operation(&test_workspace_id(), &operation, Some("alice")) + .await + .unwrap(); + publish_agent_for_operation(®istry, &operation, "sales-interrupted-approval").await; + let approval_key_name = "approval-interrupted"; + create_approval_platform_api_key(®istry, "sales-interrupted-approval", approval_key_name) + .await; + let now = OffsetDateTime::now_utc(); + let approval = ApprovalRequest { + id: ApprovalRequestId::new("approval_interrupted_mutation"), + workspace_id: test_workspace_id(), + agent_id: test_agent_id("sales-interrupted-approval"), + operation_id: operation.id.clone(), + operation_version: operation.version, + status: ApprovalRequestStatus::Pending, + risk_level: OperationApprovalRiskLevel::Dangerous, + request_payload: json!({"email": "interrupted@example.com"}), + response_payload: None, + created_at: now - time::Duration::minutes(10), + expires_at: now + time::Duration::minutes(5), + decided_at: None, + decided_by_key_id: None, + decision_note: None, + }; + registry + .create_approval_request(CreateApprovalRequest { + approval: &approval, + }) + .await + .unwrap(); + let approval_key_id = PlatformApiKeyId::new(format!("pk_{approval_key_name}")); + registry + .decide_approval_request(crank_registry::DecideApprovalRequest { + workspace_id: &approval.workspace_id, + agent_id: &approval.agent_id, + approval_id: &approval.id, + status: ApprovalRequestStatus::Approved, + decided_at: now - time::Duration::minutes(10), + decided_by_key_id: &approval_key_id, + response_payload: Some(json!({"approve": "yes"})), + decision_note: None, + }) + .await + .unwrap() + .unwrap(); + registry + .claim_approval_request( + &approval.workspace_id, + &approval.agent_id, + &approval.id, + now - time::Duration::minutes(7), + ) + .await + .unwrap() + .unwrap(); + + let _app = build_test_app_with_approval_recovery(registry.clone()); + let failed = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let current = registry + .get_approval_request_for_agent( + &approval.workspace_id, + &approval.agent_id, + &approval.id, + ) + .await + .unwrap() + .unwrap(); + if current.approval.status == ApprovalRequestStatus::Failed { + break current; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("recovery must quarantine interrupted execution"); + + assert_eq!( + failed.approval.response_payload.unwrap()["error"]["code"], + "approval_execution_outcome_unknown" + ); + assert_eq!( + upstream_calls.load(std::sync::atomic::Ordering::SeqCst), + 0, + "recovery must not repeat a mutating upstream request" + ); +} + +fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router { + crank_community_mcp::build_app_with_background_workers( + registry, + Duration::from_millis(0), + Some("https://crank.example.com".to_owned()), + SecretCrypto::new("test-master-key").unwrap(), + crank_runtime::community_with_outbound_policy( + crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + ) + .build(), + RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()), + Arc::new(InMemoryCoordinationStateStore::default()), + Arc::new(InMemorySessionStore::default()), + Arc::new(CommunityMachineCredentialVerifier), + ) +} + +async fn spawn_counted_approval_upstream() -> (String, Arc) { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let handler_calls = Arc::clone(&calls); + let app = Router::new().route( + "/crm/leads", + post(move |Json(payload): Json| { + let calls = Arc::clone(&handler_calls); + async move { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Json(json!({ + "id": "lead_123", + "email": payload["email"] + })) + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + (format!("http://{address}"), calls) +} + #[tokio::test] async fn elicitation_approval_requires_client_capability() { let registry = test_registry().await; diff --git a/apps/mcp-server/tests/integration/execution_stages.rs b/apps/mcp-server/tests/integration/execution_stages.rs new file mode 100644 index 0000000..7e15f2c --- /dev/null +++ b/apps/mcp-server/tests/integration/execution_stages.rs @@ -0,0 +1,376 @@ +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use axum::{ + Json, Router, + body::{Body, Bytes, to_bytes}, + extract::State, + http::{HeaderMap, Request, StatusCode, header}, + routing::post, +}; +use crank_core::{InvocationSource, PlatformApiKeyScope}; +use crank_observability::{ + OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider, +}; +use crank_registry::{ListInvocationLogsQuery, PublishRequest}; +use opentelemetry::global; +use opentelemetry_proto::tonic::{ + collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value, trace::v1::Span, +}; +use opentelemetry_sdk::propagation::TraceContextPropagator; +use prost::Message; +use serde_json::{Value, json}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tokio::net::TcpListener; +use tower::ServiceExt; +use tracing::instrument::WithSubscriber; +use tracing_subscriber::{Layer, filter::filter_fn, layer::SubscriberExt}; + +use super::common::{ + build_test_app, create_platform_api_key, publish_agent_for_operation, test_operation, + test_registry, test_workspace_id, +}; + +const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c"; +const REQUEST_ID: &str = "req_stage_end_to_end"; +const CANARY: &str = "dc-stage-canary-secret"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exports_real_tool_stages_without_sensitive_data() { + global::set_text_map_propagator(TraceContextPropagator::new()); + let (otlp_endpoint, request_rx, collector) = spawn_otlp_collector().await; + let trace_config = OtlpTraceConfig::try_new( + Some(otlp_endpoint), + Some("http/protobuf".to_owned()), + Duration::from_secs(2), + OtlpBatchConfig::try_new(128, 128, Duration::from_secs(300), Duration::from_secs(2)) + .unwrap(), + ) + .unwrap(); + let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "integration-test").unwrap(); + let (provider, tracer) = build_tracer_provider(&identity, &trace_config) + .unwrap() + .expect("enabled OTLP provider"); + let subscriber = tracing_subscriber::registry().with( + tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(filter_fn(|metadata| { + metadata.is_span() && metadata.target() == "crank::trace" + })), + ); + let dispatch = tracing::Dispatch::new(subscriber); + + let registry = test_registry().await; + let observed_traceparent = Arc::new(Mutex::new(None)); + let upstream_base_url = spawn_upstream(Arc::clone(&observed_traceparent)).await; + let operation = test_operation(&upstream_base_url, "stage_end_to_end"); + registry + .create_operation(&test_workspace_id(), &operation, Some("test")) + .await + .unwrap(); + registry + .publish_operation(PublishRequest { + workspace_id: &test_workspace_id(), + operation_id: &operation.id, + version: 1, + published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(), + published_by: Some("test"), + }) + .await + .unwrap(); + publish_agent_for_operation(®istry, &operation, "stage-agent").await; + let api_key = create_platform_api_key( + ®istry, + "stage-agent", + "stage-key", + &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], + ) + .await; + let app = build_test_app(registry.clone(), Duration::ZERO, None); + + let call_result = async { + let initialized = send_jsonrpc( + app.clone(), + &api_key, + None, + None, + None, + json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {} + } + }), + ) + .await; + assert_eq!(initialized.status(), StatusCode::OK); + let session_id = initialized + .headers() + .get("MCP-Session-Id") + .unwrap() + .to_str() + .unwrap() + .to_owned(); + + let notification = send_jsonrpc( + app.clone(), + &api_key, + Some(&session_id), + None, + None, + json!({ + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {} + }), + ) + .await; + assert_eq!(notification.status(), StatusCode::ACCEPTED); + + send_jsonrpc( + app, + &api_key, + Some(&session_id), + Some(REQUEST_ID), + Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"), + json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "stage_end_to_end", + "arguments": { "email": CANARY } + } + }), + ) + .await + } + .with_subscriber(dispatch) + .await; + + assert_eq!(call_result.status(), StatusCode::OK); + let body = to_bytes(call_result.into_body(), 1024 * 1024) + .await + .unwrap(); + let body: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["result"]["isError"], false); + provider.force_flush().unwrap(); + let request = tokio::time::timeout(Duration::from_secs(2), request_rx) + .await + .unwrap() + .unwrap(); + collector.abort(); + assert_eq!( + request + .headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("application/x-protobuf") + ); + assert!( + !request + .body + .windows(CANARY.len()) + .any(|window| window == CANARY.as_bytes()) + ); + let export = ExportTraceServiceRequest::decode(request.body).unwrap(); + let spans = export + .resource_spans + .iter() + .flat_map(|resource| &resource.scope_spans) + .flat_map(|scope| &scope.spans) + .collect::>(); + + let traceparent = observed_traceparent + .lock() + .unwrap() + .clone() + .expect("upstream traceparent"); + assert_eq!(&traceparent[3..35], REMOTE_TRACE_ID); + + let expected_trace_id = decode_trace_id(REMOTE_TRACE_ID); + let trace_spans = spans + .iter() + .copied() + .filter(|span| span.trace_id.as_slice() == expected_trace_id) + .collect::>(); + for expected in [ + "mcp.request", + "mcp.rate_limit", + "mcp.access.check", + "mcp.catalog.load", + "mcp.tools.resolve", + "runtime.execute", + "runtime.arguments.map", + "upstream.http", + "runtime.response.transform", + "history.write", + "db.query", + ] { + assert!( + trace_spans.iter().any(|span| span.name == expected), + "missing span {expected}" + ); + } + assert!(!trace_spans.iter().any(|span| span.name == "approval.check")); + assert!( + !trace_spans + .iter() + .any(|span| span.name == "runtime.idempotency") + ); + + let root = trace_spans + .iter() + .find(|span| span.name == "mcp.request") + .expect("mcp root"); + let runtime = trace_spans + .iter() + .find(|span| span.name == "runtime.execute") + .expect("runtime"); + assert!(!runtime.parent_span_id.is_empty()); + assert_eq!(runtime.parent_span_id, root.span_id); + + let history = trace_spans + .iter() + .find(|span| span.name == "history.write") + .expect("history write"); + let history_db = trace_spans + .iter() + .find(|span| { + span.name == "db.query" + && string_attribute(span, "db.operation") == Some("invocation_history.write") + }) + .expect("history PostgreSQL write"); + assert_eq!(history_db.parent_span_id, history.span_id); + + let logs = registry + .list_invocation_logs(ListInvocationLogsQuery { + workspace_id: &test_workspace_id(), + level: None, + search_text: None, + source: Some(InvocationSource::AgentToolCall), + operation_id: Some(&operation.id), + agent_id: None, + created_after: None, + limit: 10, + }) + .await + .unwrap(); + assert_eq!(logs.len(), 1); + assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID)); + + provider.shutdown().unwrap(); +} + +struct OtlpRequest { + headers: HeaderMap, + body: Bytes, +} + +async fn spawn_otlp_collector() -> ( + String, + tokio::sync::oneshot::Receiver, + tokio::task::JoinHandle<()>, +) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (request_tx, request_rx) = tokio::sync::oneshot::channel(); + let sender = Arc::new(Mutex::new(Some(request_tx))); + let app = Router::new().route( + "/v1/traces", + post({ + let sender = Arc::clone(&sender); + move |headers: HeaderMap, body: Bytes| { + let sender = Arc::clone(&sender); + async move { + if let Some(sender) = sender.lock().unwrap().take() { + let _ = sender.send(OtlpRequest { headers, body }); + } + StatusCode::OK + } + } + }), + ); + let collector = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{address}/v1/traces"), request_rx, collector) +} + +fn decode_trace_id(value: &str) -> [u8; 16] { + let mut bytes = [0_u8; 16]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap(); + } + bytes +} + +fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> { + span.attributes.iter().find_map(|attribute| { + let value = attribute.value.as_ref()?.value.as_ref()?; + (attribute.key == key) + .then_some(value) + .and_then(|value| match value { + any_value::Value::StringValue(value) => Some(value.as_str()), + _ => None, + }) + }) +} + +async fn send_jsonrpc( + app: Router, + api_key: &str, + session_id: Option<&str>, + request_id: Option<&str>, + traceparent: Option<&str>, + payload: Value, +) -> axum::response::Response { + let mut request = Request::builder() + .method("POST") + .uri("/v1/default/stage-agent") + .header(header::CONTENT_TYPE, "application/json") + .header(header::ACCEPT, "application/json, text/event-stream") + .header(header::AUTHORIZATION, format!("Bearer {api_key}")) + .header("MCP-Protocol-Version", "2025-11-25"); + if let Some(session_id) = session_id { + request = request.header("MCP-Session-Id", session_id); + } + if let Some(request_id) = request_id { + request = request.header("x-request-id", request_id); + } + if let Some(traceparent) = traceparent { + request = request.header("traceparent", traceparent); + } + app.oneshot(request.body(Body::from(payload.to_string())).unwrap()) + .await + .unwrap() +} + +async fn spawn_upstream(observed: Arc>>) -> String { + async fn create_lead( + State(observed): State>>>, + headers: HeaderMap, + Json(_payload): Json, + ) -> Json { + *observed.lock().unwrap() = headers + .get("traceparent") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + Json(json!({ "id": "lead_123" })) + } + + let app = Router::new() + .route("/crm/leads", post(create_lead)) + .with_state(observed); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{address}") +} diff --git a/apps/mcp-server/tests/integration/request_context.rs b/apps/mcp-server/tests/integration/request_context.rs new file mode 100644 index 0000000..7837b73 --- /dev/null +++ b/apps/mcp-server/tests/integration/request_context.rs @@ -0,0 +1,130 @@ +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; +use opentelemetry::{ + global, + trace::{TraceId, TracerProvider as _}, +}; +use opentelemetry_sdk::{ + error::OTelSdkResult, + propagation::TraceContextPropagator, + trace::{SdkTracerProvider, SpanData, SpanExporter}, +}; +use tower::ServiceExt; +use tracing::instrument::WithSubscriber; +use tracing_subscriber::layer::SubscriberExt; + +use super::common::{build_test_app, test_registry}; + +const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c"; + +#[tokio::test(flavor = "current_thread")] +async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() { + global::set_text_map_propagator(TraceContextPropagator::new()); + let exported = Arc::new(Mutex::new(Vec::new())); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(CapturingExporter(Arc::clone(&exported))) + .build(); + let tracer = provider.tracer("mcp-request-context-test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + let dispatch = tracing::Dispatch::new(subscriber); + let app = build_test_app(test_registry().await, Duration::ZERO, None); + + let (valid, invalid, absent) = async { + let valid = send_health( + app.clone(), + Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"), + Some("request-id-is-separate"), + ) + .await; + let invalid = send_health( + app.clone(), + Some("canary-invalid-traceparent"), + Some("bad,value"), + ) + .await; + let absent = send_health(app, None, None).await; + (valid, invalid, absent) + } + .with_subscriber(dispatch) + .await; + provider.force_flush().unwrap(); + + assert_eq!(valid.status, StatusCode::OK); + assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate")); + assert_eq!(invalid.status, StatusCode::OK); + assert_eq!(absent.status, StatusCode::OK); + assert!(valid.traceparent_response.is_none()); + assert!(invalid.traceparent_response.is_none()); + assert!(absent.traceparent_response.is_none()); + + let trace_ids: Vec<_> = exported + .lock() + .unwrap() + .iter() + .filter(|span| span.name.as_ref() == "mcp.request") + .map(|span| span.span_context.trace_id()) + .collect(); + assert_eq!(trace_ids.len(), 3); + assert_eq!(trace_ids[0].to_string(), REMOTE_TRACE_ID); + assert_ne!(trace_ids[1], trace_ids[0]); + assert_ne!(trace_ids[2], trace_ids[0]); + assert_ne!(trace_ids[1], trace_ids[2]); + assert!(!trace_ids.contains(&TraceId::INVALID)); + provider.shutdown().unwrap(); +} + +async fn send_health( + app: axum::Router, + traceparent: Option<&str>, + request_id: Option<&str>, +) -> ProbeResponse { + let mut request = Request::builder().uri("/health"); + if let Some(traceparent) = traceparent { + request = request.header("traceparent", traceparent); + } + if let Some(request_id) = request_id { + request = request.header("x-request-id", request_id); + } + let response = app + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + + ProbeResponse { + status: response.status(), + request_id: response + .headers() + .get("x-request-id") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned), + traceparent_response: response + .headers() + .get("traceparent") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned), + } +} + +struct ProbeResponse { + status: StatusCode, + request_id: Option, + traceparent_response: Option, +} + +#[derive(Clone, Debug)] +struct CapturingExporter(Arc>>); + +impl SpanExporter for CapturingExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + self.0.lock().unwrap().extend(batch); + Ok(()) + } +} diff --git a/apps/mcp-server/tests/integration/transport_protocol.rs b/apps/mcp-server/tests/integration/transport_protocol.rs index ad335f4..50a5e84 100644 --- a/apps/mcp-server/tests/integration/transport_protocol.rs +++ b/apps/mcp-server/tests/integration/transport_protocol.rs @@ -38,7 +38,8 @@ use sha2::{Digest, Sha256}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::net::TcpListener; use tokio::time::sleep; -use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*}; +use tracing_subscriber::fmt::MakeWriter; +use uuid::Version; use crank_community_mcp::{ auth::{CommunityMachineCredentialVerifier, SharedMachineCredentialVerifier}, @@ -46,6 +47,9 @@ use crank_community_mcp::{ catalog::PublishedToolCatalog, session::{InMemorySessionStore, SharedSessionStore, TransportSessionStore}, }; +use crank_observability::{ + ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber, +}; fn test_workspace_id() -> WorkspaceId { WorkspaceId::new("ws_default") @@ -404,7 +408,10 @@ async fn generates_request_id_for_tool_call_responses_and_logs() { .to_str() .unwrap() .to_owned(); - assert!(!request_id.is_empty()); + assert_eq!( + uuid::Uuid::parse_str(&request_id).unwrap().get_version(), + Some(Version::SortRand) + ); let call_result = response.json::().await.unwrap(); assert_eq!(call_result["result"]["isError"], false); @@ -427,7 +434,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() { assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str())); } -#[tokio::test] +#[tokio::test(flavor = "current_thread")] async fn emits_request_id_in_mcp_ingress_logs() { let registry = test_registry().await; let upstream_base_url = spawn_upstream_server().await; @@ -456,6 +463,18 @@ async fn emits_request_id_in_mcp_ingress_logs() { ) .await; + let writer = SharedLogWriter::default(); + let subscriber = build_subscriber( + ObservabilityConfig::new( + ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(), + "info", + RedactionLimits::default(), + ), + writer.clone(), + ) + .unwrap(); + let dispatch = tracing::Dispatch::new(subscriber); + let _guard = tracing::dispatcher::set_default(&dispatch); let base_url = spawn_mcp_server(build_test_app( registry, Duration::from_millis(0), @@ -464,18 +483,7 @@ async fn emits_request_id_in_mcp_ingress_logs() { .await; let client = reqwest::Client::new(); let mcp_url = agent_mcp_url(&base_url, "sales-request-trace"); - let writer = SharedLogWriter::default(); - let subscriber = tracing_subscriber::registry().with( - tracing_subscriber::fmt::layer() - .with_writer(writer.clone()) - .without_time() - .with_ansi(false) - .with_target(false) - .compact() - .with_filter(LevelFilter::INFO), - ); - let _ = tracing::subscriber::set_global_default(subscriber); let response = post_jsonrpc_response( &client, &mcp_url, @@ -500,14 +508,16 @@ async fn emits_request_id_in_mcp_ingress_logs() { assert_eq!(response.status(), reqwest::StatusCode::OK); let logs = writer.output(); - assert!( - logs.contains("mcp request received"), - "captured logs did not include ingress marker: {logs}" - ); - assert!(logs.contains("req_mcp_trace_123")); - assert!(logs.contains("sales-request-trace")); - assert!(logs.contains("default")); - assert!(logs.contains("initialize")); + let event = logs + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find(|event| event["event"] == "mcp.request.received") + .unwrap(); + assert_eq!(event["service"], "mcp-server"); + assert_eq!(event["request_id"], "req_mcp_trace_123"); + assert_eq!(event["fields"]["agent_slug"], "sales-request-trace"); + assert_eq!(event["fields"]["workspace_slug"], "default"); + assert_eq!(event["fields"]["jsonrpc_method"], "initialize"); } #[tokio::test] @@ -803,11 +813,30 @@ async fn get_requires_session_header() { .get(agent_mcp_url(&base_url, "sales-get-sse-missing")) .header(header::ACCEPT, "text/event-stream") .header(header::AUTHORIZATION, format!("Bearer {api_key}")) + .header("x-request-id", "req_early_mcp_error") .send() .await .unwrap(); assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); + assert_eq!( + response.headers()["x-request-id"].to_str().unwrap(), + "req_early_mcp_error" + ); + + let invalid_response = client + .get(agent_mcp_url(&base_url, "sales-get-sse-missing")) + .header(header::ACCEPT, "text/event-stream") + .header(header::AUTHORIZATION, format!("Bearer {api_key}")) + .header("x-request-id", "bad,value") + .send() + .await + .unwrap(); + let generated = invalid_response.headers()["x-request-id"].to_str().unwrap(); + assert_eq!( + uuid::Uuid::parse_str(generated).unwrap().get_version(), + Some(Version::SortRand) + ); } #[tokio::test] diff --git a/apps/mcp-server/tests/request_context.rs b/apps/mcp-server/tests/request_context.rs new file mode 100644 index 0000000..68ff757 --- /dev/null +++ b/apps/mcp-server/tests/request_context.rs @@ -0,0 +1,4 @@ +#[path = "integration/common.rs"] +mod common; +#[path = "integration/request_context.rs"] +mod request_context; diff --git a/apps/ui/html/workspace-setup.html b/apps/ui/html/workspace-setup.html index 33fb5aa..a577b4b 100644 --- a/apps/ui/html/workspace-setup.html +++ b/apps/ui/html/workspace-setup.html @@ -97,8 +97,8 @@
-
Export all data
-
Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.
+
Export workspace catalog
+
Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.
diff --git a/apps/ui/js/i18n.js b/apps/ui/js/i18n.js index 26a9022..80c0424 100644 --- a/apps/ui/js/i18n.js +++ b/apps/ui/js/i18n.js @@ -442,8 +442,8 @@ var TRANSLATIONS = { 'workspace_setup.create.subtitle': 'This Community installation uses one workspace for MCP operations and agents.', 'workspace_setup.create.footer': 'This Community installation uses one workspace.', 'workspace_setup.danger.title': 'Danger zone', - 'workspace_setup.danger.export_title': 'Export all data', - 'workspace_setup.danger.export_body': 'Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.', + 'workspace_setup.danger.export_title': 'Export workspace catalog', + 'workspace_setup.danger.export_body': 'Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.', 'workspace_setup.export': 'Export', 'workspace_setup.role.owner': 'Owner', 'workspace_setup.role.admin': 'Admin', @@ -1334,8 +1334,8 @@ var TRANSLATIONS = { 'workspace_setup.create.subtitle': 'В Community используется один воркспейс для MCP-операций и агентов.', 'workspace_setup.create.footer': 'В Community используется один воркспейс.', 'workspace_setup.danger.title': 'Опасная зона', - 'workspace_setup.danger.export_title': 'Экспортировать все данные', - 'workspace_setup.danger.export_body': 'Скачать JSON-снимок настроек воркспейса, операций, агентов, секретов, данных использования и ключей доступа агентов.', + 'workspace_setup.danger.export_title': 'Экспорт каталога рабочего пространства', + 'workspace_setup.danger.export_body': 'Скачать невосстанавливаемый JSON-каталог настроек рабочего пространства, сводок операций и агентов, а также метаданных ключей API.', 'workspace_setup.export': 'Экспорт', 'workspace_setup.role.owner': 'Владелец', 'workspace_setup.role.admin': 'Администратор', diff --git a/apps/ui/js/logs.js b/apps/ui/js/logs.js index db55fb1..737ab28 100644 --- a/apps/ui/js/logs.js +++ b/apps/ui/js/logs.js @@ -8,6 +8,8 @@ document.addEventListener('DOMContentLoaded', function () { openId: null, liveMode: true, timer: null, + searchTimer: null, + refreshPromise: null, workspaceId: null, loading: false, loadError: '', @@ -457,7 +459,13 @@ document.addEventListener('DOMContentLoaded', function () { } async function refreshOperationalData() { - await Promise.all([loadLogs(), loadApprovals()]); + if (state.refreshPromise) { + return state.refreshPromise; + } + state.refreshPromise = Promise.all([loadLogs(), loadApprovals()]).finally(function () { + state.refreshPromise = null; + }); + return state.refreshPromise; } async function loadLogDetail(logId) { @@ -494,10 +502,14 @@ document.addEventListener('DOMContentLoaded', function () { function startPolling() { stopPolling(); - if (!state.liveMode) { + if (!state.liveMode || document.hidden) { return; } - state.timer = setInterval(refreshOperationalData, 4000); + state.timer = setTimeout(async function poll() { + state.timer = null; + await refreshOperationalData(); + startPolling(); + }, 4000); } function toggleLive() { @@ -526,7 +538,13 @@ document.addEventListener('DOMContentLoaded', function () { if (logSearch) { logSearch.addEventListener('input', function () { state.search = this.value.trim(); - loadLogs(); + if (state.searchTimer) { + clearTimeout(state.searchTimer); + } + state.searchTimer = setTimeout(function () { + state.searchTimer = null; + loadLogs(); + }, 250); }); } @@ -572,6 +590,24 @@ document.addEventListener('DOMContentLoaded', function () { refreshOperationalData(); }); + document.addEventListener('visibilitychange', function () { + if (document.hidden) { + stopPolling(); + return; + } + if (state.liveMode) { + refreshOperationalData().finally(startPolling); + } + }); + + window.addEventListener('pagehide', function () { + stopPolling(); + if (state.searchTimer) { + clearTimeout(state.searchTimer); + state.searchTimer = null; + } + }); + setLiveState(); startPolling(); diff --git a/apps/ui/js/workspace-setup.js b/apps/ui/js/workspace-setup.js index 4faa788..e568b7a 100644 --- a/apps/ui/js/workspace-setup.js +++ b/apps/ui/js/workspace-setup.js @@ -196,7 +196,7 @@ async function exportWorkspaceSnapshot() { var slug = workspaceFormState.workspaceRecord && workspaceFormState.workspaceRecord.workspace ? workspaceFormState.workspaceRecord.workspace.slug : tKey('settings.nav.workspace'); - downloadJsonFile(slug + '-snapshot.json', snapshot); + downloadJsonFile(slug + '-catalog-snapshot.json', snapshot); } catch (error) { if (window.CrankUi) { window.CrankUi.error(error.message || tKey('workspace_setup.export_error'), tKey('workspace_setup.export_error_title')); diff --git a/apps/ui/package-lock.json b/apps/ui/package-lock.json index 23c190a..8621068 100644 --- a/apps/ui/package-lock.json +++ b/apps/ui/package-lock.json @@ -9,7 +9,7 @@ "@fontsource/inter": "5.2.8", "@fontsource/jetbrains-mono": "5.2.8", "alpinejs": "3.15.12", - "js-yaml": "5.2.1" + "js-yaml": "5.2.2" }, "devDependencies": { "@playwright/test": "1.61.1", @@ -580,9 +580,9 @@ } }, "node_modules/js-yaml": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz", - "integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz", + "integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==", "funding": [ { "type": "github", diff --git a/apps/ui/package.json b/apps/ui/package.json index 8a5d2d3..6a3a699 100644 --- a/apps/ui/package.json +++ b/apps/ui/package.json @@ -11,7 +11,7 @@ "@fontsource/inter": "5.2.8", "@fontsource/jetbrains-mono": "5.2.8", "alpinejs": "3.15.12", - "js-yaml": "5.2.1" + "js-yaml": "5.2.2" }, "devDependencies": { "@playwright/test": "1.61.1", diff --git a/apps/ui/tests/e2e/wizard.spec.js b/apps/ui/tests/e2e/wizard.spec.js index 5e24e30..85c0732 100644 --- a/apps/ui/tests/e2e/wizard.spec.js +++ b/apps/ui/tests/e2e/wizard.spec.js @@ -161,15 +161,15 @@ test('wizard builds visual request mappings from JSON sample and path params', a await page.goto('/wizard/'); await page.locator('[data-testid="wizard-protocol-rest"]').click(); - await page.evaluate(() => window.CrankWizardShell.doGoToStep(2)); + await page.evaluate(() => window.CrankWizardShell.goToStep(2)); await expect(page.locator('#step-panel-2')).toBeVisible(); await page.locator('#endpoint-path').fill('/rates/{date}'); - await page.evaluate(() => window.CrankWizardShell.doGoToStep(3)); + await page.evaluate(() => window.CrankWizardShell.goToStep(3)); await expect(page.locator('#step-panel-3-rest')).toBeVisible(); await page.locator('.method-card[data-method="GET"]').click(); - await page.evaluate(() => window.CrankWizardShell.doGoToStep(5)); + await page.evaluate(() => window.CrankWizardShell.goToStep(5)); await expect(page.locator('#step-panel-5')).toBeVisible(); await page.locator('#wizard-input-sample').fill(JSON.stringify({ diff --git a/apps/ui/tests/e2e/workspace-settings.spec.js b/apps/ui/tests/e2e/workspace-settings.spec.js index ef0652f..0f09e13 100644 --- a/apps/ui/tests/e2e/workspace-settings.spec.js +++ b/apps/ui/tests/e2e/workspace-settings.spec.js @@ -11,6 +11,12 @@ test('workspace and settings pages show live session data', async ({ page }) => await expect(page.locator('#section-members')).toHaveCount(0); await expect(page.locator('#section-invite')).toHaveCount(0); await expect(page.locator('#delete-workspace-btn')).toHaveCount(0); + await expect(page.locator('[data-i18n="workspace_setup.danger.export_title"]')).toHaveText( + localized('Export workspace catalog', 'Экспорт каталога рабочего пространства'), + ); + await expect(page.locator('[data-i18n="workspace_setup.danger.export_body"]')).not.toContainText( + localized('all data', 'все данные'), + ); await page.goto('/settings'); await expect(page.locator('.page-title')).toHaveText(localized('Account settings', 'Настройки аккаунта')); diff --git a/crates/crank-adapter-rest/Cargo.toml b/crates/crank-adapter-rest/Cargo.toml index fd01f67..d3ef6ec 100644 --- a/crates/crank-adapter-rest/Cargo.toml +++ b/crates/crank-adapter-rest/Cargo.toml @@ -3,18 +3,26 @@ name = "crank-adapter-rest" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] async-trait = "0.1" crank-core = { path = "../crank-core" } +crank-trace = { path = "../crank-trace" } futures-util = "0.3" +metrics.workspace = true +opentelemetry.workspace = true reqwest = { workspace = true, features = ["stream"] } serde.workspace = true serde_json.workspace = true thiserror.workspace = true tokio.workspace = true +tracing.workspace = true +tracing-opentelemetry.workspace = true [dev-dependencies] axum.workspace = true +opentelemetry_sdk.workspace = true tokio.workspace = true +tracing-subscriber.workspace = true diff --git a/crates/crank-adapter-rest/src/client.rs b/crates/crank-adapter-rest/src/client.rs index f9b0d9f..7234246 100644 --- a/crates/crank-adapter-rest/src/client.rs +++ b/crates/crank-adapter-rest/src/client.rs @@ -7,7 +7,9 @@ use std::{ }; use crank_core::{HttpMethod, RestTarget}; +use crank_trace::{ErrorCategory, Stage, StageOutcome}; use futures_util::StreamExt; +use opentelemetry::{global, propagation::Injector, trace::TraceContextExt}; use reqwest::{ Client, dns::{Addrs, Name, Resolve, Resolving}, @@ -15,6 +17,8 @@ use reqwest::{ redirect, }; use serde_json::Value; +use tracing::{Instrument, Span}; +use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::{RestAdapterError, RestRequest, RestResponse}; @@ -66,10 +70,37 @@ impl RestAdapter { &self, target: &RestTarget, request: &RestRequest, + ) -> Result { + let started_at = std::time::Instant::now(); + let result = self.execute_inner(target, request).await; + let outcome = match &result { + Ok(_) => "success", + Err(error) => upstream_outcome(error), + }; + metrics::counter!( + "crank_upstream_requests_total", + "operation_kind" => "rest", + "outcome" => outcome + ) + .increment(1); + metrics::histogram!( + "crank_upstream_request_duration_seconds", + "operation_kind" => "rest", + "outcome" => outcome + ) + .record(started_at.elapsed().as_secs_f64()); + result + } + + async fn execute_inner( + &self, + target: &RestTarget, + request: &RestRequest, ) -> Result { let url = build_url(target, request)?; self.policy.validate_url(&url)?; - let headers = build_headers(target, request)?; + let mut headers = build_headers(target, request)?; + apply_current_trace_context(&mut headers); let client = self.client .as_ref() @@ -85,23 +116,60 @@ impl RestAdapter { builder = builder.json(body); } - let response = builder.send().await?; - let status = response.status(); - let headers = normalize_headers(response.headers()); - let body = decode_body(response, self.policy.max_response_bytes).await?; + let upstream_span = Stage::UpstreamHttp.span(); + let result = async { + let response = builder.send().await?; + let status = response.status(); + let headers = normalize_headers(response.headers()); + let body = decode_body(response, self.policy.max_response_bytes).await?; - if !status.is_success() { - return Err(RestAdapterError::UnexpectedStatus { - status: status.as_u16(), + if !status.is_success() { + return Err(RestAdapterError::UnexpectedStatus { + status: status.as_u16(), + body, + }); + } + + Ok(RestResponse { + status_code: status.as_u16(), + headers, body, - }); + }) } + .instrument(upstream_span.clone()) + .await; + match &result { + Ok(_) => StageOutcome::Success.record(&upstream_span), + Err(_) => { + StageOutcome::Error.record(&upstream_span); + ErrorCategory::Upstream.record(&upstream_span); + } + } + result + } +} - Ok(RestResponse { - status_code: status.as_u16(), - headers, - body, - }) +fn upstream_outcome(error: &RestAdapterError) -> &'static str { + match error { + RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => { + "client_error" + } + RestAdapterError::UnexpectedStatus { status, .. } if (500..600).contains(status) => { + "server_error" + } + RestAdapterError::UnexpectedStatus { .. } => "unexpected_status", + RestAdapterError::Transport(error) if error.is_timeout() => "timeout", + RestAdapterError::Transport(_) => "transport_error", + RestAdapterError::ResponseTooLarge { .. } => "response_too_large", + RestAdapterError::TargetNotAllowed { .. } => "rejected", + RestAdapterError::WindowExpired => "window_expired", + RestAdapterError::InvalidSseEvent => "invalid_response", + RestAdapterError::InvalidBaseUrl { .. } + | RestAdapterError::InvalidPathParameter { .. } + | RestAdapterError::InvalidQueryParameter { .. } + | RestAdapterError::InvalidHeaderName { .. } + | RestAdapterError::InvalidHeaderValue { .. } => "invalid_request", + RestAdapterError::InvalidConfiguration { .. } => "configuration", } } @@ -407,6 +475,9 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), HeaderName::try_from(name).map_err(|_| RestAdapterError::InvalidHeaderName { header: name.to_owned(), })?; + if is_trace_propagation_header(&header_name) { + return Ok(()); + } let header_value = HeaderValue::try_from(value).map_err(|_| RestAdapterError::InvalidHeaderValue { header: name.to_owned(), @@ -416,6 +487,38 @@ fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), Ok(()) } +fn is_trace_propagation_header(name: &HeaderName) -> bool { + matches!(name.as_str(), "traceparent" | "tracestate" | "baggage") +} + +fn apply_current_trace_context(headers: &mut HeaderMap) { + for header in ["traceparent", "tracestate", "baggage"] { + headers.remove(header); + } + + let context = Span::current().context(); + if !context.span().span_context().is_valid() { + return; + } + global::get_text_map_propagator(|propagator| { + propagator.inject_context(&context, &mut ReqwestHeaderInjector(headers)); + }); +} + +struct ReqwestHeaderInjector<'a>(&'a mut HeaderMap); + +impl Injector for ReqwestHeaderInjector<'_> { + fn set(&mut self, key: &str, value: String) { + let Ok(name) = HeaderName::try_from(key) else { + return; + }; + let Ok(value) = HeaderValue::try_from(value) else { + return; + }; + self.0.insert(name, value); + } +} + async fn decode_body( response: reqwest::Response, max_response_bytes: usize, diff --git a/crates/crank-adapter-rest/src/lib.rs b/crates/crank-adapter-rest/src/lib.rs index 0f9d5b1..10d1be6 100644 --- a/crates/crank-adapter-rest/src/lib.rs +++ b/crates/crank-adapter-rest/src/lib.rs @@ -26,13 +26,15 @@ impl ProtocolAdapter for RestAdapter { &self, target: &Target, prepared: &PreparedRequest, - _context: &RuntimeRequestContext, + context: &RuntimeRequestContext, ) -> Result { let target = rest_target(target)?; + let mut headers = prepared.headers.clone(); + headers.extend(context.outbound_headers()); let request = RestRequest { path_params: prepared.path_params.clone(), query_params: prepared.query_params.clone(), - headers: prepared.headers.clone(), + headers, body: prepared.body.clone(), timeout_ms: prepared.timeout_ms, }; diff --git a/crates/crank-adapter-rest/tests/integration/client.rs b/crates/crank-adapter-rest/tests/integration/client.rs index d15eee2..5edc816 100644 --- a/crates/crank-adapter-rest/tests/integration/client.rs +++ b/crates/crank-adapter-rest/tests/integration/client.rs @@ -8,9 +8,19 @@ use axum::{ routing::{get, post}, }; use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest}; -use crank_core::{HttpMethod, RestTarget}; +use crank_core::{ + HttpMethod, PreparedRequest, ProtocolAdapter, RestTarget, RuntimeRequestContext, Target, +}; +use opentelemetry::{ + global, + trace::{TraceContextExt, TracerProvider as _}, +}; +use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider}; use serde_json::{Value, json}; use tokio::net::TcpListener; +use tracing::Instrument; +use tracing_opentelemetry::OpenTelemetrySpanExt; +use tracing_subscriber::layer::SubscriberExt; #[tokio::test] async fn executes_rest_request_and_normalizes_json_response() { @@ -45,6 +55,120 @@ async fn executes_rest_request_and_normalizes_json_response() { ); } +#[tokio::test] +async fn protocol_context_overrides_mapped_correlation_headers() { + let base_url = spawn_test_server().await; + let adapter = test_adapter(); + let target = Target::Rest(RestTarget { + base_url, + method: HttpMethod::Post, + path_template: "/users/{user_id}".to_owned(), + static_headers: BTreeMap::from([ + ("x-request-id".to_owned(), "static-request".to_owned()), + ( + "x-correlation-id".to_owned(), + "static-correlation".to_owned(), + ), + ]), + }); + let prepared = PreparedRequest { + path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]), + headers: BTreeMap::from([ + ("x-request-id".to_owned(), "mapped-request".to_owned()), + ( + "x-correlation-id".to_owned(), + "mapped-correlation".to_owned(), + ), + ]), + body: Some(json!({ "name": "Ada" })), + timeout_ms: 1_000, + ..PreparedRequest::default() + }; + let context = RuntimeRequestContext::new("req-runtime", "corr-runtime"); + + let response = adapter + .invoke_unary(&target, &prepared, &context) + .await + .unwrap(); + + assert_eq!(response.body["request_id"], "req-runtime"); + assert_eq!(response.body["correlation_id"], "corr-runtime"); +} + +#[tokio::test(flavor = "current_thread")] +async fn current_trace_context_overrides_mapped_traceparent() { + global::set_text_map_propagator(TraceContextPropagator::new()); + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("rest-propagation-test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + let dispatch = tracing::Dispatch::new(subscriber); + let _dispatch_guard = tracing::dispatcher::set_default(&dispatch); + let span = tracing::info_span!("runtime.execute"); + let context = span.context(); + let expected_trace_id = context.span().span_context().trace_id().to_string(); + let base_url = spawn_test_server().await; + let target = RestTarget { + base_url, + method: HttpMethod::Post, + path_template: "/users/{user_id}".to_owned(), + static_headers: BTreeMap::new(), + }; + let request = RestRequest { + path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]), + query_params: BTreeMap::new(), + headers: BTreeMap::from([( + "traceparent".to_owned(), + "00-11111111111111111111111111111111-2222222222222222-01".to_owned(), + )]), + body: Some(json!({ "name": "Ada" })), + timeout_ms: 1_000, + }; + + let response = test_adapter() + .execute(&target, &request) + .instrument(span) + .await + .unwrap(); + + assert_eq!( + &response.body["traceparent"].as_str().unwrap()[3..35], + expected_trace_id + ); + provider.shutdown().unwrap(); +} + +#[tokio::test] +async fn user_configured_propagation_headers_are_removed_without_trusted_context() { + let base_url = spawn_test_server().await; + let target = RestTarget { + base_url, + method: HttpMethod::Post, + path_template: "/users/{user_id}".to_owned(), + static_headers: BTreeMap::from([ + ( + "traceparent".to_owned(), + "untrusted\ninvalid-value".to_owned(), + ), + ("tracestate".to_owned(), "vendor=value".to_owned()), + ("baggage".to_owned(), "secret=must-not-leave".to_owned()), + ]), + }; + let request = RestRequest { + path_params: BTreeMap::from([("user_id".to_owned(), "42".to_owned())]), + query_params: BTreeMap::new(), + headers: BTreeMap::new(), + body: Some(json!({ "name": "Ada" })), + timeout_ms: 1_000, + }; + + let response = test_adapter().execute(&target, &request).await.unwrap(); + + assert!(response.body.get("traceparent").is_none()); + assert!(response.body.get("tracestate").is_none()); + assert!(response.body.get("baggage").is_none()); +} + #[tokio::test] async fn returns_unexpected_status_with_normalized_body() { let base_url = spawn_test_server().await; @@ -186,14 +310,57 @@ async fn create_user( .get("x-static") .and_then(|value| value.to_str().ok()) .unwrap_or_default(); + let request_id = headers + .get("x-request-id") + .and_then(|value| value.to_str().ok()); + let correlation_id = headers + .get("x-correlation-id") + .and_then(|value| value.to_str().ok()); + let traceparent = headers + .get("traceparent") + .and_then(|value| value.to_str().ok()); + let tracestate = headers + .get("tracestate") + .and_then(|value| value.to_str().ok()); + let baggage = headers.get("baggage").and_then(|value| value.to_str().ok()); - Json(json!({ + let mut response = json!({ "id": user_id, "query": query.get("expand").cloned().unwrap_or_default(), "trace": trace, "static": static_header, "payload": payload - })) + }); + let response = response.as_object_mut().unwrap(); + if let Some(request_id) = request_id { + response.insert( + "request_id".to_owned(), + Value::String(request_id.to_owned()), + ); + } + if let Some(correlation_id) = correlation_id { + response.insert( + "correlation_id".to_owned(), + Value::String(correlation_id.to_owned()), + ); + } + if let Some(traceparent) = traceparent { + response.insert( + "traceparent".to_owned(), + Value::String(traceparent.to_owned()), + ); + } + if let Some(tracestate) = tracestate { + response.insert( + "tracestate".to_owned(), + Value::String(tracestate.to_owned()), + ); + } + if let Some(baggage) = baggage { + response.insert("baggage".to_owned(), Value::String(baggage.to_owned())); + } + + Json(Value::Object(response.clone())) } async fn fail() -> (axum::http::StatusCode, Json) { diff --git a/crates/crank-community-auth/Cargo.toml b/crates/crank-community-auth/Cargo.toml index 3b32613..8cad6a6 100644 --- a/crates/crank-community-auth/Cargo.toml +++ b/crates/crank-community-auth/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-community-auth" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] diff --git a/crates/crank-community-auth/src/password_provider.rs b/crates/crank-community-auth/src/password_provider.rs index 6f1ca6f..f369bed 100644 --- a/crates/crank-community-auth/src/password_provider.rs +++ b/crates/crank-community-auth/src/password_provider.rs @@ -52,7 +52,11 @@ impl IdentityProvider for PasswordIdentityProvider { &self.password_pepper, &user.password_hash, ) { - debug!(email = %payload.email, "password identity provider rejected credentials"); + debug!( + name: "auth.password.rejected", + identity_provider = "password", + "password identity provider rejected credentials" + ); return Err(IdentityError::BadCredentials); } diff --git a/crates/crank-community-mcp/Cargo.toml b/crates/crank-community-mcp/Cargo.toml index b6171e6..bcf11f3 100644 --- a/crates/crank-community-mcp/Cargo.toml +++ b/crates/crank-community-mcp/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-community-mcp" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] @@ -11,10 +12,13 @@ axum.workspace = true base64.workspace = true crank-adapter-rest = { path = "../crank-adapter-rest" } crank-core = { path = "../crank-core" } +crank-observability = { path = "../crank-observability" } crank-registry = { path = "../crank-registry" } crank-runtime = { path = "../crank-runtime" } crank-schema = { path = "../crank-schema" } +crank-trace = { path = "../crank-trace" } futures-util = "0.3" +metrics.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true @@ -29,3 +33,7 @@ uuid.workspace = true [dev-dependencies] crank-mapping = { path = "../crank-mapping" } crank-test-support = { path = "../crank-test-support" } +opentelemetry.workspace = true +opentelemetry_sdk.workspace = true +tracing-opentelemetry.workspace = true +tracing-subscriber.workspace = true diff --git a/crates/crank-community-mcp/src/access.rs b/crates/crank-community-mcp/src/access.rs index 9315f50..90d0504 100644 --- a/crates/crank-community-mcp/src/access.rs +++ b/crates/crank-community-mcp/src/access.rs @@ -1,27 +1,54 @@ use std::sync::Arc; -use axum::http::{HeaderMap, StatusCode, header::AUTHORIZATION}; +use axum::{ + http::{HeaderMap, StatusCode, header::AUTHORIZATION}, + response::{IntoResponse, Response}, +}; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use crank_core::{OperationSecurityLevel, PlatformApiKeyScope}; +use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query}; use sha2::{Digest, Sha256}; use time::OffsetDateTime; +use tracing::Instrument; use crate::{ app::{AgentRoutePath, AppState}, auth::VerifiedMachineCredential, }; +#[derive(Clone, Copy, Debug)] +pub(super) enum MachineAccessError { + Denied(StatusCode), + Unavailable, +} + +impl MachineAccessError { + pub(super) fn is_denied(self) -> bool { + matches!(self, Self::Denied(_)) + } +} + +impl IntoResponse for MachineAccessError { + fn into_response(self) -> Response { + match self { + Self::Denied(status) => status.into_response(), + Self::Unavailable => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } +} + pub(super) async fn require_machine_access( state: &Arc, path: &AgentRoutePath, headers: &HeaderMap, required_scope: PlatformApiKeyScope, -) -> Result { - let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?; +) -> Result { + let secret = + bearer_token(headers).ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))?; let credential = resolve_machine_credential(state, path, secret).await?; if !allows_scope(&credential.scopes, required_scope) { - return Err(StatusCode::FORBIDDEN); + return Err(MachineAccessError::Denied(StatusCode::FORBIDDEN)); } Ok(credential) @@ -35,15 +62,18 @@ pub(super) async fn require_approval_access( ) -> Result { let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?; let secret_hash = hash_access_secret(secret); - let Some(api_key) = state - .registry - .get_approval_api_key_by_secret_for_agent_slug( - &path.workspace_slug, - &path.agent_slug, - &secret_hash, - ) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? + let Some(api_key) = observe_db_query( + DbOperation::MachineAccessRead, + state + .registry + .get_approval_api_key_by_secret_for_agent_slug( + &path.workspace_slug, + &path.agent_slug, + &secret_hash, + ), + ) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? else { return Err(StatusCode::UNAUTHORIZED); }; @@ -53,11 +83,16 @@ pub(super) async fn require_approval_access( } let used_at = OffsetDateTime::now_utc(); - state - .registry - .touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + observe_db_query( + DbOperation::MachineAccessTouch, + state.registry.touch_platform_api_key( + &api_key.api_key.workspace_id, + &api_key.api_key.id, + &used_at, + ), + ) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(api_key) } @@ -100,7 +135,7 @@ async fn resolve_machine_credential( state: &Arc, path: &AgentRoutePath, token: &str, -) -> Result { +) -> Result { if let Some(credential) = verify_static_agent_key(state, path, token).await? { return Ok(credential); } @@ -109,35 +144,68 @@ async fn resolve_machine_credential( .credential_verifier .verify_bearer_token(&path.workspace_slug, &path.agent_slug, token) .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? - .ok_or(StatusCode::UNAUTHORIZED) + .map_err(|_| MachineAccessError::Unavailable)? + .ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED)) } async fn verify_static_agent_key( state: &Arc, path: &AgentRoutePath, secret: &str, -) -> Result, StatusCode> { +) -> Result, MachineAccessError> { let secret_hash = hash_access_secret(secret); - let Some(api_key) = state + let read_span = Stage::DbQuery + .db_span(DbOperation::MachineAccessRead) + .expect("database stage"); + let api_key_result = state .registry .get_platform_api_key_by_secret_for_agent_slug( &path.workspace_slug, &path.agent_slug, &secret_hash, ) + .instrument(read_span.clone()) .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? - else { + .map_err(|_| MachineAccessError::Unavailable); + let api_key = match api_key_result { + Ok(api_key) => { + StageOutcome::Success.record(&read_span); + drop(read_span); + api_key + } + Err(status) => { + StageOutcome::Error.record(&read_span); + ErrorCategory::Database.record(&read_span); + drop(read_span); + return Err(status); + } + }; + let Some(api_key) = api_key else { return Ok(None); }; let used_at = OffsetDateTime::now_utc(); - state + let touch_span = Stage::DbQuery + .db_span(DbOperation::MachineAccessTouch) + .expect("database stage"); + let touch_result = state .registry .touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at) + .instrument(touch_span.clone()) .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + .map_err(|_| MachineAccessError::Unavailable); + match touch_result { + Ok(()) => { + StageOutcome::Success.record(&touch_span); + drop(touch_span); + } + Err(status) => { + StageOutcome::Error.record(&touch_span); + ErrorCategory::Database.record(&touch_span); + drop(touch_span); + return Err(status); + } + } Ok(Some(VerifiedMachineCredential { machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey, diff --git a/crates/crank-community-mcp/src/app.rs b/crates/crank-community-mcp/src/app.rs index 76d3514..ea56b63 100644 --- a/crates/crank-community-mcp/src/app.rs +++ b/crates/crank-community-mcp/src/app.rs @@ -7,36 +7,38 @@ use std::{ use axum::{ Json, Router, - extract::{Path, State}, + extract::{Extension, Path, State}, http::{HeaderMap, StatusCode}, response::{IntoResponse, Response, sse::Event}, routing::{get, post}, }; use crank_core::{ ApprovalRequest, ApprovalRequestId, ApprovalRequestStatus, AuthProfile, CoordinationStateStore, - InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus, - OperationApprovalMode, PlatformApiKeyScope, SecretId, + InvocationLevel, InvocationSource, InvocationStatus, OperationApprovalMode, + PlatformApiKeyScope, SecretId, }; use crank_registry::{ - CreateApprovalRequest, CreateInvocationLogRequest, DecideApprovalRequest, - ExpireApprovalRequest, PostgresRegistry, PublishedAgentTool, + CreateApprovalRequest, DecideApprovalRequest, ExpireApprovalRequest, PostgresRegistry, + PublishedAgentTool, }; use crank_runtime::{ RequestRateLimiter, ResolvedAuth, RuntimeError, RuntimeExecutionRequest, RuntimeExecutor, RuntimeOperation, RuntimeRequestContext, SecretCrypto, }; +use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query}; use futures_util::stream; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use time::OffsetDateTime; -use tracing::{info, warn}; +use tokio::sync::Semaphore; +use tracing::{Instrument, info, warn}; use crate::{ access::{ - credential_allows_security_level, require_approval_access, require_machine_access, - serialize_machine_access_mode, serialize_security_level, + credential_allows_security_level, serialize_machine_access_mode, serialize_security_level, }, approval_execution::{execute_approved_request, spawn_approval_recovery}, + approval_response::approval_required_response, auth::{SharedMachineCredentialVerifier, VerifiedMachineCredential}, catalog::PublishedToolCatalog, jsonrpc::{ @@ -44,10 +46,8 @@ use crate::{ jsonrpc_result, method_name, negotiated_protocol_version, params, request_id, }, manifest::catalog_tool_definitions, - rate_limit::{ - enforce_post_rate_limit, enforce_transport_rate_limit, rate_limited_jsonrpc_response, - rate_limited_status_response, - }, + rate_limit::{rate_limited_jsonrpc_response, rate_limited_status_response}, + request_context::{RequestContext, apply_request_context}, session::{SessionState, SharedSessionStore}, tool_error::{ ToolErrorContract, generic_tool_error_contract, runtime_error_code, @@ -56,13 +56,25 @@ use crate::{ tool_search::handle_catalog_tool_call, transport::{ AllowedOrigins, HEADER_MCP_SESSION_ID, ResponseMode, json_response, - negotiate_post_response_mode, protocol_version_from_headers, resolve_request_id, - session_id_from_headers, sse_response, transport_response, validate_get_accept_header, - validate_origin, validate_session_protocol_version, with_request_id_header, + negotiate_post_response_mode, protocol_version_from_headers, session_id_from_headers, + sse_response, transport_response, validate_get_accept_header, validate_origin, + validate_session_protocol_version, with_request_id_header, }, }; +mod invocation_history; +mod metrics; +mod stages; +use self::metrics::{ActiveSessionGuard, McpRequestMetrics}; +use self::stages::{ + enforce_traced_rate_limit, require_traced_approval_access, require_traced_machine_access, +}; +#[cfg(test)] +use invocation_history::observe_invocation_history_outcome; +pub(super) use invocation_history::{InvocationRecord, persist_invocation}; const TRANSPORT_SESSION_TTL_MS: u64 = 86_400_000; +const DEFAULT_MAX_CONCURRENT_SESSIONS: usize = 16; +const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60); #[derive(Clone)] pub(super) struct AppState { @@ -72,6 +84,7 @@ pub(super) struct AppState { pub(super) api_rate_limiter: RequestRateLimiter, secret_crypto: SecretCrypto, sessions: SharedSessionStore, + session_slots: Arc, pub(super) credential_verifier: SharedMachineCredentialVerifier, allowed_origins: AllowedOrigins, } @@ -144,6 +157,7 @@ pub fn build_app( coordination_store, sessions, credential_verifier, + DEFAULT_MAX_CONCURRENT_SESSIONS, false, ) } @@ -159,6 +173,33 @@ pub fn build_app_with_background_workers( coordination_store: Arc, sessions: SharedSessionStore, credential_verifier: SharedMachineCredentialVerifier, +) -> Router { + build_app_with_background_workers_and_limits( + registry, + refresh_interval, + public_base_url, + secret_crypto, + runtime, + api_rate_limiter, + coordination_store, + sessions, + credential_verifier, + DEFAULT_MAX_CONCURRENT_SESSIONS, + ) +} + +#[allow(clippy::too_many_arguments)] +pub fn build_app_with_background_workers_and_limits( + registry: PostgresRegistry, + refresh_interval: Duration, + public_base_url: Option, + secret_crypto: SecretCrypto, + runtime: RuntimeExecutor, + api_rate_limiter: RequestRateLimiter, + coordination_store: Arc, + sessions: SharedSessionStore, + credential_verifier: SharedMachineCredentialVerifier, + max_concurrent_sessions: usize, ) -> Router { build_app_inner( registry, @@ -170,6 +211,7 @@ pub fn build_app_with_background_workers( coordination_store, sessions, credential_verifier, + max_concurrent_sessions, true, ) } @@ -185,6 +227,7 @@ fn build_app_inner( coordination_store: Arc, sessions: SharedSessionStore, credential_verifier: SharedMachineCredentialVerifier, + max_concurrent_sessions: usize, start_background_workers: bool, ) -> Router { let state = Arc::new(AppState { @@ -194,15 +237,18 @@ fn build_app_inner( api_rate_limiter, secret_crypto, sessions, + session_slots: Arc::new(Semaphore::new(max_concurrent_sessions)), credential_verifier, allowed_origins: AllowedOrigins::new(public_base_url), }); if start_background_workers { spawn_approval_recovery(Arc::clone(&state)); + spawn_session_cleanup(Arc::clone(&state)); } Router::new() .route("/health", get(health)) + .route("/ready", get(readiness)) .route( "/v1/{workspace_slug}/{agent_slug}", get(mcp_get).post(mcp_post).delete(mcp_delete), @@ -224,6 +270,10 @@ fn build_app_inner( post(deny_request), ) .with_state(state) + .layer(axum::middleware::from_fn(apply_request_context)) + .layer(axum::middleware::from_fn( + crank_observability::record_http_request, + )) } async fn health() -> Json { @@ -233,27 +283,81 @@ async fn health() -> Json { })) } +fn spawn_session_cleanup(state: Arc) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(SESSION_CLEANUP_INTERVAL); + loop { + interval.tick().await; + match state + .sessions + .cleanup_expired(OffsetDateTime::now_utc()) + .await + { + Ok(removed) if removed > 0 => { + info!(name: "mcp.session_cleanup.completed", removed); + } + Ok(_) => {} + Err(_) => { + warn!(name: "mcp.session_cleanup.failed", error_category = "session_store"); + } + } + } + }); +} + +async fn readiness(State(state): State>) -> Response { + match state.registry.ping().await { + Ok(()) => Json(json!({ + "service": "mcp-server", + "status": "ready", + "checks": { "postgres": "ready" } + })) + .into_response(), + Err(error) => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "service": "mcp-server", + "status": "not_ready", + "checks": { "postgres": "not_ready" }, + "error": error.to_string() + })), + ) + .into_response(), + } +} + async fn list_pending_approvals( Path(path): Path, State(state): State>, headers: HeaderMap, ) -> Response { - let key = - match require_approval_access(&state, &path, &headers, PlatformApiKeyScope::ReadPending) - .await - { - Ok(key) => key, - Err(status) => return status.into_response(), - }; + if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await { + return rate_limited_status_response(rejection); + } + + let key = match require_traced_approval_access( + &state, + &path, + &headers, + PlatformApiKeyScope::ReadPending, + ) + .await + { + Ok(key) => key, + Err(status) => return status.into_response(), + }; let Some(agent_id) = key.api_key.agent_id.as_ref() else { return StatusCode::FORBIDDEN.into_response(); }; - match state - .registry - .list_pending_approval_requests_for_agent(&key.api_key.workspace_id, agent_id) - .await + match observe_db_query( + DbOperation::ApprovalRead, + state + .registry + .list_pending_approval_requests_for_agent(&key.api_key.workspace_id, agent_id), + ) + .await { Ok(items) => Json(json!({ "items": items })).into_response(), Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), @@ -263,6 +367,7 @@ async fn list_pending_approvals( async fn approve_request( Path(path): Path, State(state): State>, + Extension(request_context): Extension, headers: HeaderMap, Json(payload): Json, ) -> Response { @@ -273,6 +378,7 @@ async fn approve_request( payload, PlatformApiKeyScope::Approve, ApprovalRequestStatus::Approved, + Some(request_context.request_id), ) .await } @@ -286,7 +392,11 @@ async fn get_approval_request( workspace_slug: path.workspace_slug, agent_slug: path.agent_slug, }; - let key = match require_approval_access( + if let Err(rejection) = enforce_traced_rate_limit(&state, &agent_path, &headers).await { + return rate_limited_status_response(rejection); + } + + let key = match require_traced_approval_access( &state, &agent_path, &headers, @@ -318,6 +428,7 @@ async fn deny_request( payload, PlatformApiKeyScope::Deny, ApprovalRequestStatus::Denied, + None, ) .await } @@ -329,15 +440,21 @@ async fn decide_approval_request( payload: ApprovalDecisionPayload, required_scope: PlatformApiKeyScope, status: ApprovalRequestStatus, + execution_request_id: Option, ) -> Response { let agent_path = AgentRoutePath { workspace_slug: path.workspace_slug, agent_slug: path.agent_slug, }; - let key = match require_approval_access(&state, &agent_path, &headers, required_scope).await { - Ok(key) => key, - Err(status) => return status.into_response(), - }; + if let Err(rejection) = enforce_traced_rate_limit(&state, &agent_path, &headers).await { + return rate_limited_status_response(rejection); + } + + let key = + match require_traced_approval_access(&state, &agent_path, &headers, required_scope).await { + Ok(key) => key, + Err(status) => return status.into_response(), + }; if (status == ApprovalRequestStatus::Approved && !payload.approve.eq_ignore_ascii_case("yes")) || (status == ApprovalRequestStatus::Denied && !payload.approve.eq_ignore_ascii_case("no")) @@ -357,36 +474,47 @@ async fn decide_approval_request( }; let approval_id = ApprovalRequestId::new(path.approval_id); - match state - .registry - .decide_approval_request(DecideApprovalRequest { - workspace_id: &key.api_key.workspace_id, - agent_id, - approval_id: &approval_id, - status, - decided_at: OffsetDateTime::now_utc(), - decided_by_key_id: &key.api_key.id, - response_payload: Some(json!({ "approve": payload.approve })), - decision_note: payload.note.as_deref(), - }) - .await + match observe_db_query( + DbOperation::ApprovalWrite, + state + .registry + .decide_approval_request(DecideApprovalRequest { + workspace_id: &key.api_key.workspace_id, + agent_id, + approval_id: &approval_id, + status, + decided_at: OffsetDateTime::now_utc(), + decided_by_key_id: &key.api_key.id, + response_payload: Some(json!({ "approve": payload.approve })), + decision_note: payload.note.as_deref(), + }), + ) + .await { Ok(Some(record)) if status == ApprovalRequestStatus::Approved => { - let claimed = match state - .registry - .claim_approval_request( + let claimed = match observe_db_query( + DbOperation::ApprovalWrite, + state.registry.claim_approval_request( &record.approval.workspace_id, &record.approval.agent_id, &record.approval.id, OffsetDateTime::now_utc(), - ) - .await + ), + ) + .await { Ok(Some(claimed)) => claimed, Ok(None) => return StatusCode::CONFLICT.into_response(), Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), }; - match execute_approved_request(&state, &agent_path, claimed).await { + match execute_approved_request( + &state, + &agent_path, + claimed, + execution_request_id.as_deref(), + ) + .await + { Ok(record) => Json(json!(record)).into_response(), Err(response) => response, } @@ -406,10 +534,13 @@ async fn approval_record_response( agent_id: &crank_core::AgentId, approval_id: &ApprovalRequestId, ) -> Response { - match state - .registry - .get_approval_request_for_agent(workspace_id, agent_id, approval_id) - .await + match observe_db_query( + DbOperation::ApprovalRead, + state + .registry + .get_approval_request_for_agent(workspace_id, agent_id, approval_id), + ) + .await { Ok(Some(record)) if record.approval.status == ApprovalRequestStatus::Pending @@ -429,10 +560,13 @@ async fn terminal_decision_response( agent_id: &crank_core::AgentId, approval_id: &ApprovalRequestId, ) -> Response { - match state - .registry - .get_approval_request_for_agent(workspace_id, agent_id, approval_id) - .await + match observe_db_query( + DbOperation::ApprovalRead, + state + .registry + .get_approval_request_for_agent(workspace_id, agent_id, approval_id), + ) + .await { Ok(Some(record)) if record.approval.status == ApprovalRequestStatus::Pending @@ -463,15 +597,18 @@ async fn expire_approval_response( agent_id: &crank_core::AgentId, approval_id: &ApprovalRequestId, ) -> Response { - match state - .registry - .expire_approval_request(ExpireApprovalRequest { - workspace_id, - agent_id, - approval_id, - expired_at: OffsetDateTime::now_utc(), - }) - .await + match observe_db_query( + DbOperation::ApprovalWrite, + state + .registry + .expire_approval_request(ExpireApprovalRequest { + workspace_id, + agent_id, + approval_id, + expired_at: OffsetDateTime::now_utc(), + }), + ) + .await { Ok(Some(record)) => Json(json!(record)).into_response(), Ok(None) => StatusCode::CONFLICT.into_response(), @@ -493,12 +630,12 @@ async fn mcp_get( } if let Err(status) = - require_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await + require_traced_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await { return status.into_response(); } - if let Err(rejection) = enforce_transport_rate_limit(&state, &path, &headers).await { + if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await { return rate_limited_status_response(rejection); } @@ -527,9 +664,19 @@ async fn mcp_get( return status.into_response(); } + let Ok(permit) = ActiveSessionGuard::try_acquire(&state.session_slots) else { + return StatusCode::TOO_MANY_REQUESTS.into_response(); + }; + + let stream = stream::unfold(permit, |permit| async move { + tokio::time::sleep(Duration::from_millis(TRANSPORT_SESSION_TTL_MS)).await; + drop(permit); + None::<(Result, _)> + }); + sse_response( StatusCode::OK, - stream::pending::>(), + stream, Some(&session_id), Some(&session.protocol_version), ) @@ -541,12 +688,12 @@ async fn mcp_delete( headers: HeaderMap, ) -> Response { if let Err(status) = - require_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await + require_traced_machine_access(&state, &path, &headers, PlatformApiKeyScope::Read).await { return status.into_response(); } - if let Err(rejection) = enforce_transport_rate_limit(&state, &path, &headers).await { + if let Err(rejection) = enforce_traced_rate_limit(&state, &path, &headers).await { return rate_limited_status_response(rejection); } @@ -574,11 +721,14 @@ async fn mcp_delete( async fn mcp_post( Path(path): Path, State(state): State>, + Extension(request_context): Extension, headers: HeaderMap, Json(message): Json, ) -> Response { - let transport_request_id = resolve_request_id(&headers); + let mut request_metrics = McpRequestMetrics::new(&message); + let transport_request_id = request_context.request_id; info!( + name: "mcp.request.received", request_id = %transport_request_id, workspace_slug = %path.workspace_slug, agent_slug = %path.agent_slug, @@ -591,7 +741,7 @@ async fn mcp_post( } let response_mode = match negotiate_post_response_mode(&headers) { - Ok(mode) => mode, + Ok(mode) => request_metrics.set_response_mode(mode), Err(status) => { return with_request_id_header(status.into_response(), &transport_request_id); } @@ -608,13 +758,13 @@ async fn mcp_post( } }; - if let Err(rejection) = enforce_post_rate_limit(&state, &path, &headers).await { + let rate_limit_result = enforce_traced_rate_limit(&state, &path, &headers).await; + if let Err(error) = rate_limit_result { return with_request_id_header( - rate_limited_jsonrpc_response(&message, response_mode, &protocol_version, rejection), + rate_limited_jsonrpc_response(&message, response_mode, &protocol_version, error), &transport_request_id, ); } - if let Some(session_id) = headers.get(HEADER_MCP_SESSION_ID) && let Ok(session_id) = session_id.to_str() { @@ -639,10 +789,12 @@ async fn mcp_post( Some("tools/call") => PlatformApiKeyScope::Write, _ => PlatformApiKeyScope::Read, }; - let credential = match require_machine_access(&state, &path, &headers, required_scope).await { + let access_result = + require_traced_machine_access(&state, &path, &headers, required_scope).await; + let credential = match access_result { Ok(credential) => credential, - Err(status) => { - return with_request_id_header(status.into_response(), &transport_request_id); + Err(error) => { + return with_request_id_header(error.into_response(), &transport_request_id); } }; @@ -768,6 +920,7 @@ async fn mcp_post( ), }; + request_metrics.complete(response.status()); with_request_id_header(response, &transport_request_id) } @@ -822,13 +975,26 @@ pub(super) async fn resolve_operation_auth( workspace_id: &crank_core::WorkspaceId, execution_config: &crank_core::ExecutionConfig, ) -> Result, RuntimeError> { - resolve_runtime_auth_for_task( + if execution_config.auth_profile_ref.is_none() { + return Ok(None); + } + let span = Stage::AuthResolve.span(); + let result = resolve_runtime_auth_for_task( &state.registry, &state.secret_crypto, workspace_id, execution_config, ) - .await + .instrument(span.clone()) + .await; + match &result { + Ok(_) => StageOutcome::Success.record(&span), + Err(_) => { + StageOutcome::Error.record(&span); + ErrorCategory::Configuration.record(&span); + } + } + result } async fn resolve_runtime_auth_for_task( @@ -841,16 +1007,18 @@ async fn resolve_runtime_auth_for_task( return Ok(None); }; - let auth_profile = registry - .get_auth_profile(workspace_id, auth_profile_id) - .await - .map_err(|error| RuntimeError::SecretCrypto { - operation: "load auth profile", - details: error.to_string(), - })? - .ok_or_else(|| RuntimeError::MissingAuthProfile { - auth_profile_id: auth_profile_id.as_str().to_owned(), - })?; + let auth_profile = observe_db_query( + DbOperation::AuthProfileRead, + registry.get_auth_profile(workspace_id, auth_profile_id), + ) + .await + .map_err(|error| RuntimeError::SecretCrypto { + operation: "load auth profile", + details: error.to_string(), + })? + .ok_or_else(|| RuntimeError::MissingAuthProfile { + auth_profile_id: auth_profile_id.as_str().to_owned(), + })?; resolve_auth_profile(registry, secret_crypto, workspace_id, &auth_profile) .await @@ -867,38 +1035,44 @@ async fn resolve_auth_profile( let used_at = OffsetDateTime::now_utc(); for secret_id in auth_profile.config.secret_ids() { - let secret = registry - .get_secret(workspace_id, secret_id) - .await - .map_err(|error| RuntimeError::SecretCrypto { - operation: "load secret", - details: error.to_string(), - })? - .ok_or_else(|| RuntimeError::MissingSecret { - secret_id: secret_id.as_str().to_owned(), - })?; - let version = registry - .get_current_secret_version(workspace_id, secret_id) - .await - .map_err(|error| RuntimeError::SecretCrypto { - operation: "load current secret version", - details: error.to_string(), - })? - .ok_or_else(|| RuntimeError::MissingSecretVersion { - secret_id: secret_id.as_str().to_owned(), - version: secret.secret.current_version, - })?; + let secret = observe_db_query( + DbOperation::SecretRead, + registry.get_secret(workspace_id, secret_id), + ) + .await + .map_err(|error| RuntimeError::SecretCrypto { + operation: "load secret", + details: error.to_string(), + })? + .ok_or_else(|| RuntimeError::MissingSecret { + secret_id: secret_id.as_str().to_owned(), + })?; + let version = observe_db_query( + DbOperation::SecretRead, + registry.get_current_secret_version(workspace_id, secret_id), + ) + .await + .map_err(|error| RuntimeError::SecretCrypto { + operation: "load current secret version", + details: error.to_string(), + })? + .ok_or_else(|| RuntimeError::MissingSecretVersion { + secret_id: secret_id.as_str().to_owned(), + version: secret.secret.current_version, + })?; let plaintext = secret_crypto.decrypt( &version.secret_version.key_version, &version.secret_version.ciphertext, )?; - registry - .touch_secret(workspace_id, secret_id, &used_at) - .await - .map_err(|error| RuntimeError::SecretCrypto { - operation: "touch secret", - details: error.to_string(), - })?; + observe_db_query( + DbOperation::SecretTouch, + registry.touch_secret(workspace_id, secret_id, &used_at), + ) + .await + .map_err(|error| RuntimeError::SecretCrypto { + operation: "touch secret", + details: error.to_string(), + })?; secrets.insert(SecretId::new(secret_id.as_str()), plaintext); } @@ -916,18 +1090,39 @@ async fn handle_base_tool_call( let tool = execution.tool; let arguments = execution.arguments; let operation = runtime_operation(&tool); - if let Some(response) = maybe_handle_approval_policy( - &state, - session, - message, - response_mode, - &tool, - &arguments, - transport_request_id, - ) - .await + if tool + .operation + .execution_config + .approval_policy + .as_ref() + .is_some_and(|policy| policy.required) { - return response; + let approval_span = Stage::ApprovalCheck.span(); + let response = maybe_handle_approval_policy( + &state, + session, + message, + response_mode, + &tool, + &arguments, + transport_request_id, + ) + .instrument(approval_span.clone()) + .await; + if let Some(result) = response { + return match result { + ApprovalPolicyResult::Required(response) => { + StageOutcome::Required.record(&approval_span); + response + } + ApprovalPolicyResult::Error(response) => { + StageOutcome::Error.record(&approval_span); + ErrorCategory::Approval.record(&approval_span); + response + } + }; + } + StageOutcome::Allowed.record(&approval_span); } let mut runtime_request_context = RuntimeRequestContext::from_request_id(transport_request_id) @@ -964,7 +1159,7 @@ async fn handle_base_tool_call( match result { Ok(output) => { - if let Err(error) = persist_invocation( + persist_invocation( &state, &tool, InvocationRecord { @@ -980,15 +1175,12 @@ async fn handle_base_tool_call( response_preview: output.clone(), }, ) - .await - { - warn!(error = %error, "successful invocation log write failed"); - } + .await; success_tool_response(message, response_mode, &session.protocol_version, output) } Err(error) => { - if let Err(log_error) = persist_invocation( + persist_invocation( &state, &tool, InvocationRecord { @@ -1004,10 +1196,7 @@ async fn handle_base_tool_call( response_preview: Value::Null, }, ) - .await - { - warn!(error = %log_error, "failed invocation log write failed"); - } + .await; tool_error_response( message, @@ -1019,6 +1208,11 @@ async fn handle_base_tool_call( } } +enum ApprovalPolicyResult { + Required(Response), + Error(Response), +} + async fn maybe_handle_approval_policy( state: &Arc, session: &SessionState, @@ -1027,7 +1221,7 @@ async fn maybe_handle_approval_policy( tool: &PublishedAgentTool, arguments: &Value, transport_request_id: &str, -) -> Option { +) -> Option { let policy = tool.operation.execution_config.approval_policy.as_ref()?; if !policy.required { return None; @@ -1066,35 +1260,12 @@ async fn maybe_create_custom_pending_approval( tool: &PublishedAgentTool, arguments: &Value, transport_request_id: &str, -) -> Option { +) -> Option { let policy = tool.operation.execution_config.approval_policy.as_ref()?; let approval_id = ApprovalRequestId::new(format!("approval_{}", uuid::Uuid::now_v7().simple())); let now = OffsetDateTime::now_utc(); let expires_at = now + time::Duration::seconds(i64::from(policy.ttl_seconds)); - let approval_url = approval_url_for(tool, &approval_id); - let response_payload = json!({ - "status": "approval_required", - "approval_id": approval_id.as_str(), - "approval_url": approval_url, - "approve": { - "method": "POST", - "url": format!("{approval_url}/approve"), - "body": { "approve": "yes" } - }, - "deny": { - "method": "POST", - "url": format!("{approval_url}/deny"), - "body": { "approve": "no" } - }, - "expires_at": expires_at, - "risk_level": policy.risk_level, - "payload_preview": if policy.show_payload_preview { - arguments.clone() - } else { - Value::Null - }, - }); let approval = ApprovalRequest { id: approval_id, workspace_id: tool.workspace_id.clone(), @@ -1112,22 +1283,26 @@ async fn maybe_create_custom_pending_approval( decision_note: None, }; - let persisted_approval = match state - .registry - .create_approval_request(CreateApprovalRequest { - approval: &approval, - }) - .await + let persisted_approval = match observe_db_query( + DbOperation::ApprovalWrite, + state + .registry + .create_approval_request(CreateApprovalRequest { + approval: &approval, + }), + ) + .await { Ok(approval) => approval, - Err(error) => return Some(internal_jsonrpc_error(message, error)), + Err(error) => { + return Some(ApprovalPolicyResult::Error(internal_jsonrpc_error( + message, error, + ))); + } }; - let response_payload = persisted_approval - .approval - .response_payload - .unwrap_or(response_payload); + let response_payload = approval_required_response(tool, &persisted_approval.approval, policy); - if let Err(error) = persist_invocation( + persist_invocation( state, tool, InvocationRecord { @@ -1143,17 +1318,14 @@ async fn maybe_create_custom_pending_approval( response_preview: response_payload.clone(), }, ) - .await - { - warn!(error = %error, "pending approval invocation log write failed"); - } + .await; - Some(success_tool_response( + Some(ApprovalPolicyResult::Required(success_tool_response( message, response_mode, &session.protocol_version, response_payload, - )) + ))) } fn handle_elicitation_approval( @@ -1164,9 +1336,9 @@ fn handle_elicitation_approval( arguments: &Value, elicitation_message: Option<&str>, transport_request_id: &str, -) -> Response { +) -> ApprovalPolicyResult { if !session.supports_elicitation { - return tool_error_response( + return ApprovalPolicyResult::Error(tool_error_response( message, response_mode, &session.protocol_version, @@ -1179,7 +1351,7 @@ fn handle_elicitation_approval( "Выберите Custom MCP Approval или подключите MCP-клиент с поддержкой elicitation.", ), ), - ); + )); } let payload_preview = tool @@ -1190,7 +1362,7 @@ fn handle_elicitation_approval( .and_then(|policy| policy.show_payload_preview.then(|| arguments.clone())) .unwrap_or(Value::Null); - success_tool_response( + ApprovalPolicyResult::Required(success_tool_response( message, response_mode, &session.protocol_version, @@ -1201,16 +1373,7 @@ fn handle_elicitation_approval( "payload_preview": payload_preview, "note": "This MCP client advertised elicitation support. Full elicitation/create continuation is handled by compatible client integrations.", }), - ) -} - -fn approval_url_for(tool: &PublishedAgentTool, approval_id: &ApprovalRequestId) -> String { - format!( - "/v1/{}/{}/approvals/{}", - tool.workspace_slug, - tool.agent_slug, - approval_id.as_str() - ) + )) } async fn handle_initialize( @@ -1394,51 +1557,6 @@ pub(super) fn build_request_preview( } } -pub(super) struct InvocationRecord<'a> { - pub(super) request_id: Option<&'a str>, - pub(super) tool_name: &'a str, - pub(super) status: InvocationStatus, - pub(super) level: InvocationLevel, - pub(super) message: &'a str, - pub(super) status_code: Option, - pub(super) error_kind: Option<&'a str>, - pub(super) duration: Duration, - pub(super) request_preview: Value, - pub(super) response_preview: Value, -} - -pub(super) async fn persist_invocation( - state: &Arc, - tool: &PublishedAgentTool, - record: InvocationRecord<'_>, -) -> Result<(), crank_registry::RegistryError> { - let created_at = OffsetDateTime::now_utc(); - let duration_ms = u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX); - let log = InvocationLog { - id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())), - workspace_id: tool.workspace_id.clone(), - agent_id: Some(tool.agent_id.clone()), - operation_id: tool.operation.id.clone(), - source: InvocationSource::AgentToolCall, - level: record.level, - status: record.status, - tool_name: record.tool_name.to_owned(), - message: record.message.to_owned(), - request_id: record.request_id.map(ToOwned::to_owned), - status_code: record.status_code, - duration_ms, - error_kind: record.error_kind.map(ToOwned::to_owned), - request_preview: record.request_preview, - response_preview: record.response_preview, - created_at, - }; - - state - .registry - .create_invocation_log(CreateInvocationLogRequest { log: &log }) - .await -} - fn success_tool_response( message: &Value, response_mode: ResponseMode, @@ -1526,42 +1644,4 @@ pub(super) fn runtime_operation(tool: &PublishedAgentTool) -> RuntimeOperation { } #[cfg(test)] -mod tests { - use axum::body::to_bytes; - use serde_json::{Value, json}; - - use super::{ResponseMode, tool_error_response}; - use crate::jsonrpc::CURRENT_PROTOCOL_VERSION; - use crate::tool_error::generic_tool_error_contract; - - #[tokio::test] - async fn tool_error_response_includes_structured_context() { - let response = tool_error_response( - &json!({"jsonrpc": "2.0", "id": "req-1"}), - ResponseMode::Json, - CURRENT_PROTOCOL_VERSION, - generic_tool_error_contract( - "streaming_payload_error", - "request root must be an object", - "req-1", - false, - Some("Проверьте параметры вызова инструмента."), - ), - ); - - let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); - let payload: Value = serde_json::from_slice(&body).unwrap(); - - assert_eq!( - payload["result"]["structuredContent"]["error"], - json!({ - "code": "streaming_payload_error", - "error_code": "streaming_payload_error", - "message": "request root must be an object", - "recoverable": false, - "request_id": "req-1", - "suggested_action": "Проверьте параметры вызова инструмента." - }) - ); - } -} +mod tests; diff --git a/crates/crank-community-mcp/src/app/invocation_history.rs b/crates/crank-community-mcp/src/app/invocation_history.rs new file mode 100644 index 0000000..ee20fbd --- /dev/null +++ b/crates/crank-community-mcp/src/app/invocation_history.rs @@ -0,0 +1,117 @@ +use std::{sync::Arc, time::Duration}; + +use crank_core::{ + InvocationLevel, InvocationLog, InvocationLogId, InvocationSource, InvocationStatus, +}; +use crank_registry::{ + CreateInvocationLogRequest, InvocationHistoryWriteOutcome, PublishedAgentTool, +}; +use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome}; +use serde_json::Value; +use time::OffsetDateTime; +use tracing::{Instrument, warn}; + +use super::AppState; + +pub(crate) struct InvocationRecord<'a> { + pub(crate) request_id: Option<&'a str>, + pub(crate) tool_name: &'a str, + pub(crate) status: InvocationStatus, + pub(crate) level: InvocationLevel, + pub(crate) message: &'a str, + pub(crate) status_code: Option, + pub(crate) error_kind: Option<&'a str>, + pub(crate) duration: Duration, + pub(crate) request_preview: Value, + pub(crate) response_preview: Value, +} + +pub(crate) async fn persist_invocation( + state: &Arc, + tool: &PublishedAgentTool, + record: InvocationRecord<'_>, +) -> InvocationHistoryWriteOutcome { + let log = InvocationLog { + id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())), + workspace_id: tool.workspace_id.clone(), + agent_id: Some(tool.agent_id.clone()), + operation_id: tool.operation.id.clone(), + source: InvocationSource::AgentToolCall, + level: record.level, + status: record.status, + tool_name: record.tool_name.to_owned(), + message: record.message.to_owned(), + request_id: record.request_id.map(ToOwned::to_owned), + status_code: record.status_code, + duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX), + error_kind: record.error_kind.map(ToOwned::to_owned), + request_preview: record.request_preview, + response_preview: record.response_preview, + created_at: OffsetDateTime::now_utc(), + }; + + let history_span = Stage::HistoryWrite.span(); + let (outcome, db_span) = async { + let db_span = Stage::DbQuery + .db_span(DbOperation::InvocationHistoryWrite) + .expect("database stage"); + let outcome = state + .registry + .create_invocation_log(CreateInvocationLogRequest { log: &log }) + .instrument(db_span.clone()) + .await; + (outcome, db_span) + } + .instrument(history_span.clone()) + .await; + match outcome { + InvocationHistoryWriteOutcome::Recorded => { + StageOutcome::Success.record(&db_span); + StageOutcome::Success.record(&history_span); + } + InvocationHistoryWriteOutcome::Lost(_) => { + StageOutcome::Error.record(&db_span); + ErrorCategory::Database.record(&db_span); + StageOutcome::Error.record(&history_span); + ErrorCategory::History.record(&history_span); + } + } + drop(db_span); + drop(history_span); + observe_invocation_history_outcome( + outcome, + record.request_id, + record.status, + "agent_tool_call", + ); + outcome +} + +pub(super) fn observe_invocation_history_outcome( + outcome: InvocationHistoryWriteOutcome, + request_id: Option<&str>, + status: InvocationStatus, + source: &'static str, +) { + let Some(loss) = outcome.loss() else { + return; + }; + crank_observability::record_operational_incident( + crank_observability::OperationalIncident::InvocationHistoryLost, + ); + warn!( + name: "mcp.invocation_history.lost", + request_id = request_id.unwrap_or_default(), + source, + invocation_status = invocation_status_label(status), + error_category = loss.category.as_str(), + "invocation history was not recorded" + ); +} + +fn invocation_status_label(status: InvocationStatus) -> &'static str { + match status { + InvocationStatus::Ok => "ok", + InvocationStatus::Error => "error", + } +} diff --git a/crates/crank-community-mcp/src/app/metrics.rs b/crates/crank-community-mcp/src/app/metrics.rs new file mode 100644 index 0000000..e50f129 --- /dev/null +++ b/crates/crank-community-mcp/src/app/metrics.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use axum::http::StatusCode; +use serde_json::Value; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +use crate::{ + jsonrpc::{is_notification, is_response, method_name}, + transport::ResponseMode, +}; + +pub(super) struct McpRequestMetrics { + method: &'static str, + response_mode: &'static str, + outcome: &'static str, +} + +impl McpRequestMetrics { + pub(super) fn new(message: &Value) -> Self { + Self { + method: normalized_mcp_method(message), + response_mode: "unknown", + outcome: "rejected", + } + } + + pub(super) fn set_response_mode(&mut self, mode: ResponseMode) -> ResponseMode { + self.response_mode = match mode { + ResponseMode::Json => "json", + ResponseMode::Sse => "sse", + }; + mode + } + + pub(super) fn complete(&mut self, status: StatusCode) { + self.outcome = match status.as_u16() { + 200..=299 => "success", + 400..=499 => "client_error", + 500..=599 => "server_error", + _ => "other", + }; + } +} + +impl Drop for McpRequestMetrics { + fn drop(&mut self) { + ::metrics::counter!( + "crank_mcp_requests_total", + "method" => self.method, + "response_mode" => self.response_mode, + "outcome" => self.outcome + ) + .increment(1); + } +} + +pub(super) fn normalized_mcp_method(message: &Value) -> &'static str { + match method_name(message) { + Some("initialize") => "initialize", + Some("notifications/initialized") => "initialized", + Some("ping") => "ping", + Some("tools/list") => "tools_list", + Some("tools/call") => "tools_call", + Some(_) if is_notification(message) => "notification", + Some(_) => "unsupported", + None if is_response(message) => "response", + None => "invalid", + } +} + +pub(super) struct ActiveSessionGuard { + _permit: OwnedSemaphorePermit, +} + +impl ActiveSessionGuard { + pub(super) fn try_acquire(slots: &Arc) -> Result { + let permit = Arc::clone(slots).try_acquire_owned().map_err(|_| { + ::metrics::counter!( + "crank_runtime_limit_rejections_total", + "stage" => "mcp_session" + ) + .increment(1); + })?; + ::metrics::gauge!("crank_mcp_active_sessions").increment(1.0); + Ok(Self { _permit: permit }) + } +} + +impl Drop for ActiveSessionGuard { + fn drop(&mut self) { + ::metrics::gauge!("crank_mcp_active_sessions").decrement(1.0); + } +} diff --git a/crates/crank-community-mcp/src/app/stages.rs b/crates/crank-community-mcp/src/app/stages.rs new file mode 100644 index 0000000..1e862fd --- /dev/null +++ b/crates/crank-community-mcp/src/app/stages.rs @@ -0,0 +1,86 @@ +use std::sync::Arc; + +use axum::http::{HeaderMap, StatusCode}; +use crank_core::PlatformApiKeyScope; +use crank_registry::PlatformApiKeyRecord; +use crank_runtime::RateLimitCheckError; +use crank_trace::{ErrorCategory, Stage, StageOutcome}; +use tracing::Instrument; + +use crate::{ + access::{MachineAccessError, require_approval_access, require_machine_access}, + app::{AgentRoutePath, AppState}, + auth::VerifiedMachineCredential, + rate_limit::enforce_transport_rate_limit, +}; + +pub(super) async fn enforce_traced_rate_limit( + state: &Arc, + path: &AgentRoutePath, + headers: &HeaderMap, +) -> Result<(), RateLimitCheckError> { + let span = Stage::McpRateLimit.span(); + let result = enforce_transport_rate_limit(state, path, headers) + .instrument(span.clone()) + .await; + match &result { + Ok(()) => StageOutcome::Allowed.record(&span), + Err(RateLimitCheckError::Rejected(_)) => { + StageOutcome::Denied.record(&span); + ErrorCategory::RateLimit.record(&span); + } + Err(RateLimitCheckError::StoreUnavailable) => { + StageOutcome::Error.record(&span); + ErrorCategory::Internal.record(&span); + } + } + result +} + +pub(super) async fn require_traced_machine_access( + state: &Arc, + path: &AgentRoutePath, + headers: &HeaderMap, + required_scope: PlatformApiKeyScope, +) -> Result { + let span = Stage::McpAccessCheck.span(); + let result = require_machine_access(state, path, headers, required_scope) + .instrument(span.clone()) + .await; + match &result { + Ok(_) => StageOutcome::Allowed.record(&span), + Err(error) if error.is_denied() => { + StageOutcome::Denied.record(&span); + ErrorCategory::Access.record(&span); + } + Err(_) => { + StageOutcome::Error.record(&span); + ErrorCategory::Internal.record(&span); + } + } + result +} + +pub(super) async fn require_traced_approval_access( + state: &Arc, + path: &AgentRoutePath, + headers: &HeaderMap, + required_scope: PlatformApiKeyScope, +) -> Result { + let span = Stage::McpAccessCheck.span(); + let result = require_approval_access(state, path, headers, required_scope) + .instrument(span.clone()) + .await; + match &result { + Ok(_) => StageOutcome::Allowed.record(&span), + Err(status) if *status == StatusCode::UNAUTHORIZED || *status == StatusCode::FORBIDDEN => { + StageOutcome::Denied.record(&span); + ErrorCategory::Access.record(&span); + } + Err(_) => { + StageOutcome::Error.record(&span); + ErrorCategory::Internal.record(&span); + } + } + result +} diff --git a/crates/crank-community-mcp/src/app/tests.rs b/crates/crank-community-mcp/src/app/tests.rs new file mode 100644 index 0000000..90e4ed5 --- /dev/null +++ b/crates/crank-community-mcp/src/app/tests.rs @@ -0,0 +1,149 @@ +use std::{ + io, + sync::{Arc, Mutex}, +}; + +use axum::body::to_bytes; +use crank_core::InvocationStatus; +use crank_observability::{ + ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity, + operational_incident_total, +}; +use crank_registry::{ + InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome, +}; +use serde_json::{Value, json}; +use tracing_subscriber::fmt::MakeWriter; + +use super::{ + ResponseMode, metrics::normalized_mcp_method, observe_invocation_history_outcome, + tool_error_response, +}; +use crate::jsonrpc::CURRENT_PROTOCOL_VERSION; +use crate::tool_error::generic_tool_error_contract; + +#[tokio::test] +async fn tool_error_response_includes_structured_context() { + let response = tool_error_response( + &json!({"jsonrpc": "2.0", "id": "req-1"}), + ResponseMode::Json, + CURRENT_PROTOCOL_VERSION, + generic_tool_error_contract( + "streaming_payload_error", + "request root must be an object", + "req-1", + false, + Some("Проверьте параметры вызова инструмента."), + ), + ); + + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let payload: Value = serde_json::from_slice(&body).unwrap(); + + assert_eq!( + payload["result"]["structuredContent"]["error"], + json!({ + "code": "streaming_payload_error", + "error_code": "streaming_payload_error", + "message": "request root must be an object", + "recoverable": false, + "request_id": "req-1", + "suggested_action": "Проверьте параметры вызова инструмента." + }) + ); +} + +#[test] +fn emits_bounded_history_loss_incident() { + let writer = SharedLogWriter::default(); + let subscriber = crank_observability::build_subscriber( + ObservabilityConfig::new( + ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(), + "info", + RedactionLimits::default(), + ), + writer.clone(), + ) + .unwrap(); + let before = operational_incident_total(OperationalIncident::InvocationHistoryLost); + let dispatch = tracing::Dispatch::new(subscriber); + let _guard = tracing::dispatcher::set_default(&dispatch); + + observe_invocation_history_outcome( + InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss { + category: InvocationHistoryLossCategory::Unavailable, + }), + Some("req_mcp_dc08"), + InvocationStatus::Ok, + "agent_tool_call", + ); + + let output = writer.output(); + assert!(!output.contains("dc08-canary-secret")); + let event: Value = output + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .find(|event: &Value| event["event"] == "mcp.invocation_history.lost") + .unwrap(); + assert_eq!(event["request_id"], "req_mcp_dc08"); + assert_eq!(event["fields"]["source"], "agent_tool_call"); + assert_eq!(event["fields"]["invocation_status"], "ok"); + assert_eq!(event["fields"]["error_category"], "unavailable"); + assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before); +} + +#[test] +fn mcp_metric_method_is_always_from_a_closed_set() { + assert_eq!( + normalized_mcp_method(&json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call"})), + "tools_call" + ); + assert_eq!( + normalized_mcp_method( + &json!({"jsonrpc": "2.0", "id": 2, "method": "customer-controlled-method"}) + ), + "unsupported" + ); + assert_eq!( + normalized_mcp_method( + &json!({"jsonrpc": "2.0", "method": "customer-controlled-notification"}) + ), + "notification" + ); +} + +#[derive(Clone, Default)] +struct SharedLogWriter { + buffer: Arc>>, +} + +impl SharedLogWriter { + fn output(&self) -> String { + String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap() + } +} + +impl<'a> MakeWriter<'a> for SharedLogWriter { + type Writer = SharedLogGuard; + + fn make_writer(&'a self) -> Self::Writer { + SharedLogGuard { + buffer: Arc::clone(&self.buffer), + } + } +} + +struct SharedLogGuard { + buffer: Arc>>, +} + +impl io::Write for SharedLogGuard { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.buffer.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/crates/crank-community-mcp/src/approval_execution.rs b/crates/crank-community-mcp/src/approval_execution.rs index e8e4e79..7459f4b 100644 --- a/crates/crank-community-mcp/src/approval_execution.rs +++ b/crates/crank-community-mcp/src/approval_execution.rs @@ -5,11 +5,13 @@ use axum::{ response::{IntoResponse, Response}, }; use crank_core::{ApprovalRequestStatus, InvocationLevel, InvocationSource, InvocationStatus}; +use crank_observability::RequestId; use crank_registry::{ApprovalRequestRecord, FinishApprovalRequest}; use crank_runtime::{RuntimeExecutionRequest, RuntimeRequestContext}; +use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query}; use serde_json::json; use time::OffsetDateTime; -use tracing::warn; +use tracing::{Instrument, warn}; use crate::{ app::{ @@ -35,32 +37,81 @@ pub(super) fn spawn_approval_recovery(state: Arc) { } async fn recover_approved_requests(state: &Arc) { + fail_interrupted_requests(state).await; + for _ in 0..32 { let now = OffsetDateTime::now_utc(); - let approval = match state - .registry - .claim_next_recoverable_approval_request( - now, - now - RECOVERY_GRACE, - now - EXECUTION_LEASE, - ) - .await + let approval = match observe_db_query( + DbOperation::ApprovalWrite, + state + .registry + .claim_next_recoverable_approval_request(now, now - RECOVERY_GRACE), + ) + .await { Ok(Some(approval)) => approval, Ok(None) => break, - Err(error) => { - warn!(error = %error, "approval recovery query failed"); + Err(_) => { + warn!( + name: "mcp.approval_recovery.query_failed", + error_category = "registry", + "approval recovery query failed" + ); break; } }; let Some(path) = approval_agent_path(state, &approval).await else { continue; }; - if execute_approved_request(state, &path, approval) - .await - .is_err() + let recovery_span = Stage::ApprovalRecovery.span(); + let result = execute_approved_request(state, &path, approval, None) + .instrument(recovery_span.clone()) + .await; + match &result { + Ok(_) => StageOutcome::Success.record(&recovery_span), + Err(_) => { + StageOutcome::Error.record(&recovery_span); + ErrorCategory::Approval.record(&recovery_span); + } + } + drop(recovery_span); + if result.is_err() { + warn!( + name: "mcp.approval_recovery.execution_failed", + error_category = "runtime", + "recovered approval execution did not finish" + ); + } + } +} + +async fn fail_interrupted_requests(state: &Arc) { + for _ in 0..32 { + let stale_before = OffsetDateTime::now_utc() - EXECUTION_LEASE; + match observe_db_query( + DbOperation::ApprovalWrite, + state + .registry + .fail_next_interrupted_approval_request(stale_before), + ) + .await { - warn!("recovered approval execution did not finish"); + Ok(Some(approval)) => { + warn!( + name: "mcp.approval_recovery.interrupted", + approval_id = approval.approval.id.as_str(), + "interrupted approval execution was not retried because its outcome is unknown" + ); + } + Ok(None) => break, + Err(_) => { + warn!( + name: "mcp.approval_recovery.interrupted_query_failed", + error_category = "registry", + "interrupted approval recovery query failed" + ); + break; + } } } } @@ -76,8 +127,12 @@ async fn approval_agent_path( { Ok(Some(workspace)) => workspace, Ok(None) => return None, - Err(error) => { - warn!(error = %error, "approval workspace lookup failed"); + Err(_) => { + warn!( + name: "mcp.approval_recovery.workspace_lookup_failed", + error_category = "registry", + "approval workspace lookup failed" + ); return None; } }; @@ -88,8 +143,12 @@ async fn approval_agent_path( { Ok(Some(agent)) => agent, Ok(None) => return None, - Err(error) => { - warn!(error = %error, "approval agent lookup failed"); + Err(_) => { + warn!( + name: "mcp.approval_recovery.agent_lookup_failed", + error_category = "registry", + "approval agent lookup failed" + ); return None; } }; @@ -103,7 +162,9 @@ pub(super) async fn execute_approved_request( state: &Arc, path: &AgentRoutePath, approval: ApprovalRequestRecord, + request_id: Option<&str>, ) -> Result { + let request_id = RequestId::resolve(request_id).into_string(); let tools = state .catalog .list_tools(&path.workspace_slug, &path.agent_slug) @@ -123,18 +184,17 @@ pub(super) async fn execute_approved_request( &approval.approval.request_payload, ); let started_at = Instant::now(); - let runtime_request_context = - RuntimeRequestContext::from_request_id(approval.approval.id.as_str().to_owned()) - .with_response_cache_scope( - tool.workspace_id.as_str().to_owned(), - tool.agent_id.as_str().to_owned(), - ) - .with_metering_context( - tool.workspace_id.clone(), - Some(tool.agent_id.clone()), - InvocationSource::AgentToolCall, - ) - .with_approval_granted(); + let runtime_request_context = RuntimeRequestContext::from_request_id(request_id.clone()) + .with_response_cache_scope( + tool.workspace_id.as_str().to_owned(), + tool.agent_id.as_str().to_owned(), + ) + .with_metering_context( + tool.workspace_id.clone(), + Some(tool.agent_id.clone()), + InvocationSource::AgentToolCall, + ) + .with_approval_granted(); let resolved_auth = resolve_operation_auth(state, &tool.workspace_id, &operation.execution_config).await; let result = match resolved_auth { @@ -176,11 +236,11 @@ pub(super) async fn execute_approved_request( ), }; - if let Err(error) = persist_invocation( + persist_invocation( state, &tool, InvocationRecord { - request_id: Some(approval.approval.id.as_str()), + request_id: Some(&request_id), tool_name: &tool.tool_name, status: invocation_status, level: invocation_level, @@ -192,46 +252,49 @@ pub(super) async fn execute_approved_request( response_preview: response_payload.clone(), }, ) - .await - { - warn!(error = %error, "approved invocation log write failed"); - } + .await; - state - .registry - .finish_approval_request(FinishApprovalRequest { - workspace_id: &approval.approval.workspace_id, - agent_id: &approval.approval.agent_id, - approval_id: &approval.approval.id, - status, - response_payload: Some(response_payload), - decision_note: None, - }) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? - .ok_or_else(|| StatusCode::CONFLICT.into_response()) + observe_db_query( + DbOperation::ApprovalWrite, + state + .registry + .finish_approval_request(FinishApprovalRequest { + workspace_id: &approval.approval.workspace_id, + agent_id: &approval.approval.agent_id, + approval_id: &approval.approval.id, + status, + response_payload: Some(response_payload), + decision_note: None, + }), + ) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? + .ok_or_else(|| StatusCode::CONFLICT.into_response()) } async fn finish_unavailable_approval( state: &Arc, approval: &ApprovalRequestRecord, ) -> Result { - state - .registry - .finish_approval_request(FinishApprovalRequest { - workspace_id: &approval.approval.workspace_id, - agent_id: &approval.approval.agent_id, - approval_id: &approval.approval.id, - status: ApprovalRequestStatus::Failed, - response_payload: Some(json!({ - "error": { - "code": "approved_operation_unavailable", - "message": "the approved operation version is no longer published" - } - })), - decision_note: None, - }) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? - .ok_or_else(|| StatusCode::CONFLICT.into_response()) + observe_db_query( + DbOperation::ApprovalWrite, + state + .registry + .finish_approval_request(FinishApprovalRequest { + workspace_id: &approval.approval.workspace_id, + agent_id: &approval.approval.agent_id, + approval_id: &approval.approval.id, + status: ApprovalRequestStatus::Failed, + response_payload: Some(json!({ + "error": { + "code": "approved_operation_unavailable", + "message": "the approved operation version is no longer published" + } + })), + decision_note: None, + }), + ) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())? + .ok_or_else(|| StatusCode::CONFLICT.into_response()) } diff --git a/crates/crank-community-mcp/src/approval_response.rs b/crates/crank-community-mcp/src/approval_response.rs new file mode 100644 index 0000000..4c12ce6 --- /dev/null +++ b/crates/crank-community-mcp/src/approval_response.rs @@ -0,0 +1,38 @@ +use crank_core::{ApprovalRequest, OperationApprovalPolicy}; +use crank_registry::PublishedAgentTool; +use serde_json::{Value, json}; + +pub(super) fn approval_required_response( + tool: &PublishedAgentTool, + approval: &ApprovalRequest, + policy: &OperationApprovalPolicy, +) -> Value { + let approval_url = format!( + "/v1/{}/{}/approvals/{}", + tool.workspace_slug, + tool.agent_slug, + approval.id.as_str() + ); + json!({ + "status": "approval_required", + "approval_id": approval.id.as_str(), + "approval_url": approval_url, + "approve": { + "method": "POST", + "url": format!("{approval_url}/approve"), + "body": { "approve": "yes" } + }, + "deny": { + "method": "POST", + "url": format!("{approval_url}/deny"), + "body": { "approve": "no" } + }, + "expires_at": approval.expires_at, + "risk_level": approval.risk_level, + "payload_preview": if policy.show_payload_preview { + approval.request_payload.clone() + } else { + Value::Null + }, + }) +} diff --git a/crates/crank-community-mcp/src/catalog.rs b/crates/crank-community-mcp/src/catalog.rs index 13a3faa..534bcb8 100644 --- a/crates/crank-community-mcp/src/catalog.rs +++ b/crates/crank-community-mcp/src/catalog.rs @@ -1,24 +1,27 @@ use std::{ collections::HashMap, - sync::Arc, + sync::{Arc, Weak}, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use crank_core::{CacheScope, CoordinationStateStore, CoordinationStateValue}; use crank_registry::{PostgresRegistry, PublishedAgentCatalog, PublishedAgentTool, RegistryError}; +use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome}; use serde::{Deserialize, Serialize}; use tokio::sync::{Mutex, RwLock}; -use tracing::{info, warn}; +use tracing::{Instrument, info, warn}; use crate::manifest::analyze_published_tool_catalog; +const MAX_LOCAL_CATALOGS: usize = 1_024; + #[derive(Clone)] pub struct PublishedToolCatalog { registry: PostgresRegistry, refresh_interval: Duration, coordination_store: Arc, cached: Arc>>, - refresh_locks: Arc>>>>, + refresh_locks: Arc>>>>, } #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -30,6 +33,14 @@ struct CatalogKey { struct CachedCatalog { loaded_at: Option, catalog: PublishedAgentCatalog, + metrics: CatalogMetrics, +} + +#[derive(Clone, Copy, Debug, Default)] +struct CatalogMetrics { + tool_count: usize, + estimated_context_tokens: usize, + warning_count: usize, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -66,15 +77,29 @@ impl PublishedToolCatalog { workspace_slug: &str, agent_slug: &str, ) -> Result { - self.refresh_if_stale(workspace_slug, agent_slug).await?; - let guard = self.cached.read().await; - guard - .get(&CatalogKey::new(workspace_slug, agent_slug)) - .map(|entry| entry.catalog.clone()) - .ok_or_else(|| RegistryError::PublishedAgentNotFound { - workspace_slug: workspace_slug.to_owned(), - agent_slug: agent_slug.to_owned(), - }) + let span = Stage::McpCatalogLoad.span(); + let result = async { + self.refresh_if_stale(workspace_slug, agent_slug).await?; + let guard = self.cached.read().await; + guard + .get(&CatalogKey::new(workspace_slug, agent_slug)) + .map(|entry| entry.catalog.clone()) + .ok_or_else(|| RegistryError::PublishedAgentNotFound { + workspace_slug: workspace_slug.to_owned(), + agent_slug: agent_slug.to_owned(), + }) + } + .instrument(span.clone()) + .await; + match &result { + Ok(_) => StageOutcome::Success.record(&span), + Err(_) => { + StageOutcome::Error.record(&span); + ErrorCategory::Catalog.record(&span); + } + } + drop(span); + result } async fn refresh_if_stale( @@ -98,11 +123,14 @@ impl PublishedToolCatalog { let refresh_lock = { let mut locks = self.refresh_locks.lock().await; - Arc::clone( - locks - .entry(key.clone()) - .or_insert_with(|| Arc::new(Mutex::new(()))), - ) + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) { + lock + } else { + let lock = Arc::new(Mutex::new(())); + locks.insert(key.clone(), Arc::downgrade(&lock)); + lock + } }; let _refresh_guard = refresh_lock.lock().await; let still_stale = { @@ -117,50 +145,59 @@ impl PublishedToolCatalog { } if let Some((catalog, age)) = self.load_shared_snapshot(workspace_slug, agent_slug).await { - log_catalog_analysis(workspace_slug, agent_slug, "shared_cache", &catalog.tools); - let mut guard = self.cached.write().await; - guard.insert( + let metrics = + log_catalog_analysis(workspace_slug, agent_slug, "shared_cache", &catalog.tools); + self.store_local_catalog( key, CachedCatalog { loaded_at: Instant::now().checked_sub(age), catalog, + metrics, }, - ); + ) + .await; return Ok(()); } - let catalog = match self + let db_span = Stage::DbQuery + .db_span(DbOperation::CatalogLoad) + .expect("database stage"); + let catalog_result = self .registry .get_published_agent_catalog_by_slug(workspace_slug, agent_slug) - .await - { + .instrument(db_span.clone()) + .await; + let catalog = match catalog_result { Ok(catalog) => catalog, - Err(error) => return Err(error), + Err(error) => { + StageOutcome::Error.record(&db_span); + ErrorCategory::Database.record(&db_span); + drop(db_span); + return Err(error); + } }; - log_catalog_analysis(workspace_slug, agent_slug, "postgres", &catalog.tools); + StageOutcome::Success.record(&db_span); + drop(db_span); + let metrics = log_catalog_analysis(workspace_slug, agent_slug, "postgres", &catalog.tools); self.store_shared_snapshot(workspace_slug, agent_slug, &catalog) .await; - let mut guard = self.cached.write().await; - let previous_count = guard - .get(&key) - .map(|entry| entry.catalog.tools.len()) - .unwrap_or_default(); - - guard.insert( - key, - CachedCatalog { - loaded_at: Some(Instant::now()), - catalog, - }, - ); + let published_tool_count = catalog.tools.len(); + let previous_count = self + .store_local_catalog( + key, + CachedCatalog { + loaded_at: Some(Instant::now()), + catalog, + metrics, + }, + ) + .await; info!( + name: "mcp.catalog.refreshed", workspace_slug, agent_slug, - published_tool_count = guard - .get(&CatalogKey::new(workspace_slug, agent_slug)) - .map(|entry| entry.catalog.tools.len()) - .unwrap_or_default(), + published_tool_count, previous_published_tool_count = previous_count, "published agent catalog refreshed" ); @@ -168,6 +205,27 @@ impl PublishedToolCatalog { Ok(()) } + async fn store_local_catalog(&self, key: CatalogKey, entry: CachedCatalog) -> usize { + let mut guard = self.cached.write().await; + let previous_count = guard + .get(&key) + .map(|current| current.catalog.tools.len()) + .unwrap_or_default(); + + if guard.len() >= MAX_LOCAL_CATALOGS && !guard.contains_key(&key) { + let oldest = guard + .iter() + .min_by_key(|(_, current)| current.loaded_at) + .map(|(candidate, _)| candidate.clone()); + if let Some(oldest) = oldest { + guard.remove(&oldest); + } + } + guard.insert(key, entry); + record_catalog_metrics(guard.values().map(|entry| entry.metrics)); + previous_count + } + async fn load_shared_snapshot( &self, workspace_slug: &str, @@ -226,12 +284,19 @@ fn log_catalog_analysis( agent_slug: &str, source: &str, tools: &[PublishedAgentTool], -) { +) -> CatalogMetrics { let analysis = match analyze_published_tool_catalog(tools) { Ok(analysis) => analysis, - Err(error) => { - warn!(workspace_slug, agent_slug, source, %error, "published catalog analysis failed"); - return; + Err(_) => { + warn!( + name: "mcp.catalog.analysis_failed", + workspace_slug, + agent_slug, + source, + error_category = "catalog_validation", + "published catalog analysis failed" + ); + return CatalogMetrics::default(); } }; let warning_count = analysis @@ -242,6 +307,7 @@ fn log_catalog_analysis( .count(); info!( + name: "mcp.catalog.analyzed", workspace_slug, agent_slug, source, @@ -255,6 +321,30 @@ fn log_catalog_analysis( catalog_quality_warning_count = warning_count, "published agent catalog analyzed" ); + + CatalogMetrics { + tool_count: analysis.budget.tool_count, + estimated_context_tokens: analysis.budget.estimated_context_tokens, + warning_count, + } +} + +fn record_catalog_metrics(metrics: impl Iterator) { + let aggregate = metrics.fold(CatalogMetrics::default(), |mut aggregate, current| { + aggregate.tool_count = aggregate.tool_count.saturating_add(current.tool_count); + aggregate.estimated_context_tokens = aggregate + .estimated_context_tokens + .saturating_add(current.estimated_context_tokens); + aggregate.warning_count = aggregate + .warning_count + .saturating_add(current.warning_count); + aggregate + }); + + metrics::gauge!("crank_catalog_tools").set(aggregate.tool_count as f64); + metrics::gauge!("crank_catalog_estimated_context_tokens") + .set(aggregate.estimated_context_tokens as f64); + metrics::gauge!("crank_catalog_warnings").set(aggregate.warning_count as f64); } fn now_unix_ms() -> u64 { diff --git a/crates/crank-community-mcp/src/lib.rs b/crates/crank-community-mcp/src/lib.rs index 6c82045..986e296 100644 --- a/crates/crank-community-mcp/src/lib.rs +++ b/crates/crank-community-mcp/src/lib.rs @@ -1,14 +1,18 @@ mod access; mod app; mod approval_execution; +mod approval_response; pub mod auth; pub mod catalog; pub mod jsonrpc; pub mod manifest; mod rate_limit; +mod request_context; pub mod session; pub mod tool_error; mod tool_search; mod transport; -pub use app::{build_app, build_app_with_background_workers}; +pub use app::{ + build_app, build_app_with_background_workers, build_app_with_background_workers_and_limits, +}; diff --git a/crates/crank-community-mcp/src/rate_limit.rs b/crates/crank-community-mcp/src/rate_limit.rs index 63f9978..446a77f 100644 --- a/crates/crank-community-mcp/src/rate_limit.rs +++ b/crates/crank-community-mcp/src/rate_limit.rs @@ -4,7 +4,7 @@ use axum::{ http::{HeaderMap, HeaderValue, StatusCode, header::RETRY_AFTER}, response::{IntoResponse, Response}, }; -use crank_runtime::RateLimitRejection; +use crank_runtime::RateLimitCheckError; use serde_json::{Value, json}; use crate::{ @@ -14,19 +14,11 @@ use crate::{ transport::{ResponseMode, session_id_from_headers, transport_response}, }; -pub(super) async fn enforce_post_rate_limit( - state: &Arc, - path: &AgentRoutePath, - headers: &HeaderMap, -) -> Result<(), RateLimitRejection> { - enforce_transport_rate_limit(state, path, headers).await -} - pub(super) async fn enforce_transport_rate_limit( state: &Arc, path: &AgentRoutePath, headers: &HeaderMap, -) -> Result<(), RateLimitRejection> { +) -> Result<(), RateLimitCheckError> { let key = rate_limit_key(path, headers); state.api_rate_limiter.check(&key).await } @@ -35,8 +27,25 @@ pub(super) fn rate_limited_jsonrpc_response( message: &Value, response_mode: ResponseMode, protocol_version: &str, - rejection: RateLimitRejection, + error: RateLimitCheckError, ) -> Response { + let RateLimitCheckError::Rejected(rejection) = error else { + return transport_response( + StatusCode::SERVICE_UNAVAILABLE, + json!({ + "jsonrpc": "2.0", + "id": request_id(message), + "error": { + "code": -32603, + "message": "rate limit service unavailable", + "data": { "code": "rate_limit_unavailable" } + } + }), + response_mode, + None, + Some(protocol_version), + ); + }; let payload = json!({ "jsonrpc": "2.0", "id": request_id(message), @@ -61,10 +70,15 @@ pub(super) fn rate_limited_jsonrpc_response( response } -pub(super) fn rate_limited_status_response(rejection: RateLimitRejection) -> Response { - let mut response = StatusCode::TOO_MANY_REQUESTS.into_response(); - attach_retry_after_header(&mut response, rejection.retry_after_ms); - response +pub(super) fn rate_limited_status_response(error: RateLimitCheckError) -> Response { + match error { + RateLimitCheckError::Rejected(rejection) => { + let mut response = StatusCode::TOO_MANY_REQUESTS.into_response(); + attach_retry_after_header(&mut response, rejection.retry_after_ms); + response + } + RateLimitCheckError::StoreUnavailable => StatusCode::SERVICE_UNAVAILABLE.into_response(), + } } fn rate_limit_key(path: &AgentRoutePath, headers: &HeaderMap) -> String { diff --git a/crates/crank-community-mcp/src/request_context.rs b/crates/crank-community-mcp/src/request_context.rs new file mode 100644 index 0000000..5895dc0 --- /dev/null +++ b/crates/crank-community-mcp/src/request_context.rs @@ -0,0 +1,39 @@ +use axum::{extract::Request, http::HeaderValue, middleware::Next, response::Response}; +use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation}; +use tracing::{Instrument, info_span}; + +use crate::transport::HEADER_X_REQUEST_ID; + +#[derive(Clone, Debug)] +pub(super) struct RequestContext { + pub(super) request_id: String, +} + +pub(super) async fn apply_request_context(mut request: Request, next: Next) -> Response { + let request_id = RequestId::resolve( + request + .headers() + .get(&HEADER_X_REQUEST_ID) + .and_then(|value| value.to_str().ok()), + ) + .into_string(); + let context = RequestContext { + request_id: request_id.clone(), + }; + let span = info_span!( + target: "crank::trace", + "mcp.request", + request_id = %request_id, + ); + set_remote_trace_parent(&span, request.headers()); + request.extensions_mut().insert(context); + + with_request_correlation(request_id.clone(), async move { + let mut response = next.run(request).instrument(span).await; + if let Ok(value) = HeaderValue::from_str(&request_id) { + response.headers_mut().insert(HEADER_X_REQUEST_ID, value); + } + response + }) + .await +} diff --git a/crates/crank-community-mcp/src/session.rs b/crates/crank-community-mcp/src/session.rs index 010ea60..4aa5402 100644 --- a/crates/crank-community-mcp/src/session.rs +++ b/crates/crank-community-mcp/src/session.rs @@ -52,6 +52,8 @@ pub trait TransportSessionStore: Send + Sync { ) -> Result; async fn delete(&self, session_id: &str) -> Result; + + async fn cleanup_expired(&self, now: OffsetDateTime) -> Result; } pub type SharedSessionStore = Arc; @@ -62,6 +64,11 @@ pub struct PostgresTransportSessionStore { } impl PostgresTransportSessionStore { + pub async fn from_pool(pool: PgPool) -> Result { + apply_postgres_migrations(&pool).await?; + Ok(Self { pool }) + } + pub async fn connect_with_options_and_pool_config( connect_options: PgConnectOptions, pool_config: PostgresPoolConfig, @@ -84,9 +91,7 @@ impl PostgresTransportSessionStore { details: error.to_string(), })?; - apply_postgres_migrations(&pool).await?; - - Ok(Self { pool }) + Self::from_pool(pool).await } } @@ -164,6 +169,13 @@ impl TransportSessionStore for InMemorySessionStore { let mut guard = self.inner.write().await; Ok(guard.remove(session_id).is_some()) } + + async fn cleanup_expired(&self, now: OffsetDateTime) -> Result { + let mut guard = self.inner.write().await; + let before = guard.len(); + guard.retain(|_, session| !is_expired(session, now)); + Ok(u64::try_from(before.saturating_sub(guard.len())).unwrap_or(u64::MAX)) + } } #[async_trait] @@ -292,9 +304,68 @@ impl TransportSessionStore for PostgresTransportSessionStore { Ok(result.rows_affected() > 0) } + + async fn cleanup_expired(&self, now: OffsetDateTime) -> Result { + let result = query( + "delete from mcp_transport_sessions + where expires_at is not null and expires_at <= $1::timestamptz", + ) + .bind(now) + .execute(&self.pool) + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + + Ok(result.rows_affected()) + } } async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreError> { + let mut transaction = pool.begin().await.map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + query("select pg_advisory_xact_lock($1)") + .bind(0x4352_414E_4B4D_4350_i64) + .execute(&mut *transaction) + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + query( + "create table if not exists __crank_mcp_migrations ( + version integer primary key, + checksum text not null, + applied_at timestamptz not null default now() + )", + ) + .execute(&mut *transaction) + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + let applied = query("select checksum from __crank_mcp_migrations where version = 1") + .fetch_optional(&mut *transaction) + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + if let Some(row) = applied { + let checksum = row.get::("checksum"); + if checksum != "mcp-transport-sessions-v1" { + return Err(SessionStoreError { + details: format!("modified MCP migration version 1: {checksum}"), + }); + } + transaction + .commit() + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + return Ok(()); + } + query( "create table if not exists mcp_transport_sessions ( id text primary key, @@ -308,14 +379,14 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro expires_at timestamptz null )", ) - .execute(pool) + .execute(&mut *transaction) .await .map_err(|error| SessionStoreError { details: error.to_string(), })?; query("alter table mcp_transport_sessions add column if not exists supports_elicitation boolean not null default false") - .execute(pool) + .execute(&mut *transaction) .await .map_err(|error| SessionStoreError { details: error.to_string(), @@ -324,7 +395,7 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro query( "alter table mcp_transport_sessions add column if not exists expires_at timestamptz null", ) - .execute(pool) + .execute(&mut *transaction) .await .map_err(|error| SessionStoreError { details: error.to_string(), @@ -334,12 +405,37 @@ async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreErro "create index if not exists mcp_transport_sessions_workspace_agent_idx on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc)", ) - .execute(pool) + .execute(&mut *transaction) .await .map_err(|error| SessionStoreError { details: error.to_string(), })?; + query( + "create index if not exists mcp_transport_sessions_expires_at_idx + on mcp_transport_sessions(expires_at) + where expires_at is not null", + ) + .execute(&mut *transaction) + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + + query("insert into __crank_mcp_migrations (version, checksum) values (1, $1)") + .bind("mcp-transport-sessions-v1") + .execute(&mut *transaction) + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + transaction + .commit() + .await + .map_err(|error| SessionStoreError { + details: error.to_string(), + })?; + Ok(()) } diff --git a/crates/crank-community-mcp/src/tool_error.rs b/crates/crank-community-mcp/src/tool_error.rs index 7db2d24..a1d224f 100644 --- a/crates/crank-community-mcp/src/tool_error.rs +++ b/crates/crank-community-mcp/src/tool_error.rs @@ -86,6 +86,10 @@ pub fn runtime_error_code(error: &RuntimeError) -> &'static str { RuntimeError::ConfirmationRequired { .. } => "confirmation_required", RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token", RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable", + RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable", + RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress", + RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict", + RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown", RuntimeError::MissingAuthProfile { .. } => "auth_profile_not_found", RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => { "secret_not_found" @@ -146,6 +150,19 @@ fn safe_runtime_error_message(error: &RuntimeError) -> String { RuntimeError::ConfirmationStoreUnavailable { .. } => { "Хранилище подтверждений временно недоступно.".to_owned() } + RuntimeError::IdempotencyStoreUnavailable { .. } => { + "Хранилище идемпотентности временно недоступно.".to_owned() + } + RuntimeError::IdempotencyInProgress { .. } => { + "Операция с этим ключом идемпотентности уже выполняется.".to_owned() + } + RuntimeError::IdempotencyConflict { .. } => { + "Ключ идемпотентности уже использован с другими параметрами.".to_owned() + } + RuntimeError::IdempotencyOutcomeUnknown { .. } => { + "Результат предыдущего выполнения неизвестен; автоматический повтор заблокирован." + .to_owned() + } RuntimeError::MissingAuthProfile { .. } => "Профиль авторизации не найден.".to_owned(), RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => { "Секрет авторизации не найден.".to_owned() @@ -169,6 +186,8 @@ fn is_recoverable_runtime_error(error: &RuntimeError) -> bool { | RuntimeError::ConcurrencyLimitExceeded { .. } | RuntimeError::SecretCrypto { .. } | RuntimeError::ConfirmationRequired { .. } + | RuntimeError::IdempotencyStoreUnavailable { .. } + | RuntimeError::IdempotencyInProgress { .. } ) } @@ -198,6 +217,14 @@ fn suggested_action(error: &RuntimeError) -> Option<&'static str> { Some("Запросите новый токен подтверждения.") } RuntimeError::ConfirmationStoreUnavailable { .. } => Some("Повторите запрос позже."), + RuntimeError::IdempotencyStoreUnavailable { .. } + | RuntimeError::IdempotencyInProgress { .. } => Some("Повторите запрос позже."), + RuntimeError::IdempotencyConflict { .. } => { + Some("Используйте новый ключ идемпотентности для изменённого запроса.") + } + RuntimeError::IdempotencyOutcomeUnknown { .. } => { + Some("Проверьте результат во внешней системе перед ручным повтором.") + } RuntimeError::MissingAuthProfile { .. } | RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } diff --git a/crates/crank-community-mcp/src/tool_search.rs b/crates/crank-community-mcp/src/tool_search.rs index 57f26a7..29e83f0 100644 --- a/crates/crank-community-mcp/src/tool_search.rs +++ b/crates/crank-community-mcp/src/tool_search.rs @@ -3,6 +3,7 @@ use std::{collections::BTreeSet, sync::Arc}; use axum::{http::StatusCode, response::Response}; use crank_core::{ToolAccessMode, search_tool_catalog}; use crank_registry::PublishedAgentCatalog; +use crank_trace::{ErrorCategory, Stage, StageOutcome}; use serde::Deserialize; use serde_json::{Value, json}; @@ -127,13 +128,25 @@ async fn execute_catalog_tool( mut arguments: Value, transport_request_id: &str, ) -> Response { - let Some(resolved) = resolve_generated_tool(&catalog.tools, tool_name) else { - return tool_not_found_response( - message, - response_mode, - &session.protocol_version, - tool_name, - ); + let resolve_span = Stage::McpToolsResolve.span(); + let resolved = resolve_span.in_scope(|| resolve_generated_tool(&catalog.tools, tool_name)); + let resolved = match resolved { + Some(resolved) => { + StageOutcome::Success.record(&resolve_span); + drop(resolve_span); + resolved + } + None => { + StageOutcome::Error.record(&resolve_span); + ErrorCategory::Catalog.record(&resolve_span); + drop(resolve_span); + return tool_not_found_response( + message, + response_mode, + &session.protocol_version, + tool_name, + ); + } }; let confirmation_token = take_confirmation_token(&mut arguments); handle_tool_call( diff --git a/crates/crank-community-mcp/src/transport.rs b/crates/crank-community-mcp/src/transport.rs index 191359a..88d1606 100644 --- a/crates/crank-community-mcp/src/transport.rs +++ b/crates/crank-community-mcp/src/transport.rs @@ -22,7 +22,6 @@ use crate::jsonrpc::{ pub(super) const HEADER_MCP_SESSION_ID: &str = "MCP-Session-Id"; pub(super) const HEADER_MCP_PROTOCOL_VERSION: &str = "MCP-Protocol-Version"; pub(super) const HEADER_X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id"); -const MAX_REQUEST_ID_LEN: usize = 128; #[derive(Clone, Copy)] pub(super) enum ResponseMode { @@ -303,24 +302,6 @@ where response } -pub(super) fn resolve_request_id(headers: &HeaderMap) -> String { - headers - .get(&HEADER_X_REQUEST_ID) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| is_valid_request_id(value)) - .map(ToOwned::to_owned) - .unwrap_or_else(|| uuid::Uuid::now_v7().to_string()) -} - -pub(super) fn is_valid_request_id(value: &str) -> bool { - !value.is_empty() - && value.len() <= MAX_REQUEST_ID_LEN - && value - .bytes() - .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';') -} - pub(super) fn extract_origin(url: &str) -> Option { Some(parse_origin(url, false)?.origin().ascii_serialization()) } diff --git a/crates/crank-community-mcp/tests/integration/session.rs b/crates/crank-community-mcp/tests/integration/session.rs index 1c59d92..1d2e237 100644 --- a/crates/crank-community-mcp/tests/integration/session.rs +++ b/crates/crank-community-mcp/tests/integration/session.rs @@ -97,3 +97,41 @@ async fn postgres_transport_sessions_evict_expired_rows_on_read() { assert_eq!(remaining, 0); } + +#[tokio::test] +async fn postgres_transport_session_cleanup_removes_abandoned_expired_rows() { + let database_url = crank_test_support::postgres_schema_url("test_mcp_cleanup").await; + let store = PostgresTransportSessionStore::connect_with_options_and_pool_config( + database_url.parse::().unwrap(), + PostgresPoolConfig::default(), + ) + .await + .unwrap(); + let now = OffsetDateTime::now_utc(); + + store + .create( + "2025-11-25", + "default", + "sales", + false, + now - time::Duration::hours(2), + Some(now - time::Duration::hours(1)), + ) + .await + .unwrap(); + let active = store + .create( + "2025-11-25", + "default", + "sales", + false, + now, + Some(now + time::Duration::hours(1)), + ) + .await + .unwrap(); + + assert_eq!(store.cleanup_expired(now).await.unwrap(), 1); + assert!(store.get(&active).await.unwrap().is_some()); +} diff --git a/crates/crank-community-mcp/tests/unit/session.rs b/crates/crank-community-mcp/tests/unit/session.rs index 1052cec..1b57e69 100644 --- a/crates/crank-community-mcp/tests/unit/session.rs +++ b/crates/crank-community-mcp/tests/unit/session.rs @@ -72,6 +72,31 @@ async fn drops_expired_in_memory_transport_sessions_on_read() { assert!(store.get(&session_id).await.unwrap().is_none()); } +#[tokio::test] +async fn cleanup_removes_only_expired_sessions() { + let store = InMemorySessionStore::default(); + let now = time::OffsetDateTime::now_utc(); + let expired = store + .create("2025-11-25", "default", "sales", false, now, Some(now)) + .await + .unwrap(); + let active = store + .create( + "2025-11-25", + "default", + "sales", + false, + now, + Some(now + time::Duration::hours(1)), + ) + .await + .unwrap(); + + assert_eq!(store.cleanup_expired(now).await.unwrap(), 1); + assert!(store.get(&expired).await.unwrap().is_none()); + assert!(store.get(&active).await.unwrap().is_some()); +} + #[test] fn formats_transport_session_store_error() { let error = SessionStoreError { diff --git a/crates/crank-core/Cargo.toml b/crates/crank-core/Cargo.toml index 773f4c5..4df892f 100644 --- a/crates/crank-core/Cargo.toml +++ b/crates/crank-core/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-core" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] diff --git a/crates/crank-core/src/cache.rs b/crates/crank-core/src/cache.rs index c3badd8..7fe8f83 100644 --- a/crates/crank-core/src/cache.rs +++ b/crates/crank-core/src/cache.rs @@ -74,6 +74,12 @@ pub struct RateLimitBucketState { pub last_refill_unix_ms: i64, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RateLimitDecision { + Allowed, + Rejected { retry_after_ms: u64 }, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ReplayGuardStatus { @@ -86,6 +92,12 @@ pub struct CoordinationStateValue { pub payload: Value, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CoordinationStateReservation { + Reserved, + Existing(CoordinationStateValue), +} + #[async_trait] pub trait ResponseCacheStore: Send + Sync { async fn get(&self, key: &str) -> Result, CacheStoreError>; @@ -108,6 +120,14 @@ pub trait RateLimitStateStore: Send + Sync { ttl: Duration, ) -> Result<(), CacheStoreError>; async fn delete_bucket(&self, key: &str) -> Result<(), CacheStoreError>; + async fn consume_token( + &self, + key: &str, + burst_tokens_micros: u64, + refill_per_second_micros: u64, + now_unix_ms: i64, + ttl: Duration, + ) -> Result; } #[async_trait] @@ -135,6 +155,26 @@ pub trait CoordinationStateStore: Send + Sync { ttl: Duration, ) -> Result<(), CacheStoreError>; async fn delete_value(&self, scope: CacheScope, key: &str) -> Result<(), CacheStoreError>; + async fn take_value( + &self, + scope: CacheScope, + key: &str, + ) -> Result, CacheStoreError>; + async fn reserve_value( + &self, + scope: CacheScope, + key: &str, + value: CoordinationStateValue, + ttl: Duration, + ) -> Result; + async fn compare_and_set_value( + &self, + scope: CacheScope, + key: &str, + expected: &CoordinationStateValue, + value: CoordinationStateValue, + ttl: Duration, + ) -> Result; } #[derive(Debug, Error, PartialEq, Eq)] diff --git a/crates/crank-core/src/lib.rs b/crates/crank-core/src/lib.rs index 66b1157..d350929 100644 --- a/crates/crank-core/src/lib.rs +++ b/crates/crank-core/src/lib.rs @@ -66,8 +66,9 @@ pub mod domain { pub mod ports { pub use crate::cache::{ - CacheStoreError, CoordinationStateStore, CoordinationStateValue, RateLimitStateStore, - ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore, + CacheStoreError, CoordinationStateReservation, CoordinationStateStore, + CoordinationStateValue, RateLimitDecision, RateLimitStateStore, ReplayGuardStatus, + ReplayGuardStore, ResponseCacheStore, }; pub use crate::ext::access::{ OwnerOnlyPolicyEngine, PolicyAction, PolicyDecision, PolicyEngine, PolicyScope, @@ -108,8 +109,9 @@ pub use auth::{ }; pub use cache::{ CacheBackend, CacheScope, CacheStoreError, CachedHeader, CachedResponse, - CoordinationStateStore, CoordinationStateValue, ParseCacheBackendError, RateLimitBucketState, - RateLimitStateStore, ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore, + CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue, + ParseCacheBackendError, RateLimitBucketState, RateLimitDecision, RateLimitStateStore, + ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore, }; pub use edition::{ EditionCapabilities, EditionLimits, MachineAccessMode, OperationSecurityLevel, ProductEdition, diff --git a/crates/crank-import/Cargo.toml b/crates/crank-import/Cargo.toml index 5b438ad..c19dc00 100644 --- a/crates/crank-import/Cargo.toml +++ b/crates/crank-import/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-import" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] diff --git a/crates/crank-mapping/Cargo.toml b/crates/crank-mapping/Cargo.toml index c81f60d..42bb226 100644 --- a/crates/crank-mapping/Cargo.toml +++ b/crates/crank-mapping/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-mapping" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] diff --git a/crates/crank-observability/Cargo.toml b/crates/crank-observability/Cargo.toml new file mode 100644 index 0000000..db55b0c --- /dev/null +++ b/crates/crank-observability/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "crank-observability" +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish.workspace = true +version.workspace = true + +[dependencies] +axum.workspace = true +metrics.workspace = true +metrics-exporter-prometheus.workspace = true +opentelemetry.workspace = true +opentelemetry-otlp.workspace = true +opentelemetry_sdk.workspace = true +percent-encoding.workspace = true +serde.workspace = true +serde_json.workspace = true +sentry.workspace = true +sha2.workspace = true +subtle.workspace = true +thiserror.workspace = true +time.workspace = true +tokio = { workspace = true, features = ["net"] } +tracing.workspace = true +tracing-opentelemetry.workspace = true +tracing-subscriber.workspace = true +url.workspace = true +uuid.workspace = true + +[dev-dependencies] +opentelemetry-proto.workspace = true +prost.workspace = true +sentry = { workspace = true, features = ["test"] } +tower.workspace = true diff --git a/crates/crank-observability/src/config.rs b/crates/crank-observability/src/config.rs new file mode 100644 index 0000000..bac9c35 --- /dev/null +++ b/crates/crank-observability/src/config.rs @@ -0,0 +1,170 @@ +use std::env; + +use thiserror::Error; + +use crate::RedactionLimits; + +const DEFAULT_ENVIRONMENT: &str = "development"; +const MAX_IDENTITY_LABEL_BYTES: usize = 64; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ServiceIdentity { + service: String, + version: String, + environment: String, +} + +impl ServiceIdentity { + pub fn try_new( + service: impl Into, + version: impl Into, + environment: impl Into, + ) -> Result { + let identity = Self { + service: service.into(), + version: version.into(), + environment: environment.into(), + }; + validate_label("service", &identity.service)?; + validate_label("version", &identity.version)?; + validate_label("environment", &identity.environment)?; + Ok(identity) + } + + pub fn service(&self) -> &str { + &self.service + } + + pub fn version(&self) -> &str { + &self.version + } + + pub fn environment(&self) -> &str { + &self.environment + } +} + +#[derive(Clone, Debug)] +pub struct ObservabilityConfig { + identity: ServiceIdentity, + filter: String, + redaction_limits: RedactionLimits, +} + +impl ObservabilityConfig { + pub fn new( + identity: ServiceIdentity, + filter: impl Into, + redaction_limits: RedactionLimits, + ) -> Self { + Self { + identity, + filter: filter.into(), + redaction_limits, + } + } + + pub fn from_env( + service: &'static str, + version: &'static str, + default_filter: &'static str, + ) -> Result { + let environment = env_value_or_default( + "CRANK_ENVIRONMENT", + env::var("CRANK_ENVIRONMENT"), + DEFAULT_ENVIRONMENT, + )?; + let filter = env_value_or_default( + "CRANK_LOG_LEVEL", + env::var("CRANK_LOG_LEVEL"), + default_filter, + )?; + let identity = ServiceIdentity::try_new(service, version, environment)?; + + Ok(Self::new(identity, filter, RedactionLimits::default())) + } + + pub(crate) fn into_parts(self) -> (ServiceIdentity, String, RedactionLimits) { + (self.identity, self.filter, self.redaction_limits) + } + + pub(crate) fn identity(&self) -> &ServiceIdentity { + &self.identity + } + + pub(crate) fn redaction_limits(&self) -> RedactionLimits { + self.redaction_limits + } +} + +#[derive(Debug, Error)] +pub enum ObservabilityConfigError { + #[error("invalid observability identity field: {field}")] + InvalidIdentity { field: &'static str }, + #[error("observability environment variable is not valid UTF-8: {field}")] + InvalidEnvironmentEncoding { field: &'static str }, +} + +fn env_value_or_default( + field: &'static str, + value: Result, + default: &'static str, +) -> Result { + match value { + Ok(value) => Ok(value), + Err(env::VarError::NotPresent) => Ok(default.to_owned()), + Err(env::VarError::NotUnicode(_)) => { + Err(ObservabilityConfigError::InvalidEnvironmentEncoding { field }) + } + } +} + +fn validate_label(field: &'static str, value: &str) -> Result<(), ObservabilityConfigError> { + let valid = !value.is_empty() + && value.len() <= MAX_IDENTITY_LABEL_BYTES + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+')); + + if valid { + Ok(()) + } else { + Err(ObservabilityConfigError::InvalidIdentity { field }) + } +} + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + + use super::{ObservabilityConfigError, ServiceIdentity, env_value_or_default}; + + #[test] + fn accepts_release_and_environment_labels() { + let identity = ServiceIdentity::try_new("admin-api", "0.3.1+build.7", "production") + .expect("identity must be valid"); + + assert_eq!(identity.service(), "admin-api"); + assert_eq!(identity.version(), "0.3.1+build.7"); + assert_eq!(identity.environment(), "production"); + } + + #[test] + fn rejects_non_utf8_environment_values() { + let error = env_value_or_default( + "CRANK_ENVIRONMENT", + Err(std::env::VarError::NotUnicode(OsString::from( + "invalid-environment", + ))), + "development", + ) + .expect_err("non-UTF-8 values must not be replaced with defaults"); + + assert!(matches!( + error, + ObservabilityConfigError::InvalidEnvironmentEncoding { + field: "CRANK_ENVIRONMENT" + } + )); + } +} diff --git a/crates/crank-observability/src/correlation.rs b/crates/crank-observability/src/correlation.rs new file mode 100644 index 0000000..4d969fe --- /dev/null +++ b/crates/crank-observability/src/correlation.rs @@ -0,0 +1,39 @@ +use std::fmt; + +use uuid::Uuid; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct RequestId(String); + +impl RequestId { + pub const MAX_LEN: usize = 128; + + pub fn resolve(candidate: Option<&str>) -> Self { + candidate + .filter(|value| Self::is_valid(value)) + .map(|value| Self(value.to_owned())) + .unwrap_or_else(|| Self(Uuid::now_v7().to_string())) + } + + pub fn is_valid(value: &str) -> bool { + !value.is_empty() + && value.len() <= Self::MAX_LEN + && value + .bytes() + .all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';') + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +impl fmt::Display for RequestId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} diff --git a/crates/crank-observability/src/error_reporting.rs b/crates/crank-observability/src/error_reporting.rs new file mode 100644 index 0000000..5f6e810 --- /dev/null +++ b/crates/crank-observability/src/error_reporting.rs @@ -0,0 +1,467 @@ +use std::{borrow::Cow, collections::BTreeMap, env, fmt, future::Future, time::Duration}; + +use sentry::{ + ClientInitGuard, ClientOptions, + protocol::{Event, Level}, + types::Dsn, +}; +use thiserror::Error; + +use crate::{ + RedactionLimits, ServiceIdentity, propagation::current_trace_id, redaction::truncate_string, +}; + +const SENTRY_DSN_ENV: &str = "CRANK_SENTRY_DSN"; +const CRITICAL_ERROR_MESSAGE: &str = "critical error"; +const SENTRY_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); + +tokio::task_local! { + static REQUEST_ID: String; +} + +pub struct SentryConfig { + dsn: Option, +} + +impl SentryConfig { + pub fn parse(value: Option<&str>) -> Result { + let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else { + return Ok(Self { dsn: None }); + }; + + let dsn = value + .parse::() + .map_err(|_| SentryConfigError::InvalidDsn)?; + Ok(Self { dsn: Some(dsn) }) + } + + pub fn from_env() -> Result { + match env::var(SENTRY_DSN_ENV) { + Ok(value) => Self::parse(Some(&value)), + Err(env::VarError::NotPresent) => Self::parse(None), + Err(env::VarError::NotUnicode(_)) => Err(SentryConfigError::InvalidEnvironmentEncoding), + } + } + + pub fn enabled(&self) -> bool { + self.dsn.is_some() + } +} + +impl fmt::Debug for SentryConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("SentryConfig") + .field("enabled", &self.enabled()) + .finish() + } +} + +#[derive(Debug, Error)] +pub enum SentryConfigError { + #[error("CRANK_SENTRY_DSN is not a valid Sentry DSN")] + InvalidDsn, + #[error("CRANK_SENTRY_DSN is not valid UTF-8")] + InvalidEnvironmentEncoding, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CriticalErrorCategory { + Panic, + Startup, + Internal, + DataIntegrity, +} + +impl CriticalErrorCategory { + pub const fn as_str(self) -> &'static str { + match self { + Self::Panic => "panic", + Self::Startup => "startup", + Self::Internal => "internal", + Self::DataIntegrity => "data_integrity", + } + } + + fn parse(value: &str) -> Option { + match value { + "panic" => Some(Self::Panic), + "startup" => Some(Self::Startup), + "internal" => Some(Self::Internal), + "data_integrity" => Some(Self::DataIntegrity), + _ => None, + } + } +} + +pub fn capture_critical_error(category: CriticalErrorCategory) { + let mut tags = correlation_tags(); + tags.insert("category".to_owned(), category.as_str().to_owned()); + sentry::capture_event(Event { + level: Level::Error, + message: Some(CRITICAL_ERROR_MESSAGE.to_owned()), + fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]), + tags, + ..Event::default() + }); +} + +pub async fn with_request_correlation(request_id: String, future: F) -> F::Output +where + F: Future, +{ + REQUEST_ID.scope(request_id, future).await +} + +pub(crate) fn init_sentry( + identity: &ServiceIdentity, + limits: RedactionLimits, + config: SentryConfig, +) -> Option { + let dsn = config.dsn?; + let identity = identity.clone(); + let options = client_options(identity, limits); + Some(sentry::init((dsn, options))) +} + +fn client_options(identity: ServiceIdentity, limits: RedactionLimits) -> ClientOptions { + let release = identity.version().to_owned(); + let environment = identity.environment().to_owned(); + let service = identity.service().to_owned(); + let sanitizer_identity = identity.clone(); + + let mut options = ClientOptions::default(); + options.release = Some(Cow::Owned(release)); + options.environment = Some(Cow::Owned(environment)); + options.server_name = Some(Cow::Owned(service)); + options.traces_sampling_strategy = sentry::TracesSamplingStrategy::Disabled; + options.max_breadcrumbs = 0; + options.attach_stacktrace = false; + options.send_default_pii = false; + options.before_send = Some(std::sync::Arc::new(move |event| { + Some(sanitize_event(event, &sanitizer_identity, limits)) + })); + options.shutdown_timeout = SENTRY_SHUTDOWN_TIMEOUT; + options.auto_session_tracking = false; + options.enable_logs = false; + options.enable_metrics = false; + options +} + +fn sanitize_event( + event: Event<'static>, + identity: &ServiceIdentity, + limits: RedactionLimits, +) -> Event<'static> { + let category = event + .tags + .get("category") + .and_then(|value| CriticalErrorCategory::parse(value)) + .unwrap_or_else(|| { + if event.exception.is_empty() { + CriticalErrorCategory::Internal + } else { + CriticalErrorCategory::Panic + } + }); + let mut tags = correlation_tags() + .into_iter() + .map(|(key, value)| (key, truncate_string(&value, limits.max_string_bytes))) + .collect::>(); + for key in ["request_id", "trace_id"] { + if let Some(value) = event.tags.get(key) { + tags.entry(key.to_owned()) + .or_insert_with(|| truncate_string(value, limits.max_string_bytes)); + } + } + tags.insert("service".to_owned(), identity.service().to_owned()); + tags.insert("category".to_owned(), category.as_str().to_owned()); + + enforce_event_budget( + Event { + event_id: event.event_id, + level: Level::Error, + fingerprint: Cow::Owned(vec![Cow::Borrowed(category.as_str())]), + message: Some(CRITICAL_ERROR_MESSAGE.to_owned()), + timestamp: event.timestamp, + server_name: Some(Cow::Owned(identity.service().to_owned())), + release: Some(Cow::Owned(identity.version().to_owned())), + environment: Some(Cow::Owned(identity.environment().to_owned())), + tags, + ..Event::default() + }, + limits.max_event_bytes, + ) +} + +fn correlation_tags() -> BTreeMap { + let mut tags = BTreeMap::new(); + if let Ok(request_id) = REQUEST_ID.try_with(Clone::clone) { + tags.insert("request_id".to_owned(), request_id); + } + if let Some(trace_id) = current_trace_id() { + tags.insert("trace_id".to_owned(), trace_id); + } + tags +} + +fn enforce_event_budget(mut event: Event<'static>, max_event_bytes: usize) -> Event<'static> { + if serialized_event_len(&event) <= max_event_bytes { + return event; + } + + event.tags.remove("request_id"); + event.tags.remove("trace_id"); + event +} + +fn serialized_event_len(event: &Event<'_>) -> usize { + serde_json::to_vec(event).map_or(usize::MAX, |serialized| serialized.len()) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + use opentelemetry::trace::TracerProvider as _; + use opentelemetry_sdk::trace::SdkTracerProvider; + use sentry::{ + Envelope, Hub, + protocol::{Breadcrumb, Context, Exception, Request, User, Value, Values}, + }; + + use super::{ + CriticalErrorCategory, capture_critical_error, client_options, sanitize_event, + with_request_correlation, + }; + use crate::{ + ObservabilityConfig, RedactionLimits, ServiceIdentity, + logging::build_subscriber_with_tracer, + }; + + fn identity() -> ServiceIdentity { + ServiceIdentity::try_new("admin-api", "1.2.3", "test").expect("valid identity") + } + + #[test] + fn client_disables_non_error_telemetry() { + let options = client_options(identity(), RedactionLimits::default()); + + assert_eq!(options.max_breadcrumbs, 0); + assert!(!options.attach_stacktrace); + assert!(!options.send_default_pii); + assert!(!options.auto_session_tracking); + assert!(!options.enable_logs); + assert!(!options.enable_metrics); + assert!(options.before_send.is_some()); + } + + #[test] + fn sanitizer_uses_a_strict_allowlist() { + let mut tags = BTreeMap::new(); + tags.insert("category".to_owned(), "data_integrity".to_owned()); + tags.insert("secret".to_owned(), "must-not-leak".to_owned()); + let mut contexts = BTreeMap::new(); + contexts.insert( + "secret".to_owned(), + Context::Other(BTreeMap::from([( + "token".to_owned(), + Value::String("must-not-leak".to_owned()), + )])), + ); + let event = sentry::protocol::Event { + message: Some("password=must-not-leak".to_owned()), + request: Some(Request::default()), + user: Some(User::default()), + breadcrumbs: Values { + values: vec![Breadcrumb::default()], + }, + exception: Values { + values: vec![Exception { + value: Some("must-not-leak".to_owned()), + ..Exception::default() + }], + }, + contexts, + extra: BTreeMap::from([( + "payload".to_owned(), + Value::String("must-not-leak".to_owned()), + )]), + tags, + ..sentry::protocol::Event::default() + }; + + let cleaned = sanitize_event(event, &identity(), RedactionLimits::default()); + let serialized = serde_json::to_string(&cleaned).expect("serialize event"); + + assert_eq!(cleaned.message.as_deref(), Some("critical error")); + assert_eq!( + cleaned.tags.get("category").map(String::as_str), + Some(CriticalErrorCategory::DataIntegrity.as_str()) + ); + assert!(cleaned.request.is_none()); + assert!(cleaned.user.is_none()); + assert!(cleaned.breadcrumbs.is_empty()); + assert!(cleaned.exception.is_empty()); + assert!(cleaned.contexts.is_empty()); + assert!(cleaned.extra.is_empty()); + assert!(!serialized.contains("must-not-leak")); + assert!(!serialized.contains("password")); + } + + #[test] + fn sanitizer_honours_total_event_budget() { + let limits = RedactionLimits { + max_string_bytes: 8 * 1024, + max_event_bytes: 512, + ..RedactionLimits::default() + }; + let event = sentry::protocol::Event { + tags: BTreeMap::from([ + ("category".to_owned(), "internal".to_owned()), + ("request_id".to_owned(), "r".repeat(8 * 1024)), + ("trace_id".to_owned(), "t".repeat(8 * 1024)), + ]), + ..sentry::protocol::Event::default() + }; + + let cleaned = sanitize_event(event, &identity(), limits); + let serialized = serde_json::to_vec(&cleaned).expect("serialize event"); + + assert!(serialized.len() <= limits.max_event_bytes); + assert_eq!( + cleaned.tags.get("category").map(String::as_str), + Some("internal") + ); + assert_eq!( + cleaned.tags.get("service").map(String::as_str), + Some("admin-api") + ); + assert!(!cleaned.tags.contains_key("request_id")); + assert!(!cleaned.tags.contains_key("trace_id")); + } + + #[test] + fn expected_application_errors_do_not_create_critical_events() { + let options = + sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); + let events = sentry::test::with_captured_events_options( + || tracing::error!("ordinary product error"), + options, + ); + + assert!(events.is_empty()); + } + + #[test] + fn explicit_critical_error_is_correlated_and_sanitized() { + let options = + sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime"); + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("critical-error-test"); + let subscriber = build_subscriber_with_tracer( + ObservabilityConfig::new(identity(), "info", RedactionLimits::default()), + std::io::sink, + Some(tracer), + ) + .expect("subscriber"); + let dispatch = tracing::Dispatch::new(subscriber); + let events = sentry::test::with_captured_events_options( + || { + tracing::dispatcher::with_default(&dispatch, || { + runtime.block_on(with_request_correlation("request-123".to_owned(), async { + let span = tracing::info_span!(target: "crank::trace", "http.request"); + let _span_guard = span.enter(); + capture_critical_error(CriticalErrorCategory::DataIntegrity); + })); + }); + }, + options, + ); + + assert_eq!(events.len(), 1); + let event = &events[0]; + assert_eq!(event.message.as_deref(), Some("critical error")); + assert_eq!( + event.tags.get("category").map(String::as_str), + Some("data_integrity") + ); + assert_eq!( + event.tags.get("request_id").map(String::as_str), + Some("request-123") + ); + assert_eq!(event.tags.get("trace_id").map(String::len), Some(32)); + assert_eq!(event.release.as_deref(), Some("1.2.3")); + assert_eq!(event.environment.as_deref(), Some("test")); + assert_eq!(event.server_name.as_deref(), Some("admin-api")); + } + + #[test] + fn panic_creates_exactly_one_sanitized_critical_event() { + let options = + sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); + let events = sentry::test::with_captured_events_options( + || { + let result = std::panic::catch_unwind(|| { + panic!("password=must-not-leak"); + }); + assert!(result.is_err()); + }, + options, + ); + + assert_eq!(events.len(), 1); + let event = &events[0]; + assert_eq!( + event.tags.get("category").map(String::as_str), + Some("panic") + ); + let serialized = serde_json::to_string(event).expect("serialize event"); + assert!(!serialized.contains("must-not-leak")); + assert!(!serialized.contains("password")); + } + + #[test] + fn receiver_failure_does_not_change_product_result_or_recurse() { + struct DroppingTransport { + attempts: AtomicUsize, + } + + impl sentry::Transport for DroppingTransport { + fn send_envelope(&self, _envelope: Envelope) { + self.attempts.fetch_add(1, Ordering::Relaxed); + } + } + + let transport = Arc::new(DroppingTransport { + attempts: AtomicUsize::new(0), + }); + let mut options = + sentry::apply_defaults(client_options(identity(), RedactionLimits::default())); + options.dsn = Some( + "https://public@example.invalid/1" + .parse() + .expect("valid test DSN"), + ); + options.transport = Some(Arc::new(transport.clone())); + let client = Arc::new(sentry::Client::from(options)); + let hub = Arc::new(Hub::new(Some(client), Arc::new(Default::default()))); + + let product_result = Hub::run(hub, || { + capture_critical_error(CriticalErrorCategory::Internal); + 42 + }); + + assert_eq!(product_result, 42); + assert_eq!(transport.attempts.load(Ordering::Relaxed), 1); + } +} diff --git a/crates/crank-observability/src/incidents.rs b/crates/crank-observability/src/incidents.rs new file mode 100644 index 0000000..070782e --- /dev/null +++ b/crates/crank-observability/src/incidents.rs @@ -0,0 +1,36 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperationalIncident { + InvocationHistoryLost, +} + +static INVOCATION_HISTORY_LOST_TOTAL: AtomicU64 = AtomicU64::new(0); + +pub fn record_operational_incident(incident: OperationalIncident) { + let counter = counter(incident); + let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + value.checked_add(1) + }); + match incident { + OperationalIncident::InvocationHistoryLost => { + metrics::counter!("crank_invocation_history_lost_total").increment(1); + metrics::counter!( + "crank_telemetry_export_failures_total", + "signal_type" => "invocation_history", + "exporter" => "postgres" + ) + .increment(1); + } + } +} + +pub fn operational_incident_total(incident: OperationalIncident) -> u64 { + counter(incident).load(Ordering::Relaxed) +} + +fn counter(incident: OperationalIncident) -> &'static AtomicU64 { + match incident { + OperationalIncident::InvocationHistoryLost => &INVOCATION_HISTORY_LOST_TOTAL, + } +} diff --git a/crates/crank-observability/src/instrumentation.rs b/crates/crank-observability/src/instrumentation.rs new file mode 100644 index 0000000..7901c20 --- /dev/null +++ b/crates/crank-observability/src/instrumentation.rs @@ -0,0 +1,133 @@ +use std::time::Instant; + +use axum::{ + extract::{MatchedPath, Request}, + middleware::Next, + response::Response, +}; +use metrics::{Gauge, Unit}; + +use crate::{MetricKind, MetricUnit, metric_schema}; + +pub async fn record_http_request(request: Request, next: Next) -> Response { + let route = request + .extensions() + .get::() + .map_or("unmatched", MatchedPath::as_str) + .to_owned(); + let method = normalized_http_method(request.method().as_str()); + let started_at = Instant::now(); + let _inflight = GaugeGuard::increment("crank_http_inflight"); + + let response = next.run(request).await; + let status_class = status_class(response.status().as_u16()); + + metrics::counter!( + "crank_http_requests_total", + "route" => route.clone(), + "method" => method, + "status_class" => status_class + ) + .increment(1); + metrics::histogram!( + "crank_http_request_duration_seconds", + "route" => route, + "method" => method + ) + .record(started_at.elapsed().as_secs_f64()); + + response +} + +pub fn record_db_pool_connections(total: u32, idle: usize) { + let idle = idle.min(total as usize) as f64; + metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(idle); + metrics::gauge!("crank_db_pool_connections", "state" => "used").set(f64::from(total) - idle); +} + +pub(crate) fn register_metric_schema() { + for definition in metric_schema() { + let unit = match definition.unit { + MetricUnit::Count => Unit::Count, + MetricUnit::Seconds => Unit::Seconds, + }; + match definition.kind { + MetricKind::Counter => { + metrics::describe_counter!(definition.name, unit, definition.description); + } + MetricKind::Gauge => { + metrics::describe_gauge!(definition.name, unit, definition.description); + } + MetricKind::Histogram => { + metrics::describe_histogram!(definition.name, unit, definition.description); + } + } + } + + metrics::gauge!("crank_http_inflight").set(0.0); + metrics::gauge!("crank_mcp_active_sessions").set(0.0); + metrics::gauge!("crank_runtime_inflight").set(0.0); + metrics::gauge!("crank_db_pool_connections", "state" => "idle").set(0.0); + metrics::gauge!("crank_db_pool_connections", "state" => "used").set(0.0); + metrics::gauge!("crank_catalog_tools").set(0.0); + metrics::gauge!("crank_catalog_estimated_context_tokens").set(0.0); + metrics::gauge!("crank_catalog_warnings").set(0.0); +} + +fn normalized_http_method(method: &str) -> &'static str { + match method { + "GET" => "GET", + "POST" => "POST", + "PUT" => "PUT", + "PATCH" => "PATCH", + "DELETE" => "DELETE", + "OPTIONS" => "OPTIONS", + "HEAD" => "HEAD", + "CONNECT" => "CONNECT", + "TRACE" => "TRACE", + _ => "OTHER", + } +} + +fn status_class(status: u16) -> &'static str { + match status { + 100..=199 => "1xx", + 200..=299 => "2xx", + 300..=399 => "3xx", + 400..=499 => "4xx", + 500..=599 => "5xx", + _ => "other", + } +} + +struct GaugeGuard { + gauge: Gauge, +} + +impl GaugeGuard { + fn increment(name: &'static str) -> Self { + let gauge = metrics::gauge!(name); + gauge.increment(1.0); + Self { gauge } + } +} + +impl Drop for GaugeGuard { + fn drop(&mut self) { + self.gauge.decrement(1.0); + } +} + +#[cfg(test)] +mod tests { + use super::{normalized_http_method, status_class}; + + #[test] + fn normalizes_unbounded_http_values() { + assert_eq!(normalized_http_method("GET"), "GET"); + assert_eq!(normalized_http_method("CUSTOM-user-controlled"), "OTHER"); + assert_eq!(status_class(204), "2xx"); + assert_eq!(status_class(429), "4xx"); + assert_eq!(status_class(999), "other"); + } +} diff --git a/crates/crank-observability/src/lib.rs b/crates/crank-observability/src/lib.rs new file mode 100644 index 0000000..d5f987f --- /dev/null +++ b/crates/crank-observability/src/lib.rs @@ -0,0 +1,37 @@ +mod config; +mod correlation; +mod error_reporting; +mod incidents; +mod instrumentation; +mod lifecycle; +mod logging; +mod metrics_schema; +mod otlp; +mod prometheus; +mod propagation; +mod redaction; +mod schema; + +pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity}; +pub use correlation::RequestId; +pub use error_reporting::{ + CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error, + with_request_correlation, +}; +pub use incidents::{OperationalIncident, operational_incident_total, record_operational_incident}; +pub use instrumentation::{record_db_pool_connections, record_http_request}; +pub use lifecycle::{ObservabilityInitError, ObservabilityLifecycle, init}; +pub use logging::build_subscriber; +pub use metrics_schema::{ + DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricUnit, metric_schema, +}; +pub use otlp::{ + OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError, build_tracer_provider, +}; +pub use prometheus::{ + MetricsConfig, MetricsConfigError, MetricsServeError, MetricsSurface, MetricsSurfaceError, +}; +pub use propagation::{inject_current_trace_context, set_remote_trace_parent}; +pub use redaction::{ + REDACTED_MARKER, RedactionLimits, RedactionLimitsError, SafeJsonError, redact_value, safe_json, +}; diff --git a/crates/crank-observability/src/lifecycle.rs b/crates/crank-observability/src/lifecycle.rs new file mode 100644 index 0000000..fca56c6 --- /dev/null +++ b/crates/crank-observability/src/lifecycle.rs @@ -0,0 +1,90 @@ +use std::{fmt, io}; + +use thiserror::Error; +use tracing_subscriber::util::SubscriberInitExt; + +use crate::{ + MetricsConfig, MetricsSurface, MetricsSurfaceError, ObservabilityConfig, + ObservabilityConfigError, OtlpTraceConfig, OtlpTraceConfigError, OtlpTraceError, + RedactionLimitsError, SentryConfig, SentryConfigError, error_reporting::init_sentry, + instrumentation::register_metric_schema, logging::build_subscriber_with_tracer, + otlp::build_tracer_provider, prometheus::install_prometheus_recorder, + propagation::install_trace_context_propagator, +}; + +#[must_use = "observability resources must be retained until process shutdown"] +pub struct ObservabilityLifecycle { + metrics_handle: metrics_exporter_prometheus::PrometheusHandle, + tracer_provider: Option, + sentry_guard: Option, +} + +impl ObservabilityLifecycle { + pub fn init(config: ObservabilityConfig) -> Result { + let identity = config.identity().clone(); + let redaction_limits = config.redaction_limits(); + let sentry_config = SentryConfig::from_env()?; + let trace_config = OtlpTraceConfig::from_env()?; + let tracing = build_tracer_provider(&identity, &trace_config)?; + let tracer = tracing.as_ref().map(|(_, tracer)| tracer.clone()); + install_trace_context_propagator(); + build_subscriber_with_tracer(config, io::stdout, tracer)? + .try_init() + .map_err(|_| ObservabilityInitError::SubscriberAlreadyInitialized)?; + let metrics_handle = install_prometheus_recorder(&identity)?; + register_metric_schema(); + let sentry_guard = init_sentry(&identity, redaction_limits, sentry_config); + + Ok(Self { + metrics_handle, + tracer_provider: tracing.map(|(provider, _)| provider), + sentry_guard, + }) + } + + pub fn metrics_surface(&self, config: MetricsConfig) -> MetricsSurface { + MetricsSurface::new(config, self.metrics_handle.clone()) + } + + pub fn traces_enabled(&self) -> bool { + self.tracer_provider.is_some() + } + + pub fn critical_errors_enabled(&self) -> bool { + self.sentry_guard.is_some() + } +} + +impl fmt::Debug for ObservabilityLifecycle { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ObservabilityLifecycle") + .field("traces_enabled", &self.traces_enabled()) + .field("critical_errors_enabled", &self.critical_errors_enabled()) + .finish_non_exhaustive() + } +} + +pub fn init(config: ObservabilityConfig) -> Result { + ObservabilityLifecycle::init(config) +} + +#[derive(Debug, Error)] +pub enum ObservabilityInitError { + #[error(transparent)] + InvalidConfig(#[from] ObservabilityConfigError), + #[error(transparent)] + InvalidRedactionLimits(#[from] RedactionLimitsError), + #[error("invalid log filter")] + InvalidFilter, + #[error("global tracing subscriber is already initialized")] + SubscriberAlreadyInitialized, + #[error(transparent)] + Metrics(#[from] MetricsSurfaceError), + #[error(transparent)] + OtlpConfig(#[from] OtlpTraceConfigError), + #[error(transparent)] + Otlp(#[from] OtlpTraceError), + #[error(transparent)] + SentryConfig(#[from] SentryConfigError), +} diff --git a/crates/crank-observability/src/logging.rs b/crates/crank-observability/src/logging.rs new file mode 100644 index 0000000..c96d6ad --- /dev/null +++ b/crates/crank-observability/src/logging.rs @@ -0,0 +1,314 @@ +use std::fmt; + +use serde_json::{Map, Number, Value}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tracing::{Event, Subscriber, field::Visit}; +use tracing_subscriber::{ + EnvFilter, Layer, + filter::filter_fn, + fmt::{FmtContext, FormatEvent, FormatFields, MakeWriter, format::Writer}, + layer::SubscriberExt, + registry::LookupSpan, +}; + +use crate::{ + ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity, + propagation::current_trace_id, + redaction::{redact_value, truncate_string}, + schema::LogEnvelope, +}; + +pub fn build_subscriber( + config: ObservabilityConfig, + writer: W, +) -> Result +where + W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static, +{ + build_subscriber_with_tracer(config, writer, None) +} + +pub(crate) fn build_subscriber_with_tracer( + config: ObservabilityConfig, + writer: W, + tracer: Option, +) -> Result +where + W: for<'writer> MakeWriter<'writer> + Send + Sync + 'static, +{ + let (identity, filter, limits) = config.into_parts(); + limits.validate()?; + let filter = EnvFilter::try_new(filter).map_err(|_| ObservabilityInitError::InvalidFilter)?; + let formatter = JsonEventFormatter::new(identity, limits); + let fmt_layer = tracing_subscriber::fmt::layer() + .with_ansi(false) + .event_format(formatter) + .with_writer(writer) + .with_filter(filter); + let otel_layer = tracer.map(|tracer| { + tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(filter_fn(|metadata| { + metadata.is_span() && metadata.target() == "crank::trace" + })) + }); + + Ok(tracing_subscriber::registry() + .with(fmt_layer) + .with(otel_layer)) +} + +#[derive(Clone, Debug)] +struct JsonEventFormatter { + identity: ServiceIdentity, + limits: RedactionLimits, +} + +impl JsonEventFormatter { + fn new(identity: ServiceIdentity, limits: RedactionLimits) -> Self { + Self { identity, limits } + } + + fn envelope(&self, event: &Event<'_>) -> Result { + let metadata = event.metadata(); + let mut visitor = JsonFieldVisitor::default(); + event.record(&mut visitor); + let mut raw_fields = visitor.fields; + let request_id = take_correlation_id(&mut raw_fields, "request_id") + .map(|value| truncate_string(&value, self.limits.max_string_bytes)); + let trace_id = take_correlation_id(&mut raw_fields, "trace_id") + .or_else(current_trace_id) + .map(|value| truncate_string(&value, self.limits.max_string_bytes)); + let cleaned = redact_value(&Value::Object(raw_fields), self.limits); + let fields = cleaned.as_object().cloned().unwrap_or_default(); + let timestamp = OffsetDateTime::now_utc() + .format(&Rfc3339) + .map_err(|_| fmt::Error)?; + + Ok(LogEnvelope { + timestamp, + level: metadata.level().as_str().to_owned(), + service: self.identity.service().to_owned(), + version: self.identity.version().to_owned(), + environment: self.identity.environment().to_owned(), + target: truncate_string(metadata.target(), self.limits.max_string_bytes), + event: truncate_string(metadata.name(), self.limits.max_string_bytes), + request_id, + trace_id, + fields, + }) + } + + fn serialize_bounded(&self, mut envelope: LogEnvelope) -> Result { + let line_budget = self.limits.max_event_bytes.saturating_sub(1); + let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?; + if serialized.len() <= line_budget { + return Ok(serialized); + } + + envelope.fields = Map::from_iter([("truncated".to_owned(), Value::Bool(true))]); + let fallback_string_limit = self.limits.max_string_bytes.min(64); + envelope.target = truncate_string(&envelope.target, fallback_string_limit); + envelope.event = truncate_string(&envelope.event, fallback_string_limit); + envelope.request_id = envelope + .request_id + .map(|value| truncate_string(&value, fallback_string_limit)); + envelope.trace_id = envelope + .trace_id + .map(|value| truncate_string(&value, fallback_string_limit)); + let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?; + if serialized.len() <= line_budget { + return Ok(serialized); + } + + envelope.request_id = None; + envelope.trace_id = None; + envelope.target = truncate_string(&envelope.target, 16); + envelope.event = truncate_string(&envelope.event, 16); + let serialized = serde_json::to_string(&envelope).map_err(|_| fmt::Error)?; + (serialized.len() <= line_budget) + .then_some(serialized) + .ok_or(fmt::Error) + } +} + +impl FormatEvent for JsonEventFormatter +where + S: Subscriber + for<'lookup> LookupSpan<'lookup>, + N: for<'writer> FormatFields<'writer> + 'static, +{ + fn format_event( + &self, + _ctx: &FmtContext<'_, S, N>, + mut writer: Writer<'_>, + event: &Event<'_>, + ) -> fmt::Result { + let serialized = self.serialize_bounded(self.envelope(event)?)?; + writer.write_str(&serialized)?; + writer.write_char('\n') + } +} + +#[derive(Default)] +struct JsonFieldVisitor { + fields: Map, +} + +impl JsonFieldVisitor { + fn insert(&mut self, field: &tracing::field::Field, value: Value) { + self.fields.insert(field.name().to_owned(), value); + } +} + +impl Visit for JsonFieldVisitor { + fn record_i64(&mut self, field: &tracing::field::Field, value: i64) { + self.insert(field, Value::Number(value.into())); + } + + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + self.insert(field, Value::Number(value.into())); + } + + fn record_bool(&mut self, field: &tracing::field::Field, value: bool) { + self.insert(field, Value::Bool(value)); + } + + fn record_f64(&mut self, field: &tracing::field::Field, value: f64) { + let value = Number::from_f64(value) + .map(Value::Number) + .unwrap_or(Value::Null); + self.insert(field, value); + } + + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.insert(field, Value::String(value.to_owned())); + } + + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) { + let rendered = format!("{value:?}"); + let value = if is_correlation_field(field.name()) { + Value::String(debug_scalar(&rendered)) + } else { + match serde_json::from_str(&rendered) { + Ok(value @ (Value::Object(_) | Value::Array(_))) => value, + _ if field.name() == "message" || is_safe_display_scalar(&rendered) => { + Value::String(rendered) + } + _ => Value::String(crate::REDACTED_MARKER.to_owned()), + } + }; + self.insert(field, value); + } +} + +fn take_correlation_id(fields: &mut Map, name: &str) -> Option { + let value = fields.remove(name)?; + let value = match value { + Value::String(value) => value, + Value::Number(value) => value.to_string(), + Value::Bool(value) => value.to_string(), + Value::Null | Value::Array(_) | Value::Object(_) => return None, + }; + (!value.is_empty()).then_some(value) +} + +fn is_correlation_field(name: &str) -> bool { + matches!(name, "request_id" | "trace_id" | "correlation_id") +} + +fn debug_scalar(rendered: &str) -> String { + serde_json::from_str::(rendered).unwrap_or_else(|_| rendered.to_owned()) +} + +fn is_safe_display_scalar(rendered: &str) -> bool { + !rendered.is_empty() + && rendered.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'+' | b'@') + }) +} + +#[cfg(test)] +mod tests { + use std::{ + io, + sync::{Arc, Mutex}, + }; + + use opentelemetry::{global, trace::TracerProvider as _}; + use opentelemetry_sdk::{ + error::OTelSdkResult, + propagation::TraceContextPropagator, + trace::{SdkTracerProvider, SpanData, SpanExporter}, + }; + use tracing::info; + + use super::build_subscriber_with_tracer; + use crate::{ + ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context, + }; + + #[test] + fn trace_spans_ignore_the_log_level_filter() { + global::set_text_map_propagator(TraceContextPropagator::new()); + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("trace-filter-test"); + let subscriber = build_subscriber_with_tracer(test_config("warn"), io::sink, Some(tracer)) + .expect("subscriber must build"); + let dispatch = tracing::Dispatch::new(subscriber); + let _dispatch_guard = tracing::dispatcher::set_default(&dispatch); + let span = tracing::info_span!(target: "crank::trace", "http.request"); + let _span_guard = span.enter(); + let mut headers = axum::http::HeaderMap::new(); + + assert!(inject_current_trace_context(&mut headers)); + assert!(headers.contains_key("traceparent")); + } + + #[test] + fn otel_layer_does_not_export_events() { + let exported = Arc::new(Mutex::new(Vec::new())); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(CapturingExporter(Arc::clone(&exported))) + .build(); + let tracer = provider.tracer("event-filter-test"); + let subscriber = build_subscriber_with_tracer(test_config("info"), io::sink, Some(tracer)) + .expect("subscriber must build"); + let dispatch = tracing::Dispatch::new(subscriber); + + tracing::dispatcher::with_default(&dispatch, || { + let span = tracing::info_span!(target: "crank::trace", "http.request"); + let _span_guard = span.enter(); + info!(password = "canary-secret", "sensitive event"); + }); + provider.force_flush().expect("span must be exported"); + + let spans = exported.lock().expect("capture lock"); + assert_eq!(spans.len(), 1); + assert!(spans[0].events.is_empty()); + assert!( + !format!("{:?}", spans[0]) + .as_bytes() + .windows(b"canary-secret".len()) + .any(|window| window == b"canary-secret") + ); + } + + fn test_config(filter: &str) -> ObservabilityConfig { + ObservabilityConfig::new( + ServiceIdentity::try_new("admin-api", "test", "test").unwrap(), + filter, + RedactionLimits::default(), + ) + } + + #[derive(Clone, Debug)] + struct CapturingExporter(Arc>>); + + impl SpanExporter for CapturingExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + self.0.lock().expect("capture lock").extend(batch); + Ok(()) + } + } +} diff --git a/crates/crank-observability/src/metrics_schema.rs b/crates/crank-observability/src/metrics_schema.rs new file mode 100644 index 0000000..505be01 --- /dev/null +++ b/crates/crank-observability/src/metrics_schema.rs @@ -0,0 +1,159 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MetricKind { + Counter, + Gauge, + Histogram, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MetricUnit { + Count, + Seconds, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MetricDefinition { + pub name: &'static str, + pub kind: MetricKind, + pub unit: MetricUnit, + pub labels: &'static [&'static str], + pub description: &'static str, +} + +pub const DURATION_BUCKETS_SECONDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, +]; + +const METRIC_SCHEMA: &[MetricDefinition] = &[ + counter( + "crank_http_requests_total", + &["route", "method", "status_class"], + "Total HTTP requests.", + ), + histogram( + "crank_http_request_duration_seconds", + &["route", "method"], + "HTTP request duration in seconds.", + ), + gauge( + "crank_http_inflight", + &[], + "HTTP requests currently being processed.", + ), + counter( + "crank_mcp_requests_total", + &["method", "response_mode", "outcome"], + "Total MCP JSON-RPC requests.", + ), + gauge( + "crank_mcp_active_sessions", + &[], + "Active MCP transport sessions.", + ), + counter( + "crank_tool_invocations_total", + &["source", "outcome", "error_kind"], + "Total tool invocations.", + ), + histogram( + "crank_tool_invocation_duration_seconds", + &["source", "outcome"], + "Tool invocation duration in seconds.", + ), + counter( + "crank_upstream_requests_total", + &["operation_kind", "outcome"], + "Total upstream requests.", + ), + histogram( + "crank_upstream_request_duration_seconds", + &["operation_kind", "outcome"], + "Upstream request duration in seconds.", + ), + gauge( + "crank_runtime_inflight", + &[], + "Runtime executions currently in progress.", + ), + counter( + "crank_runtime_limit_rejections_total", + &["stage"], + "Runtime executions rejected by a bounded limit.", + ), + gauge( + "crank_db_pool_connections", + &["state"], + "PostgreSQL pool connections by state.", + ), + gauge( + "crank_catalog_tools", + &[], + "Tools in the current published catalog.", + ), + gauge( + "crank_catalog_estimated_context_tokens", + &[], + "Estimated context tokens in the current published catalog.", + ), + gauge( + "crank_catalog_warnings", + &[], + "Warnings in the current published catalog.", + ), + counter( + "crank_invocation_history_lost_total", + &[], + "Invocation history records lost after an action completed.", + ), + counter( + "crank_telemetry_export_failures_total", + &["signal_type", "exporter"], + "Telemetry export failures.", + ), +]; + +pub const fn metric_schema() -> &'static [MetricDefinition] { + METRIC_SCHEMA +} + +const fn counter( + name: &'static str, + labels: &'static [&'static str], + description: &'static str, +) -> MetricDefinition { + MetricDefinition { + name, + kind: MetricKind::Counter, + unit: MetricUnit::Count, + labels, + description, + } +} + +const fn gauge( + name: &'static str, + labels: &'static [&'static str], + description: &'static str, +) -> MetricDefinition { + MetricDefinition { + name, + kind: MetricKind::Gauge, + unit: MetricUnit::Count, + labels, + description, + } +} + +const fn histogram( + name: &'static str, + labels: &'static [&'static str], + description: &'static str, +) -> MetricDefinition { + MetricDefinition { + name, + kind: MetricKind::Histogram, + unit: MetricUnit::Seconds, + labels, + description, + } +} diff --git a/crates/crank-observability/src/otlp.rs b/crates/crank-observability/src/otlp.rs new file mode 100644 index 0000000..96b3b46 --- /dev/null +++ b/crates/crank-observability/src/otlp.rs @@ -0,0 +1,954 @@ +use std::{collections::HashMap, env, fmt, time::Duration}; + +use axum::http::{HeaderName, HeaderValue}; +use opentelemetry::{ + KeyValue, Value, + trace::{Status, TracerProvider as _}, +}; +use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig, WithHttpConfig}; +use opentelemetry_sdk::{ + Resource, + error::OTelSdkResult, + trace::{ + BatchConfigBuilder, BatchSpanProcessor, SdkTracer, SdkTracerProvider, SpanData, + SpanExporter as SpanExporterTrait, + }, +}; +use percent_encoding::percent_decode_str; +use thiserror::Error; +use url::Url; + +use crate::ServiceIdentity; + +const DEFAULT_EXPORT_TIMEOUT: Duration = Duration::from_secs(10); +const DEFAULT_MAX_QUEUE_SIZE: usize = 2_048; +const DEFAULT_MAX_EXPORT_BATCH_SIZE: usize = 512; +const DEFAULT_SCHEDULE_DELAY: Duration = Duration::from_secs(5); +const DEFAULT_BATCH_EXPORT_TIMEOUT: Duration = Duration::from_secs(30); +const MAX_QUEUE_SIZE: usize = 65_536; +const MAX_DURATION: Duration = Duration::from_secs(300); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OtlpBatchConfig { + max_queue_size: usize, + max_export_batch_size: usize, + scheduled_delay: Duration, + export_timeout: Duration, +} + +impl OtlpBatchConfig { + pub fn try_new( + max_queue_size: usize, + max_export_batch_size: usize, + scheduled_delay: Duration, + export_timeout: Duration, + ) -> Result { + let valid = max_queue_size > 0 + && max_queue_size <= MAX_QUEUE_SIZE + && max_export_batch_size > 0 + && max_export_batch_size <= max_queue_size + && duration_is_bounded(scheduled_delay) + && duration_is_bounded(export_timeout); + if !valid { + return Err(OtlpTraceConfigError::InvalidBatchLimits); + } + + Ok(Self { + max_queue_size, + max_export_batch_size, + scheduled_delay, + export_timeout, + }) + } + + pub fn max_queue_size(&self) -> usize { + self.max_queue_size + } + + pub fn max_export_batch_size(&self) -> usize { + self.max_export_batch_size + } + + pub fn scheduled_delay(&self) -> Duration { + self.scheduled_delay + } + + pub fn export_timeout(&self) -> Duration { + self.export_timeout + } + + fn sdk_config(&self) -> opentelemetry_sdk::trace::BatchConfig { + BatchConfigBuilder::default() + .with_max_queue_size(self.max_queue_size) + .with_max_export_batch_size(self.max_export_batch_size) + .with_scheduled_delay(self.scheduled_delay) + .build() + } +} + +impl Default for OtlpBatchConfig { + fn default() -> Self { + Self { + max_queue_size: DEFAULT_MAX_QUEUE_SIZE, + max_export_batch_size: DEFAULT_MAX_EXPORT_BATCH_SIZE, + scheduled_delay: DEFAULT_SCHEDULE_DELAY, + export_timeout: DEFAULT_BATCH_EXPORT_TIMEOUT, + } + } +} + +#[derive(Clone, Eq, PartialEq)] +pub struct OtlpTraceConfig { + endpoint: Option, + export_timeout: Duration, + batch: OtlpBatchConfig, + headers: HashMap, +} + +impl fmt::Debug for OtlpTraceConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OtlpTraceConfig") + .field("enabled", &self.is_enabled()) + .field("export_timeout", &self.export_timeout) + .field("batch", &self.batch) + .field("header_count", &self.headers.len()) + .finish() + } +} + +impl OtlpTraceConfig { + pub fn from_env() -> Result { + OtlpEnvSettings::from_env()?.into_config() + } + + fn from_settings(settings: OtlpEnvSettings) -> Result { + let endpoint = match settings.traces_endpoint { + Some(endpoint) => Some(validate_endpoint(endpoint, EndpointKind::Trace)?), + None => settings + .generic_endpoint + .map(|endpoint| validate_endpoint(endpoint, EndpointKind::Generic)) + .transpose()?, + }; + if endpoint.is_none() { + return Ok(Self { + endpoint: None, + export_timeout: DEFAULT_EXPORT_TIMEOUT, + batch: OtlpBatchConfig::default(), + headers: HashMap::new(), + }); + } + + let protocol = settings.traces_protocol.or(settings.generic_protocol); + let export_timeout = match settings.traces_timeout { + Some(timeout) => duration_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", Some(timeout))?, + None => duration_env("OTEL_EXPORTER_OTLP_TIMEOUT", settings.generic_timeout)?, + } + .unwrap_or(DEFAULT_EXPORT_TIMEOUT); + let batch = OtlpBatchConfig::try_new( + usize_env("OTEL_BSP_MAX_QUEUE_SIZE", settings.max_queue_size)? + .unwrap_or(DEFAULT_MAX_QUEUE_SIZE), + usize_env( + "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + settings.max_export_batch_size, + )? + .unwrap_or(DEFAULT_MAX_EXPORT_BATCH_SIZE), + duration_env("OTEL_BSP_SCHEDULE_DELAY", settings.scheduled_delay)? + .unwrap_or(DEFAULT_SCHEDULE_DELAY), + duration_env("OTEL_BSP_EXPORT_TIMEOUT", settings.batch_export_timeout)? + .unwrap_or(DEFAULT_BATCH_EXPORT_TIMEOUT), + )?; + let headers = settings + .traces_headers + .filter(|value| !value.is_empty()) + .or(settings.generic_headers.filter(|value| !value.is_empty())) + .map(|value| parse_headers(&value)) + .transpose()? + .unwrap_or_default(); + + Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, headers) + } + + pub fn try_new( + endpoint: Option, + protocol: Option, + export_timeout: Duration, + batch: OtlpBatchConfig, + ) -> Result { + Self::try_new_with_headers(endpoint, protocol, export_timeout, batch, HashMap::new()) + } + + fn try_new_with_headers( + endpoint: Option, + protocol: Option, + export_timeout: Duration, + batch: OtlpBatchConfig, + headers: HashMap, + ) -> Result { + let endpoint = endpoint + .map(|endpoint| validate_endpoint(endpoint, EndpointKind::Trace)) + .transpose()?; + if endpoint.is_some() && protocol.as_deref().unwrap_or("http/protobuf") != "http/protobuf" { + return Err(OtlpTraceConfigError::UnsupportedProtocol); + } + if !duration_is_bounded(export_timeout) { + return Err(OtlpTraceConfigError::InvalidDuration { + field: "OTEL_EXPORTER_OTLP_TIMEOUT", + }); + } + + Ok(Self { + endpoint, + export_timeout, + batch, + headers, + }) + } + + pub fn is_enabled(&self) -> bool { + self.endpoint.is_some() + } + + pub fn export_timeout(&self) -> Duration { + self.export_timeout + } + + pub fn batch(&self) -> &OtlpBatchConfig { + &self.batch + } + + fn effective_export_timeout(&self) -> Duration { + self.export_timeout.min(self.batch.export_timeout) + } + + #[cfg(test)] + fn endpoint(&self) -> Option<&str> { + self.endpoint.as_deref() + } + + #[cfg(test)] + fn header(&self, name: &str) -> Option<&str> { + self.headers.get(name).map(String::as_str) + } +} + +#[derive(Default)] +struct OtlpEnvSettings { + traces_endpoint: Option, + generic_endpoint: Option, + traces_protocol: Option, + generic_protocol: Option, + traces_timeout: Option, + generic_timeout: Option, + traces_headers: Option, + generic_headers: Option, + max_queue_size: Option, + max_export_batch_size: Option, + scheduled_delay: Option, + batch_export_timeout: Option, +} + +impl OtlpEnvSettings { + fn from_env() -> Result { + Ok(Self { + traces_endpoint: optional_env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")?, + generic_endpoint: optional_env("OTEL_EXPORTER_OTLP_ENDPOINT")?, + traces_protocol: optional_env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")?, + generic_protocol: optional_env("OTEL_EXPORTER_OTLP_PROTOCOL")?, + traces_timeout: optional_env("OTEL_EXPORTER_OTLP_TRACES_TIMEOUT")?, + generic_timeout: optional_env("OTEL_EXPORTER_OTLP_TIMEOUT")?, + traces_headers: optional_env("OTEL_EXPORTER_OTLP_TRACES_HEADERS")?, + generic_headers: optional_env("OTEL_EXPORTER_OTLP_HEADERS")?, + max_queue_size: optional_env("OTEL_BSP_MAX_QUEUE_SIZE")?, + max_export_batch_size: optional_env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE")?, + scheduled_delay: optional_env("OTEL_BSP_SCHEDULE_DELAY")?, + batch_export_timeout: optional_env("OTEL_BSP_EXPORT_TIMEOUT")?, + }) + } + + fn into_config(self) -> Result { + OtlpTraceConfig::from_settings(self) + } +} + +#[derive(Debug, Error, Eq, PartialEq)] +pub enum OtlpTraceConfigError { + #[error("OTLP environment variable is not valid UTF-8: {field}")] + InvalidEnvironmentEncoding { field: &'static str }, + #[error("OTLP trace endpoint is invalid: {reason}")] + InvalidEndpoint { reason: &'static str }, + #[error("OTLP trace protocol must be http/protobuf")] + UnsupportedProtocol, + #[error("OTLP numeric setting is invalid: {field}")] + InvalidNumber { field: &'static str }, + #[error("OTLP duration setting is invalid: {field}")] + InvalidDuration { field: &'static str }, + #[error("OTLP batch limits are invalid")] + InvalidBatchLimits, + #[error("OTLP trace headers are invalid")] + InvalidHeaders, +} + +#[derive(Debug, Error)] +pub enum OtlpTraceError { + #[error("failed to configure OTLP trace exporter")] + ExporterConfiguration, +} + +pub fn build_tracer_provider( + identity: &ServiceIdentity, + config: &OtlpTraceConfig, +) -> Result, OtlpTraceError> { + let Some(endpoint) = config.endpoint.as_deref() else { + return Ok(None); + }; + let exporter = SpanExporter::builder() + .with_http() + .with_protocol(Protocol::HttpBinary) + .with_endpoint(endpoint) + .with_timeout(config.effective_export_timeout()) + .with_headers(config.headers.clone()) + .build() + .map_err(|_| OtlpTraceError::ExporterConfiguration)?; + let processor = BatchSpanProcessor::builder(ObservedSpanExporter(exporter)) + .with_batch_config(config.batch.sdk_config()) + .build(); + let resource = Resource::builder_empty() + .with_attributes([ + KeyValue::new("service.name", identity.service().to_owned()), + KeyValue::new("service.version", identity.version().to_owned()), + KeyValue::new( + "deployment.environment.name", + identity.environment().to_owned(), + ), + ]) + .build(); + let provider = SdkTracerProvider::builder() + .with_span_processor(processor) + .with_resource(resource) + .build(); + let tracer = provider.tracer("crank"); + + Ok(Some((provider, tracer))) +} + +#[derive(Debug)] +struct ObservedSpanExporter(SpanExporter); + +impl SpanExporterTrait for ObservedSpanExporter { + async fn export(&self, mut batch: Vec) -> OTelSdkResult { + sanitize_trace_batch(&mut batch); + let result = self.0.export(batch).await; + if result.is_err() { + metrics::counter!( + "crank_telemetry_export_failures_total", + "signal_type" => "trace", + "exporter" => "otlp" + ) + .increment(1); + } + result + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.0.shutdown_with_timeout(timeout) + } + + fn force_flush(&self) -> OTelSdkResult { + self.0.force_flush() + } + + fn set_resource(&mut self, resource: &Resource) { + self.0.set_resource(resource); + } +} + +fn sanitize_trace_batch(batch: &mut Vec) { + batch.retain(|span| is_allowed_span_name(span.name.as_ref())); + for span in batch { + let original_attribute_count = span.attributes.len(); + span.attributes.retain(is_allowed_span_attribute); + span.dropped_attributes_count = span + .dropped_attributes_count + .saturating_add((original_attribute_count - span.attributes.len()) as u32); + span.events = Default::default(); + span.links = Default::default(); + if matches!(span.status, Status::Error { .. }) { + span.status = Status::error(""); + } + } +} + +fn is_allowed_span_name(name: &str) -> bool { + matches!( + name, + "http.request" + | "mcp.request" + | "mcp.rate_limit" + | "mcp.access.check" + | "mcp.catalog.load" + | "mcp.tools.resolve" + | "approval.check" + | "runtime.execute" + | "runtime.arguments.map" + | "runtime.idempotency" + | "upstream.http" + | "runtime.response.transform" + | "auth.resolve" + | "approval.recovery" + | "history.write" + | "db.query" + ) +} + +fn is_allowed_span_attribute(attribute: &KeyValue) -> bool { + let Value::String(value) = &attribute.value else { + return false; + }; + let value = value.as_str(); + match attribute.key.as_str() { + "request_id" => crate::RequestId::is_valid(value), + "stage" => is_allowed_span_name(value), + "outcome" => matches!( + value, + "success" + | "error" + | "allowed" + | "denied" + | "required" + | "replay" + | "execute" + | "skipped" + | "cache_hit" + ), + "error.category" => matches!( + value, + "access" + | "rate_limit" + | "catalog" + | "approval" + | "idempotency" + | "schema" + | "mapping" + | "upstream" + | "transformation" + | "history" + | "database" + | "concurrency" + | "configuration" + | "internal" + ), + "db.system" => value == "postgresql", + "db.operation" => matches!( + value, + "machine_access.read" + | "machine_access.touch" + | "catalog.load" + | "approval.read" + | "approval.write" + | "auth_profile.read" + | "secret.read" + | "secret.touch" + | "invocation_history.write" + ), + _ => false, + } +} + +#[derive(Clone, Copy)] +enum EndpointKind { + Trace, + Generic, +} + +fn validate_endpoint(endpoint: String, kind: EndpointKind) -> Result { + let mut url = Url::parse(&endpoint).map_err(|_| OtlpTraceConfigError::InvalidEndpoint { + reason: "invalid URL", + })?; + if !matches!(url.scheme(), "http" | "https") { + return Err(OtlpTraceConfigError::InvalidEndpoint { + reason: "unsupported scheme", + }); + } + if url.host_str().is_none() { + return Err(OtlpTraceConfigError::InvalidEndpoint { + reason: "host is required", + }); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(OtlpTraceConfigError::InvalidEndpoint { + reason: "credentials are forbidden", + }); + } + if url.query().is_some() || url.fragment().is_some() { + return Err(OtlpTraceConfigError::InvalidEndpoint { + reason: "query and fragment are forbidden", + }); + } + if matches!(kind, EndpointKind::Generic) { + let path = url.path().trim_end_matches('/'); + url.set_path(&format!("{path}/v1/traces")); + } + + Ok(url.into()) +} + +fn parse_headers(value: &str) -> Result, OtlpTraceConfigError> { + value + .split_terminator(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .try_fold(HashMap::new(), |mut headers, item| { + let (name, encoded_value) = item + .split_once('=') + .ok_or(OtlpTraceConfigError::InvalidHeaders)?; + let name = HeaderName::from_bytes(name.trim().as_bytes()) + .map_err(|_| OtlpTraceConfigError::InvalidHeaders)?; + let value = percent_decode_str(encoded_value.trim()) + .decode_utf8() + .map_err(|_| OtlpTraceConfigError::InvalidHeaders)? + .into_owned(); + if value.is_empty() || HeaderValue::from_str(&value).is_err() { + return Err(OtlpTraceConfigError::InvalidHeaders); + } + headers.insert(name.as_str().to_owned(), value); + Ok(headers) + }) +} + +fn optional_env(field: &'static str) -> Result, OtlpTraceConfigError> { + match env::var(field) { + Ok(value) if value.is_empty() => Ok(None), + Ok(value) => Ok(Some(value)), + Err(env::VarError::NotPresent) => Ok(None), + Err(env::VarError::NotUnicode(_)) => { + Err(OtlpTraceConfigError::InvalidEnvironmentEncoding { field }) + } + } +} + +fn usize_env( + field: &'static str, + value: Option, +) -> Result, OtlpTraceConfigError> { + value + .map(|value| { + value + .parse() + .map_err(|_| OtlpTraceConfigError::InvalidNumber { field }) + }) + .transpose() +} + +fn duration_env( + field: &'static str, + value: Option, +) -> Result, OtlpTraceConfigError> { + value + .map(|value| { + value + .parse::() + .map(Duration::from_millis) + .map_err(|_| OtlpTraceConfigError::InvalidDuration { field }) + }) + .transpose() +} + +fn duration_is_bounded(duration: Duration) -> bool { + !duration.is_zero() && duration <= MAX_DURATION +} + +#[cfg(test)] +mod tests { + use std::{ + io::{Read, Write}, + net::TcpListener, + sync::mpsc, + thread, + time::{Duration, Instant}, + }; + + use axum::{ + Router, + body::{Body, to_bytes}, + extract::Request, + middleware::Next, + response::Response, + routing::get, + }; + use opentelemetry::{ + KeyValue, + trace::{Span as _, Status, Tracer as _}, + }; + use opentelemetry_proto::tonic::{ + collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value, + }; + use prost::Message; + use tower::ServiceExt; + use tracing::{Instrument, info_span}; + use tracing_subscriber::layer::SubscriberExt; + + use super::{OtlpBatchConfig, OtlpEnvSettings, OtlpTraceConfig, build_tracer_provider}; + use crate::ServiceIdentity; + + #[test] + fn signal_specific_settings_override_generic_settings() { + let config = OtlpEnvSettings { + traces_endpoint: Some("https://traces.example.test/custom".to_owned()), + generic_endpoint: Some("https://generic.example.test/otel".to_owned()), + traces_protocol: Some("http/protobuf".to_owned()), + generic_protocol: Some("grpc".to_owned()), + traces_timeout: Some("2500".to_owned()), + generic_timeout: Some("invalid-unused-fallback".to_owned()), + ..OtlpEnvSettings::default() + } + .into_config() + .unwrap(); + + assert_eq!( + config.endpoint(), + Some("https://traces.example.test/custom") + ); + assert_eq!(config.export_timeout(), Duration::from_millis(2500)); + } + + #[test] + fn disabled_export_ignores_inactive_settings() { + let config = OtlpEnvSettings { + traces_protocol: Some("grpc".to_owned()), + generic_protocol: Some("grpc".to_owned()), + traces_timeout: Some("invalid".to_owned()), + generic_timeout: Some("invalid".to_owned()), + max_queue_size: Some("invalid".to_owned()), + max_export_batch_size: Some("invalid".to_owned()), + scheduled_delay: Some("invalid".to_owned()), + batch_export_timeout: Some("invalid".to_owned()), + ..OtlpEnvSettings::default() + } + .into_config() + .unwrap(); + + assert!(!config.is_enabled()); + } + + #[test] + fn empty_signal_headers_use_generic_headers() { + let config = OtlpEnvSettings { + traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()), + traces_headers: Some(String::new()), + generic_headers: Some( + "authorization=Bearer%20canary-token,x-tenant=community".to_owned(), + ), + ..OtlpEnvSettings::default() + } + .into_config() + .unwrap(); + + assert_eq!(config.header("authorization"), Some("Bearer canary-token")); + assert_eq!(config.header("x-tenant"), Some("community")); + assert!(!format!("{config:?}").contains("canary-token")); + } + + #[test] + fn invalid_headers_return_a_safe_error() { + let config = OtlpEnvSettings { + traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()), + traces_headers: Some("authorization=canary-secret%0Ainjected".to_owned()), + ..OtlpEnvSettings::default() + }; + + let error = config.into_config().unwrap_err(); + assert!(matches!(error, super::OtlpTraceConfigError::InvalidHeaders)); + assert!(!error.to_string().contains("canary-secret")); + } + + #[test] + fn stricter_batch_timeout_bounds_http_export() { + let config = OtlpEnvSettings { + traces_endpoint: Some("https://traces.example.test/v1/traces".to_owned()), + traces_protocol: Some("http/protobuf".to_owned()), + traces_timeout: Some("9000".to_owned()), + batch_export_timeout: Some("2500".to_owned()), + ..OtlpEnvSettings::default() + } + .into_config() + .unwrap(); + + assert_eq!( + config.effective_export_timeout(), + Duration::from_millis(2500) + ); + } + + #[test] + fn generic_endpoint_receives_standard_trace_path() { + let config = OtlpEnvSettings { + generic_endpoint: Some("https://generic.example.test/otel/".to_owned()), + generic_protocol: Some("http/protobuf".to_owned()), + ..OtlpEnvSettings::default() + } + .into_config() + .unwrap(); + + assert_eq!( + config.endpoint(), + Some("https://generic.example.test/otel/v1/traces") + ); + } + + #[test] + fn disabled_export_does_not_build_a_provider() { + let config = OtlpTraceConfig::try_new( + None, + None, + Duration::from_secs(1), + OtlpBatchConfig::default(), + ) + .unwrap(); + let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "test").unwrap(); + + assert!(build_tracer_provider(&identity, &config).unwrap().is_none()); + } + + #[test] + fn real_http_protobuf_export_contains_resource_and_trace() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (request_tx, request_rx) = mpsc::sync_channel(1); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let request = read_http_request(&mut stream); + stream + .write_all( + b"HTTP/1.1 200 OK\r\ncontent-type: application/x-protobuf\r\ncontent-length: 0\r\nconnection: close\r\n\r\n", + ) + .unwrap(); + request_tx.send(request).unwrap(); + }); + let config = OtlpEnvSettings { + traces_endpoint: Some(format!("http://{address}/v1/traces")), + traces_protocol: Some("http/protobuf".to_owned()), + traces_timeout: Some("2000".to_owned()), + traces_headers: Some(String::new()), + generic_headers: Some("authorization=Bearer%20canary-token".to_owned()), + max_queue_size: Some("16".to_owned()), + max_export_batch_size: Some("8".to_owned()), + scheduled_delay: Some("10".to_owned()), + batch_export_timeout: Some("2000".to_owned()), + ..OtlpEnvSettings::default() + } + .into_config() + .unwrap(); + let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "integration-test").unwrap(); + let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap(); + let mut span = tracer.start("http.request"); + let trace_id = span.span_context().trace_id().to_bytes(); + span.set_attribute(KeyValue::new("request_id", "req_otlp_contract")); + span.set_attribute(KeyValue::new("authorization", "Bearer canary-span-secret")); + span.add_event( + "canary-span-event", + vec![KeyValue::new("payload", "canary-span-secret")], + ); + span.set_status(Status::error("canary-span-secret")); + span.end(); + + provider.force_flush().unwrap(); + provider.shutdown().unwrap(); + let request = request_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + server.join().unwrap(); + let (headers, body) = split_http_request(&request); + + assert!(headers.contains("POST /v1/traces HTTP/1.1")); + assert!( + headers + .to_ascii_lowercase() + .contains("content-type: application/x-protobuf") + ); + assert!( + headers + .to_ascii_lowercase() + .contains("authorization: bearer canary-token") + ); + let export = ExportTraceServiceRequest::decode(body).unwrap(); + let resource_spans = export.resource_spans.first().unwrap(); + let attributes = &resource_spans.resource.as_ref().unwrap().attributes; + assert_eq!( + string_attribute(attributes, "service.name"), + Some("admin-api") + ); + assert_eq!( + string_attribute(attributes, "service.version"), + Some("0.3.1") + ); + assert_eq!( + string_attribute(attributes, "deployment.environment.name"), + Some("integration-test") + ); + assert_eq!( + resource_spans.scope_spans[0].spans[0].trace_id.as_slice(), + trace_id + ); + let exported_span = &resource_spans.scope_spans[0].spans[0]; + assert_eq!(exported_span.name, "http.request"); + assert_eq!( + string_attribute(&exported_span.attributes, "request_id"), + Some("req_otlp_contract") + ); + assert!( + exported_span + .attributes + .iter() + .all(|attribute| attribute.key != "authorization") + ); + assert!(exported_span.events.is_empty()); + assert_eq!( + exported_span + .status + .as_ref() + .map(|status| status.message.as_str()), + Some("") + ); + assert!( + !body + .windows(b"canary-span-secret".len()) + .any(|window| { window == b"canary-span-secret" }) + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn unavailable_receiver_does_not_change_product_result() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + drop(stream); + }); + let config = OtlpTraceConfig::try_new( + Some(format!("http://{address}/v1/traces")), + Some("http/protobuf".to_owned()), + Duration::from_millis(250), + OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(250)) + .unwrap(), + ) + .unwrap(); + let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "fault-test").unwrap(); + let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap(); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + let dispatch = tracing::Dispatch::new(subscriber); + let _dispatch_guard = tracing::dispatcher::set_default(&dispatch); + let app = Router::new() + .route("/product", get(|| async { "product-success" })) + .layer(axum::middleware::from_fn(trace_product_request)); + + let response = app + .oneshot( + Request::builder() + .uri("/product") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), 64).await.unwrap(); + + assert_eq!(status, axum::http::StatusCode::OK); + assert_eq!(body.as_ref(), b"product-success"); + assert!(provider.force_flush().is_err()); + let _ = provider.shutdown(); + server.join().unwrap(); + } + + async fn trace_product_request(request: Request, next: Next) -> Response { + next.run(request) + .instrument(info_span!(target: "crank::trace", "http.request")) + .await + } + + #[test] + fn hanging_receiver_respects_the_stricter_export_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + thread::sleep(Duration::from_millis(750)); + drop(stream); + }); + let config = OtlpTraceConfig::try_new( + Some(format!("http://{address}/v1/traces")), + Some("http/protobuf".to_owned()), + Duration::from_secs(2), + OtlpBatchConfig::try_new(8, 4, Duration::from_millis(10), Duration::from_millis(100)) + .unwrap(), + ) + .unwrap(); + let identity = ServiceIdentity::try_new("admin-api", "0.3.1", "timeout-test").unwrap(); + let (provider, tracer) = build_tracer_provider(&identity, &config).unwrap().unwrap(); + let mut span = tracer.start("http.request"); + span.end(); + let started_at = Instant::now(); + + assert!(provider.force_flush().is_err()); + assert!(started_at.elapsed() < Duration::from_millis(500)); + let _ = provider.shutdown(); + server.join().unwrap(); + } + + fn read_http_request(stream: &mut std::net::TcpStream) -> Vec { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let read = stream.read(&mut buffer).unwrap(); + request.extend_from_slice(&buffer[..read]); + let Some(header_end) = find_bytes(&request, b"\r\n\r\n") else { + continue; + }; + let headers = String::from_utf8_lossy(&request[..header_end]); + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + if request.len() >= header_end + 4 + content_length { + return request; + } + } + } + + fn split_http_request(request: &[u8]) -> (&str, &[u8]) { + let header_end = find_bytes(request, b"\r\n\r\n").unwrap(); + ( + std::str::from_utf8(&request[..header_end]).unwrap(), + &request[header_end + 4..], + ) + } + + fn string_attribute<'a>( + attributes: &'a [opentelemetry_proto::tonic::common::v1::KeyValue], + key: &str, + ) -> Option<&'a str> { + attributes.iter().find_map(|attribute| { + let value = attribute.value.as_ref()?.value.as_ref()?; + (attribute.key == key) + .then_some(value) + .and_then(|value| match value { + any_value::Value::StringValue(value) => Some(value.as_str()), + _ => None, + }) + }) + } + + fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|candidate| candidate == needle) + } +} diff --git a/crates/crank-observability/src/prometheus.rs b/crates/crank-observability/src/prometheus.rs new file mode 100644 index 0000000..e9bd7fa --- /dev/null +++ b/crates/crank-observability/src/prometheus.rs @@ -0,0 +1,283 @@ +use std::{env, net::SocketAddr}; + +use axum::{ + Router, + extract::{Request, State}, + http::{ + HeaderMap, StatusCode, + header::{self, HeaderValue}, + }, + middleware::{self, Next}, + response::{IntoResponse, Response}, + routing::get, +}; +use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle, PrometheusRecorder}; +use sha2::{Digest, Sha256}; +use subtle::ConstantTimeEq; +use thiserror::Error; +use tokio::net::TcpListener; + +use crate::{DURATION_BUCKETS_SECONDS, ServiceIdentity}; + +const METRICS_ENABLED_ENV: &str = "CRANK_METRICS_ENABLED"; +const METRICS_TOKEN_ENV: &str = "CRANK_METRICS_BEARER_TOKEN"; +const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; + +#[derive(Clone)] +pub struct MetricsConfig { + enabled: bool, + bind_addr: SocketAddr, + token_digest: Option<[u8; 32]>, +} + +impl std::fmt::Debug for MetricsConfig { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("MetricsConfig") + .field("enabled", &self.enabled) + .field("bind_addr", &self.bind_addr) + .field("authentication_configured", &self.token_digest.is_some()) + .finish() + } +} + +impl MetricsConfig { + pub fn new( + enabled: bool, + bind_addr: SocketAddr, + bearer_token: Option, + ) -> Result { + let token_digest = bearer_token + .filter(|token| !token.is_empty()) + .map(|token| token_digest(token.as_bytes())); + + if enabled && !bind_addr.ip().is_loopback() && token_digest.is_none() { + return Err(MetricsConfigError::MissingTokenForExternalBind); + } + + Ok(Self { + enabled, + bind_addr, + token_digest, + }) + } + + pub fn from_env( + bind_env: &'static str, + default_bind: SocketAddr, + ) -> Result { + let enabled = parse_enabled(env::var(METRICS_ENABLED_ENV))?; + let bind_addr = match env::var(bind_env) { + Ok(raw) => raw + .parse() + .map_err(|_| MetricsConfigError::InvalidBindAddress { field: bind_env })?, + Err(env::VarError::NotPresent) => default_bind, + Err(env::VarError::NotUnicode(_)) => { + return Err(MetricsConfigError::InvalidEnvironmentEncoding { field: bind_env }); + } + }; + let bearer_token = match env::var(METRICS_TOKEN_ENV) { + Ok(token) => Some(token), + Err(env::VarError::NotPresent) => None, + Err(env::VarError::NotUnicode(_)) => { + return Err(MetricsConfigError::InvalidEnvironmentEncoding { + field: METRICS_TOKEN_ENV, + }); + } + }; + + Self::new(enabled, bind_addr, bearer_token) + } + + pub fn enabled(&self) -> bool { + self.enabled + } + + pub fn bind_addr(&self) -> SocketAddr { + self.bind_addr + } + + pub fn requires_authentication(&self) -> bool { + !self.bind_addr.ip().is_loopback() + } +} + +#[derive(Debug, Error)] +pub enum MetricsConfigError { + #[error("metrics environment variable is not valid UTF-8: {field}")] + InvalidEnvironmentEncoding { field: &'static str }, + #[error("metrics bind address is invalid: {field}")] + InvalidBindAddress { field: &'static str }, + #[error("metrics enabled flag must be one of true, false, 1, 0")] + InvalidEnabledFlag, + #[error("external metrics bind requires a bearer token")] + MissingTokenForExternalBind, +} + +#[derive(Clone)] +struct MetricsState { + handle: PrometheusHandle, + token_digest: Option<[u8; 32]>, + requires_authentication: bool, +} + +pub struct MetricsSurface { + config: MetricsConfig, + state: MetricsState, + _recorder: Option, +} + +impl MetricsSurface { + pub(crate) fn new(config: MetricsConfig, handle: PrometheusHandle) -> Self { + Self { + state: MetricsState { + handle, + token_digest: config.token_digest, + requires_authentication: config.requires_authentication(), + }, + config, + _recorder: None, + } + } + + pub fn for_test( + config: MetricsConfig, + identity: ServiceIdentity, + ) -> Result { + let recorder = prometheus_builder(&identity)?.build_recorder(); + let handle = recorder.handle(); + let mut surface = Self::new(config, handle); + surface._recorder = Some(recorder); + Ok(surface) + } + + pub fn router(&self) -> Router { + Router::new() + .route("/metrics", get(render_metrics)) + .route("/health", get(metrics_health)) + .layer(middleware::from_fn_with_state( + self.state.clone(), + authorize_metrics, + )) + .with_state(self.state.clone()) + } + + pub async fn bind(self) -> Result { + let listener = TcpListener::bind(self.config.bind_addr) + .await + .map_err(|_| MetricsServeError::Bind)?; + Ok(MetricsServer { + listener, + router: self.router(), + }) + } +} + +pub struct MetricsServer { + listener: TcpListener, + router: Router, +} + +impl MetricsServer { + pub async fn serve(self) -> Result<(), MetricsServeError> { + axum::serve(self.listener, self.router) + .await + .map_err(|_| MetricsServeError::Serve) + } +} + +#[derive(Debug, Error)] +pub enum MetricsSurfaceError { + #[error("failed to configure Prometheus recorder")] + RecorderConfiguration, +} + +#[derive(Debug, Error)] +pub enum MetricsServeError { + #[error("failed to bind metrics listener")] + Bind, + #[error("metrics listener stopped unexpectedly")] + Serve, +} + +pub(crate) fn install_prometheus_recorder( + identity: &ServiceIdentity, +) -> Result { + prometheus_builder(identity)? + .install_recorder() + .map_err(|_| MetricsSurfaceError::RecorderConfiguration) +} + +fn prometheus_builder( + identity: &ServiceIdentity, +) -> Result { + PrometheusBuilder::new() + .set_buckets(DURATION_BUCKETS_SECONDS) + .map(|builder| { + builder + .add_global_label("service", identity.service()) + .add_global_label("version", identity.version()) + .add_global_label("environment", identity.environment()) + }) + .map_err(|_| MetricsSurfaceError::RecorderConfiguration) +} + +async fn render_metrics(State(state): State) -> Response { + let mut response = state.handle.render().into_response(); + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static(PROMETHEUS_CONTENT_TYPE), + ); + response +} + +async fn metrics_health() -> impl IntoResponse { + (StatusCode::OK, "ok\n") +} + +async fn authorize_metrics( + State(state): State, + request: Request, + next: Next, +) -> Response { + if !state.requires_authentication { + return next.run(request).await; + } + + let authorized = bearer_token(request.headers()) + .map(token_digest) + .zip(state.token_digest) + .is_some_and(|(actual, expected)| bool::from(actual.ct_eq(&expected))); + + if authorized { + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + } +} + +fn bearer_token(headers: &HeaderMap) -> Option<&[u8]> { + headers + .get(header::AUTHORIZATION)? + .as_bytes() + .strip_prefix(b"Bearer ") + .filter(|token| !token.is_empty()) +} + +fn token_digest(token: &[u8]) -> [u8; 32] { + Sha256::digest(token).into() +} + +fn parse_enabled(value: Result) -> Result { + match value { + Ok(raw) => match raw.to_ascii_lowercase().as_str() { + "true" | "1" => Ok(true), + "false" | "0" => Ok(false), + _ => Err(MetricsConfigError::InvalidEnabledFlag), + }, + Err(env::VarError::NotPresent) => Ok(true), + Err(env::VarError::NotUnicode(_)) => Err(MetricsConfigError::InvalidEnvironmentEncoding { + field: METRICS_ENABLED_ENV, + }), + } +} diff --git a/crates/crank-observability/src/propagation.rs b/crates/crank-observability/src/propagation.rs new file mode 100644 index 0000000..3cd116b --- /dev/null +++ b/crates/crank-observability/src/propagation.rs @@ -0,0 +1,70 @@ +use axum::http::{HeaderMap, HeaderName, HeaderValue}; +use opentelemetry::{ + Context, global, + propagation::{Extractor, Injector}, + trace::TraceContextExt, +}; +use opentelemetry_sdk::propagation::TraceContextPropagator; +use tracing::Span; +use tracing_opentelemetry::OpenTelemetrySpanExt; + +pub fn set_remote_trace_parent(span: &Span, headers: &HeaderMap) -> bool { + let context = + global::get_text_map_propagator(|propagator| propagator.extract(&HeaderExtractor(headers))); + let span_context = context.span().span_context().clone(); + if !span_context.is_valid() || !span_context.is_remote() { + return false; + } + + span.set_parent(context).is_ok() +} + +pub fn inject_current_trace_context(headers: &mut HeaderMap) -> bool { + let context = Span::current().context(); + if !context.span().span_context().is_valid() { + return false; + } + + global::get_text_map_propagator(|propagator| { + propagator.inject_context(&context, &mut HeaderInjector(headers)); + }); + true +} + +pub(crate) fn install_trace_context_propagator() { + global::set_text_map_propagator(TraceContextPropagator::new()); +} + +struct HeaderExtractor<'a>(&'a HeaderMap); + +impl Extractor for HeaderExtractor<'_> { + fn get(&self, key: &str) -> Option<&str> { + self.0.get(key).and_then(|value| value.to_str().ok()) + } + + fn keys(&self) -> Vec<&str> { + self.0.keys().map(HeaderName::as_str).collect() + } +} + +struct HeaderInjector<'a>(&'a mut HeaderMap); + +impl Injector for HeaderInjector<'_> { + fn set(&mut self, key: &str, value: String) { + let Ok(name) = HeaderName::try_from(key) else { + return; + }; + let Ok(value) = HeaderValue::try_from(value) else { + return; + }; + self.0.insert(name, value); + } +} + +pub(crate) fn current_trace_id() -> Option { + let context: Context = Span::current().context(); + let span_context = context.span().span_context().clone(); + span_context + .is_valid() + .then(|| span_context.trace_id().to_string()) +} diff --git a/crates/crank-observability/src/redaction.rs b/crates/crank-observability/src/redaction.rs new file mode 100644 index 0000000..133dc55 --- /dev/null +++ b/crates/crank-observability/src/redaction.rs @@ -0,0 +1,292 @@ +use serde_json::{Map, Value}; +use thiserror::Error; + +pub const REDACTED_MARKER: &str = "[REDACTED]"; +const TRUNCATED_MARKER: &str = "[TRUNCATED]"; +const MIN_EVENT_BYTES: usize = 512; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RedactionLimits { + pub max_string_bytes: usize, + pub max_array_items: usize, + pub max_object_fields: usize, + pub max_depth: usize, + pub max_event_bytes: usize, +} + +impl Default for RedactionLimits { + fn default() -> Self { + Self { + max_string_bytes: 1024, + max_array_items: 32, + max_object_fields: 64, + max_depth: 8, + max_event_bytes: 16 * 1024, + } + } +} + +impl RedactionLimits { + pub fn validate(self) -> Result<(), RedactionLimitsError> { + for (field, value, minimum) in [ + ( + "max_string_bytes", + self.max_string_bytes, + TRUNCATED_MARKER.len(), + ), + ("max_array_items", self.max_array_items, 1), + ("max_object_fields", self.max_object_fields, 1), + ("max_depth", self.max_depth, 1), + ("max_event_bytes", self.max_event_bytes, MIN_EVENT_BYTES), + ] { + if value < minimum { + return Err(RedactionLimitsError::TooSmall { field, minimum }); + } + } + Ok(()) + } +} + +#[derive(Debug, Error)] +pub enum RedactionLimitsError { + #[error("invalid redaction limit {field}: minimum is {minimum}")] + TooSmall { field: &'static str, minimum: usize }, +} + +#[derive(Debug, Error)] +pub enum SafeJsonError { + #[error(transparent)] + InvalidLimits(#[from] RedactionLimitsError), + #[error(transparent)] + Serialization(#[from] serde_json::Error), +} + +pub fn redact_value(value: &Value, limits: RedactionLimits) -> Value { + redact_at_depth(value, limits, 0) +} + +pub fn safe_json(value: &Value, limits: RedactionLimits) -> Result { + limits.validate()?; + let serialized = serde_json::to_string(&redact_value(value, limits))?; + if serialized.len() <= limits.max_event_bytes { + return Ok(serialized); + } + + Ok(serde_json::to_string(&serde_json::json!({ + "truncated": true + }))?) +} + +pub(crate) fn truncate_string(value: &str, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value.to_owned(); + } + if max_bytes == 0 { + return String::new(); + } + + let marker = if max_bytes >= TRUNCATED_MARKER.len() { + TRUNCATED_MARKER + } else { + "" + }; + let content_budget = max_bytes.saturating_sub(marker.len()); + let mut boundary = content_budget.min(value.len()); + while boundary > 0 && !value.is_char_boundary(boundary) { + boundary -= 1; + } + + let mut truncated = String::with_capacity(max_bytes); + truncated.push_str(&value[..boundary]); + if marker.is_empty() { + let mut marker_boundary = max_bytes.min(TRUNCATED_MARKER.len()); + while marker_boundary > 0 && !TRUNCATED_MARKER.is_char_boundary(marker_boundary) { + marker_boundary -= 1; + } + truncated.clear(); + truncated.push_str(&TRUNCATED_MARKER[..marker_boundary]); + } else { + truncated.push_str(marker); + } + truncated +} + +fn redact_at_depth(value: &Value, limits: RedactionLimits, depth: usize) -> Value { + if depth >= limits.max_depth { + return Value::String(TRUNCATED_MARKER.to_owned()); + } + + match value { + Value::Null | Value::Bool(_) | Value::Number(_) => value.clone(), + Value::String(value) => Value::String(truncate_string(value, limits.max_string_bytes)), + Value::Array(values) => redact_array(values, limits, depth), + Value::Object(values) => redact_object(values, limits, depth), + } +} + +fn redact_array(values: &[Value], limits: RedactionLimits, depth: usize) -> Value { + if limits.max_array_items == 0 { + return Value::Array(Vec::new()); + } + + let truncated = values.len() > limits.max_array_items; + let value_limit = if truncated { + limits.max_array_items.saturating_sub(1) + } else { + limits.max_array_items + }; + let mut output: Vec<_> = values + .iter() + .take(value_limit) + .map(|value| redact_at_depth(value, limits, depth + 1)) + .collect(); + if truncated { + output.push(Value::String(TRUNCATED_MARKER.to_owned())); + } + Value::Array(output) +} + +fn redact_object(values: &Map, limits: RedactionLimits, depth: usize) -> Value { + if limits.max_object_fields == 0 { + return Value::Object(Map::new()); + } + + let truncated = values.len() > limits.max_object_fields; + let value_limit = if truncated { + limits.max_object_fields.saturating_sub(1) + } else { + limits.max_object_fields + }; + let mut output = Map::new(); + + for (index, (key, value)) in values.iter().take(value_limit).enumerate() { + let cleaned = if is_sensitive_key(key) { + Value::String(REDACTED_MARKER.to_owned()) + } else if is_url_key(key) { + value + .as_str() + .map(sanitize_url) + .map(|value| truncate_string(&value, limits.max_string_bytes)) + .map(Value::String) + .unwrap_or_else(|| redact_at_depth(value, limits, depth + 1)) + } else { + redact_at_depth(value, limits, depth + 1) + }; + output.insert( + bounded_object_key(key, limits.max_string_bytes, index), + cleaned, + ); + } + if truncated { + output.insert( + "_truncated".to_owned(), + Value::String(TRUNCATED_MARKER.to_owned()), + ); + } + + Value::Object(output) +} + +fn normalized_key(key: &str) -> String { + key.bytes() + .filter(|byte| !matches!(byte, b'_' | b'-' | b'.')) + .map(|byte| byte.to_ascii_lowercase() as char) + .collect() +} + +fn is_sensitive_key(key: &str) -> bool { + let key = normalized_key(key); + let exact_match = matches!( + key.as_str(), + "password" + | "passwd" + | "secret" + | "token" + | "apikey" + | "accesskey" + | "secretkey" + | "authorization" + | "proxyauthorization" + | "cookie" + | "setcookie" + | "query" + | "querystring" + | "rawquery" + | "urlquery" + | "payload" + | "body" + | "requestbody" + | "arguments" + | "result" + | "response" + | "context" + | "error" + | "errormessage" + ); + let contains_high_risk_name = [ + "password", + "passwd", + "secret", + "token", + "apikey", + "accesskey", + "authorization", + "cookie", + ] + .iter() + .any(|part| key.contains(part)); + + exact_match + || contains_high_risk_name + || key.ends_with("payload") + || key.ends_with("body") + || key.ends_with("arguments") + || key.ends_with("result") + || key.ends_with("response") + || key.ends_with("query") + || key.ends_with("context") + || key.starts_with("query") +} + +fn is_url_key(key: &str) -> bool { + let key = normalized_key(key); + matches!( + key.as_str(), + "url" | "uri" | "endpoint" | "endpointurl" | "endpointuri" | "requesturl" | "targeturl" + ) || key.ends_with("url") + || key.ends_with("uri") + || key.ends_with("endpoint") +} + +fn sanitize_url(value: &str) -> String { + let without_query = value + .find(['?', '#']) + .map(|index| &value[..index]) + .unwrap_or(value); + let Some(scheme_end) = without_query.find("://") else { + return without_query.to_owned(); + }; + let authority_start = scheme_end + 3; + let authority_end = without_query[authority_start..] + .find('/') + .map(|index| authority_start + index) + .unwrap_or(without_query.len()); + let authority = &without_query[authority_start..authority_end]; + let Some(userinfo_end) = authority.rfind('@') else { + return without_query.to_owned(); + }; + + format!( + "{}{}", + &without_query[..authority_start], + &without_query[authority_start + userinfo_end + 1..] + ) +} + +fn bounded_object_key(key: &str, max_bytes: usize, index: usize) -> String { + if key.len() <= max_bytes { + return key.to_owned(); + } + + truncate_string(&format!("_truncated_key_{index}"), max_bytes) +} diff --git a/crates/crank-observability/src/schema.rs b/crates/crank-observability/src/schema.rs new file mode 100644 index 0000000..852b6fc --- /dev/null +++ b/crates/crank-observability/src/schema.rs @@ -0,0 +1,18 @@ +use serde::Serialize; +use serde_json::{Map, Value}; + +#[derive(Debug, Serialize)] +pub(crate) struct LogEnvelope { + pub timestamp: String, + pub level: String, + pub service: String, + pub version: String, + pub environment: String, + pub target: String, + pub event: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub request_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub trace_id: Option, + pub fields: Map, +} diff --git a/crates/crank-observability/tests/correlation.rs b/crates/crank-observability/tests/correlation.rs new file mode 100644 index 0000000..c5c85c9 --- /dev/null +++ b/crates/crank-observability/tests/correlation.rs @@ -0,0 +1,42 @@ +use crank_observability::RequestId; +use uuid::Version; + +#[test] +fn preserves_valid_opaque_request_id() { + let request_id = RequestId::resolve(Some("req_test-123/abc")); + + assert_eq!(request_id.as_str(), "req_test-123/abc"); +} + +#[test] +fn replaces_missing_and_invalid_values_with_uuid_v7() { + for candidate in [ + None, + Some(""), + Some("bad value"), + Some(" leading"), + Some("trailing "), + Some("bad,value"), + Some("bad;value"), + Some("я"), + ] { + let request_id = RequestId::resolve(candidate); + let parsed = uuid::Uuid::parse_str(request_id.as_str()).expect("generated UUID"); + + assert_eq!(parsed.get_version(), Some(Version::SortRand)); + } +} + +#[test] +fn rejects_values_over_the_shared_limit() { + let oversized = "x".repeat(RequestId::MAX_LEN + 1); + let request_id = RequestId::resolve(Some(&oversized)); + + assert_ne!(request_id.as_str(), oversized); + assert_eq!( + uuid::Uuid::parse_str(request_id.as_str()) + .expect("generated UUID") + .get_version(), + Some(Version::SortRand) + ); +} diff --git a/crates/crank-observability/tests/critical_errors.rs b/crates/crank-observability/tests/critical_errors.rs new file mode 100644 index 0000000..56da8b5 --- /dev/null +++ b/crates/crank-observability/tests/critical_errors.rs @@ -0,0 +1,33 @@ +use crank_observability::{CriticalErrorCategory, SentryConfig, SentryConfigError}; + +#[test] +fn missing_or_blank_dsn_disables_critical_error_channel() { + assert!(!SentryConfig::parse(None).expect("missing DSN").enabled()); + assert!(!SentryConfig::parse(Some("")).expect("empty DSN").enabled()); + assert!( + !SentryConfig::parse(Some(" ")) + .expect("blank DSN") + .enabled() + ); +} + +#[test] +fn invalid_explicit_dsn_is_rejected_without_echoing_the_value() { + let secret_value = "not-a-dsn?token=control-secret"; + let error = SentryConfig::parse(Some(secret_value)).expect_err("invalid DSN must fail"); + + assert!(matches!(error, SentryConfigError::InvalidDsn)); + assert!(!error.to_string().contains(secret_value)); + assert!(!error.to_string().contains("control-secret")); +} + +#[test] +fn critical_error_categories_are_closed_and_stable() { + assert_eq!(CriticalErrorCategory::Panic.as_str(), "panic"); + assert_eq!(CriticalErrorCategory::Startup.as_str(), "startup"); + assert_eq!(CriticalErrorCategory::Internal.as_str(), "internal"); + assert_eq!( + CriticalErrorCategory::DataIntegrity.as_str(), + "data_integrity" + ); +} diff --git a/crates/crank-observability/tests/http_metrics.rs b/crates/crank-observability/tests/http_metrics.rs new file mode 100644 index 0000000..5f3eec6 --- /dev/null +++ b/crates/crank-observability/tests/http_metrics.rs @@ -0,0 +1,92 @@ +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use axum::{ + Router, + body::{Body, to_bytes}, + http::{Request, StatusCode}, + middleware, + routing::get, +}; +use crank_observability::{ + MetricsConfig, ObservabilityConfig, RedactionLimits, ServiceIdentity, record_http_request, +}; +use tower::ServiceExt; + +#[tokio::test] +async fn http_metrics_use_matched_routes_and_closed_labels() { + let identity = + ServiceIdentity::try_new("metrics-test", "0.3.1", "test").expect("valid identity"); + let lifecycle = crank_observability::init(ObservabilityConfig::new( + identity, + "off", + RedactionLimits::default(), + )) + .expect("observability lifecycle"); + let config = MetricsConfig::new( + true, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464), + None, + ) + .expect("loopback metrics"); + let metrics = lifecycle.metrics_surface(config).router(); + let app = Router::new() + .route( + "/documents/{document_id}", + get(|| async { StatusCode::NO_CONTENT }), + ) + .layer(middleware::from_fn(record_http_request)); + + let sensitive_path_segment = "customer-secret-document-id"; + for index in 0..100 { + let response = app + .clone() + .oneshot( + Request::get(format!("/documents/{sensitive_path_segment}-{index}")) + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("response"); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + } + + let response = app + .oneshot( + Request::get("/unknown/customer-controlled-path") + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("response"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + let response = metrics + .oneshot( + Request::get("/metrics") + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("metrics response"); + let body = to_bytes(response.into_body(), 1024 * 1024) + .await + .expect("bounded metrics body"); + let body = String::from_utf8(body.to_vec()).expect("utf-8 metrics"); + + assert!(body.contains("crank_http_requests_total")); + assert!(body.contains("route=\"/documents/{document_id}\"")); + assert!(body.contains("method=\"GET\"")); + assert!(body.contains("status_class=\"2xx\"")); + assert!(body.contains("crank_http_request_duration_seconds_bucket")); + assert!(!body.contains(sensitive_path_segment)); + assert_eq!( + body.lines() + .filter(|line| { + line.starts_with("crank_http_requests_total{") + && line.contains("route=\"/documents/{document_id}\"") + }) + .count(), + 1, + "different entity ids must not create additional series" + ); +} diff --git a/crates/crank-observability/tests/incidents.rs b/crates/crank-observability/tests/incidents.rs new file mode 100644 index 0000000..e829b1c --- /dev/null +++ b/crates/crank-observability/tests/incidents.rs @@ -0,0 +1,15 @@ +use crank_observability::{ + OperationalIncident, operational_incident_total, record_operational_incident, +}; + +#[test] +fn history_loss_counter_has_no_dynamic_dimensions() { + let before = operational_incident_total(OperationalIncident::InvocationHistoryLost); + + record_operational_incident(OperationalIncident::InvocationHistoryLost); + + assert!( + operational_incident_total(OperationalIncident::InvocationHistoryLost) > before, + "the closed incident counter must increase" + ); +} diff --git a/crates/crank-observability/tests/json_logging.rs b/crates/crank-observability/tests/json_logging.rs new file mode 100644 index 0000000..a403668 --- /dev/null +++ b/crates/crank-observability/tests/json_logging.rs @@ -0,0 +1,392 @@ +use std::{ + io, + sync::{Arc, Mutex}, +}; + +use crank_observability::{ + ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber, safe_json, +}; +use serde_json::Value; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; +use tracing_subscriber::fmt::MakeWriter; + +#[derive(Clone, Default)] +struct SharedWriter { + buffer: Arc>>, +} + +impl SharedWriter { + fn output(&self) -> String { + String::from_utf8(self.buffer.lock().expect("test writer lock").clone()) + .expect("log output must be UTF-8") + } +} + +impl<'a> MakeWriter<'a> for SharedWriter { + type Writer = SharedWriterGuard; + + fn make_writer(&'a self) -> Self::Writer { + SharedWriterGuard { + buffer: Arc::clone(&self.buffer), + } + } +} + +struct SharedWriterGuard { + buffer: Arc>>, +} + +impl io::Write for SharedWriterGuard { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.buffer + .lock() + .map_err(|_| io::Error::other("test writer lock poisoned"))? + .extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn capture(service: &'static str, emit: impl FnOnce()) -> Vec { + capture_with_limits(service, RedactionLimits::default(), emit) +} + +fn capture_with_limits( + service: &'static str, + limits: RedactionLimits, + emit: impl FnOnce(), +) -> Vec { + let writer = SharedWriter::default(); + let config = ObservabilityConfig::new( + ServiceIdentity::try_new(service, "0.3.1", "test").expect("valid test identity"), + "info", + limits, + ); + let subscriber = + build_subscriber(config, writer.clone()).expect("test subscriber must be built"); + + tracing::subscriber::with_default(subscriber, emit); + + writer + .output() + .lines() + .map(|line| { + assert!(!line.contains('\u{1b}'), "ANSI is forbidden: {line}"); + serde_json::from_str(line).expect("every line must be one JSON object") + }) + .collect() +} + +#[test] +fn schema_contract_is_identical_for_both_services() { + let outputs = ["admin-api", "mcp-server"].map(|service| { + capture(service, || { + tracing::info!( + name: "service.started", + target: "crank::startup", + port = 3101_u64, + "service started" + ); + }) + }); + + for (service, events) in ["admin-api", "mcp-server"].into_iter().zip(outputs.iter()) { + assert_eq!(events.len(), 1); + let event = &events[0]; + assert_eq!(event["service"], service); + assert_eq!(event["version"], "0.3.1"); + assert_eq!(event["environment"], "test"); + assert_eq!(event["level"], "INFO"); + assert_eq!(event["target"], "crank::startup"); + assert_eq!(event["event"], "service.started"); + assert!(event["fields"].is_object()); + assert_eq!(event["fields"]["port"], 3101); + assert_eq!(event["fields"]["message"], "service started"); + let timestamp = event["timestamp"].as_str().expect("timestamp string"); + let parsed = + OffsetDateTime::parse(timestamp, &Rfc3339).expect("timestamp must be RFC 3339"); + assert_eq!(parsed.offset(), time::UtcOffset::UTC); + } + + let first_keys: Vec<_> = outputs[0][0] + .as_object() + .expect("object") + .keys() + .cloned() + .collect(); + let second_keys: Vec<_> = outputs[1][0] + .as_object() + .expect("object") + .keys() + .cloned() + .collect(); + assert_eq!(first_keys, second_keys); +} + +#[test] +fn correlation_fields_are_distinct_and_only_present_when_recorded() { + let present = capture("admin-api", || { + tracing::info!( + name: "admin.request.completed", + request_id = "req-123", + trace_id = "trace-456" + ); + }); + assert_eq!(present[0]["request_id"], "req-123"); + assert_eq!(present[0]["trace_id"], "trace-456"); + assert!(present[0]["fields"].get("request_id").is_none()); + assert!(present[0]["fields"].get("trace_id").is_none()); + + let absent = capture("mcp-server", || { + tracing::info!(name: "mcp.request.completed", status = 200_u64); + }); + assert!(absent[0].get("request_id").is_none()); + assert!(absent[0].get("trace_id").is_none()); +} + +#[test] +fn correlation_fields_preserve_scalar_display_values_before_field_limits() { + let limits = RedactionLimits { + max_object_fields: 1, + ..RedactionLimits::default() + }; + let request_id = "123"; + let events = capture_with_limits("admin-api", limits, || { + tracing::info!( + name: "admin.request.completed", + alpha = "field that consumes the object budget", + request_id = %request_id, + trace_id = true, + ); + }); + + assert_eq!(events[0]["request_id"], "123"); + assert_eq!(events[0]["trace_id"], "true"); +} + +#[test] +fn empty_correlation_fields_are_omitted() { + let events = capture("admin-api", || { + tracing::info!( + name: "admin.request.completed", + request_id = "", + trace_id = "" + ); + }); + + assert!(events[0].get("request_id").is_none()); + assert!(events[0].get("trace_id").is_none()); +} + +#[test] +fn formatter_redacts_fields_before_serialization() { + let context = safe_json( + &serde_json::json!({ + "nested": { + "access_token": "nested-canary-secret", + "endpoint": "https://example.test/private?key=nested-canary-secret" + } + }), + RedactionLimits::default(), + ) + .expect("safe nested context"); + let events = capture("admin-api", || { + tracing::warn!( + name: "admin.request.rejected", + password = "canary-secret", + url = "https://example.test/path?token=canary-secret", + safe_fields = %context, + unsafe_context = ?serde_json::json!({"password": "debug-canary-secret"}), + error_code = "invalid_request" + ); + }); + let serialized = serde_json::to_string(&events[0]).expect("event JSON"); + + assert_eq!(events[0]["fields"]["password"], "[REDACTED]"); + assert_eq!(events[0]["fields"]["url"], "https://example.test/path"); + assert_eq!( + events[0]["fields"]["safe_fields"]["nested"]["access_token"], + "[REDACTED]" + ); + assert_eq!( + events[0]["fields"]["safe_fields"]["nested"]["endpoint"], + "https://example.test/private" + ); + assert_eq!(events[0]["fields"]["unsafe_context"], "[REDACTED]"); + assert_eq!(events[0]["fields"]["error_code"], "invalid_request"); + assert!(!serialized.contains("canary-secret")); +} + +#[test] +fn arbitrary_debug_text_is_never_written_verbatim() { + struct Credentials; + + impl std::fmt::Debug for Credentials { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("Credentials { password: \"debug-canary-secret\" }") + } + } + + let events = capture("admin-api", || { + tracing::warn!( + name: "admin.debug.inspected", + details = ?Credentials + ); + }); + let serialized = serde_json::to_string(&events[0]).expect("event JSON"); + + assert_eq!(events[0]["fields"]["details"], "[REDACTED]"); + assert!(!serialized.contains("debug-canary-secret")); +} + +#[test] +fn compound_sensitive_event_fields_are_redacted() { + let events = capture("admin-api", || { + tracing::warn!( + name: "admin.request.rejected", + client_api_key = "client-canary-secret", + authorization_header = "Bearer auth-canary-secret", + response_body = "response-canary-secret", + tool_arguments = "argument-canary-secret", + query_params = "query-canary-secret", + ); + }); + let serialized = serde_json::to_string(&events[0]).expect("event JSON"); + + for key in [ + "client_api_key", + "authorization_header", + "response_body", + "tool_arguments", + "query_params", + ] { + assert_eq!(events[0]["fields"][key], "[REDACTED]"); + } + assert!(!serialized.contains("canary-secret")); +} + +#[test] +fn oversized_event_falls_back_to_valid_bounded_json() { + let limits = RedactionLimits { + max_event_bytes: 512, + ..RedactionLimits::default() + }; + let events = capture_with_limits("admin-api", limits, || { + tracing::info!( + name: "admin.payload.inspected", + description = %"x".repeat(1024) + ); + }); + let serialized = serde_json::to_vec(&events[0]).expect("bounded event JSON"); + + assert!(serialized.len() < limits.max_event_bytes); + assert_eq!(events[0]["fields"]["truncated"], true); +} + +#[test] +fn safe_json_honours_the_total_event_budget() { + let limits = RedactionLimits { + max_event_bytes: 512, + ..RedactionLimits::default() + }; + + let serialized = safe_json( + &serde_json::json!({"description": "x".repeat(4096)}), + limits, + ) + .expect("safe JSON must remain serializable"); + + assert!(serialized.len() <= limits.max_event_bytes); + assert_eq!( + serde_json::from_str::(&serialized).expect("valid JSON")["truncated"], + true + ); +} + +#[test] +fn subscriber_rejects_limits_that_cannot_hold_an_event() { + let config = ObservabilityConfig::new( + ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"), + "info", + RedactionLimits { + max_event_bytes: 16, + ..RedactionLimits::default() + }, + ); + + assert!(build_subscriber(config, SharedWriter::default()).is_err()); +} + +#[test] +fn minimum_event_budget_handles_maximum_identity_labels() { + let writer = SharedWriter::default(); + let config = ObservabilityConfig::new( + ServiceIdentity::try_new("s".repeat(64), "v".repeat(64), "e".repeat(64)) + .expect("maximum identity labels are valid"), + "info", + RedactionLimits { + max_event_bytes: 512, + ..RedactionLimits::default() + }, + ); + let subscriber = + build_subscriber(config, writer.clone()).expect("minimum valid budget must be usable"); + + tracing::subscriber::with_default(subscriber, || { + tracing::info!( + name: "event-name-that-is-intentionally-longer-than-the-fallback-limit", + description = %"x".repeat(4096), + ); + }); + + let output = writer.output(); + assert!(output.len() <= 512); + assert_eq!(output.lines().count(), 1); + serde_json::from_str::(output.trim_end()).expect("bounded line must remain valid JSON"); +} + +#[test] +fn env_filter_is_applied_and_invalid_filter_is_safe() { + let writer = SharedWriter::default(); + let config = ObservabilityConfig::new( + ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"), + "warn", + RedactionLimits::default(), + ); + let subscriber = + build_subscriber(config, writer.clone()).expect("test subscriber must be built"); + tracing::subscriber::with_default(subscriber, || { + tracing::info!(name: "filtered.info", "filtered"); + tracing::warn!(name: "visible.warning", "visible"); + }); + let output = writer.output(); + + assert!(!output.contains("filtered.info")); + assert!(output.contains("visible.warning")); + + let invalid = ObservabilityConfig::new( + ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity"), + "[not a valid filter", + RedactionLimits::default(), + ); + let error = build_subscriber(invalid, SharedWriter::default()) + .err() + .expect("invalid filter must fail"); + assert_eq!(error.to_string(), "invalid log filter"); + assert!(!error.to_string().contains("not a valid filter")); +} + +#[test] +fn service_identity_rejects_empty_or_unsafe_labels() { + for (service, version, environment) in [ + ("", "0.3.1", "test"), + ("admin api", "0.3.1", "test"), + ("admin-api", "", "test"), + ("admin-api", "0.3.1", "prod\nsecret"), + ] { + assert!(ServiceIdentity::try_new(service, version, environment).is_err()); + } +} diff --git a/crates/crank-observability/tests/lifecycle.rs b/crates/crank-observability/tests/lifecycle.rs new file mode 100644 index 0000000..0e62ce8 --- /dev/null +++ b/crates/crank-observability/tests/lifecycle.rs @@ -0,0 +1,22 @@ +use crank_observability::{ + ObservabilityConfig, ObservabilityInitError, RedactionLimits, ServiceIdentity, init, +}; + +fn config() -> ObservabilityConfig { + ObservabilityConfig::new( + ServiceIdentity::try_new("lifecycle-test", "0.3.1", "test").expect("valid test identity"), + "info", + RedactionLimits::default(), + ) +} + +#[test] +fn repeated_global_initialization_returns_typed_error() { + let _lifecycle = init(config()).expect("first initialization must succeed"); + let error = init(config()).expect_err("second initialization must fail"); + + assert!(matches!( + error, + ObservabilityInitError::SubscriberAlreadyInitialized + )); +} diff --git a/crates/crank-observability/tests/otlp.rs b/crates/crank-observability/tests/otlp.rs new file mode 100644 index 0000000..17cd498 --- /dev/null +++ b/crates/crank-observability/tests/otlp.rs @@ -0,0 +1,56 @@ +use std::time::Duration; + +use crank_observability::{OtlpBatchConfig, OtlpTraceConfig, OtlpTraceConfigError}; + +#[test] +fn absent_endpoint_disables_export_without_background_resources() { + let config = OtlpTraceConfig::try_new( + None, + None, + Duration::from_secs(10), + OtlpBatchConfig::default(), + ) + .expect("missing endpoint must be valid"); + + assert!(!config.is_enabled()); +} + +#[test] +fn explicit_config_accepts_only_bounded_http_protobuf() { + let config = OtlpTraceConfig::try_new( + Some("https://collector.example.test/v1/traces".to_owned()), + Some("http/protobuf".to_owned()), + Duration::from_secs(3), + OtlpBatchConfig::try_new(256, 64, Duration::from_millis(500), Duration::from_secs(3)) + .unwrap(), + ) + .expect("bounded HTTP protobuf config must be valid"); + + assert!(config.is_enabled()); + assert_eq!(config.export_timeout(), Duration::from_secs(3)); + assert_eq!(config.batch().max_queue_size(), 256); + assert_eq!(config.batch().max_export_batch_size(), 64); +} + +#[test] +fn invalid_values_return_safe_typed_errors() { + let secret_endpoint = "https://user:canary-secret@collector.example.test/v1/traces"; + let error = OtlpTraceConfig::try_new( + Some(secret_endpoint.to_owned()), + Some("grpc".to_owned()), + Duration::ZERO, + OtlpBatchConfig::default(), + ) + .expect_err("credentials in endpoint must be rejected"); + + assert!(matches!( + error, + OtlpTraceConfigError::InvalidEndpoint { .. } + )); + assert!(!error.to_string().contains(secret_endpoint)); + assert!(!error.to_string().contains("canary-secret")); + + let error = OtlpBatchConfig::try_new(8, 9, Duration::from_millis(1), Duration::from_secs(1)) + .expect_err("batch cannot exceed queue"); + assert!(matches!(error, OtlpTraceConfigError::InvalidBatchLimits)); +} diff --git a/crates/crank-observability/tests/prometheus.rs b/crates/crank-observability/tests/prometheus.rs new file mode 100644 index 0000000..95768aa --- /dev/null +++ b/crates/crank-observability/tests/prometheus.rs @@ -0,0 +1,134 @@ +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode, header}, +}; +use crank_observability::{ + DURATION_BUCKETS_SECONDS, MetricsConfig, MetricsConfigError, MetricsSurface, ServiceIdentity, + metric_schema, +}; +use tower::ServiceExt; + +fn identity() -> ServiceIdentity { + ServiceIdentity::try_new("admin-api", "0.3.1", "test").expect("valid identity") +} + +#[test] +fn loopback_is_allowed_without_a_token() { + let config = MetricsConfig::new( + true, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9464), + None, + ) + .expect("loopback metrics must be safe by default"); + + assert_eq!(config.bind_addr().to_string(), "127.0.0.1:9464"); + assert!(!config.requires_authentication()); +} + +#[test] +fn non_loopback_without_a_token_is_rejected_without_secret_data() { + let error = MetricsConfig::new( + true, + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9464), + None, + ) + .expect_err("external metrics must require authentication"); + + assert!(matches!( + error, + MetricsConfigError::MissingTokenForExternalBind + )); + assert!(!error.to_string().contains("token=")); +} + +#[test] +fn schema_is_closed_and_uses_fixed_duration_buckets() { + let schema = metric_schema(); + let names: Vec<_> = schema.iter().map(|metric| metric.name).collect(); + + assert!(names.contains(&"crank_http_requests_total")); + assert!(names.contains(&"crank_http_request_duration_seconds")); + assert!(names.contains(&"crank_mcp_requests_total")); + assert!(names.contains(&"crank_tool_invocations_total")); + assert!(names.contains(&"crank_runtime_inflight")); + assert!(names.contains(&"crank_db_pool_connections")); + assert!(names.contains(&"crank_catalog_tools")); + assert!(names.contains(&"crank_invocation_history_lost_total")); + assert!(names.contains(&"crank_telemetry_export_failures_total")); + + for metric in schema { + for forbidden in [ + "workspace", + "agent_id", + "operation_id", + "request_id", + "url", + "error_message", + "text", + ] { + assert!( + !metric.labels.contains(&forbidden), + "{} exposes forbidden label {forbidden}", + metric.name + ); + } + } + + assert_eq!( + DURATION_BUCKETS_SECONDS, + &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0 + ] + ); +} + +#[tokio::test] +async fn external_surface_protects_both_routes_and_exposes_nothing_else() { + let token = "metrics-canary-secret"; + let config = MetricsConfig::new( + true, + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9464), + Some(token.to_owned()), + ) + .expect("external metrics with token"); + let surface = MetricsSurface::for_test(config, identity()).expect("test metrics surface"); + let app = surface.router(); + + for path in ["/metrics", "/health"] { + let unauthorized = app + .clone() + .oneshot(Request::get(path).body(Body::empty()).expect("request")) + .await + .expect("response"); + assert_eq!(unauthorized.status(), StatusCode::UNAUTHORIZED); + + let authorized = app + .clone() + .oneshot( + Request::get(path) + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("response"); + assert_eq!(authorized.status(), StatusCode::OK); + let body = to_bytes(authorized.into_body(), 1024 * 1024) + .await + .expect("bounded body"); + assert!(!String::from_utf8_lossy(&body).contains(token)); + } + + let absent = app + .oneshot( + Request::get("/api/operations") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("response"); + assert_eq!(absent.status(), StatusCode::NOT_FOUND); +} diff --git a/crates/crank-observability/tests/propagation.rs b/crates/crank-observability/tests/propagation.rs new file mode 100644 index 0000000..c3c58d3 --- /dev/null +++ b/crates/crank-observability/tests/propagation.rs @@ -0,0 +1,65 @@ +use axum::http::{HeaderMap, HeaderValue}; +use crank_observability::{inject_current_trace_context, set_remote_trace_parent}; +use opentelemetry::{global, trace::TracerProvider as _}; +use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider}; +use tracing::info_span; +use tracing_subscriber::layer::SubscriberExt; + +const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c"; + +#[test] +fn valid_remote_parent_is_continued_and_request_id_is_unrelated() { + with_trace_dispatch(|| { + let mut incoming = HeaderMap::new(); + incoming.insert( + "traceparent", + HeaderValue::from_static("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"), + ); + incoming.insert("x-request-id", HeaderValue::from_static("req-unrelated")); + + let span = info_span!("http.request", request_id = "req-unrelated"); + assert!(set_remote_trace_parent(&span, &incoming)); + let _guard = span.enter(); + + let mut outgoing = HeaderMap::new(); + assert!(inject_current_trace_context(&mut outgoing)); + let propagated = outgoing["traceparent"].to_str().unwrap(); + assert_eq!(&propagated[3..35], REMOTE_TRACE_ID); + assert!(!propagated.contains("req-unrelated")); + assert!(!outgoing.contains_key("baggage")); + }); +} + +#[test] +fn invalid_parent_is_ignored_and_a_new_trace_is_created() { + with_trace_dispatch(|| { + let mut incoming = HeaderMap::new(); + incoming.insert( + "traceparent", + HeaderValue::from_static("canary-invalid-traceparent"), + ); + + let span = info_span!("mcp.request"); + assert!(!set_remote_trace_parent(&span, &incoming)); + let _guard = span.enter(); + + let mut outgoing = HeaderMap::new(); + assert!(inject_current_trace_context(&mut outgoing)); + let propagated = outgoing["traceparent"].to_str().unwrap(); + assert!(propagated.starts_with("00-")); + assert!(!propagated.contains("canary-invalid-traceparent")); + }); +} + +fn with_trace_dispatch(test: impl FnOnce()) { + global::set_text_map_propagator(TraceContextPropagator::new()); + let provider = SdkTracerProvider::builder().build(); + let tracer = provider.tracer("propagation-test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + let dispatch = tracing::Dispatch::new(subscriber); + let _guard = tracing::dispatcher::set_default(&dispatch); + + test(); + provider.shutdown().unwrap(); +} diff --git a/crates/crank-observability/tests/redaction.rs b/crates/crank-observability/tests/redaction.rs new file mode 100644 index 0000000..1b8b83e --- /dev/null +++ b/crates/crank-observability/tests/redaction.rs @@ -0,0 +1,206 @@ +use crank_observability::{RedactionLimits, redact_value}; +use serde_json::{Value, json}; + +const REDACTED: &str = "[REDACTED]"; +const TRUNCATED: &str = "[TRUNCATED]"; + +#[test] +fn sensitive_keys_are_redacted_case_insensitively() { + let keys = [ + "password", + "PassWord", + "api_key", + "access-token", + "authorization", + "Proxy.Authorization", + "cookie", + "set_cookie", + "payload", + "request_body", + "arguments", + "result", + "response", + ]; + + for key in keys { + let cleaned = redact_value(&json!({ key: "canary-secret" }), RedactionLimits::default()); + assert_eq!(cleaned[key], REDACTED, "key {key} was not redacted"); + assert!(!cleaned.to_string().contains("canary-secret")); + } +} + +#[test] +fn generated_sensitive_key_variants_are_redacted() { + for canonical in [ + "password", + "api_key", + "access_key", + "secret_key", + "proxy_authorization", + "set_cookie", + "query_string", + "request_body", + ] { + for separator in ["_", "-", "."] { + let variant = canonical + .split('_') + .collect::>() + .join(separator) + .to_ascii_uppercase(); + let cleaned = redact_value( + &json!({ variant.clone(): "canary-secret" }), + RedactionLimits::default(), + ); + + assert_eq!( + cleaned[&variant], REDACTED, + "key {variant} was not redacted" + ); + assert!(!cleaned.to_string().contains("canary-secret")); + } + } +} + +#[test] +fn compound_sensitive_key_names_are_redacted() { + for key in [ + "client_api_key", + "aws_access_key_id", + "http_authorization_header", + "response_body", + "tool_arguments", + "query_params", + "secret_value", + ] { + let cleaned = redact_value(&json!({ key: "canary-secret" }), RedactionLimits::default()); + + assert_eq!(cleaned[key], REDACTED, "key {key} was not redacted"); + assert!(!cleaned.to_string().contains("canary-secret")); + } +} + +#[test] +fn url_query_and_fragment_are_removed_at_every_depth() { + let input = json!({ + "url": "https://example.test/path?token=canary-secret#fragment", + "nested": [{ + "endpoint_uri": "https://example.test/other?q=canary-secret" + }] + }); + + let cleaned = redact_value(&input, RedactionLimits::default()); + + assert_eq!(cleaned["url"], "https://example.test/path"); + assert_eq!( + cleaned["nested"][0]["endpoint_uri"], + "https://example.test/other" + ); + assert!(!cleaned.to_string().contains("canary-secret")); +} + +#[test] +fn url_credentials_and_compound_endpoint_fields_are_removed() { + let input = json!({ + "upstream_endpoint": "https://user:canary-secret@example.test/path?token=canary-secret", + }); + + let cleaned = redact_value(&input, RedactionLimits::default()); + + assert_eq!(cleaned["upstream_endpoint"], "https://example.test/path"); + assert!(!cleaned.to_string().contains("user")); + assert!(!cleaned.to_string().contains("canary-secret")); +} + +#[test] +fn nested_values_and_collections_respect_all_limits() { + let limits = RedactionLimits { + max_string_bytes: 16, + max_array_items: 3, + max_object_fields: 3, + max_depth: 2, + max_event_bytes: 256, + }; + let input = json!({ + "long": "абвгдежзийклмнопрсту", + "array": [1, 2, 3, 4, 5], + "object": {"a": 1, "b": 2, "c": 3, "d": 4}, + "nested": {"level2": {"level3": "must not survive"}} + }); + + let cleaned = redact_value(&input, limits); + let object = cleaned + .as_object() + .expect("cleaned root must remain an object"); + + assert!(object.len() <= limits.max_object_fields); + assert!( + cleaned["long"] + .as_str() + .map(|value| value.len() <= limits.max_string_bytes) + .unwrap_or(true) + ); + assert!( + cleaned["array"] + .as_array() + .map(|value| value.len() <= limits.max_array_items) + .unwrap_or(true) + ); + assert!(!cleaned.to_string().contains("must not survive")); + assert!(cleaned.to_string().contains(TRUNCATED)); +} + +#[test] +fn truncation_preserves_utf8_and_never_reveals_secret_fragments() { + let input = json!({ + "secret_key": "секретное-значение", + "description": "я".repeat(2048), + }); + + let cleaned = redact_value(&input, RedactionLimits::default()); + let serialized = serde_json::to_string(&cleaned).expect("cleaned value must be valid JSON"); + + assert_eq!(cleaned["secret_key"], REDACTED); + assert!(!serialized.contains("секретное")); + assert!(cleaned["description"].as_str().is_some()); +} + +#[test] +fn redacted_value_does_not_mutate_input() { + let input = json!({"password": "canary-secret"}); + let original = input.clone(); + + let _ = redact_value(&input, RedactionLimits::default()); + + assert_eq!(input, original); +} + +#[test] +fn object_keys_respect_the_string_limit() { + let limits = RedactionLimits { + max_string_bytes: 16, + ..RedactionLimits::default() + }; + let long_key = format!("field-{}", "x".repeat(128)); + + let cleaned = redact_value(&json!({ long_key: "value" }), limits); + + assert!( + cleaned + .as_object() + .expect("cleaned object") + .keys() + .all(|key| key.len() <= limits.max_string_bytes) + ); +} + +#[test] +fn limits_have_finite_safe_defaults() { + let limits = RedactionLimits::default(); + + assert_eq!(limits.max_string_bytes, 1024); + assert_eq!(limits.max_array_items, 32); + assert_eq!(limits.max_object_fields, 64); + assert_eq!(limits.max_depth, 8); + assert_eq!(limits.max_event_bytes, 16 * 1024); + assert!(Value::Null.is_null()); +} diff --git a/crates/crank-registry/Cargo.toml b/crates/crank-registry/Cargo.toml index 559efc1..fcd8dc5 100644 --- a/crates/crank-registry/Cargo.toml +++ b/crates/crank-registry/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-registry" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] diff --git a/crates/crank-registry/src/error.rs b/crates/crank-registry/src/error.rs index 49b3449..876eac5 100644 --- a/crates/crank-registry/src/error.rs +++ b/crates/crank-registry/src/error.rs @@ -74,6 +74,8 @@ pub enum RegistryError { YamlImportJobNotFound { job_id: String }, #[error("import job {job_id} was not found")] ImportJobNotFound { job_id: String }, + #[error("import job {job_id} was already applied with different parameters")] + ImportJobAlreadyApplied { job_id: String }, #[error("unsupported enum representation for field {field}")] InvalidEnumRepresentation { field: &'static str }, #[error("invalid numeric value for field {field}: {value}")] diff --git a/crates/crank-registry/src/lib.rs b/crates/crank-registry/src/lib.rs index 5d07486..945a693 100644 --- a/crates/crank-registry/src/lib.rs +++ b/crates/crank-registry/src/lib.rs @@ -9,12 +9,14 @@ pub use ext::{ExtensionMigration, RegistryExtension, apply_extension_migrations} pub mod records { pub use crate::model::{ - AgentSummary, AgentVersionRecord, ApprovalRequestRecord, AuthUserRecord, DescriptorKind, - DescriptorMetadata, ImportJob, ImportJobId, ImportJobKind, ImportJobStatus, - InvitationRecord, InvocationLogRecord, MembershipRecord, OperationAgentRef, - OperationSampleMetadata, OperationSummary, OperationUsageSummary, OperationVersionRecord, - Page, PlatformApiKeyRecord, PublishedAgentCatalog, PublishedAgentTool, RegistryOperation, - SampleKind, SecretRecord, SecretVersionRecord, SessionRecord, UsageAgentBreakdown, + AgentSummary, AgentVersionRecord, AppliedImportOperation, ApprovalRequestRecord, + AuthUserRecord, DescriptorKind, DescriptorMetadata, ImportJob, ImportJobApplyResult, + ImportJobId, ImportJobKind, ImportJobStatus, InvitationRecord, InvocationHistoryLoss, + InvocationHistoryLossCategory, InvocationHistoryWriteOutcome, InvocationLogRecord, + MembershipRecord, OperationAgentRef, OperationSampleMetadata, OperationSummary, + OperationUsageSummary, OperationVersionRecord, Page, PlatformApiKeyRecord, + PublishedAgentCatalog, PublishedAgentTool, RegistryOperation, SampleKind, SecretRecord, + SecretVersionRecord, SessionRecord, SkippedImportOperation, UsageAgentBreakdown, UsageBucket, UsageOperationBreakdown, UsageRollupRecord, UsageSummary, UsageTimelinePoint, WorkspaceMembershipRecord, WorkspaceRecord, WorkspaceUpstream, WorkspaceUpstreamId, YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus, @@ -23,11 +25,12 @@ pub mod records { pub mod requests { pub use crate::model::{ - CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest, - CreateImportJobRequest, CreateInvitationRequest, CreateInvocationLogRequest, - CreatePlatformApiKeyRequest, CreateSecretRequest, CreateVersionRequest, - CreateWorkspaceRequest, CreateYamlImportJobRequest, DecideApprovalRequest, - ExpireApprovalRequest, FinishApprovalRequest, FinishImportJobRequest, + ApplyImportJobRequest, CreateAgentDraftVersionRequest, CreateAgentRequest, + CreateApprovalRequest, CreateImportJobRequest, CreateInvitationRequest, + CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest, + CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, + DecideApprovalRequest, ExpireApprovalRequest, FinishApprovalRequest, + FinishImportJobRequest, ImportConflictMode, ImportOperationDraft, ListApprovalRequestsQuery, ListInvocationLogsQuery, PublishAgentRequest, PublishRequest, RotateSecretRequest, SaveAgentBindingsRequest, SaveAgentCatalogConfigRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest, @@ -41,20 +44,23 @@ pub mod infrastructure { } pub use model::{ - AgentSummary, AgentVersionRecord, ApprovalRequestRecord, AuthUserRecord, - CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest, - CreateImportJobRequest, CreateInvitationRequest, CreateInvocationLogRequest, - CreatePlatformApiKeyRequest, CreateSecretRequest, CreateVersionRequest, CreateWorkspaceRequest, - CreateYamlImportJobRequest, DecideApprovalRequest, DescriptorKind, DescriptorMetadata, - ExpireApprovalRequest, FinishApprovalRequest, FinishImportJobRequest, ImportJob, ImportJobId, - ImportJobKind, ImportJobStatus, InvitationRecord, InvocationLogRecord, - ListApprovalRequestsQuery, ListInvocationLogsQuery, MembershipRecord, OperationAgentRef, - OperationSampleMetadata, OperationSummary, OperationUsageSummary, OperationVersionRecord, Page, - PlatformApiKeyRecord, PublishAgentRequest, PublishRequest, PublishedAgentCatalog, - PublishedAgentTool, RegistryOperation, RotateSecretRequest, SampleKind, - SaveAgentBindingsRequest, SaveAgentCatalogConfigRequest, SaveAuthProfileRequest, - SaveDescriptorMetadataRequest, SaveSampleMetadataRequest, SaveWorkspaceUpstreamRequest, - SecretRecord, SecretVersionRecord, SessionRecord, UpdateWorkspaceRequest, UsageAgentBreakdown, + AgentSummary, AgentVersionRecord, AppliedImportOperation, ApplyImportJobRequest, + ApprovalRequestRecord, AuthUserRecord, CreateAgentDraftVersionRequest, CreateAgentRequest, + CreateApprovalRequest, CreateImportJobRequest, CreateInvitationRequest, + CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest, + CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, + DecideApprovalRequest, DescriptorKind, DescriptorMetadata, ExpireApprovalRequest, + FinishApprovalRequest, FinishImportJobRequest, ImportConflictMode, ImportJob, + ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobStatus, ImportOperationDraft, + InvitationRecord, InvocationHistoryLoss, InvocationHistoryLossCategory, + InvocationHistoryWriteOutcome, InvocationLogRecord, ListApprovalRequestsQuery, + ListInvocationLogsQuery, MembershipRecord, OperationAgentRef, OperationSampleMetadata, + OperationSummary, OperationUsageSummary, OperationVersionRecord, Page, PlatformApiKeyRecord, + PublishAgentRequest, PublishRequest, PublishedAgentCatalog, PublishedAgentTool, + RegistryOperation, RotateSecretRequest, SampleKind, SaveAgentBindingsRequest, + SaveAgentCatalogConfigRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest, + SaveSampleMetadataRequest, SaveWorkspaceUpstreamRequest, SecretRecord, SecretVersionRecord, + SessionRecord, SkippedImportOperation, UpdateWorkspaceRequest, UsageAgentBreakdown, UsageBucket, UsageOperationBreakdown, UsageQuery, UsageRollupRecord, UsageSummary, UsageTimelinePoint, WorkspaceMembershipRecord, WorkspaceRecord, WorkspaceUpstream, WorkspaceUpstreamId, YamlImportJob, YamlImportJobCompletion, YamlImportJobId, diff --git a/crates/crank-registry/src/migrations.rs b/crates/crank-registry/src/migrations.rs index 47408b4..4b950b7 100644 --- a/crates/crank-registry/src/migrations.rs +++ b/crates/crank-registry/src/migrations.rs @@ -1,6 +1,61 @@ -use sqlx::{PgPool, query}; +use sqlx::{PgPool, Postgres, Row, Transaction, query}; + +const CORE_MIGRATION_LOCK_ID: i64 = 0x43_52_41_4E_4B; +const BASELINE_VERSION: i32 = 1; +const BASELINE_CHECKSUM: &str = "crank-community-baseline-v1"; pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { + let mut transaction = pool.begin().await?; + query("select pg_advisory_xact_lock($1)") + .bind(CORE_MIGRATION_LOCK_ID) + .execute(&mut *transaction) + .await?; + query( + "create table if not exists __crank_core_migrations ( + version integer primary key, + description text not null, + checksum text not null, + applied_at timestamptz not null default now() + )", + ) + .execute(&mut *transaction) + .await?; + + let applied = query( + "select version, checksum + from __crank_core_migrations + order by version", + ) + .fetch_all(&mut *transaction) + .await?; + for row in &applied { + let version = row.try_get::("version")?; + let checksum = row.try_get::("checksum")?; + if version != BASELINE_VERSION || checksum != BASELINE_CHECKSUM { + return Err(sqlx::Error::Protocol(format!( + "unsupported or modified core migration: version={version}, checksum={checksum}" + ))); + } + } + + if applied.is_empty() { + apply_baseline(&mut transaction).await?; + query( + "insert into __crank_core_migrations (version, description, checksum) + values ($1, $2, $3)", + ) + .bind(BASELINE_VERSION) + .bind("community baseline") + .bind(BASELINE_CHECKSUM) + .execute(&mut *transaction) + .await?; + } + + transaction.commit().await?; + Ok(()) +} + +async fn apply_baseline(transaction: &mut Transaction<'_, Postgres>) -> Result<(), sqlx::Error> { query( "create table if not exists workspaces ( id text primary key, @@ -12,7 +67,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { updated_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -25,11 +80,11 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { created_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table users add column if not exists password_hash text null") - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -48,7 +103,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { ) on conflict (id) do nothing", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -60,7 +115,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { primary key (workspace_id, user_id) )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -75,14 +130,14 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { created_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "alter table user_sessions add column if not exists current_workspace_id text null references workspaces(id) on delete set null", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -105,7 +160,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { ) on conflict (id) do nothing", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -122,7 +177,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { ) on conflict (workspace_id, user_id) do nothing", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -137,7 +192,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { created_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -158,29 +213,29 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { allowed_origins_json jsonb not null default '[]'::jsonb )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists platform_api_keys_workspace_name_idx on platform_api_keys(workspace_id, name)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table platform_api_keys add column if not exists agent_id text null") - .execute(pool) + .execute(&mut **transaction) .await?; query( "alter table platform_api_keys add column if not exists key_kind text not null default 'mcp_client'", ) - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table platform_api_keys add column if not exists expires_at timestamptz null") - .execute(pool) + .execute(&mut **transaction) .await?; query( "alter table platform_api_keys add column if not exists allowed_origins_json jsonb not null default '[]'::jsonb", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -203,7 +258,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { ) on conflict (id) do nothing", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -223,35 +278,35 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { published_at timestamptz null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table operations add column if not exists workspace_id text null references workspaces(id) on delete cascade") - .execute(pool) + .execute(&mut **transaction) .await?; query( "alter table operations add column if not exists category text not null default 'general'", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "alter table operations add column if not exists security_level text not null default 'standard'", ) - .execute(pool) + .execute(&mut **transaction) .await?; query("update operations set workspace_id = 'ws_default' where workspace_id is null") - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table operations alter column workspace_id set not null") - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table operations drop constraint if exists operations_name_key") - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists operations_workspace_name_idx on operations(workspace_id, name)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -276,11 +331,11 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { primary key (operation_id, version) )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table operation_versions add column if not exists wizard_state_json jsonb null") - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -292,7 +347,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { foreign key (operation_id, version) references operation_versions(operation_id, version) on delete cascade )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -308,7 +363,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { foreign key (operation_id, version) references operation_versions(operation_id, version) on delete cascade )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -324,7 +379,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { foreign key (operation_id, version) references operation_versions(operation_id, version) on delete cascade )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -338,25 +393,25 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { updated_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table auth_profiles add column if not exists workspace_id text null references workspaces(id) on delete cascade") - .execute(pool) + .execute(&mut **transaction) .await?; query("update auth_profiles set workspace_id = 'ws_default' where workspace_id is null") - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table auth_profiles alter column workspace_id set not null") - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table auth_profiles drop constraint if exists auth_profiles_name_key") - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists auth_profiles_workspace_name_idx on auth_profiles(workspace_id, name)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -371,17 +426,17 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { updated_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists workspace_upstreams_workspace_name_idx on workspace_upstreams(workspace_id, name)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists workspace_upstreams_workspace_base_auth_idx on workspace_upstreams(workspace_id, base_url, coalesce(auth_profile_id, ''))", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "insert into workspace_upstreams ( @@ -411,7 +466,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { and wu.name = 'Frankfurter' )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -427,13 +482,13 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { updated_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists secrets_workspace_name_idx on secrets(workspace_id, name)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -447,7 +502,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { primary key (secret_id, version) )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -464,7 +519,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { finished_at timestamptz null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -483,7 +538,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { finished_at timestamptz null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -501,13 +556,13 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { published_at timestamptz null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists agents_workspace_slug_idx on agents(workspace_id, slug)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -521,7 +576,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { primary key (agent_id, version) )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -538,13 +593,13 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { foreign key (operation_id, operation_version) references operation_versions(operation_id, version) on delete cascade )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists agent_bindings_tool_name_idx on agent_operation_bindings(agent_id, agent_version, tool_name)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -556,7 +611,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { foreign key (agent_id, version) references agent_versions(agent_id, version) on delete cascade )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -577,36 +632,30 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { decision_note text null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; - query("alter table approval_requests drop column if exists confirmation_title") - .execute(pool) - .await?; - query("alter table approval_requests drop column if exists confirmation_body") - .execute(pool) - .await?; query("alter table approval_requests add column if not exists execution_started_at timestamptz null") - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table approval_requests add column if not exists execution_attempts integer not null default 0") - .execute(pool) + .execute(&mut **transaction) .await?; query("alter table approval_requests add column if not exists request_fingerprint text null") - .execute(pool) + .execute(&mut **transaction) .await?; query( "create unique index if not exists approval_requests_pending_fingerprint_idx on approval_requests(agent_id, operation_id, operation_version, request_fingerprint) where status = 'pending' and request_fingerprint is not null", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create index if not exists approval_requests_agent_status_idx on approval_requests(workspace_id, agent_id, status, expires_at)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -629,25 +678,25 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { created_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create index if not exists invocation_logs_workspace_created_idx on invocation_logs(workspace_id, created_at desc)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create index if not exists invocation_logs_workspace_operation_created_idx on invocation_logs(workspace_id, operation_id, created_at desc)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( "create index if not exists invocation_logs_workspace_agent_created_idx on invocation_logs(workspace_id, agent_id, created_at desc)", ) - .execute(pool) + .execute(&mut **transaction) .await?; query( @@ -665,7 +714,7 @@ pub async fn apply_postgres(pool: &PgPool) -> Result<(), sqlx::Error> { updated_at timestamptz not null )", ) - .execute(pool) + .execute(&mut **transaction) .await?; Ok(()) diff --git a/crates/crank-registry/src/model.rs b/crates/crank-registry/src/model.rs index 62343e1..992d474 100644 --- a/crates/crank-registry/src/model.rs +++ b/crates/crank-registry/src/model.rs @@ -395,11 +395,92 @@ pub struct FinishImportJobRequest<'a> { pub finished_at: &'a OffsetDateTime, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportConflictMode { + Skip, + Rename, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImportOperationDraft { + pub operation_key: String, + pub operation: RegistryOperation, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AppliedImportOperation { + pub operation_key: String, + pub operation_id: OperationId, + pub name: String, + pub version: u32, + pub renamed_from: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SkippedImportOperation { + pub operation_key: String, + pub name: String, + pub reason: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImportJobApplyResult { + pub application_key: String, + pub created: Vec, + pub skipped: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ApplyImportJobRequest<'a> { + pub id: &'a ImportJobId, + pub workspace_id: &'a WorkspaceId, + pub application_key: &'a str, + pub conflict_mode: ImportConflictMode, + pub operations: &'a [ImportOperationDraft], + pub finished_at: &'a OffsetDateTime, +} + #[derive(Clone, Debug, PartialEq)] pub struct CreateInvocationLogRequest<'a> { pub log: &'a InvocationLog, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum InvocationHistoryLossCategory { + Unavailable, + InvalidRecord, +} + +impl InvocationHistoryLossCategory { + pub fn as_str(self) -> &'static str { + match self { + Self::Unavailable => "unavailable", + Self::InvalidRecord => "invalid_record", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct InvocationHistoryLoss { + pub category: InvocationHistoryLossCategory, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum InvocationHistoryWriteOutcome { + Recorded, + Lost(InvocationHistoryLoss), +} + +impl InvocationHistoryWriteOutcome { + pub fn loss(self) -> Option { + match self { + Self::Recorded => None, + Self::Lost(loss) => Some(loss), + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ListInvocationLogsQuery<'a> { pub workspace_id: &'a WorkspaceId, diff --git a/crates/crank-registry/src/postgres/api_key.rs b/crates/crank-registry/src/postgres/api_key.rs index ebe9782..20a771d 100644 --- a/crates/crank-registry/src/postgres/api_key.rs +++ b/crates/crank-registry/src/postgres/api_key.rs @@ -397,18 +397,31 @@ impl PostgresRegistry { key_id: &PlatformApiKeyId, used_at: &time::OffsetDateTime, ) -> Result<(), RegistryError> { - let result = sqlx::query( - "update platform_api_keys - set last_used_at = $3::timestamptz - where workspace_id = $1 and id = $2", + let exists = sqlx::query_scalar::<_, bool>( + "with target as ( + select id + from platform_api_keys + where workspace_id = $1 and id = $2 + ), updated as ( + update platform_api_keys + set last_used_at = $3::timestamptz + where workspace_id = $1 + and id = $2 + and ( + last_used_at is null + or last_used_at < $3::timestamptz - interval '1 minute' + ) + returning id + ) + select exists(select 1 from target)", ) .bind(workspace_id.as_str()) .bind(key_id.as_str()) .bind(used_at) - .execute(&self.pool) + .fetch_one(&self.pool) .await?; - if result.rows_affected() == 0 { + if !exists { return Err(RegistryError::PlatformApiKeyNotFound { key_id: key_id.as_str().to_owned(), }); diff --git a/crates/crank-registry/src/postgres/approval.rs b/crates/crank-registry/src/postgres/approval.rs index 4b553e9..f3f0ee3 100644 --- a/crates/crank-registry/src/postgres/approval.rs +++ b/crates/crank-registry/src/postgres/approval.rs @@ -357,21 +357,20 @@ impl PostgresRegistry { &self, started_at: OffsetDateTime, approved_before: OffsetDateTime, - stale_before: OffsetDateTime, ) -> Result, RegistryError> { let row = sqlx::query( "with candidate as ( select id from approval_requests - where (status = 'approved' and decided_at <= $1) - or (status = 'executing' and execution_started_at < $2) + where status = 'approved' + and decided_at <= $1 order by decided_at asc nulls last, created_at asc for update skip locked limit 1 ) update approval_requests as approval set status = 'executing', - execution_started_at = $3, + execution_started_at = $2, execution_attempts = approval.execution_attempts + 1 from candidate where approval.id = candidate.id @@ -383,7 +382,6 @@ impl PostgresRegistry { approval.decided_at, approval.decided_by_key_id, approval.decision_note", ) .bind(approved_before) - .bind(stale_before) .bind(started_at) .fetch_optional(&self.pool) .await?; @@ -391,6 +389,50 @@ impl PostgresRegistry { row.map(map_approval_request_row).transpose() } + pub async fn fail_next_interrupted_approval_request( + &self, + stale_before: OffsetDateTime, + ) -> Result, RegistryError> { + let response_payload = serde_json::json!({ + "error": { + "code": "approval_execution_outcome_unknown", + "message": "execution was interrupted; the operation was not retried automatically" + } + }); + let row = sqlx::query( + "with candidate as ( + select id + from approval_requests + where status = 'executing' + and execution_started_at < $1 + order by execution_started_at asc, created_at asc + for update skip locked + limit 1 + ) + update approval_requests as approval + set status = 'failed', + response_payload_json = $2, + decision_note = coalesce( + approval.decision_note, + 'execution interrupted; outcome unknown' + ) + from candidate + where approval.id = candidate.id + returning + approval.id, approval.workspace_id, approval.agent_id, + approval.operation_id, approval.operation_version, approval.status, + approval.risk_level, approval.request_payload_json, + approval.response_payload_json, approval.created_at, approval.expires_at, + approval.decided_at, approval.decided_by_key_id, approval.decision_note", + ) + .bind(stale_before) + .bind(Json(response_payload)) + .fetch_optional(&self.pool) + .await?; + + row.map(map_approval_request_row).transpose() + } + pub async fn expire_approval_request( &self, request: ExpireApprovalRequest<'_>, diff --git a/crates/crank-registry/src/postgres/auth.rs b/crates/crank-registry/src/postgres/auth.rs index 9d45d61..0683f2a 100644 --- a/crates/crank-registry/src/postgres/auth.rs +++ b/crates/crank-registry/src/postgres/auth.rs @@ -1,15 +1,31 @@ use super::*; use time::OffsetDateTime; +fn map_auth_user_row(row: &PgRow) -> Result { + let status = row.try_get::("status")?; + Ok(AuthUserRecord { + user: User { + id: UserId::new(row.try_get::("id")?), + email: row.try_get("email")?, + display_name: row.try_get("display_name")?, + status: deserialize_enum_text(&status, "status")?, + created_at: row.try_get::("created_at")?, + }, + password_hash: row + .try_get::, _>("password_hash")? + .unwrap_or_default(), + }) +} + impl PostgresRegistry { - pub async fn upsert_bootstrap_user( + pub async fn ensure_bootstrap_user( &self, email: &str, display_name: &str, password_hash: &str, ) -> Result { let user_id = format!("user_{}", uuid::Uuid::now_v7().simple()); - let row = sqlx::query!( + if let Some(id) = sqlx::query_scalar::<_, String>( "insert into users ( id, email, @@ -24,16 +40,40 @@ impl PostgresRegistry { set display_name = excluded.display_name, password_hash = excluded.password_hash, status = 'active' + where users.password_hash is null returning id", - user_id, - email, - display_name, - password_hash, ) + .bind(user_id) + .bind(email) + .bind(display_name) + .bind(password_hash) + .fetch_optional(&self.pool) + .await? + { + return Ok(UserId::new(id)); + } + + let existing = sqlx::query_scalar::<_, String>( + "select id + from users + where email = $1 + limit 1", + ) + .bind(email) .fetch_one(&self.pool) .await?; - Ok(UserId::new(row.id)) + Ok(UserId::new(existing)) + } + + pub async fn upsert_bootstrap_user( + &self, + email: &str, + display_name: &str, + password_hash: &str, + ) -> Result { + self.ensure_bootstrap_user(email, display_name, password_hash) + .await } pub async fn ensure_membership( @@ -67,70 +107,46 @@ impl PostgresRegistry { &self, email: &str, ) -> Result, RegistryError> { - let row = sqlx::query!( + let row = sqlx::query( "select id, email, display_name, - password_hash as \"password_hash!\", + password_hash, status, - created_at as \"created_at!: OffsetDateTime\" + created_at from users where email = $1 limit 1", - email, ) + .bind(email) .fetch_optional(&self.pool) .await?; - row.map(|row| { - Ok(AuthUserRecord { - user: User { - id: UserId::new(row.id), - email: row.email, - display_name: row.display_name, - status: deserialize_enum_text(&row.status, "status")?, - created_at: row.created_at, - }, - password_hash: row.password_hash, - }) - }) - .transpose() + row.as_ref().map(map_auth_user_row).transpose() } pub async fn get_auth_user_by_id( &self, user_id: &UserId, ) -> Result, RegistryError> { - let row = sqlx::query!( + let row = sqlx::query( "select id, email, display_name, - password_hash as \"password_hash!\", + password_hash, status, - created_at as \"created_at!: OffsetDateTime\" + created_at from users where id = $1 limit 1", - user_id.as_str(), ) + .bind(user_id.as_str()) .fetch_optional(&self.pool) .await?; - row.map(|row| { - Ok(AuthUserRecord { - user: User { - id: UserId::new(row.id), - email: row.email, - display_name: row.display_name, - status: deserialize_enum_text(&row.status, "status")?, - created_at: row.created_at, - }, - password_hash: row.password_hash, - }) - }) - .transpose() + row.as_ref().map(map_auth_user_row).transpose() } pub async fn update_user_profile( @@ -191,6 +207,45 @@ impl PostgresRegistry { Ok(()) } + pub async fn update_user_password_and_revoke_other_sessions( + &self, + user_id: &UserId, + current_session_id: &UserSessionId, + password_hash: &str, + ) -> Result<(), RegistryError> { + let mut tx = self.pool.begin().await?; + let result = sqlx::query( + "update users + set password_hash = $2 + where id = $1", + ) + .bind(user_id.as_str()) + .bind(password_hash) + .execute(&mut *tx) + .await?; + + if result.rows_affected() == 0 { + return Err(RegistryError::UserNotFound { + user_id: user_id.as_str().to_owned(), + }); + } + + sqlx::query( + "update user_sessions + set status = 'revoked' + where user_id = $1 + and id <> $2 + and status = 'active'", + ) + .bind(user_id.as_str()) + .bind(current_session_id.as_str()) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(()) + } + pub async fn create_user_session( &self, session_id: &UserSessionId, @@ -305,7 +360,11 @@ impl PostgresRegistry { sqlx::query( "update user_sessions set last_seen_at = now() - where id = $1", + where id = $1 + and ( + last_seen_at is null + or last_seen_at < now() - interval '1 minute' + )", ) .bind(session_id.as_str()) .execute(&self.pool) diff --git a/crates/crank-registry/src/postgres/connection.rs b/crates/crank-registry/src/postgres/connection.rs index 1f911d6..8a1ae7e 100644 --- a/crates/crank-registry/src/postgres/connection.rs +++ b/crates/crank-registry/src/postgres/connection.rs @@ -41,6 +41,11 @@ impl PostgresRegistry { &self.pool } + pub async fn ping(&self) -> Result<(), RegistryError> { + sqlx::query("select 1").execute(&self.pool).await?; + Ok(()) + } + pub async fn migrate(&self) -> Result<(), RegistryError> { migrations::apply_postgres(&self.pool).await?; Ok(()) diff --git a/crates/crank-registry/src/postgres/import_job.rs b/crates/crank-registry/src/postgres/import_job.rs index f81d11e..9d10aca 100644 --- a/crates/crank-registry/src/postgres/import_job.rs +++ b/crates/crank-registry/src/postgres/import_job.rs @@ -1,5 +1,7 @@ use super::*; +const APPLICATION_RESULT_KEY: &str = "_crank_application_result"; + impl PostgresRegistry { pub async fn create_import_job( &self, @@ -97,6 +99,43 @@ impl PostgresRegistry { Ok(()) } + pub async fn apply_import_job( + &self, + request: ApplyImportJobRequest<'_>, + ) -> Result { + let mut tx = self.pool.begin().await?; + let applied = apply_import_job_transaction(&mut tx, &request).await; + + match applied { + Ok(result) => { + tx.commit().await?; + Ok(result) + } + Err(error) => { + tx.rollback().await?; + let error_text = error.to_string(); + let _ = sqlx::query( + "update import_jobs + set status = $3, + error_text = $4, + finished_at = $5::timestamptz + where id = $1 + and workspace_id = $2 + and status <> $6", + ) + .bind(request.id.as_str()) + .bind(request.workspace_id.as_str()) + .bind(serialize_enum_text(&ImportJobStatus::Failed, "status")?) + .bind(error_text) + .bind(request.finished_at) + .bind(serialize_enum_text(&ImportJobStatus::Completed, "status")?) + .execute(&self.pool) + .await; + Err(error) + } + } + } + pub async fn delete_expired_import_jobs(&self) -> Result { let result = sqlx::query("delete from import_jobs where expires_at < now()") .execute(&self.pool) @@ -105,3 +144,160 @@ impl PostgresRegistry { Ok(result.rows_affected()) } } + +async fn apply_import_job_transaction( + tx: &mut Transaction<'_, Postgres>, + request: &ApplyImportJobRequest<'_>, +) -> Result { + let row = sqlx::query( + "select status, preview_payload + from import_jobs + where id = $1 and workspace_id = $2 + for update", + ) + .bind(request.id.as_str()) + .bind(request.workspace_id.as_str()) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(|| RegistryError::ImportJobNotFound { + job_id: request.id.as_str().to_owned(), + })?; + let status = deserialize_enum_text::(row.try_get("status")?, "status")?; + let mut preview_payload = row.try_get::("preview_payload")?; + + if status == ImportJobStatus::Completed + && let Some(result) = stored_application_result(&preview_payload)? + { + if result.application_key != request.application_key { + return Err(RegistryError::ImportJobAlreadyApplied { + job_id: request.id.as_str().to_owned(), + }); + } + return Ok(result); + } + + sqlx::query("select id from workspaces where id = $1 for update") + .bind(request.workspace_id.as_str()) + .fetch_one(&mut **tx) + .await?; + + let mut result = ImportJobApplyResult { + application_key: request.application_key.to_owned(), + ..ImportJobApplyResult::default() + }; + for draft in request.operations { + if draft.operation.version != 1 { + return Err(RegistryError::InvalidInitialVersion { + operation_id: draft.operation.id.as_str().to_owned(), + version: draft.operation.version, + }); + } + + let mut operation = draft.operation.clone(); + let original_name = operation.name.clone(); + if operation_name_exists(tx, request.workspace_id, &operation.name).await? { + match request.conflict_mode { + ImportConflictMode::Skip => { + result.skipped.push(SkippedImportOperation { + operation_key: draft.operation_key.clone(), + name: operation.name, + reason: "operation_name_conflict".to_owned(), + }); + continue; + } + ImportConflictMode::Rename => { + operation.name = + next_available_operation_name(tx, request.workspace_id, &operation.name) + .await?; + } + } + } + + insert_operation_rows(tx, request.workspace_id, &operation, None).await?; + result.created.push(AppliedImportOperation { + operation_key: draft.operation_key.clone(), + operation_id: operation.id, + name: operation.name.clone(), + version: operation.version, + renamed_from: (operation.name != original_name).then_some(original_name), + }); + } + + let stored_result = serde_json::to_value(&result)?; + if let Some(object) = preview_payload.as_object_mut() { + object.insert(APPLICATION_RESULT_KEY.to_owned(), stored_result); + } else { + preview_payload = serde_json::json!({ + "preview": preview_payload, + "_crank_application_result": stored_result, + }); + } + let created_operation_ids = serde_json::to_value( + result + .created + .iter() + .map(|operation| operation.operation_id.as_str()) + .collect::>(), + )?; + sqlx::query( + "update import_jobs + set status = $3, + preview_payload = $4, + created_operation_ids = $5, + error_text = null, + finished_at = $6::timestamptz + where id = $1 and workspace_id = $2", + ) + .bind(request.id.as_str()) + .bind(request.workspace_id.as_str()) + .bind(serialize_enum_text(&ImportJobStatus::Completed, "status")?) + .bind(preview_payload) + .bind(created_operation_ids) + .bind(request.finished_at) + .execute(&mut **tx) + .await?; + + Ok(result) +} + +fn stored_application_result( + preview_payload: &Value, +) -> Result, RegistryError> { + preview_payload + .get(APPLICATION_RESULT_KEY) + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(RegistryError::from) +} + +async fn operation_name_exists( + tx: &mut Transaction<'_, Postgres>, + workspace_id: &WorkspaceId, + name: &str, +) -> Result { + Ok(sqlx::query_scalar::<_, bool>( + "select exists( + select 1 from operations where workspace_id = $1 and name = $2 + )", + ) + .bind(workspace_id.as_str()) + .bind(name) + .fetch_one(&mut **tx) + .await?) +} + +async fn next_available_operation_name( + tx: &mut Transaction<'_, Postgres>, + workspace_id: &WorkspaceId, + base_name: &str, +) -> Result { + for index in 2.. { + let candidate = format!("{base_name}_{index}"); + if !operation_name_exists(tx, workspace_id, &candidate).await? { + return Ok(candidate); + } + } + + unreachable!() +} diff --git a/crates/crank-registry/src/postgres/mod.rs b/crates/crank-registry/src/postgres/mod.rs index bb9cd5b..ce279be 100644 --- a/crates/crank-registry/src/postgres/mod.rs +++ b/crates/crank-registry/src/postgres/mod.rs @@ -30,23 +30,25 @@ pub use pool_config::{PostgresPoolConfig, PostgresPoolConfigError}; use crate::{ error::RegistryError, model::{ - AgentSummary, AgentVersionRecord, ApprovalRequestRecord, AuthUserRecord, - CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest, - CreateImportJobRequest, CreateInvitationRequest, CreateInvocationLogRequest, - CreatePlatformApiKeyRequest, CreateSecretRequest, CreateVersionRequest, - CreateWorkspaceRequest, CreateYamlImportJobRequest, DecideApprovalRequest, - DescriptorMetadata, ExpireApprovalRequest, FinishApprovalRequest, FinishImportJobRequest, - ImportJob, ImportJobId, InvitationRecord, InvocationLogRecord, ListApprovalRequestsQuery, + AgentSummary, AgentVersionRecord, AppliedImportOperation, ApplyImportJobRequest, + ApprovalRequestRecord, AuthUserRecord, CreateAgentDraftVersionRequest, CreateAgentRequest, + CreateApprovalRequest, CreateImportJobRequest, CreateInvitationRequest, + CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest, + CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest, + DecideApprovalRequest, DescriptorMetadata, ExpireApprovalRequest, FinishApprovalRequest, + FinishImportJobRequest, ImportConflictMode, ImportJob, ImportJobApplyResult, ImportJobId, + ImportJobStatus, InvitationRecord, InvocationHistoryLoss, InvocationHistoryLossCategory, + InvocationHistoryWriteOutcome, InvocationLogRecord, ListApprovalRequestsQuery, ListInvocationLogsQuery, MembershipRecord, OperationAgentRef, OperationSampleMetadata, OperationSummary, OperationUsageSummary, OperationVersionRecord, PlatformApiKeyRecord, PublishAgentRequest, PublishRequest, PublishedAgentCatalog, PublishedAgentTool, RegistryOperation, RotateSecretRequest, SaveAgentBindingsRequest, SaveAgentCatalogConfigRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest, SaveWorkspaceUpstreamRequest, SecretRecord, SecretVersionRecord, - SessionRecord, UpdateWorkspaceRequest, UsageAgentBreakdown, UsageOperationBreakdown, - UsageQuery, UsageRollupRecord, UsageSummary, UsageTimelinePoint, WorkspaceMembershipRecord, - WorkspaceRecord, WorkspaceUpstream, YamlImportJob, YamlImportJobCompletion, - YamlImportJobId, YamlImportJobStatus, + SessionRecord, SkippedImportOperation, UpdateWorkspaceRequest, UsageAgentBreakdown, + UsageOperationBreakdown, UsageQuery, UsageRollupRecord, UsageSummary, UsageTimelinePoint, + WorkspaceMembershipRecord, WorkspaceRecord, WorkspaceUpstream, YamlImportJob, + YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus, }, }; @@ -135,6 +137,61 @@ async fn insert_version_row( Ok(()) } +async fn insert_operation_rows( + tx: &mut Transaction<'_, Postgres>, + workspace_id: &WorkspaceId, + snapshot: &RegistryOperation, + created_by: Option<&str>, +) -> Result<(), RegistryError> { + sqlx::query( + "insert into operations ( + id, + workspace_id, + name, + display_name, + category, + protocol, + security_level, + status, + current_draft_version, + latest_published_version, + created_at, + updated_at, + published_at + ) values ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11::timestamptz, + $12::timestamptz, + $13::timestamptz + )", + ) + .bind(snapshot.id.as_str()) + .bind(workspace_id.as_str()) + .bind(&snapshot.name) + .bind(&snapshot.display_name) + .bind(&snapshot.category) + .bind(serialize_enum_text(&snapshot.protocol, "protocol")?) + .bind(serialize_enum_text( + &snapshot.security_level, + "security_level", + )?) + .bind(serialize_enum_text(&snapshot.status, "status")?) + .bind(to_db_version(snapshot.version)) + .bind( + snapshot + .published_at + .as_ref() + .map(|_| to_db_version(snapshot.version)), + ) + .bind(snapshot.created_at) + .bind(snapshot.updated_at) + .bind(snapshot.published_at) + .execute(&mut **tx) + .await?; + + insert_version_row(tx, snapshot, None, created_by).await +} + async fn insert_agent_version_row( tx: &mut Transaction<'_, Postgres>, version: &AgentVersion, diff --git a/crates/crank-registry/src/postgres/observability.rs b/crates/crank-registry/src/postgres/observability.rs index fb06911..4da565e 100644 --- a/crates/crank-registry/src/postgres/observability.rs +++ b/crates/crank-registry/src/postgres/observability.rs @@ -1,9 +1,47 @@ use super::*; +fn invocation_history_loss_category(error: &RegistryError) -> InvocationHistoryLossCategory { + match error { + RegistryError::Storage(error) + if error + .as_database_error() + .and_then(|database_error| database_error.code()) + .is_some_and(|code| code.starts_with("22") || code.starts_with("23")) => + { + InvocationHistoryLossCategory::InvalidRecord + } + RegistryError::Storage(_) => InvocationHistoryLossCategory::Unavailable, + _ => InvocationHistoryLossCategory::InvalidRecord, + } +} + impl PostgresRegistry { + pub async fn delete_invocation_logs_before( + &self, + cutoff: OffsetDateTime, + ) -> Result { + let result = sqlx::query("delete from invocation_logs where created_at < $1::timestamptz") + .bind(cutoff) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + pub async fn create_invocation_log( &self, request: CreateInvocationLogRequest<'_>, + ) -> InvocationHistoryWriteOutcome { + match self.try_create_invocation_log(request).await { + Ok(()) => InvocationHistoryWriteOutcome::Recorded, + Err(error) => InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss { + category: invocation_history_loss_category(&error), + }), + } + } + + async fn try_create_invocation_log( + &self, + request: CreateInvocationLogRequest<'_>, ) -> Result<(), RegistryError> { let request_preview = crank_core::sanitize_invocation_preview(&request.log.request_preview); let response_preview = diff --git a/crates/crank-registry/src/postgres/operation.rs b/crates/crank-registry/src/postgres/operation.rs index db0191f..71219a7 100644 --- a/crates/crank-registry/src/postgres/operation.rs +++ b/crates/crank-registry/src/postgres/operation.rs @@ -26,53 +26,7 @@ impl PostgresRegistry { let mut tx = self.pool.begin().await?; - sqlx::query( - "insert into operations ( - id, - workspace_id, - name, - display_name, - category, - protocol, - security_level, - status, - current_draft_version, - latest_published_version, - created_at, - updated_at, - published_at - ) values ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, - $11::timestamptz, - $12::timestamptz, - $13::timestamptz - )", - ) - .bind(snapshot.id.as_str()) - .bind(workspace_id.as_str()) - .bind(&snapshot.name) - .bind(&snapshot.display_name) - .bind(&snapshot.category) - .bind(serialize_enum_text(&snapshot.protocol, "protocol")?) - .bind(serialize_enum_text( - &snapshot.security_level, - "security_level", - )?) - .bind(serialize_enum_text(&snapshot.status, "status")?) - .bind(to_db_version(snapshot.version)) - .bind( - snapshot - .published_at - .as_ref() - .map(|_| to_db_version(snapshot.version)), - ) - .bind(snapshot.created_at) - .bind(snapshot.updated_at) - .bind(snapshot.published_at) - .execute(&mut *tx) - .await?; - - insert_version_row(&mut tx, snapshot, None, created_by).await?; + insert_operation_rows(&mut tx, workspace_id, snapshot, created_by).await?; tx.commit().await?; Ok(()) diff --git a/crates/crank-registry/src/postgres/secret.rs b/crates/crank-registry/src/postgres/secret.rs index 9aab09c..41eee65 100644 --- a/crates/crank-registry/src/postgres/secret.rs +++ b/crates/crank-registry/src/postgres/secret.rs @@ -282,18 +282,31 @@ impl PostgresRegistry { secret_id: &SecretId, used_at: &OffsetDateTime, ) -> Result<(), RegistryError> { - let result = sqlx::query( - "update secrets - set last_used_at = $3::timestamptz - where workspace_id = $1 and id = $2", + let exists = sqlx::query_scalar::<_, bool>( + "with target as ( + select id + from secrets + where workspace_id = $1 and id = $2 + ), updated as ( + update secrets + set last_used_at = $3::timestamptz + where workspace_id = $1 + and id = $2 + and ( + last_used_at is null + or last_used_at < $3::timestamptz - interval '1 minute' + ) + returning id + ) + select exists(select 1 from target)", ) .bind(workspace_id.as_str()) .bind(secret_id.as_str()) .bind(used_at) - .execute(&self.pool) + .fetch_one(&self.pool) .await?; - if result.rows_affected() == 0 { + if !exists { return Err(RegistryError::SecretNotFound { secret_id: secret_id.as_str().to_owned(), }); diff --git a/crates/crank-registry/tests/integration.rs b/crates/crank-registry/tests/integration.rs index 27257d8..737019f 100644 --- a/crates/crank-registry/tests/integration.rs +++ b/crates/crank-registry/tests/integration.rs @@ -1,6 +1,8 @@ mod integration { mod agents_usage; mod common; + mod migrations; + mod observability; mod operations_artifacts; mod workspace_access; } diff --git a/crates/crank-registry/tests/integration/agents_usage.rs b/crates/crank-registry/tests/integration/agents_usage.rs index 9062f5c..509e3c9 100644 --- a/crates/crank-registry/tests/integration/agents_usage.rs +++ b/crates/crank-registry/tests/integration/agents_usage.rs @@ -228,32 +228,36 @@ async fn manages_operation_usage_and_agent_ref_reads() { }) .await .unwrap(); - registry - .create_invocation_log(CreateInvocationLogRequest { - log: &test_invocation_log( - "log_usage_ok", - &operation.id, - Some(agent.id.clone()), - crank_core::InvocationStatus::Ok, - 120, - "2026-03-25T12:20:00Z", - ), - }) - .await - .unwrap(); - registry - .create_invocation_log(CreateInvocationLogRequest { - log: &test_invocation_log( - "log_usage_err", - &operation.id, - Some(agent.id.clone()), - crank_core::InvocationStatus::Error, - 240, - "2026-03-25T12:21:00Z", - ), - }) - .await - .unwrap(); + assert_eq!( + registry + .create_invocation_log(CreateInvocationLogRequest { + log: &test_invocation_log( + "log_usage_ok", + &operation.id, + Some(agent.id.clone()), + crank_core::InvocationStatus::Ok, + 120, + "2026-03-25T12:20:00Z", + ), + }) + .await, + crank_registry::InvocationHistoryWriteOutcome::Recorded + ); + assert_eq!( + registry + .create_invocation_log(CreateInvocationLogRequest { + log: &test_invocation_log( + "log_usage_err", + &operation.id, + Some(agent.id.clone()), + crank_core::InvocationStatus::Error, + 240, + "2026-03-25T12:21:00Z", + ), + }) + .await, + crank_registry::InvocationHistoryWriteOutcome::Recorded + ); let has_bindings = registry .has_published_agent_bindings_for_operation(&test_workspace_id(), &operation.id) diff --git a/crates/crank-registry/tests/integration/migrations.rs b/crates/crank-registry/tests/integration/migrations.rs new file mode 100644 index 0000000..2b1b8e7 --- /dev/null +++ b/crates/crank-registry/tests/integration/migrations.rs @@ -0,0 +1,44 @@ +use crank_registry::PostgresRegistry; +use sqlx::Row; + +#[tokio::test] +async fn core_migration_is_versioned_and_safe_under_concurrent_startup() { + let database_url = crank_test_support::postgres_schema_url("test_core_migration").await; + + let (first, second) = tokio::join!( + PostgresRegistry::connect(&database_url), + PostgresRegistry::connect(&database_url), + ); + let first = first.expect("first service startup must apply the migration"); + second.expect("second service startup must observe the applied migration"); + + let rows = sqlx::query( + "select version, description, checksum from __crank_core_migrations order by version", + ) + .fetch_all(first.pool()) + .await + .expect("migration ledger must be readable"); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].get::("version"), 1); + assert_eq!( + rows[0].get::("description"), + "community baseline" + ); + assert_eq!( + rows[0].get::("checksum"), + "crank-community-baseline-v1" + ); + + let approval_columns = sqlx::query( + "select column_name + from information_schema.columns + where table_schema = current_schema() + and table_name = 'approval_requests' + and column_name in ('execution_started_at', 'execution_attempts', 'request_fingerprint')", + ) + .fetch_all(first.pool()) + .await + .expect("approval schema must be readable"); + assert_eq!(approval_columns.len(), 3); +} diff --git a/crates/crank-registry/tests/integration/observability.rs b/crates/crank-registry/tests/integration/observability.rs new file mode 100644 index 0000000..db2e3fa --- /dev/null +++ b/crates/crank-registry/tests/integration/observability.rs @@ -0,0 +1,31 @@ +use crank_registry::{ + CreateInvocationLogRequest, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome, +}; + +use super::common::{TestDatabase, test_invocation_log}; + +#[tokio::test] +async fn invocation_history_write_returns_typed_loss_without_error_details() { + let database = TestDatabase::new().await; + let registry = database.registry().await; + let log = test_invocation_log( + "log_missing_owner", + &crank_core::OperationId::new("op_missing"), + None, + crank_core::InvocationStatus::Ok, + 10, + "2026-03-25T12:20:00Z", + ); + + let outcome = registry + .create_invocation_log(CreateInvocationLogRequest { log: &log }) + .await; + + assert_eq!( + outcome, + InvocationHistoryWriteOutcome::Lost(crank_registry::InvocationHistoryLoss { + category: InvocationHistoryLossCategory::InvalidRecord, + }) + ); + database.cleanup().await; +} diff --git a/crates/crank-registry/tests/integration/workspace_access.rs b/crates/crank-registry/tests/integration/workspace_access.rs index 5e2a3e8..c402214 100644 --- a/crates/crank-registry/tests/integration/workspace_access.rs +++ b/crates/crank-registry/tests/integration/workspace_access.rs @@ -38,6 +38,38 @@ fn timestamp(value: &str) -> OffsetDateTime { OffsetDateTime::parse(value, &Rfc3339).unwrap() } +#[tokio::test] +async fn bootstrap_user_does_not_overwrite_an_existing_password() { + let database = TestDatabase::new().await; + let registry = database.registry().await; + let email = "bootstrap-owner@example.com"; + + let user_id = registry + .upsert_bootstrap_user(email, "Bootstrap Owner", "initial-bootstrap-hash") + .await + .unwrap(); + registry + .update_user_password(&user_id, "user-selected-hash") + .await + .unwrap(); + + let repeated_user_id = registry + .upsert_bootstrap_user(email, "Changed Bootstrap Name", "changed-bootstrap-hash") + .await + .unwrap(); + let stored = registry + .get_auth_user_by_email(email) + .await + .unwrap() + .unwrap(); + + assert_eq!(repeated_user_id, user_id); + assert_eq!(stored.password_hash, "user-selected-hash"); + assert_eq!(stored.user.display_name, "Bootstrap Owner"); + + database.cleanup().await; +} + #[tokio::test] async fn stores_and_finishes_yaml_import_jobs() { let database = TestDatabase::new().await; @@ -352,6 +384,24 @@ async fn creates_and_loads_user_sessions_with_typed_expiration() { assert_eq!(session.user.id, user_id); assert!(session.user.created_at.unix_timestamp() > 0); + registry.touch_user_session(&session_id).await.unwrap(); + let first_seen = sqlx::query_scalar::<_, OffsetDateTime>( + "select last_seen_at from user_sessions where id = $1", + ) + .bind(session_id.as_str()) + .fetch_one(registry.pool()) + .await + .unwrap(); + registry.touch_user_session(&session_id).await.unwrap(); + let second_seen = sqlx::query_scalar::<_, OffsetDateTime>( + "select last_seen_at from user_sessions where id = $1", + ) + .bind(session_id.as_str()) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(second_seen, first_seen); + database.cleanup().await; } @@ -454,6 +504,40 @@ async fn manages_platform_api_key_read_paths() { Some(timestamp("2026-03-25T12:05:00Z")) ); + registry + .touch_platform_api_key( + &workspace.id, + &PlatformApiKeyId::new("key_01"), + ×tamp("2026-03-25T12:05:30Z"), + ) + .await + .unwrap(); + let throttled = registry + .list_platform_api_keys(&workspace.id) + .await + .unwrap(); + assert_eq!( + throttled[0].api_key.last_used_at, + Some(timestamp("2026-03-25T12:05:00Z")) + ); + + registry + .touch_platform_api_key( + &workspace.id, + &PlatformApiKeyId::new("key_01"), + ×tamp("2026-03-25T12:06:01Z"), + ) + .await + .unwrap(); + let refreshed = registry + .list_platform_api_keys(&workspace.id) + .await + .unwrap(); + assert_eq!( + refreshed[0].api_key.last_used_at, + Some(timestamp("2026-03-25T12:06:01Z")) + ); + database.cleanup().await; } @@ -587,7 +671,6 @@ async fn manages_approval_request_lifecycle() { .claim_next_recoverable_approval_request( timestamp("2026-03-25T12:02:01Z"), timestamp("2026-03-25T12:01:59Z"), - timestamp("2026-03-25T11:55:00Z"), ) .await .unwrap(); @@ -608,23 +691,10 @@ async fn manages_approval_request_lifecycle() { .claim_next_recoverable_approval_request( timestamp("2026-03-25T12:02:10Z"), timestamp("2026-03-25T12:02:09Z"), - timestamp("2026-03-25T12:01:59Z"), ) .await .unwrap(); assert!(fresh_claim.is_none()); - let recovered = registry - .claim_next_recoverable_approval_request( - timestamp("2026-03-25T12:03:00Z"), - timestamp("2026-03-25T12:02:59Z"), - timestamp("2026-03-25T12:02:30Z"), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(recovered.approval.id, approval.id); - assert_eq!(recovered.approval.status, ApprovalRequestStatus::Executing); - let completed = registry .finish_approval_request(FinishApprovalRequest { workspace_id: &workspace_id, @@ -637,7 +707,6 @@ async fn manages_approval_request_lifecycle() { .await .unwrap() .unwrap(); - assert_eq!(completed.approval.status, ApprovalRequestStatus::Completed); assert_eq!( completed.approval.response_payload, @@ -653,9 +722,81 @@ async fn manages_approval_request_lifecycle() { .await .unwrap(); assert_eq!(completed_by_status.len(), 1); + + let mut interrupted_approval = approval.clone(); + interrupted_approval.id = ApprovalRequestId::new("approval_interrupted_01"); + interrupted_approval.request_payload = json!({"amount": 150}); + interrupted_approval.created_at = timestamp("2026-03-25T12:02:10Z"); + registry + .create_approval_request(CreateApprovalRequest { + approval: &interrupted_approval, + }) + .await + .unwrap(); + registry + .decide_approval_request(DecideApprovalRequest { + workspace_id: &workspace_id, + agent_id: &agent.id, + approval_id: &interrupted_approval.id, + status: ApprovalRequestStatus::Approved, + decided_at: timestamp("2026-03-25T12:02:20Z"), + decided_by_key_id: &approval_key.id, + response_payload: Some(json!({"approve": "yes"})), + decision_note: None, + }) + .await + .unwrap() + .unwrap(); + registry + .claim_approval_request( + &workspace_id, + &agent.id, + &interrupted_approval.id, + timestamp("2026-03-25T12:02:21Z"), + ) + .await + .unwrap() + .unwrap(); + + let recovered = registry + .claim_next_recoverable_approval_request( + timestamp("2026-03-25T12:03:00Z"), + timestamp("2026-03-25T12:02:59Z"), + ) + .await + .unwrap(); + assert!(recovered.is_none()); + + let interrupted = registry + .fail_next_interrupted_approval_request(timestamp("2026-03-25T12:02:30Z")) + .await + .unwrap() + .unwrap(); + + assert_eq!(interrupted.approval.id, interrupted_approval.id); + assert_eq!(interrupted.approval.status, ApprovalRequestStatus::Failed); assert_eq!( - completed_by_status[0].approval.status, - ApprovalRequestStatus::Completed + interrupted.approval.response_payload, + Some(json!({ + "error": { + "code": "approval_execution_outcome_unknown", + "message": "execution was interrupted; the operation was not retried automatically" + } + })) + ); + + let failed_by_status = registry + .list_approval_requests(ListApprovalRequestsQuery { + workspace_id: &workspace_id, + status: Some(ApprovalRequestStatus::Failed), + limit: 10, + }) + .await + .unwrap(); + assert_eq!(failed_by_status.len(), 1); + assert_eq!( + failed_by_status[0].approval.status, + ApprovalRequestStatus::Failed ); let pending_after_decision = registry diff --git a/crates/crank-runtime/Cargo.toml b/crates/crank-runtime/Cargo.toml index de54471..39f7386 100644 --- a/crates/crank-runtime/Cargo.toml +++ b/crates/crank-runtime/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-runtime" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [lib] @@ -19,19 +20,22 @@ crank-adapter-rest = { path = "../crank-adapter-rest" } crank-core = { path = "../crank-core" } crank-mapping = { path = "../crank-mapping" } crank-schema = { path = "../crank-schema" } +crank-trace = { path = "../crank-trace" } hkdf.workspace = true +metrics.workspace = true redis = { version = "0.29", features = ["tokio-comp", "connection-manager"] } serde.workspace = true serde_json.workspace = true sha2.workspace = true thiserror.workspace = true time.workspace = true -tokio = { workspace = true, features = ["sync"] } +tokio = { workspace = true, features = ["sync", "time"] } tracing.workspace = true uuid.workspace = true [dev-dependencies] axum.workspace = true futures-util = "0.3" +testcontainers.workspace = true time.workspace = true tracing-subscriber.workspace = true diff --git a/crates/crank-runtime/src/cache.rs b/crates/crank-runtime/src/cache.rs index 789676c..e52ce4c 100644 --- a/crates/crank-runtime/src/cache.rs +++ b/crates/crank-runtime/src/cache.rs @@ -8,9 +8,9 @@ use std::{ use async_trait::async_trait; use crank_core::{ - CacheBackend, CacheScope, CacheStoreError, CachedResponse, CoordinationStateStore, - CoordinationStateValue, RateLimitBucketState, RateLimitStateStore, ReplayGuardStatus, - ReplayGuardStore, ResponseCacheStore, + CacheBackend, CacheScope, CacheStoreError, CachedResponse, CoordinationStateReservation, + CoordinationStateStore, CoordinationStateValue, RateLimitBucketState, RateLimitDecision, + RateLimitStateStore, ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore, }; use redis::{Client, aio::ConnectionManager}; use serde::de::DeserializeOwned; @@ -307,6 +307,43 @@ impl RateLimitStateStore for InMemoryRateLimitStateStore { entries.remove(key); Ok(()) } + + async fn consume_token( + &self, + key: &str, + burst_tokens_micros: u64, + refill_per_second_micros: u64, + now_unix_ms: i64, + ttl: Duration, + ) -> Result { + validate_key(key)?; + validate_rate_limit_parameters(burst_tokens_micros, refill_per_second_micros)?; + let expires_at = expiry_from_ttl(ttl)?; + let now = Instant::now(); + let mut entries = self.entries.write().await; + retain_unexpired(&mut entries, now); + let state = entries + .get(key) + .map(|entry| entry.value) + .unwrap_or(RateLimitBucketState { + tokens_micros: burst_tokens_micros, + last_refill_unix_ms: now_unix_ms, + }); + let (state, decision) = consume_bucket_token( + state, + burst_tokens_micros, + refill_per_second_micros, + now_unix_ms, + ); + entries.insert( + key.to_owned(), + ExpiringValue { + value: state, + expires_at, + }, + ); + Ok(decision) + } } #[async_trait] @@ -373,6 +410,64 @@ impl CoordinationStateStore for InMemoryCoordinationStateStore { entries.remove(&storage_key); Ok(()) } + + async fn take_value( + &self, + scope: CacheScope, + key: &str, + ) -> Result, CacheStoreError> { + validate_key(key)?; + let storage_key = scoped_key(scope, key); + let now = Instant::now(); + let mut entries = self.entries.write().await; + retain_unexpired(&mut entries, now); + Ok(entries.remove(&storage_key).map(|entry| entry.value)) + } + + async fn reserve_value( + &self, + scope: CacheScope, + key: &str, + value: CoordinationStateValue, + ttl: Duration, + ) -> Result { + validate_key(key)?; + let expires_at = expiry_from_ttl(ttl)?; + let storage_key = scoped_key(scope, key); + let now = Instant::now(); + let mut entries = self.entries.write().await; + retain_unexpired(&mut entries, now); + if let Some(existing) = entries.get(&storage_key) { + return Ok(CoordinationStateReservation::Existing( + existing.value.clone(), + )); + } + entries.insert(storage_key, ExpiringValue { value, expires_at }); + Ok(CoordinationStateReservation::Reserved) + } + + async fn compare_and_set_value( + &self, + scope: CacheScope, + key: &str, + expected: &CoordinationStateValue, + value: CoordinationStateValue, + ttl: Duration, + ) -> Result { + validate_key(key)?; + let expires_at = expiry_from_ttl(ttl)?; + let storage_key = scoped_key(scope, key); + let now = Instant::now(); + let mut entries = self.entries.write().await; + retain_unexpired(&mut entries, now); + let matches = entries + .get(&storage_key) + .is_some_and(|entry| entry.value == *expected); + if matches { + entries.insert(storage_key, ExpiringValue { value, expires_at }); + } + Ok(matches) + } } #[async_trait] @@ -413,6 +508,65 @@ impl RateLimitStateStore for RedisCacheStore { async fn delete_bucket(&self, key: &str) -> Result<(), CacheStoreError> { self.delete_kind_key("rate_limit", key).await } + + async fn consume_token( + &self, + key: &str, + burst_tokens_micros: u64, + refill_per_second_micros: u64, + now_unix_ms: i64, + ttl: Duration, + ) -> Result { + validate_key(key)?; + validate_rate_limit_parameters(burst_tokens_micros, refill_per_second_micros)?; + let storage_key = self.prefixed_key("rate_limit", key); + let ttl_ms = self.ttl_ms(ttl)?; + let mut connection = self.connection_manager.clone(); + let script = r#" +local current = redis.call('GET', KEYS[1]) +local tokens = tonumber(ARGV[1]) +local last_refill = tonumber(ARGV[3]) +if current then + local decoded = cjson.decode(current) + tokens = tonumber(decoded.tokens_micros) + last_refill = tonumber(decoded.last_refill_unix_ms) +end +local effective_now = math.max(tonumber(ARGV[3]), last_refill) +local elapsed = effective_now - last_refill +local replenished = math.floor(elapsed * tonumber(ARGV[2]) / 1000) +tokens = math.min(tonumber(ARGV[1]), tokens + replenished) +local allowed = 0 +local retry_after = 0 +if tokens >= 1000000 then + tokens = tokens - 1000000 + allowed = 1 +else + local missing = 1000000 - tokens + retry_after = math.max(1, math.ceil(missing * 1000 / tonumber(ARGV[2]))) +end +redis.call('PSETEX', KEYS[1], ARGV[4], cjson.encode({ + tokens_micros = tokens, + last_refill_unix_ms = effective_now +})) +return {allowed, retry_after} +"#; + let (allowed, retry_after_ms): (u8, u64) = redis::cmd("EVAL") + .arg(script) + .arg(1) + .arg(storage_key) + .arg(burst_tokens_micros) + .arg(refill_per_second_micros) + .arg(now_unix_ms) + .arg(ttl_ms) + .query_async(&mut connection) + .await + .map_err(|source| self.unavailable(source))?; + Ok(if allowed == 1 { + RateLimitDecision::Allowed + } else { + RateLimitDecision::Rejected { retry_after_ms } + }) + } } #[async_trait] @@ -472,6 +626,126 @@ impl CoordinationStateStore for RedisCacheStore { self.delete_kind_key(Self::coordination_kind(scope), key) .await } + + async fn take_value( + &self, + scope: CacheScope, + key: &str, + ) -> Result, CacheStoreError> { + validate_key(key)?; + let storage_key = self.prefixed_key(Self::coordination_kind(scope), key); + let mut connection = self.connection_manager.clone(); + let encoded: Option> = redis::cmd("EVAL") + .arg("local value = redis.call('GET', KEYS[1]); if value then redis.call('DEL', KEYS[1]); end; return value") + .arg(1) + .arg(storage_key) + .query_async(&mut connection) + .await + .map_err(|source| self.unavailable(source))?; + encoded + .map(|bytes| self.deserialize_value(&bytes)) + .transpose() + } + + async fn reserve_value( + &self, + scope: CacheScope, + key: &str, + value: CoordinationStateValue, + ttl: Duration, + ) -> Result { + validate_key(key)?; + let storage_key = self.prefixed_key(Self::coordination_kind(scope), key); + let encoded = self.serialize_value(&value)?; + let ttl_ms = self.ttl_ms(ttl)?; + let mut connection = self.connection_manager.clone(); + let (reserved, existing): (u8, Vec) = redis::cmd("EVAL") + .arg("local current = redis.call('GET', KEYS[1]); if current then return {0, current}; end; redis.call('PSETEX', KEYS[1], ARGV[2], ARGV[1]); return {1, ''}") + .arg(1) + .arg(storage_key) + .arg(encoded) + .arg(ttl_ms) + .query_async(&mut connection) + .await + .map_err(|source| self.unavailable(source))?; + if reserved == 1 { + Ok(CoordinationStateReservation::Reserved) + } else { + Ok(CoordinationStateReservation::Existing( + self.deserialize_value(&existing)?, + )) + } + } + + async fn compare_and_set_value( + &self, + scope: CacheScope, + key: &str, + expected: &CoordinationStateValue, + value: CoordinationStateValue, + ttl: Duration, + ) -> Result { + validate_key(key)?; + let storage_key = self.prefixed_key(Self::coordination_kind(scope), key); + let expected = self.serialize_value(expected)?; + let value = self.serialize_value(&value)?; + let ttl_ms = self.ttl_ms(ttl)?; + let mut connection = self.connection_manager.clone(); + let replaced: u8 = redis::cmd("EVAL") + .arg("local current = redis.call('GET', KEYS[1]); if current ~= ARGV[1] then return 0; end; redis.call('PSETEX', KEYS[1], ARGV[3], ARGV[2]); return 1") + .arg(1) + .arg(storage_key) + .arg(expected) + .arg(value) + .arg(ttl_ms) + .query_async(&mut connection) + .await + .map_err(|source| self.unavailable(source))?; + Ok(replaced == 1) + } +} + +fn consume_bucket_token( + mut state: RateLimitBucketState, + burst_tokens_micros: u64, + refill_per_second_micros: u64, + now_unix_ms: i64, +) -> (RateLimitBucketState, RateLimitDecision) { + let effective_now = now_unix_ms.max(state.last_refill_unix_ms); + let elapsed_ms = + u64::try_from(effective_now.saturating_sub(state.last_refill_unix_ms)).unwrap_or(u64::MAX); + let replenished = + u128::from(elapsed_ms).saturating_mul(u128::from(refill_per_second_micros)) / 1000; + state.tokens_micros = u128::from(state.tokens_micros) + .saturating_add(replenished) + .min(u128::from(burst_tokens_micros)) + .try_into() + .unwrap_or(burst_tokens_micros); + state.last_refill_unix_ms = effective_now; + if state.tokens_micros >= 1_000_000 { + state.tokens_micros -= 1_000_000; + return (state, RateLimitDecision::Allowed); + } + let missing = 1_000_000_u64.saturating_sub(state.tokens_micros); + let retry_after_ms = u128::from(missing) + .saturating_mul(1000) + .div_ceil(u128::from(refill_per_second_micros)) + .max(1) + .try_into() + .unwrap_or(u64::MAX); + (state, RateLimitDecision::Rejected { retry_after_ms }) +} + +fn validate_rate_limit_parameters( + burst_tokens_micros: u64, + refill_per_second_micros: u64, +) -> Result<(), CacheStoreError> { + if burst_tokens_micros < 1_000_000 || refill_per_second_micros == 0 { + return Err(CacheStoreError::InvalidKey { + message: "rate limit capacity and refill rate must be positive".to_owned(), + }); + } + Ok(()) } fn validate_key(key: &str) -> Result<(), CacheStoreError> { @@ -530,7 +804,11 @@ fn parse_optional_string(name: &'static str) -> Result, RuntimeCa fn parse_optional_u64(name: &'static str) -> Result, RuntimeCacheConfigError> { match env::var(name) { Ok(raw) => { - let value = raw + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Ok(None); + } + let value = trimmed .parse::() .map_err(|source| RuntimeCacheConfigError::InvalidTtl { value: raw, source })?; if value == 0 { diff --git a/crates/crank-runtime/src/confirmation.rs b/crates/crank-runtime/src/confirmation.rs index 8e58f39..c4fc20d 100644 --- a/crates/crank-runtime/src/confirmation.rs +++ b/crates/crank-runtime/src/confirmation.rs @@ -50,6 +50,12 @@ pub async fn confirm_operation( consume_confirmation_token(store, &scope, provided_token, &input_hash).await } +pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool { + effective_safety_policy(operation) + .class + .requires_confirmation() +} + fn effective_safety_policy(operation: &RuntimeOperation) -> OperationSafetyPolicy { operation .execution_config @@ -137,13 +143,12 @@ async fn consume_confirmation_token( ) -> Result<(), RuntimeError> { let key = confirmation_cache_key(operation_scope, token); let stored = store - .get_value(CacheScope::Coordination, &key) + .take_value(CacheScope::Coordination, &key) .await .map_err(|error| RuntimeError::InvalidPreparedRequest { field: "confirmation_token".to_owned(), reason: error.to_string(), })?; - let _ = store.delete_value(CacheScope::Coordination, &key).await; let Some(stored) = stored else { return Err(RuntimeError::InvalidConfirmationToken { diff --git a/crates/crank-runtime/src/error.rs b/crates/crank-runtime/src/error.rs index 2042438..1884e1a 100644 --- a/crates/crank-runtime/src/error.rs +++ b/crates/crank-runtime/src/error.rs @@ -38,6 +38,14 @@ pub enum RuntimeError { InvalidConfirmationToken { operation_id: String }, #[error("confirmation store is unavailable for operation {operation_id}")] ConfirmationStoreUnavailable { operation_id: String }, + #[error("idempotency store is unavailable for operation {operation_id}")] + IdempotencyStoreUnavailable { operation_id: String }, + #[error("operation {operation_id} is already executing for this idempotency key")] + IdempotencyInProgress { operation_id: String }, + #[error("idempotency key for operation {operation_id} was reused with different input")] + IdempotencyConflict { operation_id: String }, + #[error("the outcome of operation {operation_id} is unknown; automatic retry is unsafe")] + IdempotencyOutcomeUnknown { operation_id: String }, #[error("auth profile {auth_profile_id} was not found")] MissingAuthProfile { auth_profile_id: String }, #[error("secret {secret_id} was not found")] diff --git a/crates/crank-runtime/src/executor.rs b/crates/crank-runtime/src/executor.rs index 17f0252..06294ca 100644 --- a/crates/crank-runtime/src/executor.rs +++ b/crates/crank-runtime/src/executor.rs @@ -1,14 +1,16 @@ use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; use crank_core::{ - AdapterRegistry, CoordinationStateStore, ExecutionMode, InvocationStatus, MeteringEvent, - ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter, + AdapterRegistry, CoordinationStateStore, ExecutionMode, InvocationSource, InvocationStatus, + MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter, }; +use crank_trace::{ErrorCategory, Stage, StageOutcome}; +use metrics::Gauge; use serde_json::{Map, Value, json}; use time::OffsetDateTime; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; -use tracing::debug; +use tracing::{Instrument, Span, debug}; use crate::{ AdapterResponse, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeLimits, RuntimeOperation, @@ -176,18 +178,31 @@ impl RuntimeExecutor { request: RuntimeExecutionRequest<'_>, ) -> Result { log_runtime_event("unary.execute", request.operation, request.request_context); - let _permit = self.acquire_unary_permit(request.operation)?; let started_at = Instant::now(); - let prepared_request = self.prepare_request(request.operation, request.input)?; - let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth); - let result = self - .execute_prepared( + let runtime_span = Stage::RuntimeExecute.span(); + let result = async { + let _permit = self.acquire_unary_permit(request.operation)?; + let _inflight = RuntimeInFlightGuard::new(); + let mapping_span = Stage::RuntimeArgumentsMap.span(); + let prepared_request = + mapping_span.in_scope(|| self.prepare_request(request.operation, request.input)); + record_runtime_result(&mapping_span, &prepared_request); + drop(mapping_span); + let prepared_request = prepared_request?; + let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth); + self.execute_prepared( request.operation, request.input, prepared_request, request.request_context, ) - .await; + .await + } + .instrument(runtime_span.clone()) + .await; + record_runtime_result(&runtime_span, &result); + drop(runtime_span); + record_execution_metrics(request.request_context, &result, started_at); self.record_metering( request.operation, request.request_context, @@ -221,59 +236,132 @@ impl RuntimeExecutor { request_context: Option<&RuntimeRequestContext>, ) -> Result { let mut prepared_request = prepared_request; - let idempotency_key = - crate::idempotency::prepare_idempotency(operation, input, &mut prepared_request)?; - crate::confirmation::confirm_operation( - self.coordination_store.as_deref(), + let idempotency_applicable = crate::idempotency::policy(operation).is_some(); + let idempotency_key = match crate::idempotency::prepare_idempotency( operation, input, - request_context, - ) - .await?; - if let Some(response) = self - .load_idempotent_adapter_response( + &mut prepared_request, + ) { + Ok(key) => key, + Err(error) if idempotency_applicable => { + let span = Stage::RuntimeIdempotency.span(); + StageOutcome::Error.record(&span); + ErrorCategory::Idempotency.record(&span); + return Err(error); + } + Err(error) => return Err(error), + }; + + if crate::confirmation::is_applicable(operation) { + let approval_span = Stage::ApprovalCheck.span(); + let approval_result = crate::confirmation::confirm_operation( + self.coordination_store.as_deref(), operation, + input, + request_context, + ) + .instrument(approval_span.clone()) + .await; + match &approval_result { + Ok(()) => StageOutcome::Success.record(&approval_span), + Err(RuntimeError::ConfirmationRequired { .. }) => { + StageOutcome::Required.record(&approval_span); + ErrorCategory::Approval.record(&approval_span); + } + Err(error) => { + StageOutcome::Error.record(&approval_span); + runtime_error_category(error).record(&approval_span); + } + } + drop(approval_span); + approval_result?; + } + + let idempotency = if idempotency_applicable { + let idempotency_span = Stage::RuntimeIdempotency.span(); + let result = crate::idempotency::begin( + self.coordination_store.as_deref(), + operation, + input, idempotency_key.as_deref(), request_context, ) - .await - { - let finalized_output = finalize_output(operation, &response)?; - operation.output_schema.validate_shape(&finalized_output)?; - return Ok(finalized_output); + .instrument(idempotency_span.clone()) + .await; + match &result { + Ok(crate::idempotency::IdempotencyAction::Execute(_)) => { + StageOutcome::Execute.record(&idempotency_span); + } + Ok(crate::idempotency::IdempotencyAction::Replay(_)) => { + StageOutcome::Replay.record(&idempotency_span); + } + Ok(crate::idempotency::IdempotencyAction::Disabled) => { + StageOutcome::Skipped.record(&idempotency_span); + } + Err(error) => { + StageOutcome::Error.record(&idempotency_span); + runtime_error_category(error).record(&idempotency_span); + } + } + drop(idempotency_span); + result? + } else { + crate::idempotency::IdempotencyAction::Disabled + }; + if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency { + return transform_response(operation, response); } - let adapter_response = match self + let adapter_result = match self .load_cached_adapter_response(operation, &prepared_request, request_context) .await { - Some(response) => response, + Some(response) => Ok(response), None => { let adapter_response = self .execute_adapter(operation, prepared_request.clone(), request_context) - .await?; - self.store_cached_adapter_response( - operation, - &prepared_request, - request_context, - &adapter_response, - ) - .await; - self.store_idempotent_adapter_response( - operation, - idempotency_key.as_deref(), - request_context, - &adapter_response, - ) - .await; + .await; + if let Ok(response) = &adapter_response { + self.store_cached_adapter_response( + operation, + &prepared_request, + request_context, + response, + ) + .await; + } adapter_response } }; - let finalized_output = finalize_output(operation, &adapter_response)?; - - operation.output_schema.validate_shape(&finalized_output)?; - - Ok(finalized_output) + let adapter_response = match adapter_result { + Ok(response) => response, + Err(error) => { + if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency + && let Some(store) = self.coordination_store.as_deref() + { + let idempotency_span = Stage::RuntimeIdempotency.span(); + let cleanup_result = + crate::idempotency::mark_outcome_unknown(store, operation, reservation) + .instrument(idempotency_span.clone()) + .await; + record_runtime_result(&idempotency_span, &cleanup_result); + } + return Err(error); + } + }; + if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency + && let Some(store) = self.coordination_store.as_deref() + { + let idempotency_span = Stage::RuntimeIdempotency.span(); + let completion_result = + crate::idempotency::complete(store, operation, reservation, &adapter_response) + .instrument(idempotency_span.clone()) + .await; + record_runtime_result(&idempotency_span, &completion_result); + drop(idempotency_span); + completion_result?; + } + transform_response(operation, &adapter_response) } async fn record_metering( @@ -323,7 +411,6 @@ impl RuntimeExecutor { let prepared_request = adapter_prepared_request( operation, &prepared_request, - request_context, operation.execution_config.timeout_ms, ); let adapter_context = adapter_request_context(request_context); @@ -360,11 +447,11 @@ impl RuntimeExecutor { let cache_key = response_cache_key(operation, prepared_request, request_context)?; let cached = match response_cache.get(&cache_key).await { Ok(cached) => cached?, - Err(error) => { + Err(_) => { debug!( - operation_id = %operation.operation_id, - cache_key, - error = %error, + name: "runtime.response_cache.read_failed", + operation_id = operation.operation_id.as_str(), + error_category = "response_cache", "response cache lookup skipped" ); return None; @@ -373,11 +460,11 @@ impl RuntimeExecutor { match adapter_response_from_cached(cached) { Ok(response) => Some(response), - Err(error) => { + Err(_) => { debug!( - operation_id = %operation.operation_id, - cache_key, - error, + name: "runtime.response_cache.decode_failed", + operation_id = operation.operation_id.as_str(), + error_category = "cached_response", "cached response payload was invalid" ); let _ = response_cache.delete(&cache_key).await; @@ -412,87 +499,19 @@ impl RuntimeExecutor { return; }; - if let Err(error) = response_cache + if response_cache .put(&cache_key, cached_response, cache_ttl) .await + .is_err() { debug!( - operation_id = %operation.operation_id, - cache_key, - error = %error, + name: "runtime.response_cache.write_failed", + operation_id = operation.operation_id.as_str(), + error_category = "response_cache", "response cache write skipped" ); } } - - async fn load_idempotent_adapter_response( - &self, - operation: &RuntimeOperation, - idempotency_key: Option<&str>, - request_context: Option<&RuntimeRequestContext>, - ) -> Option { - let response_cache = self.response_cache.as_ref()?; - let cache_key = - crate::idempotency::cache_key(operation, idempotency_key?, request_context)?; - let cached = match response_cache.get(&cache_key).await { - Ok(cached) => cached?, - Err(error) => { - debug!( - operation_id = %operation.operation_id, - cache_key, - error = %error, - "idempotency cache lookup skipped" - ); - return None; - } - }; - - adapter_response_from_cached(cached).ok() - } - - async fn store_idempotent_adapter_response( - &self, - operation: &RuntimeOperation, - idempotency_key: Option<&str>, - request_context: Option<&RuntimeRequestContext>, - adapter_response: &AdapterResponse, - ) { - let Some(response_cache) = self.response_cache.as_ref() else { - return; - }; - if !(200..=299).contains(&adapter_response.status_code) { - return; - } - let Some(policy) = crate::idempotency::policy(operation) else { - return; - }; - let Some(cache_key) = crate::idempotency::cache_key( - operation, - idempotency_key.unwrap_or_default(), - request_context, - ) else { - return; - }; - let Some(cached_response) = cached_response_from_adapter(adapter_response) else { - return; - }; - - if let Err(error) = response_cache - .put( - &cache_key, - cached_response, - Duration::from_millis(policy.ttl_ms), - ) - .await - { - debug!( - operation_id = %operation.operation_id, - cache_key, - error = %error, - "idempotency cache write skipped" - ); - } - } } fn adapter_request_context( @@ -552,14 +571,149 @@ fn finalize_output( .unwrap_or_else(|| Value::Object(Map::new()))) } +fn transform_response( + operation: &RuntimeOperation, + response: &AdapterResponse, +) -> Result { + let span = Stage::RuntimeResponseTransform.span(); + let result = span.in_scope(|| { + let finalized_output = finalize_output(operation, response)?; + operation.output_schema.validate_shape(&finalized_output)?; + Ok(finalized_output) + }); + match &result { + Ok(_) => StageOutcome::Success.record(&span), + Err(_) => { + StageOutcome::Error.record(&span); + ErrorCategory::Transformation.record(&span); + } + } + result +} + +fn record_runtime_result(span: &Span, result: &Result) { + match result { + Ok(_) => StageOutcome::Success.record(span), + Err(error) => { + StageOutcome::Error.record(span); + runtime_error_category(error).record(span); + } + } +} + +fn runtime_error_category(error: &RuntimeError) -> ErrorCategory { + match error { + RuntimeError::Schema(_) => ErrorCategory::Schema, + RuntimeError::Mapping(_) | RuntimeError::InvalidPreparedRequest { .. } => { + ErrorCategory::Mapping + } + RuntimeError::RestAdapter(_) + | RuntimeError::ProtocolAdapter(_) + | RuntimeError::UnsupportedProtocol { .. } + | RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream, + RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency, + RuntimeError::ConfirmationRequired { .. } + | RuntimeError::InvalidConfirmationToken { .. } + | RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval, + RuntimeError::IdempotencyStoreUnavailable { .. } + | RuntimeError::IdempotencyInProgress { .. } + | RuntimeError::IdempotencyConflict { .. } + | RuntimeError::IdempotencyOutcomeUnknown { .. } => ErrorCategory::Idempotency, + RuntimeError::MissingAuthProfile { .. } + | RuntimeError::MissingSecret { .. } + | RuntimeError::MissingSecretVersion { .. } + | RuntimeError::InvalidAuthSecretValue { .. } + | RuntimeError::SecretCrypto { .. } => ErrorCategory::Configuration, + } +} + fn try_acquire_limit( limiter: Arc, kind: &'static str, limit: usize, ) -> Result { - limiter - .try_acquire_owned() - .map_err(|_| RuntimeError::ConcurrencyLimitExceeded { kind, limit }) + limiter.try_acquire_owned().map_err(|_| { + metrics::counter!( + "crank_runtime_limit_rejections_total", + "stage" => "concurrency" + ) + .increment(1); + RuntimeError::ConcurrencyLimitExceeded { kind, limit } + }) +} + +fn record_execution_metrics( + request_context: Option<&RuntimeRequestContext>, + result: &Result, + started_at: Instant, +) { + let source = request_context + .and_then(RuntimeRequestContext::metering_context) + .map_or("internal", |context| match context.source { + InvocationSource::AdminTestRun => "admin_test_run", + InvocationSource::AgentToolCall => "agent_tool_call", + }); + let (outcome, error_kind) = match result { + Ok(_) => ("success", "none"), + Err(error) => ("error", runtime_error_kind(error)), + }; + + metrics::counter!( + "crank_tool_invocations_total", + "source" => source, + "outcome" => outcome, + "error_kind" => error_kind + ) + .increment(1); + metrics::histogram!( + "crank_tool_invocation_duration_seconds", + "source" => source, + "outcome" => outcome + ) + .record(started_at.elapsed().as_secs_f64()); +} + +fn runtime_error_kind(error: &RuntimeError) -> &'static str { + match error { + RuntimeError::Schema(_) => "schema", + RuntimeError::Mapping(_) => "mapping", + RuntimeError::RestAdapter(_) => "rest_adapter", + RuntimeError::ProtocolAdapter(_) => "protocol_adapter", + RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol", + RuntimeError::UnsupportedExecutionMode { .. } => "unsupported_execution_mode", + RuntimeError::ConcurrencyLimitExceeded { .. } => "concurrency_limit", + RuntimeError::InvalidPreparedRequest { .. } => "invalid_prepared_request", + RuntimeError::ConfirmationRequired { .. } => "confirmation_required", + RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token", + RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_store", + RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_store", + RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress", + RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict", + RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown", + RuntimeError::MissingAuthProfile { .. } => "missing_auth_profile", + RuntimeError::MissingSecret { .. } => "missing_secret", + RuntimeError::MissingSecretVersion { .. } => "missing_secret_version", + RuntimeError::InvalidAuthSecretValue { .. } => "invalid_auth_secret", + RuntimeError::SecretCrypto { .. } => "secret_crypto", + } +} + +struct RuntimeInFlightGuard { + gauge: Gauge, +} + +impl RuntimeInFlightGuard { + fn new() -> Self { + let gauge = metrics::gauge!("crank_runtime_inflight"); + gauge.increment(1.0); + Self { gauge } + } +} + +impl Drop for RuntimeInFlightGuard { + fn drop(&mut self) { + self.gauge.decrement(1.0); + } } fn log_runtime_event( @@ -567,19 +721,26 @@ fn log_runtime_event( operation: &RuntimeOperation, request_context: Option<&RuntimeRequestContext>, ) { - let request_id = request_context - .map(|context| context.request_id.as_str()) - .unwrap_or_default(); - let correlation_id = request_context - .map(|context| context.correlation_id.as_str()) - .unwrap_or_default(); - - debug!( - stage, - operation_id = %operation.operation_id, - protocol = ?operation.protocol, - request_id, - correlation_id, - "runtime execution" - ); + let protocol = match operation.protocol { + crank_core::Protocol::Rest => "rest", + }; + if let Some(context) = request_context { + debug!( + name: "runtime.execution.stage_reached", + stage, + operation_id = operation.operation_id.as_str(), + protocol, + request_id = context.request_id.as_str(), + correlation_id = context.correlation_id.as_str(), + "runtime execution" + ); + } else { + debug!( + name: "runtime.execution.stage_reached", + stage, + operation_id = operation.operation_id.as_str(), + protocol, + "runtime execution" + ); + } } diff --git a/crates/crank-runtime/src/idempotency.rs b/crates/crank-runtime/src/idempotency.rs index 95a7fab..998ad49 100644 --- a/crates/crank-runtime/src/idempotency.rs +++ b/crates/crank-runtime/src/idempotency.rs @@ -1,9 +1,32 @@ -use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use crank_core::{HttpMethod, IdempotencyMode, IdempotencyPolicy, Target}; -use serde_json::Value; -use sha2::{Digest, Sha256}; +use std::time::{Duration, Instant}; -use crate::{PreparedRequest, RuntimeError, RuntimeOperation, RuntimeRequestContext}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use crank_core::{ + CacheScope, CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue, + HttpMethod, IdempotencyMode, IdempotencyPolicy, Target, +}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::{ + AdapterResponse, PreparedRequest, RuntimeError, RuntimeOperation, RuntimeRequestContext, +}; + +const POLL_INTERVAL: Duration = Duration::from_millis(10); + +pub(crate) enum IdempotencyAction { + Disabled, + Execute(IdempotencyReservation), + Replay(AdapterResponse), +} + +pub(crate) struct IdempotencyReservation { + key: String, + initial: CoordinationStateValue, + fingerprint: String, + result_ttl: Duration, +} pub fn prepare_idempotency( operation: &RuntimeOperation, @@ -39,6 +62,185 @@ pub fn prepare_idempotency( Ok(Some(key)) } +pub(crate) async fn begin( + store: Option<&dyn CoordinationStateStore>, + operation: &RuntimeOperation, + input: &Value, + idempotency_key: Option<&str>, + request_context: Option<&RuntimeRequestContext>, +) -> Result { + let Some(policy) = policy(operation) else { + return Ok(IdempotencyAction::Disabled); + }; + let Some(idempotency_key) = idempotency_key else { + return Ok(IdempotencyAction::Disabled); + }; + let Some(key) = cache_key(operation, idempotency_key, request_context) else { + return Err(RuntimeError::IdempotencyStoreUnavailable { + operation_id: operation.operation_id.as_str().to_owned(), + }); + }; + let Some(store) = store else { + return Err(RuntimeError::IdempotencyStoreUnavailable { + operation_id: operation.operation_id.as_str().to_owned(), + }); + }; + + let fingerprint = request_fingerprint(input)?; + let result_ttl = Duration::from_millis(policy.ttl_ms); + let reservation_ttl = result_ttl.max(Duration::from_millis( + operation.execution_config.timeout_ms.saturating_add(1_000), + )); + let initial = in_progress_value(&fingerprint); + let reservation = store + .reserve_value( + CacheScope::Coordination, + &key, + initial.clone(), + reservation_ttl, + ) + .await + .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { + operation_id: operation.operation_id.as_str().to_owned(), + })?; + + match reservation { + CoordinationStateReservation::Reserved => { + Ok(IdempotencyAction::Execute(IdempotencyReservation { + key, + initial, + fingerprint, + result_ttl, + })) + } + CoordinationStateReservation::Existing(existing) => { + resolve_existing(store, operation, &key, &fingerprint, existing).await + } + } +} + +pub(crate) async fn complete( + store: &dyn CoordinationStateStore, + operation: &RuntimeOperation, + reservation: &IdempotencyReservation, + response: &AdapterResponse, +) -> Result<(), RuntimeError> { + let completed = CoordinationStateValue { + payload: json!({ + "state": "completed", + "fingerprint": reservation.fingerprint, + "response": response, + }), + }; + let replaced = store + .compare_and_set_value( + CacheScope::Coordination, + &reservation.key, + &reservation.initial, + completed, + reservation.result_ttl, + ) + .await + .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { + operation_id: operation.operation_id.as_str().to_owned(), + })?; + if replaced { + Ok(()) + } else { + Err(RuntimeError::IdempotencyOutcomeUnknown { + operation_id: operation.operation_id.as_str().to_owned(), + }) + } +} + +pub(crate) async fn mark_outcome_unknown( + store: &dyn CoordinationStateStore, + operation: &RuntimeOperation, + reservation: &IdempotencyReservation, +) -> Result<(), RuntimeError> { + let unknown = CoordinationStateValue { + payload: json!({ + "state": "outcome_unknown", + "fingerprint": reservation.fingerprint, + }), + }; + let replaced = store + .compare_and_set_value( + CacheScope::Coordination, + &reservation.key, + &reservation.initial, + unknown, + reservation.result_ttl, + ) + .await + .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { + operation_id: operation.operation_id.as_str().to_owned(), + })?; + if replaced { + Ok(()) + } else { + Err(RuntimeError::IdempotencyOutcomeUnknown { + operation_id: operation.operation_id.as_str().to_owned(), + }) + } +} + +async fn resolve_existing( + store: &dyn CoordinationStateStore, + operation: &RuntimeOperation, + key: &str, + fingerprint: &str, + mut existing: CoordinationStateValue, +) -> Result { + let operation_id = operation.operation_id.as_str().to_owned(); + let deadline = + Instant::now() + Duration::from_millis(operation.execution_config.timeout_ms.max(1)); + loop { + let existing_fingerprint = existing.payload.get("fingerprint").and_then(Value::as_str); + if existing_fingerprint != Some(fingerprint) { + return Err(RuntimeError::IdempotencyConflict { operation_id }); + } + match existing.payload.get("state").and_then(Value::as_str) { + Some("completed") => { + let response = existing + .payload + .get("response") + .cloned() + .and_then(|value| serde_json::from_value(value).ok()) + .ok_or_else(|| RuntimeError::InvalidPreparedRequest { + field: "idempotency_state".to_owned(), + reason: "completed idempotency state has no valid response".to_owned(), + })?; + return Ok(IdempotencyAction::Replay(response)); + } + Some("outcome_unknown") => { + return Err(RuntimeError::IdempotencyOutcomeUnknown { operation_id }); + } + Some("in_progress") if Instant::now() < deadline => { + tokio::time::sleep(POLL_INTERVAL).await; + existing = store + .get_value(CacheScope::Coordination, key) + .await + .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { + operation_id: operation_id.clone(), + })? + .ok_or_else(|| RuntimeError::IdempotencyOutcomeUnknown { + operation_id: operation_id.clone(), + })?; + } + Some("in_progress") => { + return Err(RuntimeError::IdempotencyInProgress { operation_id }); + } + _ => { + return Err(RuntimeError::InvalidPreparedRequest { + field: "idempotency_state".to_owned(), + reason: "unknown idempotency state".to_owned(), + }); + } + } + } +} + pub fn policy(operation: &RuntimeOperation) -> Option<&IdempotencyPolicy> { let is_mutating_rest = matches!(&operation.target, Target::Rest(target) if target.method != HttpMethod::Get); @@ -54,7 +256,7 @@ pub fn policy(operation: &RuntimeOperation) -> Option<&IdempotencyPolicy> { Some(policy) } -pub fn cache_key( +fn cache_key( operation: &RuntimeOperation, idempotency_key: &str, request_context: Option<&RuntimeRequestContext>, @@ -76,6 +278,42 @@ pub fn cache_key( )) } +fn request_fingerprint(input: &Value) -> Result { + let canonical = canonical_json(input); + let encoded = + serde_json::to_vec(&canonical).map_err(|error| RuntimeError::InvalidPreparedRequest { + field: "idempotency_fingerprint".to_owned(), + reason: error.to_string(), + })?; + Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded))) +} + +fn canonical_json(value: &Value) -> Value { + match value { + Value::Object(object) => { + let mut entries = object.iter().collect::>(); + entries.sort_unstable_by_key(|(key, _)| *key); + let mut canonical = Map::new(); + for (key, value) in entries { + canonical.insert(key.clone(), canonical_json(value)); + } + Value::Object(canonical) + } + Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()), + _ => value.clone(), + } +} + +fn in_progress_value(fingerprint: &str) -> CoordinationStateValue { + CoordinationStateValue { + payload: json!({ + "state": "in_progress", + "fingerprint": fingerprint, + "owner": Uuid::now_v7().simple().to_string(), + }), + } +} + fn key_from_policy( policy: &IdempotencyPolicy, input: &Value, diff --git a/crates/crank-runtime/src/lib.rs b/crates/crank-runtime/src/lib.rs index e23b748..f5d182b 100644 --- a/crates/crank-runtime/src/lib.rs +++ b/crates/crank-runtime/src/lib.rs @@ -32,7 +32,8 @@ pub use executor_builder::{ pub use limits::{RuntimeLimits, RuntimeLimitsConfigError}; pub use model::{AdapterResponse, PreparedRequest, RuntimeOperation}; pub use rate_limit::{ - RateLimitRejection, RequestRateLimitConfig, RequestRateLimitConfigError, RequestRateLimiter, + RateLimitCheckError, RateLimitRejection, RequestRateLimitConfig, RequestRateLimitConfigError, + RequestRateLimiter, }; pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext}; pub use secret_crypto::SecretCrypto; diff --git a/crates/crank-runtime/src/limits.rs b/crates/crank-runtime/src/limits.rs index 1e5f411..0293033 100644 --- a/crates/crank-runtime/src/limits.rs +++ b/crates/crank-runtime/src/limits.rs @@ -3,16 +3,19 @@ use std::{env, num::ParseIntError}; use thiserror::Error; const DEFAULT_MAX_CONCURRENT_UNARY: usize = 64; +const DEFAULT_MAX_CONCURRENT_SESSIONS: usize = 16; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct RuntimeLimits { pub max_concurrent_unary: usize, + pub max_concurrent_sessions: usize, } impl Default for RuntimeLimits { fn default() -> Self { Self { max_concurrent_unary: DEFAULT_MAX_CONCURRENT_UNARY, + max_concurrent_sessions: DEFAULT_MAX_CONCURRENT_SESSIONS, } } } @@ -24,6 +27,10 @@ impl RuntimeLimits { "CRANK_RUNTIME_MAX_CONCURRENT_UNARY", DEFAULT_MAX_CONCURRENT_UNARY, )?, + max_concurrent_sessions: parse_limit( + "CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS", + DEFAULT_MAX_CONCURRENT_SESSIONS, + )?, }) } } @@ -71,6 +78,7 @@ mod tests { let limits = RuntimeLimits::default(); assert!(limits.max_concurrent_unary > 0); + assert!(limits.max_concurrent_sessions > 0); } #[test] diff --git a/crates/crank-runtime/src/rate_limit.rs b/crates/crank-runtime/src/rate_limit.rs index 44a5aeb..d8192c8 100644 --- a/crates/crank-runtime/src/rate_limit.rs +++ b/crates/crank-runtime/src/rate_limit.rs @@ -4,8 +4,9 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use crank_core::{RateLimitBucketState, RateLimitStateStore}; +use crank_core::{RateLimitDecision, RateLimitStateStore}; use thiserror::Error; +use tracing::warn; const STALE_KEY_TTL: Duration = Duration::from_secs(300); const TOKEN_SCALE: u64 = 1_000_000; @@ -47,6 +48,12 @@ pub struct RateLimitRejection { pub retry_after_ms: u64, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RateLimitCheckError { + Rejected(RateLimitRejection), + StoreUnavailable, +} + #[derive(Clone)] pub struct RequestRateLimiter { config: RequestRateLimitConfig, @@ -87,14 +94,24 @@ impl RequestRateLimiter { } } - pub async fn check(&self, key: &str) -> Result<(), RateLimitRejection> { - match &self.backend { - RequestRateLimiterBackend::Local { .. } => self.check_local_at(key, Instant::now()), + pub async fn check(&self, key: &str) -> Result<(), RateLimitCheckError> { + let result = match &self.backend { + RequestRateLimiterBackend::Local { .. } => self + .check_local_at(key, Instant::now()) + .map_err(RateLimitCheckError::Rejected), RequestRateLimiterBackend::Shared { store } => { self.check_shared_at(store.as_ref(), key, now_unix_ms()) .await } + }; + if matches!(result, Err(RateLimitCheckError::Rejected(_))) { + metrics::counter!( + "crank_runtime_limit_rejections_total", + "stage" => "rate_limit" + ) + .increment(1); } + result } fn check_local_at(&self, key: &str, now: Instant) -> Result<(), RateLimitRejection> { @@ -137,35 +154,35 @@ impl RequestRateLimiter { store: &dyn RateLimitStateStore, key: &str, now_unix_ms: i64, - ) -> Result<(), RateLimitRejection> { - let burst_tokens = u64::from(self.config.burst) * TOKEN_SCALE; - let refill_per_second = u64::from(self.config.requests_per_second) * TOKEN_SCALE; - let mut state = - store - .get_bucket(key) - .await - .unwrap_or(None) - .unwrap_or(RateLimitBucketState { - tokens_micros: burst_tokens, - last_refill_unix_ms: now_unix_ms, - }); - - let elapsed_ms = (now_unix_ms - state.last_refill_unix_ms).max(0) as u64; - let replenished = - state.tokens_micros + (elapsed_ms.saturating_mul(refill_per_second) / 1000); - state.tokens_micros = replenished.min(burst_tokens); - state.last_refill_unix_ms = now_unix_ms; - - if state.tokens_micros >= TOKEN_SCALE { - state.tokens_micros -= TOKEN_SCALE; - let _ = store.put_bucket(key, state, STALE_KEY_TTL).await; - return Ok(()); + ) -> Result<(), RateLimitCheckError> { + let decision = match store + .consume_token( + key, + u64::from(self.config.burst) * TOKEN_SCALE, + u64::from(self.config.requests_per_second) * TOKEN_SCALE, + now_unix_ms, + STALE_KEY_TTL, + ) + .await + { + Ok(decision) => decision, + Err(_) => { + warn!( + name: "runtime.rate_limit.failed_closed", + error_category = "coordination_store", + "shared rate limiter failed closed" + ); + return Err(RateLimitCheckError::StoreUnavailable); + } + }; + match decision { + RateLimitDecision::Allowed => Ok(()), + RateLimitDecision::Rejected { retry_after_ms } => { + Err(RateLimitCheckError::Rejected(RateLimitRejection { + retry_after_ms, + })) + } } - - let missing_tokens = TOKEN_SCALE.saturating_sub(state.tokens_micros); - let retry_after_ms = missing_tokens.div_ceil(refill_per_second).max(1); - let _ = store.put_bucket(key, state, STALE_KEY_TTL).await; - Err(RateLimitRejection { retry_after_ms }) } } @@ -185,6 +202,11 @@ mod tests { time::{Duration, Instant}, }; + use async_trait::async_trait; + use crank_core::{ + CacheStoreError, RateLimitBucketState, RateLimitDecision, RateLimitStateStore, + }; + use crate::InMemoryRateLimitStateStore; use super::{RequestRateLimitConfig, RequestRateLimitConfigError, RequestRateLimiter}; @@ -234,7 +256,12 @@ mod tests { assert!(limiter.check_shared_at_store("key", 0).await.is_ok()); let rejection = limiter.check_shared_at_store("key", 0).await.unwrap_err(); - assert_eq!(rejection.retry_after_ms, 1); + assert_eq!( + rejection, + super::RateLimitCheckError::Rejected(super::RateLimitRejection { + retry_after_ms: 500, + }) + ); assert!(limiter.check_shared_at_store("key", 500).await.is_ok()); } @@ -254,12 +281,120 @@ mod tests { assert!(second.check_shared_at_store("shared", 1000).await.is_ok()); } + #[tokio::test] + async fn shared_limiter_does_not_refill_when_clock_moves_backwards() { + let limiter = RequestRateLimiter::new_shared( + RequestRateLimitConfig::new(1, 1).unwrap(), + Arc::new(InMemoryRateLimitStateStore::default()), + ); + + assert!(limiter.check_shared_at_store("clock", 1_000).await.is_ok()); + assert_eq!( + limiter + .check_shared_at_store("clock", 500) + .await + .unwrap_err(), + super::RateLimitCheckError::Rejected(super::RateLimitRejection { + retry_after_ms: 1_000, + }) + ); + assert_eq!( + limiter + .check_shared_at_store("clock", 1_500) + .await + .unwrap_err(), + super::RateLimitCheckError::Rejected(super::RateLimitRejection { + retry_after_ms: 500, + }) + ); + assert!(limiter.check_shared_at_store("clock", 2_000).await.is_ok()); + } + + #[tokio::test] + async fn shared_limiter_consumes_burst_atomically_under_concurrency() { + let limiter = RequestRateLimiter::new_shared( + RequestRateLimitConfig::new(1, 8).unwrap(), + Arc::new(InMemoryRateLimitStateStore::default()), + ); + let attempts = (0..64).map(|_| limiter.check_shared_at_store("concurrent", 0)); + let results = futures_util::future::join_all(attempts).await; + + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 8); + assert!( + results + .iter() + .filter_map(|result| result.as_ref().err()) + .all(|rejection| { + *rejection + == super::RateLimitCheckError::Rejected(super::RateLimitRejection { + retry_after_ms: 1_000, + }) + }) + ); + } + + #[tokio::test] + async fn shared_limiter_fails_closed_when_store_is_unavailable() { + let limiter = RequestRateLimiter::new_shared( + RequestRateLimitConfig::new(10, 10).unwrap(), + Arc::new(UnavailableRateLimitStore), + ); + + let error = limiter + .check_shared_at_store("unavailable", 0) + .await + .unwrap_err(); + assert_eq!(error, super::RateLimitCheckError::StoreUnavailable); + } + + struct UnavailableRateLimitStore; + + #[async_trait] + impl RateLimitStateStore for UnavailableRateLimitStore { + async fn get_bucket( + &self, + _key: &str, + ) -> Result, CacheStoreError> { + Err(unavailable()) + } + + async fn put_bucket( + &self, + _key: &str, + _value: RateLimitBucketState, + _ttl: Duration, + ) -> Result<(), CacheStoreError> { + Err(unavailable()) + } + + async fn delete_bucket(&self, _key: &str) -> Result<(), CacheStoreError> { + Err(unavailable()) + } + + async fn consume_token( + &self, + _key: &str, + _burst_tokens_micros: u64, + _refill_per_second_micros: u64, + _now_unix_ms: i64, + _ttl: Duration, + ) -> Result { + Err(unavailable()) + } + } + + fn unavailable() -> CacheStoreError { + CacheStoreError::Unavailable { + message: "test store is unavailable".to_owned(), + } + } + impl RequestRateLimiter { async fn check_shared_at_store( &self, key: &str, now_unix_ms: i64, - ) -> Result<(), super::RateLimitRejection> { + ) -> Result<(), super::RateLimitCheckError> { let super::RequestRateLimiterBackend::Shared { store } = &self.backend else { panic!("check_shared_at_store called for non-shared limiter"); }; diff --git a/crates/crank-runtime/src/request_preparation.rs b/crates/crank-runtime/src/request_preparation.rs index c9eb3ba..ddbcff8 100644 --- a/crates/crank-runtime/src/request_preparation.rs +++ b/crates/crank-runtime/src/request_preparation.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use crank_core::Target; use serde_json::Value; -use crate::{PreparedRequest, RuntimeError, RuntimeOperation, RuntimeRequestContext}; +use crate::{PreparedRequest, RuntimeError, RuntimeOperation}; impl PreparedRequest { pub fn from_mapping_output(mapped: &Value) -> Result { @@ -28,7 +28,6 @@ impl PreparedRequest { pub(crate) fn adapter_prepared_request( operation: &RuntimeOperation, prepared_request: &PreparedRequest, - request_context: Option<&RuntimeRequestContext>, timeout_ms: u64, ) -> PreparedRequest { let static_headers = match &operation.target { @@ -40,7 +39,6 @@ pub(crate) fn adapter_prepared_request( static_headers, &operation.execution_config.headers, &prepared_request.headers, - &runtime_context_headers(request_context), ); prepared_request.timeout_ms = timeout_ms; prepared_request @@ -50,23 +48,13 @@ fn merge_headers( static_headers: &BTreeMap, execution_headers: &BTreeMap, request_headers: &BTreeMap, - context_headers: &BTreeMap, ) -> BTreeMap { let mut headers = static_headers.clone(); headers.extend(execution_headers.clone()); headers.extend(request_headers.clone()); - headers.extend(context_headers.clone()); headers } -fn runtime_context_headers( - request_context: Option<&RuntimeRequestContext>, -) -> BTreeMap { - request_context - .map(RuntimeRequestContext::outbound_headers) - .unwrap_or_default() -} - fn read_string_map( value: Option<&Value>, field_name: &str, diff --git a/crates/crank-runtime/tests/integration.rs b/crates/crank-runtime/tests/integration.rs index ba989f4..aa706b8 100644 --- a/crates/crank-runtime/tests/integration.rs +++ b/crates/crank-runtime/tests/integration.rs @@ -2,4 +2,6 @@ mod integration { mod confirmation; mod idempotency; mod no_input_get; + mod stages; + mod valkey; } diff --git a/crates/crank-runtime/tests/integration/confirmation.rs b/crates/crank-runtime/tests/integration/confirmation.rs index 2a4c0d4..993839e 100644 --- a/crates/crank-runtime/tests/integration/confirmation.rs +++ b/crates/crank-runtime/tests/integration/confirmation.rs @@ -16,6 +16,7 @@ use crank_runtime::{ InMemoryCoordinationStateStore, RuntimeError, RuntimeExecutorBuilder, RuntimeRequestContext, }; use crank_schema::{Schema, SchemaKind}; +use futures_util::future::join_all; use serde_json::json; use time::OffsetDateTime; @@ -89,6 +90,62 @@ async fn destructive_operation_requires_single_use_confirmation() { assert_eq!(call_count.load(Ordering::SeqCst), 2); } +#[tokio::test] +async fn confirmation_token_allows_only_one_concurrent_execution() { + let call_count = Arc::new(AtomicUsize::new(0)); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(CountingAdapter { + call_count: Arc::clone(&call_count), + })) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let operation: crank_runtime::RuntimeOperation = destructive_delete_operation().into(); + let context = RuntimeRequestContext::from_request_id("req_confirm_concurrent") + .with_response_cache_scope("workspace_1", "agent_1"); + let first = executor + .execute_with_context( + &operation, + &json!({ "order_id": "ord_123" }), + Some(&context), + ) + .await + .unwrap_err(); + let RuntimeError::ConfirmationRequired { + confirmation_token, .. + } = first + else { + panic!("expected confirmation token") + }; + + let attempts = (0..16).map(|_| { + let executor = executor.clone(); + let operation = operation.clone(); + let context = context + .clone() + .with_confirmation_token(confirmation_token.clone()); + async move { + executor + .execute_with_context( + &operation, + &json!({ "order_id": "ord_123" }), + Some(&context), + ) + .await + } + }); + let results = join_all(attempts).await; + let successful = results.iter().filter(|result| result.is_ok()).count(); + + assert_eq!(successful, 1); + assert_eq!(call_count.load(Ordering::SeqCst), 1); + assert!( + results + .iter() + .filter_map(|result| result.as_ref().err()) + .all(|error| matches!(error, RuntimeError::InvalidConfirmationToken { .. })) + ); +} + struct CountingAdapter { call_count: Arc, } diff --git a/crates/crank-runtime/tests/integration/idempotency.rs b/crates/crank-runtime/tests/integration/idempotency.rs index 141b2f0..009db35 100644 --- a/crates/crank-runtime/tests/integration/idempotency.rs +++ b/crates/crank-runtime/tests/integration/idempotency.rs @@ -12,7 +12,10 @@ use crank_core::{ ToolDescription, }; use crank_mapping::{MappingRule, MappingSet}; -use crank_runtime::{InMemoryResponseCacheStore, RuntimeExecutorBuilder}; +use crank_runtime::{ + InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeError, + RuntimeExecutorBuilder, +}; use crank_schema::{Schema, SchemaKind}; use serde_json::json; use time::OffsetDateTime; @@ -23,8 +26,10 @@ async fn replays_mutation_result_for_same_idempotency_key() { let executor = RuntimeExecutorBuilder::new() .register_adapter(Arc::new(CountingAdapter { call_count: Arc::clone(&call_count), + release: None, })) .with_response_cache(Arc::new(InMemoryResponseCacheStore::default())) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) .build(); let operation = idempotent_post_operation().into(); let context = crank_runtime::RuntimeRequestContext::from_request_id("req_1") @@ -51,8 +56,10 @@ async fn required_idempotency_rejects_missing_key_before_adapter_call() { let executor = RuntimeExecutorBuilder::new() .register_adapter(Arc::new(CountingAdapter { call_count: Arc::clone(&call_count), + release: None, })) .with_response_cache(Arc::new(InMemoryResponseCacheStore::default())) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) .build(); let mut operation: crank_runtime::RuntimeOperation = idempotent_post_operation().into(); let policy = operation.execution_config.idempotency.as_mut().unwrap(); @@ -71,8 +78,189 @@ async fn required_idempotency_rejects_missing_key_before_adapter_call() { assert_eq!(call_count.load(Ordering::SeqCst), 0); } +#[tokio::test] +async fn required_idempotency_fails_closed_without_coordination_store() { + let call_count = Arc::new(AtomicUsize::new(0)); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(CountingAdapter { + call_count: Arc::clone(&call_count), + release: None, + })) + .build(); + let operation = idempotent_post_operation().into(); + let context = crank_runtime::RuntimeRequestContext::from_request_id("req_no_store") + .with_response_cache_scope("workspace_1", "agent_1"); + + let error = executor + .execute_with_context( + &operation, + &json!({ "request_id": "must-not-run" }), + Some(&context), + ) + .await + .expect_err("required idempotency must not execute without an atomic store"); + + assert!(matches!( + error, + RuntimeError::IdempotencyStoreUnavailable { .. } + )); + assert_eq!(call_count.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn concurrent_calls_with_same_key_execute_adapter_once() { + let call_count = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(CountingAdapter { + call_count: Arc::clone(&call_count), + release: Some(Arc::clone(&release)), + })) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let operation: crank_runtime::RuntimeOperation = idempotent_post_operation().into(); + let context = crank_runtime::RuntimeRequestContext::from_request_id("req_concurrent") + .with_response_cache_scope("workspace_1", "agent_1"); + let input = json!({ "request_id": "order-concurrent" }); + + let first_executor = executor.clone(); + let first_operation = operation.clone(); + let first_context = context.clone(); + let first_input = input.clone(); + let first = tokio::spawn(async move { + first_executor + .execute_with_context(&first_operation, &first_input, Some(&first_context)) + .await + }); + wait_for_call_count(&call_count, 1).await; + + let second_executor = executor.clone(); + let second_operation = operation.clone(); + let second_context = context.clone(); + let second_input = input.clone(); + let second = tokio::spawn(async move { + second_executor + .execute_with_context(&second_operation, &second_input, Some(&second_context)) + .await + }); + for _ in 0..100 { + tokio::task::yield_now().await; + } + assert_eq!(call_count.load(Ordering::SeqCst), 1); + + release.notify_waiters(); + let first_result = first.await.unwrap().unwrap(); + let second_result = second.await.unwrap().unwrap(); + assert_eq!(first_result, second_result); + assert_eq!(call_count.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn same_key_with_different_input_is_rejected() { + let call_count = Arc::new(AtomicUsize::new(0)); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(CountingAdapter { + call_count: Arc::clone(&call_count), + release: None, + })) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let operation = idempotent_post_operation().into(); + let context = crank_runtime::RuntimeRequestContext::from_request_id("req_conflict") + .with_response_cache_scope("workspace_1", "agent_1"); + + executor + .execute_with_context( + &operation, + &json!({ "request_id": "stable-key", "amount": 10 }), + Some(&context), + ) + .await + .unwrap(); + let error = executor + .execute_with_context( + &operation, + &json!({ "request_id": "stable-key", "amount": 20 }), + Some(&context), + ) + .await + .expect_err("same key must not accept a different request fingerprint"); + + assert!(matches!(error, RuntimeError::IdempotencyConflict { .. })); + assert_eq!(call_count.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn uncertain_adapter_failure_blocks_automatic_retry() { + let call_count = Arc::new(AtomicUsize::new(0)); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(FailingAdapter { + call_count: Arc::clone(&call_count), + })) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let operation = idempotent_post_operation().into(); + let context = crank_runtime::RuntimeRequestContext::from_request_id("req_unknown") + .with_response_cache_scope("workspace_1", "agent_1"); + let input = json!({ "request_id": "uncertain-outcome" }); + + let first = executor + .execute_with_context(&operation, &input, Some(&context)) + .await + .expect_err("adapter failure must be returned"); + assert!(matches!(first, RuntimeError::ProtocolAdapter(_))); + + let retry = executor + .execute_with_context(&operation, &input, Some(&context)) + .await + .expect_err("an uncertain external outcome must not be retried automatically"); + assert!(matches!( + retry, + RuntimeError::IdempotencyOutcomeUnknown { .. } + )); + assert_eq!(call_count.load(Ordering::SeqCst), 1); +} + +async fn wait_for_call_count(call_count: &AtomicUsize, expected: usize) { + for _ in 0..1_000 { + if call_count.load(Ordering::SeqCst) >= expected { + return; + } + tokio::task::yield_now().await; + } + panic!("adapter did not receive {expected} call(s)"); +} + struct CountingAdapter { call_count: Arc, + release: Option>, +} + +struct FailingAdapter { + call_count: Arc, +} + +#[async_trait] +impl ProtocolAdapter for FailingAdapter { + fn protocol(&self) -> Protocol { + Protocol::Rest + } + + fn supports_mode(&self, mode: ExecutionMode) -> bool { + mode == ExecutionMode::Unary + } + + async fn invoke_unary( + &self, + _target: &Target, + _prepared: &crank_core::PreparedRequest, + _context: &RuntimeRequestContext, + ) -> Result { + self.call_count.fetch_add(1, Ordering::SeqCst); + Err(ProtocolAdapterError::Message( + "upstream outcome is unknown".to_owned(), + )) + } } #[async_trait] @@ -91,11 +279,11 @@ impl ProtocolAdapter for CountingAdapter { prepared: &crank_core::PreparedRequest, _context: &RuntimeRequestContext, ) -> Result { - assert_eq!( - prepared.headers.get("Idempotency-Key").map(String::as_str), - Some("order-123") - ); + assert!(prepared.headers.contains_key("Idempotency-Key")); let call_number = self.call_count.fetch_add(1, Ordering::SeqCst) + 1; + if let Some(release) = &self.release { + release.notified().await; + } Ok(AdapterResponse { status_code: 201, headers: BTreeMap::new(), diff --git a/crates/crank-runtime/tests/integration/stages.rs b/crates/crank-runtime/tests/integration/stages.rs new file mode 100644 index 0000000..0bd0348 --- /dev/null +++ b/crates/crank-runtime/tests/integration/stages.rs @@ -0,0 +1,426 @@ +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +use async_trait::async_trait; +use crank_core::{ + AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod, + IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass, + OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter, + ProtocolAdapterError, RestTarget, Target, ToolDescription, +}; +use crank_mapping::{MappingRule, MappingSet}; +use crank_runtime::{ + InMemoryCoordinationStateStore, RuntimeError, RuntimeExecutorBuilder, RuntimeRequestContext, +}; +use crank_schema::{Schema, SchemaKind}; +use crank_trace::{Stage, StageOutcome}; +use serde_json::json; +use time::OffsetDateTime; +use tracing::{Id, Instrument, Subscriber, field::Visit, instrument::WithSubscriber}; +use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan}; + +#[tokio::test] +async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() { + let capture = TraceCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(SuccessAdapter)) + .build(); + let operation = operation().into(); + let context = RuntimeRequestContext::from_request_id("req_stage_test"); + let result = async { + let root = tracing::info_span!(target: "crank::trace", "mcp.request"); + executor + .execute_with_context( + &operation, + &json!({"name": "canary-secret"}), + Some(&context), + ) + .instrument(root) + .await + } + .with_subscriber(subscriber) + .await; + + assert_eq!(result.unwrap(), json!({"accepted": true})); + let spans = capture.snapshot(); + assert_stage(&spans, "runtime.execute", "success"); + assert_stage(&spans, "runtime.arguments.map", "success"); + assert_stage(&spans, "upstream.http", "success"); + assert_stage(&spans, "runtime.response.transform", "success"); + assert!(!spans.iter().any(|span| span.name == "approval.check")); + assert!(!spans.iter().any(|span| span.name == "runtime.idempotency")); + assert!( + spans + .iter() + .flat_map(|span| span.fields.values()) + .all(|value| !value.contains("canary-secret")) + ); + + let runtime = spans + .iter() + .find(|span| span.name == "runtime.execute") + .expect("runtime span"); + assert_eq!(runtime.parent_name, Some("mcp.request")); + for child in [ + "runtime.arguments.map", + "upstream.http", + "runtime.response.transform", + ] { + assert_eq!( + spans + .iter() + .find(|span| span.name == child) + .and_then(|span| span.parent_name), + Some("runtime.execute"), + "{child} must be a runtime child" + ); + } +} + +#[tokio::test] +async fn failed_mapping_records_closed_category_and_stops_later_stages() { + let capture = TraceCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(SuccessAdapter)) + .build(); + let operation = operation().into(); + + let result = async { executor.execute(&operation, &json!({})).await } + .with_subscriber(subscriber) + .await; + + assert!(result.is_err()); + let spans = capture.snapshot(); + let runtime = spans + .iter() + .find(|span| span.name == "runtime.execute") + .expect("runtime span"); + assert_eq!(runtime.fields["outcome"], "error"); + assert_eq!(runtime.fields["error.category"], "schema"); + assert_stage(&spans, "runtime.arguments.map", "error"); + assert!(!spans.iter().any(|span| span.name == "upstream.http")); + assert!( + !spans + .iter() + .any(|span| span.name == "runtime.response.transform") + ); +} + +#[tokio::test] +async fn approval_stage_is_present_only_when_confirmation_is_required() { + let capture = TraceCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(SuccessAdapter)) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let mut source = operation(); + source.execution_config.safety = Some(OperationSafetyPolicy { + class: OperationSafetyClass::Destructive, + confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }), + }); + let operation = source.into(); + let context = RuntimeRequestContext::from_request_id("req_approval_stage") + .with_response_cache_scope("workspace", "agent"); + + let result = async { + executor + .execute_with_context( + &operation, + &json!({"name": "requires-confirmation"}), + Some(&context), + ) + .await + } + .with_subscriber(subscriber) + .await; + + assert!(matches!( + result, + Err(RuntimeError::ConfirmationRequired { .. }) + )); + let spans = capture.snapshot(); + assert_stage(&spans, "approval.check", "required"); + assert!(!spans.iter().any(|span| span.name == "upstream.http")); + assert!(!spans.iter().any(|span| span.name == "runtime.idempotency")); +} + +#[tokio::test] +async fn idempotency_stage_distinguishes_execution_from_replay() { + let capture = TraceCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let executor = RuntimeExecutorBuilder::new() + .register_adapter(Arc::new(SuccessAdapter)) + .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) + .build(); + let mut source = operation(); + source.execution_config.idempotency = Some(IdempotencyPolicy { + mode: IdempotencyMode::Required, + ttl_ms: 60_000, + input_field: Some("name".to_owned()), + header_name: Some("Idempotency-Key".to_owned()), + }); + let operation = source.into(); + let context = RuntimeRequestContext::from_request_id("req_idempotency_stage") + .with_response_cache_scope("workspace", "agent"); + + let (first, replay) = async { + let first = executor + .execute_with_context(&operation, &json!({"name": "stable-key"}), Some(&context)) + .await; + let replay = executor + .execute_with_context(&operation, &json!({"name": "stable-key"}), Some(&context)) + .await; + (first, replay) + } + .with_subscriber(subscriber) + .await; + + assert!(first.is_ok()); + assert!(replay.is_ok()); + let spans = capture.snapshot(); + let idempotency_outcomes = spans + .iter() + .filter(|span| span.name == "runtime.idempotency") + .map(|span| span.fields["outcome"].as_str()) + .collect::>(); + assert!(idempotency_outcomes.contains(&"execute")); + assert!(idempotency_outcomes.contains(&"replay")); + assert_eq!( + spans + .iter() + .filter(|span| span.name == "upstream.http") + .count(), + 1, + "replay must not pretend to call upstream" + ); +} + +fn assert_stage(spans: &[CapturedSpan], name: &str, outcome: &str) { + let span = spans + .iter() + .find(|span| span.name == name) + .unwrap_or_else(|| panic!("missing stage {name}")); + assert_eq!(span.fields["outcome"], outcome); +} + +struct SuccessAdapter; + +#[async_trait] +impl ProtocolAdapter for SuccessAdapter { + fn protocol(&self) -> Protocol { + Protocol::Rest + } + + fn supports_mode(&self, mode: ExecutionMode) -> bool { + mode == ExecutionMode::Unary + } + + async fn invoke_unary( + &self, + _target: &Target, + _prepared: &crank_core::PreparedRequest, + _context: &crank_core::RuntimeRequestContext, + ) -> Result { + let span = Stage::UpstreamHttp.span(); + let response = Ok(AdapterResponse { + status_code: 200, + headers: BTreeMap::new(), + body: json!({"accepted": true}), + data: json!({"accepted": true}), + }); + StageOutcome::Success.record(&span); + response + } +} + +fn operation() -> Operation { + Operation { + id: OperationId::new("op_stage_test"), + name: "stage_test".to_owned(), + display_name: "Stage test".to_owned(), + category: "test".to_owned(), + protocol: Protocol::Rest, + security_level: OperationSecurityLevel::Standard, + status: OperationStatus::Published, + version: 1, + target: Target::Rest(RestTarget { + base_url: "https://example.invalid".to_owned(), + method: HttpMethod::Post, + path_template: "/test".to_owned(), + static_headers: BTreeMap::new(), + }), + input_schema: object_schema(BTreeMap::from([("name".to_owned(), string_schema())])), + output_schema: object_schema(BTreeMap::from([("accepted".to_owned(), bool_schema())])), + input_mapping: MappingSet { + rules: vec![MappingRule { + source: "$.mcp.name".to_owned(), + target: "$.request.body.name".to_owned(), + required: true, + default_value: None, + transform: None, + condition: None, + notes: None, + }], + }, + output_mapping: MappingSet { + rules: vec![MappingRule { + source: "$.response.body.accepted".to_owned(), + target: "$.output.accepted".to_owned(), + required: true, + default_value: None, + transform: None, + condition: None, + notes: None, + }], + }, + execution_config: ExecutionConfig { + timeout_ms: 1_000, + retry_policy: None, + response_cache: None, + idempotency: None, + safety: None, + approval_policy: None, + auth_profile_ref: None, + headers: BTreeMap::new(), + }, + tool_description: ToolDescription { + title: "Stage test".to_owned(), + description: "Tests trace stages.".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + samples: None, + generated_draft: None, + config_export: None, + wizard_state: None, + created_at: OffsetDateTime::UNIX_EPOCH, + updated_at: OffsetDateTime::UNIX_EPOCH, + published_at: None, + } +} + +fn object_schema(fields: BTreeMap) -> Schema { + Schema { + kind: SchemaKind::Object, + description: None, + required: true, + nullable: false, + default_value: None, + fields, + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} + +fn string_schema() -> Schema { + Schema { + kind: SchemaKind::String, + description: None, + required: true, + nullable: false, + default_value: None, + fields: BTreeMap::new(), + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +} + +fn bool_schema() -> Schema { + Schema { + kind: SchemaKind::Boolean, + ..string_schema() + } +} + +#[derive(Clone, Default)] +struct TraceCapture { + spans: Arc>>, +} + +impl TraceCapture { + fn snapshot(&self) -> Vec { + self.spans.lock().expect("span lock").clone() + } +} + +#[derive(Clone, Debug)] +struct CapturedSpan { + name: &'static str, + parent_name: Option<&'static str>, + fields: BTreeMap, +} + +impl Layer for TraceCapture +where + S: Subscriber + for<'lookup> LookupSpan<'lookup>, +{ + fn on_new_span( + &self, + attributes: &tracing::span::Attributes<'_>, + id: &Id, + context: tracing_subscriber::layer::Context<'_, S>, + ) { + let parent = attributes + .parent() + .and_then(|parent| context.span(parent)) + .or_else(|| { + attributes + .is_contextual() + .then(|| context.lookup_current()) + .flatten() + }); + let mut visitor = FieldVisitor::default(); + attributes.record(&mut visitor); + let mut spans = self.spans.lock().expect("span lock"); + let index = spans.len(); + spans.push(CapturedSpan { + name: attributes.metadata().name(), + parent_name: parent.map(|span| span.metadata().name()), + fields: visitor.fields, + }); + context + .span(id) + .expect("span exists") + .extensions_mut() + .insert(index); + } + + fn on_record( + &self, + id: &Id, + values: &tracing::span::Record<'_>, + context: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = FieldVisitor::default(); + values.record(&mut visitor); + let span = context.span(id).expect("span exists"); + let index = *span.extensions().get::().expect("capture index"); + self.spans.lock().expect("span lock")[index] + .fields + .extend(visitor.fields); + } +} + +#[derive(Default)] +struct FieldVisitor { + fields: BTreeMap, +} + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.fields + .insert(field.name().to_owned(), value.to_owned()); + } + + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.fields + .insert(field.name().to_owned(), format!("{value:?}")); + } +} diff --git a/crates/crank-runtime/tests/integration/valkey.rs b/crates/crank-runtime/tests/integration/valkey.rs new file mode 100644 index 0000000..3d952c5 --- /dev/null +++ b/crates/crank-runtime/tests/integration/valkey.rs @@ -0,0 +1,167 @@ +use std::{sync::Arc, time::Duration}; + +use crank_core::{ + CacheBackend, CacheScope, CoordinationStateReservation, CoordinationStateStore, + CoordinationStateValue, RateLimitDecision, RateLimitStateStore, +}; +use crank_runtime::RedisCacheStore; +use futures_util::future::join_all; +use serde_json::json; +use testcontainers::{ + GenericImage, + core::{IntoContainerPort, WaitFor}, + runners::AsyncRunner, +}; + +#[tokio::test] +async fn valkey_coordination_and_rate_limit_operations_are_atomic() { + let container = GenericImage::new("valkey/valkey", "8-alpine") + .with_exposed_port(6379.tcp()) + .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections")) + .start() + .await + .expect("Valkey test container must start"); + let port = container + .get_host_port_ipv4(6379.tcp()) + .await + .expect("Valkey port must be mapped"); + let store = Arc::new( + RedisCacheStore::connect(CacheBackend::Valkey, &format!("redis://127.0.0.1:{port}/0")) + .await + .expect("runtime store must connect to Valkey"), + ); + + verify_atomic_coordination(store.as_ref()).await; + verify_atomic_rate_limit(store).await; +} + +async fn verify_atomic_coordination(store: &RedisCacheStore) { + let pending = CoordinationStateValue { + payload: json!({ "state": "pending" }), + }; + let attempts = (0..32).map(|_| { + store.reserve_value( + CacheScope::Coordination, + "valkey-reservation", + pending.clone(), + Duration::from_secs(30), + ) + }); + let results = join_all(attempts).await; + assert_eq!( + results + .iter() + .filter(|result| matches!(result, Ok(CoordinationStateReservation::Reserved))) + .count(), + 1 + ); + + let completed = CoordinationStateValue { + payload: json!({ "state": "completed" }), + }; + assert!( + store + .compare_and_set_value( + CacheScope::Coordination, + "valkey-reservation", + &pending, + completed.clone(), + Duration::from_secs(30), + ) + .await + .unwrap() + ); + assert_eq!( + store + .take_value(CacheScope::Coordination, "valkey-reservation") + .await + .unwrap(), + Some(completed) + ); + assert_eq!( + store + .take_value(CacheScope::Coordination, "valkey-reservation") + .await + .unwrap(), + None + ); +} + +async fn verify_atomic_rate_limit(store: Arc) { + let attempts = (0..64).map(|_| { + let store = Arc::clone(&store); + async move { + store + .consume_token( + "valkey-burst", + 8_000_000, + 1_000_000, + 0, + Duration::from_secs(30), + ) + .await + } + }); + let results = join_all(attempts).await; + assert_eq!( + results + .iter() + .filter(|result| matches!(result, Ok(RateLimitDecision::Allowed))) + .count(), + 8 + ); + assert!( + results + .iter() + .filter_map(|result| result.as_ref().ok()) + .all(|decision| matches!( + decision, + RateLimitDecision::Allowed + | RateLimitDecision::Rejected { + retry_after_ms: 1_000 + } + )) + ); + + assert_eq!( + store + .consume_token( + "valkey-retry-after", + 2_000_000, + 2_000_000, + 0, + Duration::from_secs(30), + ) + .await + .unwrap(), + RateLimitDecision::Allowed + ); + assert_eq!( + store + .consume_token( + "valkey-retry-after", + 2_000_000, + 2_000_000, + 0, + Duration::from_secs(30), + ) + .await + .unwrap(), + RateLimitDecision::Allowed + ); + assert_eq!( + store + .consume_token( + "valkey-retry-after", + 2_000_000, + 2_000_000, + 0, + Duration::from_secs(30), + ) + .await + .unwrap(), + RateLimitDecision::Rejected { + retry_after_ms: 500 + } + ); +} diff --git a/crates/crank-runtime/tests/unit/cache.rs b/crates/crank-runtime/tests/unit/cache.rs index db90993..b95dfc3 100644 --- a/crates/crank-runtime/tests/unit/cache.rs +++ b/crates/crank-runtime/tests/unit/cache.rs @@ -1,9 +1,14 @@ -use std::time::Duration; +use std::{ + ffi::OsString, + sync::{Mutex, MutexGuard}, + time::Duration, +}; use crank_core::{ CacheBackend, CacheScope, CacheStoreError, CachedHeader, CachedResponse, - CoordinationStateStore, CoordinationStateValue, RateLimitBucketState, RateLimitStateStore, - ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore, + CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue, + RateLimitBucketState, RateLimitStateStore, ReplayGuardStatus, ReplayGuardStore, + ResponseCacheStore, }; use crank_runtime::{ InMemoryCoordinationStateStore, InMemoryRateLimitStateStore, InMemoryReplayGuardStore, @@ -12,6 +17,13 @@ use crank_runtime::{ }; use serde_json::json; +const CACHE_ENV_NAMES: [&str; 3] = [ + "CRANK_CACHE_BACKEND", + "CRANK_CACHE_URL", + "CRANK_CACHE_DEFAULT_TTL_MS", +]; +static CACHE_ENV_LOCK: Mutex<()> = Mutex::new(()); + #[test] fn defaults_to_in_memory_cache_without_url() { let config = RuntimeCacheConfig::default(); @@ -21,8 +33,24 @@ fn defaults_to_in_memory_cache_without_url() { assert_eq!(config.default_ttl_ms, None); } +#[test] +fn treats_blank_optional_cache_values_as_unset() { + let _env = IsolatedCacheEnv::new(); + unsafe { + std::env::set_var("CRANK_CACHE_URL", " "); + std::env::set_var("CRANK_CACHE_DEFAULT_TTL_MS", " "); + } + + let config = RuntimeCacheConfig::from_env().unwrap(); + + assert_eq!(config.backend, CacheBackend::Memory); + assert_eq!(config.url, None); + assert_eq!(config.default_ttl_ms, None); +} + #[test] fn loads_valkey_config_from_env() { + let _env = IsolatedCacheEnv::new(); unsafe { std::env::set_var("CRANK_CACHE_BACKEND", "valkey"); std::env::set_var("CRANK_CACHE_URL", "redis://cache:6379/0"); @@ -34,16 +62,11 @@ fn loads_valkey_config_from_env() { assert_eq!(config.backend, CacheBackend::Valkey); assert_eq!(config.url.as_deref(), Some("redis://cache:6379/0")); assert_eq!(config.default_ttl_ms, Some(15_000)); - - unsafe { - std::env::remove_var("CRANK_CACHE_BACKEND"); - std::env::remove_var("CRANK_CACHE_URL"); - std::env::remove_var("CRANK_CACHE_DEFAULT_TTL_MS"); - } } #[test] fn rejects_external_backend_without_url() { + let _env = IsolatedCacheEnv::new(); unsafe { std::env::set_var("CRANK_CACHE_BACKEND", "redis"); std::env::remove_var("CRANK_CACHE_URL"); @@ -57,14 +80,11 @@ fn rejects_external_backend_without_url() { backend: CacheBackend::Redis } )); - - unsafe { - std::env::remove_var("CRANK_CACHE_BACKEND"); - } } #[test] fn rejects_zero_ttl() { + let _env = IsolatedCacheEnv::new(); unsafe { std::env::set_var("CRANK_CACHE_DEFAULT_TTL_MS", "0"); } @@ -77,9 +97,44 @@ fn rejects_zero_ttl() { name: "CRANK_CACHE_DEFAULT_TTL_MS" } )); +} - unsafe { - std::env::remove_var("CRANK_CACHE_DEFAULT_TTL_MS"); +struct IsolatedCacheEnv { + _lock: MutexGuard<'static, ()>, + previous: Vec<(&'static str, Option)>, +} + +impl IsolatedCacheEnv { + fn new() -> Self { + let lock = CACHE_ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let previous = CACHE_ENV_NAMES + .iter() + .map(|name| (*name, std::env::var_os(name))) + .collect(); + for name in CACHE_ENV_NAMES { + unsafe { + std::env::remove_var(name); + } + } + Self { + _lock: lock, + previous, + } + } +} + +impl Drop for IsolatedCacheEnv { + fn drop(&mut self) { + for (name, value) in &self.previous { + unsafe { + match value { + Some(value) => std::env::set_var(name, value), + None => std::env::remove_var(name), + } + } + } } } @@ -209,6 +264,81 @@ async fn in_memory_coordination_store_scopes_keys() { ); } +#[tokio::test] +async fn in_memory_coordination_store_atomically_takes_and_reserves_values() { + let store = InMemoryCoordinationStateStore::default(); + let value = CoordinationStateValue { + payload: json!({ "state": "pending" }), + }; + store + .put_value( + CacheScope::Coordination, + "atomic-job", + value.clone(), + Duration::from_secs(20), + ) + .await + .unwrap(); + + let (first, second) = tokio::join!( + store.take_value(CacheScope::Coordination, "atomic-job"), + store.take_value(CacheScope::Coordination, "atomic-job") + ); + assert_eq!( + usize::from(first.unwrap().is_some()) + usize::from(second.unwrap().is_some()), + 1 + ); + + assert_eq!( + store + .reserve_value( + CacheScope::Coordination, + "reservation", + value.clone(), + Duration::from_secs(20), + ) + .await + .unwrap(), + CoordinationStateReservation::Reserved + ); + assert_eq!( + store + .reserve_value( + CacheScope::Coordination, + "reservation", + CoordinationStateValue { + payload: json!({ "state": "other" }), + }, + Duration::from_secs(20), + ) + .await + .unwrap(), + CoordinationStateReservation::Existing(value.clone()) + ); + let completed = CoordinationStateValue { + payload: json!({ "state": "completed" }), + }; + assert!( + store + .compare_and_set_value( + CacheScope::Coordination, + "reservation", + &value, + completed.clone(), + Duration::from_secs(20), + ) + .await + .unwrap() + ); + assert_eq!( + store + .get_value(CacheScope::Coordination, "reservation") + .await + .unwrap(), + Some(completed) + ); +} + #[tokio::test] async fn in_memory_stores_reject_empty_keys() { let response_store = InMemoryResponseCacheStore::default(); diff --git a/crates/crank-schema/Cargo.toml b/crates/crank-schema/Cargo.toml index 1592938..5c8f1ee 100644 --- a/crates/crank-schema/Cargo.toml +++ b/crates/crank-schema/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-schema" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] diff --git a/crates/crank-test-support/Cargo.toml b/crates/crank-test-support/Cargo.toml index 6334b32..c9d6560 100644 --- a/crates/crank-test-support/Cargo.toml +++ b/crates/crank-test-support/Cargo.toml @@ -3,6 +3,7 @@ name = "crank-test-support" edition.workspace = true license.workspace = true rust-version.workspace = true +publish.workspace = true version.workspace = true [dependencies] diff --git a/crates/crank-trace/Cargo.toml b/crates/crank-trace/Cargo.toml new file mode 100644 index 0000000..e3d9817 --- /dev/null +++ b/crates/crank-trace/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "crank-trace" +edition.workspace = true +license.workspace = true +rust-version.workspace = true +publish.workspace = true +version.workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +tracing.workspace = true + +[dev-dependencies] +tracing-subscriber.workspace = true diff --git a/crates/crank-trace/src/lib.rs b/crates/crank-trace/src/lib.rs new file mode 100644 index 0000000..e4c6d27 --- /dev/null +++ b/crates/crank-trace/src/lib.rs @@ -0,0 +1,198 @@ +//! Закрытый семантический контракт spans Crank. +//! +//! Этот crate не настраивает subscriber и не знает об OTLP. Он ограничивает +//! имена и атрибуты стадий статическим словарём, чтобы продуктовые crate не +//! могли случайно экспортировать пользовательские данные. + +use std::future::Future; + +use tracing::{Instrument, Span, field::Empty, info_span}; + +macro_rules! stage_span { + ($name:literal) => { + info_span!( + target: "crank::trace", + $name, + outcome = Empty, + error.category = Empty, + ) + }; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Stage { + McpRateLimit, + McpAccessCheck, + McpCatalogLoad, + McpToolsResolve, + ApprovalCheck, + RuntimeExecute, + RuntimeArgumentsMap, + RuntimeIdempotency, + UpstreamHttp, + RuntimeResponseTransform, + AuthResolve, + ApprovalRecovery, + HistoryWrite, + DbQuery, +} + +impl Stage { + pub fn span(self) -> Span { + match self { + Self::McpRateLimit => stage_span!("mcp.rate_limit"), + Self::McpAccessCheck => stage_span!("mcp.access.check"), + Self::McpCatalogLoad => stage_span!("mcp.catalog.load"), + Self::McpToolsResolve => stage_span!("mcp.tools.resolve"), + Self::ApprovalCheck => stage_span!("approval.check"), + Self::RuntimeExecute => stage_span!("runtime.execute"), + Self::RuntimeArgumentsMap => stage_span!("runtime.arguments.map"), + Self::RuntimeIdempotency => stage_span!("runtime.idempotency"), + Self::UpstreamHttp => stage_span!("upstream.http"), + Self::RuntimeResponseTransform => stage_span!("runtime.response.transform"), + Self::AuthResolve => stage_span!("auth.resolve"), + Self::ApprovalRecovery => stage_span!("approval.recovery"), + Self::HistoryWrite => stage_span!("history.write"), + Self::DbQuery => info_span!( + target: "crank::trace", + "db.query", + outcome = Empty, + error.category = Empty, + db.system = "postgresql", + db.operation = Empty, + ), + } + } + + pub fn db_span(self, operation: DbOperation) -> Option { + if self != Self::DbQuery { + return None; + } + let span = self.span(); + span.record("db.operation", operation.as_str()); + Some(span) + } +} + +pub async fn observe_db_query( + operation: DbOperation, + future: impl Future>, +) -> Result { + let span = Stage::DbQuery + .db_span(operation) + .expect("database operation requires db.query stage"); + let result = future.instrument(span.clone()).await; + match &result { + Ok(_) => StageOutcome::Success.record(&span), + Err(_) => { + StageOutcome::Error.record(&span); + ErrorCategory::Database.record(&span); + } + } + result +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StageOutcome { + Success, + Error, + Allowed, + Denied, + Required, + Replay, + Execute, + Skipped, + CacheHit, +} + +impl StageOutcome { + pub fn record(self, span: &Span) { + span.record("outcome", self.as_str()); + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Error => "error", + Self::Allowed => "allowed", + Self::Denied => "denied", + Self::Required => "required", + Self::Replay => "replay", + Self::Execute => "execute", + Self::Skipped => "skipped", + Self::CacheHit => "cache_hit", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ErrorCategory { + Access, + RateLimit, + Catalog, + Approval, + Idempotency, + Schema, + Mapping, + Upstream, + Transformation, + History, + Database, + Concurrency, + Configuration, + Internal, +} + +impl ErrorCategory { + pub fn record(self, span: &Span) { + span.record("error.category", self.as_str()); + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::Access => "access", + Self::RateLimit => "rate_limit", + Self::Catalog => "catalog", + Self::Approval => "approval", + Self::Idempotency => "idempotency", + Self::Schema => "schema", + Self::Mapping => "mapping", + Self::Upstream => "upstream", + Self::Transformation => "transformation", + Self::History => "history", + Self::Database => "database", + Self::Concurrency => "concurrency", + Self::Configuration => "configuration", + Self::Internal => "internal", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DbOperation { + MachineAccessRead, + MachineAccessTouch, + CatalogLoad, + ApprovalRead, + ApprovalWrite, + AuthProfileRead, + SecretRead, + SecretTouch, + InvocationHistoryWrite, +} + +impl DbOperation { + pub const fn as_str(self) -> &'static str { + match self { + Self::MachineAccessRead => "machine_access.read", + Self::MachineAccessTouch => "machine_access.touch", + Self::CatalogLoad => "catalog.load", + Self::ApprovalRead => "approval.read", + Self::ApprovalWrite => "approval.write", + Self::AuthProfileRead => "auth_profile.read", + Self::SecretRead => "secret.read", + Self::SecretTouch => "secret.touch", + Self::InvocationHistoryWrite => "invocation_history.write", + } + } +} diff --git a/crates/crank-trace/tests/contract.rs b/crates/crank-trace/tests/contract.rs new file mode 100644 index 0000000..7a54b36 --- /dev/null +++ b/crates/crank-trace/tests/contract.rs @@ -0,0 +1,106 @@ +use std::sync::{Arc, Mutex}; + +use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome}; +use tracing::{Id, Subscriber, field::Visit}; +use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan}; + +#[test] +fn stage_names_and_attributes_are_closed() { + let captured = Arc::new(Mutex::new(Vec::new())); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(Arc::clone(&captured))); + + tracing::subscriber::with_default(subscriber, || { + let span = Stage::RuntimeExecute.span(); + StageOutcome::Success.record(&span); + ErrorCategory::Mapping.record(&span); + drop(span); + + let db_span = Stage::DbQuery + .db_span(DbOperation::InvocationHistoryWrite) + .expect("db stage accepts a db operation"); + StageOutcome::Error.record(&db_span); + drop(db_span); + }); + + let spans = captured.lock().expect("captured spans"); + assert_eq!(spans[0].name, "runtime.execute"); + assert_eq!(spans[0].fields["outcome"], "success"); + assert_eq!(spans[0].fields["error.category"], "mapping"); + assert_eq!(spans[1].name, "db.query"); + assert_eq!(spans[1].fields["db.system"], "postgresql"); + assert_eq!(spans[1].fields["db.operation"], "invocation_history.write"); +} + +#[test] +fn non_database_stage_rejects_database_attributes() { + assert!( + Stage::RuntimeExecute + .db_span(DbOperation::CatalogLoad) + .is_none() + ); +} + +#[derive(Clone)] +struct CaptureLayer(Arc>>); + +#[derive(Debug)] +struct CapturedSpan { + name: &'static str, + fields: std::collections::BTreeMap, +} + +impl Layer for CaptureLayer +where + S: Subscriber + for<'lookup> LookupSpan<'lookup>, +{ + fn on_new_span( + &self, + attributes: &tracing::span::Attributes<'_>, + id: &Id, + context: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = FieldVisitor::default(); + attributes.record(&mut visitor); + context + .span(id) + .expect("span exists") + .extensions_mut() + .insert(self.0.lock().expect("capture lock").len()); + self.0.lock().expect("capture lock").push(CapturedSpan { + name: attributes.metadata().name(), + fields: visitor.fields, + }); + } + + fn on_record( + &self, + id: &Id, + values: &tracing::span::Record<'_>, + context: tracing_subscriber::layer::Context<'_, S>, + ) { + let span = context.span(id).expect("span exists"); + let index = *span.extensions().get::().expect("capture index"); + let mut visitor = FieldVisitor::default(); + values.record(&mut visitor); + self.0.lock().expect("capture lock")[index] + .fields + .extend(visitor.fields); + } +} + +#[derive(Default)] +struct FieldVisitor { + fields: std::collections::BTreeMap, +} + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.fields + .insert(field.name().to_owned(), value.to_owned()); + } + + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.fields + .insert(field.name().to_owned(), format!("{value:?}")); + } +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..372b5af --- /dev/null +++ b/deny.toml @@ -0,0 +1,119 @@ +[graph] +all-features = true +targets = ["x86_64-unknown-linux-gnu"] + +[advisories] +yanked = "deny" +unmaintained = "workspace" +unsound = "all" + +[bans] +multiple-versions = "warn" +wildcards = "deny" +allow-wildcard-paths = true + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] + +[licenses] +confidence-threshold = 0.93 +unused-allowed-license = "warn" +allow = [ + "0BSD", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "MIT", +] + +[licenses.private] +ignore = true + +# Точечные исключения для обязательных транзитивных зависимостей с +# разрешительными лицензиями, не входящими в базовый список проекта. +[[licenses.exceptions]] +name = "foldhash" +allow = ["Zlib"] + +[[licenses.exceptions]] +name = "webpki-roots" +allow = ["CDLA-Permissive-2.0"] + +[[licenses.exceptions]] +name = "icu_collections" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "unicode-ident" +allow = ["MIT", "Apache-2.0", "Unicode-3.0"] + +[[licenses.exceptions]] +name = "icu_locale_core" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "icu_normalizer" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "icu_normalizer_data" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "icu_properties" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "icu_properties_data" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "icu_provider" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "litemap" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "potential_utf" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "tinystr" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "writeable" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "yoke" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "yoke-derive" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "zerofrom" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "zerofrom-derive" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "zerotrie" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "zerovec" +allow = ["Unicode-3.0"] + +[[licenses.exceptions]] +name = "zerovec-derive" +allow = ["Unicode-3.0"] diff --git a/deploy/community/.env.example b/deploy/community/.env.example index a201d07..4eb8663 100644 --- a/deploy/community/.env.example +++ b/deploy/community/.env.example @@ -30,7 +30,26 @@ CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304 CRANK_CACHE_BACKEND=memory CRANK_CACHE_URL= CRANK_CACHE_DEFAULT_TTL_MS= +CRANK_ENVIRONMENT=production CRANK_LOG_LEVEL=info +CRANK_SENTRY_DSN= +CRANK_METRICS_ENABLED=true +CRANK_ADMIN_METRICS_BIND=127.0.0.1:9464 +CRANK_MCP_METRICS_BIND=127.0.0.1:9465 +CRANK_METRICS_BEARER_TOKEN= +CRANK_INVOCATION_LOG_RETENTION_DAYS=30 +OTEL_EXPORTER_OTLP_ENDPOINT= +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_TRACES_PROTOCOL= +OTEL_EXPORTER_OTLP_TIMEOUT=10000 +OTEL_EXPORTER_OTLP_TRACES_TIMEOUT= +OTEL_EXPORTER_OTLP_HEADERS= +OTEL_EXPORTER_OTLP_TRACES_HEADERS= +OTEL_BSP_MAX_QUEUE_SIZE=2048 +OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512 +OTEL_BSP_SCHEDULE_DELAY=5000 +OTEL_BSP_EXPORT_TIMEOUT=30000 CRANK_MASTER_KEY=change-me-master-key CRANK_SESSION_SECRET=change-me-session-secret CRANK_PASSWORD_PEPPER=change-me-password-pepper diff --git a/deploy/community/.env.images.example b/deploy/community/.env.images.example index eb7c159..b430ef3 100644 --- a/deploy/community/.env.images.example +++ b/deploy/community/.env.images.example @@ -24,7 +24,24 @@ CRANK_CACHE_BACKEND=memory CRANK_CACHE_URL= CRANK_CACHE_DEFAULT_TTL_MS= +CRANK_ENVIRONMENT=production CRANK_LOG_LEVEL=info +CRANK_METRICS_ENABLED=true +CRANK_ADMIN_METRICS_BIND=127.0.0.1:9464 +CRANK_MCP_METRICS_BIND=127.0.0.1:9465 +CRANK_METRICS_BEARER_TOKEN= +OTEL_EXPORTER_OTLP_ENDPOINT= +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_TRACES_PROTOCOL= +OTEL_EXPORTER_OTLP_TIMEOUT=10000 +OTEL_EXPORTER_OTLP_TRACES_TIMEOUT= +OTEL_EXPORTER_OTLP_HEADERS= +OTEL_EXPORTER_OTLP_TRACES_HEADERS= +OTEL_BSP_MAX_QUEUE_SIZE=2048 +OTEL_BSP_MAX_EXPORT_BATCH_SIZE=512 +OTEL_BSP_SCHEDULE_DELAY=5000 +OTEL_BSP_EXPORT_TIMEOUT=30000 CRANK_MASTER_KEY=change-me-master-key CRANK_SESSION_SECRET=change-me-session-secret CRANK_PASSWORD_PEPPER=change-me-password-pepper diff --git a/deploy/community/docker-compose.images.yml b/deploy/community/docker-compose.images.yml index 373300c..f99b8eb 100644 --- a/deploy/community/docker-compose.images.yml +++ b/deploy/community/docker-compose.images.yml @@ -37,18 +37,48 @@ services: admin-api: image: ${CRANK_ADMIN_API_IMAGE:-git.itexp.me/bsodfather/crank-community-admin-api:main} restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + required: false environment: POSTGRES_HOST: ${POSTGRES_HOST:-postgres} POSTGRES_PORT: ${POSTGRES_PORT:-5432} POSTGRES_DB: ${POSTGRES_DB:-crank} POSTGRES_USER: ${POSTGRES_USER:-crank} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-crank} + POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-20} + POSTGRES_MIN_CONNECTIONS: ${POSTGRES_MIN_CONNECTIONS:-2} + POSTGRES_ACQUIRE_TIMEOUT_MS: ${POSTGRES_ACQUIRE_TIMEOUT_MS:-5000} + POSTGRES_IDLE_TIMEOUT_MS: ${POSTGRES_IDLE_TIMEOUT_MS:-600000} + POSTGRES_MAX_LIFETIME_MS: ${POSTGRES_MAX_LIFETIME_MS:-1800000} CRANK_STORAGE_ROOT: ${CRANK_STORAGE_ROOT:-/var/lib/crank/storage} CRANK_ADMIN_BIND: ${CRANK_ADMIN_BIND:-0.0.0.0:3001} CRANK_CACHE_BACKEND: ${CRANK_CACHE_BACKEND:-memory} CRANK_CACHE_URL: ${CRANK_CACHE_URL:-} CRANK_CACHE_DEFAULT_TTL_MS: ${CRANK_CACHE_DEFAULT_TTL_MS:-} + CRANK_ADMIN_RATE_LIMIT_RPS: ${CRANK_ADMIN_RATE_LIMIT_RPS:-30} + CRANK_ADMIN_RATE_LIMIT_BURST: ${CRANK_ADMIN_RATE_LIMIT_BURST:-60} + CRANK_RUNTIME_MAX_CONCURRENT_UNARY: ${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-64} + CRANK_ENVIRONMENT: ${CRANK_ENVIRONMENT:-production} CRANK_LOG_LEVEL: ${CRANK_LOG_LEVEL:-info} + CRANK_SENTRY_DSN: ${CRANK_SENTRY_DSN:-} + CRANK_METRICS_ENABLED: ${CRANK_METRICS_ENABLED:-true} + CRANK_ADMIN_METRICS_BIND: ${CRANK_ADMIN_METRICS_BIND:-127.0.0.1:9464} + CRANK_METRICS_BEARER_TOKEN: ${CRANK_METRICS_BEARER_TOKEN:-} + CRANK_INVOCATION_LOG_RETENTION_DAYS: ${CRANK_INVOCATION_LOG_RETENTION_DAYS:-30} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: ${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-} + OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-10000} + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: ${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-} + OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: ${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-512} + OTEL_BSP_SCHEDULE_DELAY: ${OTEL_BSP_SCHEDULE_DELAY:-5000} + OTEL_BSP_EXPORT_TIMEOUT: ${OTEL_BSP_EXPORT_TIMEOUT:-30000} CRANK_MASTER_KEY: ${CRANK_MASTER_KEY} CRANK_BASE_URL: ${CRANK_BASE_URL:-http://localhost:3000} CRANK_SESSION_SECRET: ${CRANK_SESSION_SECRET} @@ -67,7 +97,7 @@ services: ports: - "${CRANK_PUBLISH_BIND:-127.0.0.1}:3001:3001" healthcheck: - test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/health"] + test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"] interval: 15s timeout: 5s retries: 5 @@ -75,19 +105,49 @@ services: mcp-server: image: ${CRANK_MCP_SERVER_IMAGE:-git.itexp.me/bsodfather/crank-community-mcp-server:main} restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + required: false environment: POSTGRES_HOST: ${POSTGRES_HOST:-postgres} POSTGRES_PORT: ${POSTGRES_PORT:-5432} POSTGRES_DB: ${POSTGRES_DB:-crank} POSTGRES_USER: ${POSTGRES_USER:-crank} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-crank} + POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-20} + POSTGRES_MIN_CONNECTIONS: ${POSTGRES_MIN_CONNECTIONS:-2} + POSTGRES_ACQUIRE_TIMEOUT_MS: ${POSTGRES_ACQUIRE_TIMEOUT_MS:-5000} + POSTGRES_IDLE_TIMEOUT_MS: ${POSTGRES_IDLE_TIMEOUT_MS:-600000} + POSTGRES_MAX_LIFETIME_MS: ${POSTGRES_MAX_LIFETIME_MS:-1800000} CRANK_STORAGE_ROOT: ${CRANK_STORAGE_ROOT:-/var/lib/crank/storage} CRANK_MCP_BIND: ${CRANK_MCP_BIND:-0.0.0.0:3002} CRANK_MCP_REFRESH_MS: ${CRANK_MCP_REFRESH_MS:-5000} CRANK_CACHE_BACKEND: ${CRANK_CACHE_BACKEND:-memory} CRANK_CACHE_URL: ${CRANK_CACHE_URL:-} CRANK_CACHE_DEFAULT_TTL_MS: ${CRANK_CACHE_DEFAULT_TTL_MS:-} + CRANK_MCP_RATE_LIMIT_RPS: ${CRANK_MCP_RATE_LIMIT_RPS:-60} + CRANK_MCP_RATE_LIMIT_BURST: ${CRANK_MCP_RATE_LIMIT_BURST:-120} + CRANK_RUNTIME_MAX_CONCURRENT_UNARY: ${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-64} + CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS: ${CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS:-16} + CRANK_ENVIRONMENT: ${CRANK_ENVIRONMENT:-production} CRANK_LOG_LEVEL: ${CRANK_LOG_LEVEL:-info} + CRANK_SENTRY_DSN: ${CRANK_SENTRY_DSN:-} + CRANK_METRICS_ENABLED: ${CRANK_METRICS_ENABLED:-true} + CRANK_MCP_METRICS_BIND: ${CRANK_MCP_METRICS_BIND:-127.0.0.1:9465} + CRANK_METRICS_BEARER_TOKEN: ${CRANK_METRICS_BEARER_TOKEN:-} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: ${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-} + OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-10000} + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: ${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-} + OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: ${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-512} + OTEL_BSP_SCHEDULE_DELAY: ${OTEL_BSP_SCHEDULE_DELAY:-5000} + OTEL_BSP_EXPORT_TIMEOUT: ${OTEL_BSP_EXPORT_TIMEOUT:-30000} CRANK_MASTER_KEY: ${CRANK_MASTER_KEY} CRANK_BASE_URL: ${CRANK_BASE_URL:-http://localhost:3000} CRANK_OUTBOUND_ALLOWED_HOSTS: ${CRANK_OUTBOUND_ALLOWED_HOSTS:-} @@ -98,7 +158,7 @@ services: ports: - "${CRANK_PUBLISH_BIND:-127.0.0.1}:3002:3002" healthcheck: - test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/health"] + test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/ready"] interval: 15s timeout: 5s retries: 5 diff --git a/deploy/community/docker-compose.yml b/deploy/community/docker-compose.yml index ee11401..2d5660f 100644 --- a/deploy/community/docker-compose.yml +++ b/deploy/community/docker-compose.yml @@ -27,12 +27,38 @@ services: POSTGRES_DB: ${POSTGRES_DB:-crank} POSTGRES_USER: ${POSTGRES_USER:-crank} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-crank} + POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-20} + POSTGRES_MIN_CONNECTIONS: ${POSTGRES_MIN_CONNECTIONS:-2} + POSTGRES_ACQUIRE_TIMEOUT_MS: ${POSTGRES_ACQUIRE_TIMEOUT_MS:-5000} + POSTGRES_IDLE_TIMEOUT_MS: ${POSTGRES_IDLE_TIMEOUT_MS:-600000} + POSTGRES_MAX_LIFETIME_MS: ${POSTGRES_MAX_LIFETIME_MS:-1800000} CRANK_STORAGE_ROOT: ${CRANK_STORAGE_ROOT:-/var/lib/crank/storage} CRANK_ADMIN_BIND: ${CRANK_ADMIN_BIND:-0.0.0.0:3001} CRANK_CACHE_BACKEND: ${CRANK_CACHE_BACKEND:-memory} CRANK_CACHE_URL: ${CRANK_CACHE_URL:-} CRANK_CACHE_DEFAULT_TTL_MS: ${CRANK_CACHE_DEFAULT_TTL_MS:-} + CRANK_ADMIN_RATE_LIMIT_RPS: ${CRANK_ADMIN_RATE_LIMIT_RPS:-30} + CRANK_ADMIN_RATE_LIMIT_BURST: ${CRANK_ADMIN_RATE_LIMIT_BURST:-60} + CRANK_RUNTIME_MAX_CONCURRENT_UNARY: ${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-64} + CRANK_ENVIRONMENT: ${CRANK_ENVIRONMENT:-production} CRANK_LOG_LEVEL: ${CRANK_LOG_LEVEL:-info} + CRANK_SENTRY_DSN: ${CRANK_SENTRY_DSN:-} + CRANK_METRICS_ENABLED: ${CRANK_METRICS_ENABLED:-true} + CRANK_ADMIN_METRICS_BIND: ${CRANK_ADMIN_METRICS_BIND:-127.0.0.1:9464} + CRANK_METRICS_BEARER_TOKEN: ${CRANK_METRICS_BEARER_TOKEN:-} + CRANK_INVOCATION_LOG_RETENTION_DAYS: ${CRANK_INVOCATION_LOG_RETENTION_DAYS:-30} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: ${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-} + OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-10000} + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: ${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-} + OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: ${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-512} + OTEL_BSP_SCHEDULE_DELAY: ${OTEL_BSP_SCHEDULE_DELAY:-5000} + OTEL_BSP_EXPORT_TIMEOUT: ${OTEL_BSP_EXPORT_TIMEOUT:-30000} CRANK_MASTER_KEY: ${CRANK_MASTER_KEY} CRANK_BASE_URL: ${CRANK_BASE_URL:-http://localhost:3000} CRANK_SESSION_SECRET: ${CRANK_SESSION_SECRET} @@ -51,7 +77,7 @@ services: ports: - "${CRANK_PUBLISH_BIND:-127.0.0.1}:3001:3001" healthcheck: - test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/health"] + test: ["CMD", "curl", "--fail", "http://127.0.0.1:3001/ready"] interval: 15s timeout: 5s retries: 5 @@ -68,13 +94,39 @@ services: POSTGRES_DB: ${POSTGRES_DB:-crank} POSTGRES_USER: ${POSTGRES_USER:-crank} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-crank} + POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-20} + POSTGRES_MIN_CONNECTIONS: ${POSTGRES_MIN_CONNECTIONS:-2} + POSTGRES_ACQUIRE_TIMEOUT_MS: ${POSTGRES_ACQUIRE_TIMEOUT_MS:-5000} + POSTGRES_IDLE_TIMEOUT_MS: ${POSTGRES_IDLE_TIMEOUT_MS:-600000} + POSTGRES_MAX_LIFETIME_MS: ${POSTGRES_MAX_LIFETIME_MS:-1800000} CRANK_STORAGE_ROOT: ${CRANK_STORAGE_ROOT:-/var/lib/crank/storage} CRANK_MCP_BIND: ${CRANK_MCP_BIND:-0.0.0.0:3002} CRANK_MCP_REFRESH_MS: ${CRANK_MCP_REFRESH_MS:-5000} CRANK_CACHE_BACKEND: ${CRANK_CACHE_BACKEND:-memory} CRANK_CACHE_URL: ${CRANK_CACHE_URL:-} CRANK_CACHE_DEFAULT_TTL_MS: ${CRANK_CACHE_DEFAULT_TTL_MS:-} + CRANK_MCP_RATE_LIMIT_RPS: ${CRANK_MCP_RATE_LIMIT_RPS:-60} + CRANK_MCP_RATE_LIMIT_BURST: ${CRANK_MCP_RATE_LIMIT_BURST:-120} + CRANK_RUNTIME_MAX_CONCURRENT_UNARY: ${CRANK_RUNTIME_MAX_CONCURRENT_UNARY:-64} + CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS: ${CRANK_RUNTIME_MAX_CONCURRENT_SESSIONS:-16} + CRANK_ENVIRONMENT: ${CRANK_ENVIRONMENT:-production} CRANK_LOG_LEVEL: ${CRANK_LOG_LEVEL:-info} + CRANK_SENTRY_DSN: ${CRANK_SENTRY_DSN:-} + CRANK_METRICS_ENABLED: ${CRANK_METRICS_ENABLED:-true} + CRANK_MCP_METRICS_BIND: ${CRANK_MCP_METRICS_BIND:-127.0.0.1:9465} + CRANK_METRICS_BEARER_TOKEN: ${CRANK_METRICS_BEARER_TOKEN:-} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: ${OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:-} + OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-10000} + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: ${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-} + OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: ${OTEL_EXPORTER_OTLP_TRACES_HEADERS:-} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: ${OTEL_BSP_MAX_EXPORT_BATCH_SIZE:-512} + OTEL_BSP_SCHEDULE_DELAY: ${OTEL_BSP_SCHEDULE_DELAY:-5000} + OTEL_BSP_EXPORT_TIMEOUT: ${OTEL_BSP_EXPORT_TIMEOUT:-30000} CRANK_MASTER_KEY: ${CRANK_MASTER_KEY} CRANK_BASE_URL: ${CRANK_BASE_URL:-http://localhost:3000} CRANK_OUTBOUND_ALLOWED_HOSTS: ${CRANK_OUTBOUND_ALLOWED_HOSTS:-} @@ -85,7 +137,7 @@ services: ports: - "${CRANK_PUBLISH_BIND:-127.0.0.1}:3002:3002" healthcheck: - test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/health"] + test: ["CMD", "curl", "--fail", "http://127.0.0.1:3002/ready"] interval: 15s timeout: 5s retries: 5 diff --git a/docs/deployment.md b/docs/deployment.md index 528678a..4ccc503 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -22,6 +22,8 @@ admin-api -> PostgreSQL mcp-server -> PostgreSQL admin-api -> Valkey/Redis, опционально mcp-server -> Valkey/Redis, опционально +admin-api -> внешний OTLP endpoint, опционально +mcp-server -> внешний OTLP endpoint, опционально ``` ## Файлы запуска @@ -152,6 +154,8 @@ docker compose up -d ```bash curl http://127.0.0.1:3001/health curl http://127.0.0.1:3002/health +curl http://127.0.0.1:3001/ready +curl http://127.0.0.1:3002/ready ``` Ожидаемые ответы: @@ -169,7 +173,22 @@ curl -I http://127.0.0.1:3000/ ## Эксплуатация -- Делайте регулярные бэкапы PostgreSQL. +- `/health` проверяет, что процесс жив; `/ready` дополнительно проверяет доступность PostgreSQL и используется Docker Compose. +- CD перед каждым обновлением создаёт согласованный комплект в `backups/`: дамп PostgreSQL, том артефактов, окружение, Compose и контрольные суммы. Хранятся последние пять локальных комплектов. +- Копируйте комплекты резервных копий на отдельный хост или в объектное хранилище. +- Проверенное восстановление выполняется только с явным подтверждением: + +```bash +CRANK_RESTORE_CONFIRM=restore ./scripts/restore-community.sh /opt/crank /opt/crank/backups/20260721T120000Z +``` + +- Обновления схемы выполняются под блокировкой, одной транзакцией и фиксируются в `__crank_core_migrations`. Миграции Community должны оставаться обратно совместимыми с предыдущей версией приложения. - Не храните реальные секреты в Git. -- Для rollback используйте конкретные image tags, а не только `main`. +- CD использует неизменяемые теги коммитов и автоматически возвращает прежнюю конфигурацию и образы при провале readiness. - `CRANK_PUBLISH_BIND=0.0.0.0` нужен только если reverse proxy работает на другом host. +- OTLP Collector и хранилище трасс не входят в Community Compose. Для + внешнего приёмника задайте `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`; секретные + headers передавайте через OpenBao, а не через файлы репозитория. +- GlitchTip и другие Sentry-совместимые приёмники также не входят в Community + Compose. Для отправки только критических ошибок передайте + `CRANK_SENTRY_DSN` через OpenBao; пустое значение отключает канал. diff --git a/docs/observability.md b/docs/observability.md index 0cfec0e..8df0cd6 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -1,6 +1,286 @@ -# Журналы и использование +# Наблюдаемость -Crank сохраняет данные о тестовых запусках и вызовах опубликованных MCP-инструментов. +## Эксплуатационные журналы stdout + +`admin-api` и `mcp-server` используют один жизненный цикл наблюдаемости и +одинаковую однострочную JSON-схему: + +```json +{ + "timestamp": "2026-07-26T12:34:56.789Z", + "level": "INFO", + "service": "admin-api", + "version": "0.3.1", + "environment": "production", + "target": "admin_api::request_context", + "event": "admin.request.completed", + "request_id": "019...", + "fields": { + "method": "GET", + "route": "/api/operations", + "status": 200 + } +} +``` + +Обязательны `timestamp`, `level`, `service`, `version`, `environment`, +`target`, `event` и объект `fields`. Поля `request_id` и `trace_id` независимы +и появляются только при наличии соответствующего контекста. Имена событий +статичны и имеют вид `<компонент>.<объект>.<исход>`. + +Форматирование не использует ANSI. Одна строка stdout всегда соответствует +одному JSON-объекту. + +### Сквозная корреляция + +Admin API и MCP принимают `x-request-id` как непрозрачный идентификатор. +Допустимое входное значение сохраняется без изменений. Если заголовок +отсутствует или содержит пробелы, управляющие символы, `,`, `;`, не-ASCII +символы либо больше 128 байт, Crank создаёт UUIDv7. + +Один идентификатор: + +- возвращается в `x-request-id` успешного или ошибочного ответа; +- записывается в корневой span входного запроса; +- передаётся в runtime; +- заменяет статические или полученные из mapping значения + `x-request-id` и `x-correlation-id` перед исходящим REST-запросом; +- сохраняется в прикладной истории вызова. + +`request id` не является `trace id` и не подменяет распределённую +трассировку. + +### Очистка + +Перед сериализацией рекурсивно очищаются пароли, секреты, токены, ключи +доступа, authorization, cookie, query, полные payload, body, arguments, +result и response. Составные имена полей проверяются по тем же правилам. +Из URL удаляются учётные данные, query и fragment. + +Пределы одного события: + +- строка: 1024 байта; +- массив: 32 элемента; +- объект: 64 поля; +- вложенность: 8 уровней; +- вся строка JSON вместе с завершающим переводом строки: 16 КиБ. + +Усечение сохраняет корректный UTF-8 и JSON. Секрет заменяется целиком на +`[REDACTED]`; его часть, длина или хеш не выводятся. Если после очистки +событие превышает 16 КиБ, `fields` заменяется ограниченным признаком +усечения. Произвольное представление `Debug` не считается безопасным +структурированным значением: для вложенных данных используется +предварительно очищенный JSON. + +Уровень и окружение задаются через `CRANK_LOG_LEVEL` и +`CRANK_ENVIRONMENT`. Для stdout не требуются внешние службы, фоновые очереди +или сетевые endpoint. Их отсутствие не влияет на readiness. + +## Критические ошибки + +Необязательный Sentry-совместимый канал включается одним значением: + +```env +CRANK_SENTRY_DSN=https://public-key@errors.example.com/1 +``` + +Пустой или отсутствующий `CRANK_SENTRY_DSN` полностью отключает канал. +Неверное непустое значение останавливает запуск до приёма запросов, при этом +само значение не попадает в ошибку. Стандартная переменная `SENTRY_DSN` не +используется. + +Канал получает только необработанные panic и явно классифицированные +критические ошибки. Ожидаемые ошибки HTTP, MCP, runtime и внешних API остаются +в обычных журналах, показателях и трассах. Для критического события допустимы +только: + +- закрытая категория `panic`, `startup`, `internal` или `data_integrity`; +- `service`, `release` и `environment`; +- доступные `request_id` и `trace_id`; +- статическое сообщение без исходного текста ошибки. + +До отправки удаляются request, user, breadcrumbs, URL, query, cookie, +authorization, payload, произвольные contexts и extra, а также исходный текст +panic или ошибки. Performance tracing, журналы, показатели и отслеживание +сессий Sentry SDK отключены. Отказ внешнего приёмника не меняет результат +продуктового запроса и не создаёт рекурсивное событие. + +GlitchTip и другие Sentry-совместимые приёмники не входят в Community Compose. +Их доступность не участвует в `/health` и `/ready`. + +## Показатели Prometheus + +`admin-api` и `mcp-server` создают отдельные поверхности показателей. Они не +смешиваются с продуктовыми маршрутами: + +| Сервис | Значение по умолчанию | +|---|---| +| `admin-api` | `127.0.0.1:9464` | +| `mcp-server` | `127.0.0.1:9465` | + +На каждом адресе существуют только: + +- `/metrics` — показатели Prometheus; +- `/health` — статическая проверка самого listener-а. + +Любой другой путь возвращает `404`. `/health` этой поверхности не заменяет +`/ready` рабочего сервиса. Ошибка запуска listener-а останавливает процесс до +приёма рабочих запросов. + +Loopback доступен без авторизации. Для любого non-loopback адреса обязателен +отдельный `CRANK_METRICS_BEARER_TOKEN`; без него конфигурация отклоняется при +старте. Токен не переиспользует сессию администратора или ключ MCP. Оба +маршрута внешней поверхности требуют `Authorization: Bearer ...`. + +Community Compose не содержит Prometheus или Grafana и не публикует порты +9464/9465 на host. Для сетевого сборщика необходимо явно: + +1. задать `CRANK_ADMIN_METRICS_BIND=0.0.0.0:9464` и + `CRANK_MCP_METRICS_BIND=0.0.0.0:9465`; +2. передать непустой `CRANK_METRICS_BEARER_TOKEN`; +3. подключить сборщик к внутренней сети Compose либо отдельно опубликовать + нужные порты с ограничением firewall. + +Токен сборщика хранится в файле, а не в `prometheus.yml`. Актуальная +конфигурация Prometheus использует `authorization.credentials_file`: + +```yaml +scrape_configs: + - job_name: crank-admin + static_configs: + - targets: ["admin-api:9464"] + authorization: + type: Bearer + credentials_file: /run/secrets/crank_metrics_token + + - job_name: crank-mcp + static_configs: + - targets: ["mcp-server:9465"] + authorization: + type: Bearer + credentials_file: /run/secrets/crank_metrics_token +``` + +Семейства первой версии: + +- `crank_http_requests_total`, `crank_http_request_duration_seconds`, + `crank_http_inflight`; +- `crank_mcp_requests_total`, `crank_mcp_active_sessions`; +- `crank_tool_invocations_total`, `crank_tool_invocation_duration_seconds`; +- `crank_upstream_requests_total`, + `crank_upstream_request_duration_seconds`; +- `crank_runtime_inflight`, `crank_runtime_limit_rejections_total`; +- `crank_db_pool_connections`; +- `crank_catalog_tools`, `crank_catalog_estimated_context_tokens`, + `crank_catalog_warnings`; +- `crank_invocation_history_lost_total`, + `crank_telemetry_export_failures_total`. + +Recorder добавляет к рядам проверенные статические labels `service`, +`version`, `environment`. Остальные labels имеют закрытый набор значений. +Запрещено использовать workspace, идентификаторы агента, операции или +запроса, фактический URL, текст ошибки, payload и пользовательский текст. +HTTP route берётся только из шаблона Axum; неизвестные маршруты и методы +сворачиваются в `unmatched` и `OTHER`. + +Buckets длительности фиксированы кодом: +`0.005`, `0.01`, `0.025`, `0.05`, `0.1`, `0.25`, `0.5`, `1`, `2.5`, `5`, +`10`, `30`, `60` секунд. Это пока техническая шкала, а не SLO. Фактические +series и расход памяти измеряются в истории 1.8; произвольный численный +бюджет до замеров не назначается. + +## Распределённые трассы + +Crank принимает и передаёт стандартный W3C `traceparent` на границах Admin +API, MCP и исходящих REST-вызовов. `x-request-id` остаётся отдельным +идентификатором запроса. Baggage не извлекается и не передаётся. + +Экспорт отключён по умолчанию: при пустых +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` и `OTEL_EXPORTER_OTLP_ENDPOINT` +provider, batch processor и фоновый поток не создаются. После включения +используется только OTLP/HTTP binary protobuf. Community Compose не включает +Collector или хранилище трасс: оператор подключает внешний совместимый +приёмник. + +Уровень эксплуатационных журналов не отключает трассы. В OTLP попадают +только явно отмеченные spans с внутренней целью `crank::trace`; события +`tracing` не экспортируются, чтобы их поля не обходили очистку JSON-журналов. +Новые экспортируемые spans должны использовать ту же цель и содержать только +ограниченные безопасные атрибуты. + +Очередь экспорта конечна. Экспорт выполняется вне обработки продуктового +запроса, поэтому недоступность приёмника не меняет HTTP, MCP или runtime +результат. Ошибки доставки учитываются в +`crank_telemetry_export_failures_total{signal_type="trace",exporter="otlp"}` +без журналирования endpoint, headers и без создания новой OTLP-трассы. + +Входящий корректный `traceparent` становится родителем корневого span. +Некорректный заголовок трактуется как отсутствующий: запрос продолжается с +новой трассой, а исходное значение не возвращается и не журналируется. +Перед исходящим REST-запросом доверенный текущий контекст заменяет любой +`traceparent`, заданный в настройках операции. + +### Стадии выполнения инструмента + +Crank создаёт дочерний span только тогда, когда соответствующая стадия +фактически выполняется: + +| Span | Значение | +|---|---| +| `mcp.rate_limit` | проверка ограничения частоты MCP-запроса | +| `mcp.access.check` | машинная проверка доступа | +| `mcp.catalog.load` | получение опубликованного каталога | +| `mcp.tools.resolve` | разрешение конкретного инструмента | +| `approval.check` | применимая проверка подтверждения | +| `runtime.execute` | выполнение инструмента в runtime | +| `runtime.arguments.map` | проверка схемы и отображение аргументов | +| `runtime.idempotency` | применимая проверка идемпотентности | +| `upstream.http` | фактический HTTP-вызов внешней системы | +| `runtime.response.transform` | отображение и проверка ответа | +| `auth.resolve` | применимое разрешение профиля авторизации и секретов | +| `approval.recovery` | фоновое восстановление подтверждённого вызова | +| `history.write` | запись прикладной истории | +| `db.query` | значимая операция PostgreSQL | + +Например, ответ из кэша или повтор идемпотентного результата не создаёт +`upstream.http`. Операция без подтверждения и идемпотентности не создаёт +соответствующие spans. Отсутствующая стадия никогда не изображается успешной. + +Атрибуты стадий имеют закрытый словарь: `outcome`, `error.category`, +`db.system` и `db.operation`. Значения задаются перечислениями из +`crank-trace`; пользовательские идентификаторы, URL, заголовки, query, +аргументы, ответы и текст ошибок туда передать нельзя. Ошибка содержит только +категорию, достаточную для локализации стадии. `request_id` присутствует +только в проверенном корневом span и сопоставляется с записью прикладной +истории. + +Перед OTLP-сериализацией экспортёр повторно применяет разрешительный список +имён и атрибутов, удаляет события и ссылки span и очищает текстовое описание +статуса ошибки. Эта последняя граница действует независимо от очистки +эксплуатационных JSON-журналов и не позволяет новому инструментированию +случайно экспортировать пользовательские данные. + +Точный список переменных и правила OpenBao приведены в +[описании runtime-конфигурации](runtime-config.md). + +## Прикладные журналы и использование + +Crank сохраняет в PostgreSQL данные о тестовых запусках и вызовах +опубликованных MCP-инструментов. Это отдельная прикладная история, а не +копия stdout. + +Если внешнее действие завершилось, но PostgreSQL отклонил запись истории, +Crank не меняет фактический результат и не повторяет действие. Вместо +синтетической записи создаётся эксплуатационный инцидент `DC-08`: + +- `admin.invocation_history.lost` или `mcp.invocation_history.lost` в stdout; +- внутренний монотонный счётчик без пользовательских меток; +- `crank_invocation_history_lost_total` и закрытый показатель потери + экспорта; +- только закрытые поля `source`, `invocation_status` и `error_category`. + +Текст ошибки PostgreSQL, payload и секреты в событие не передаются. Внешняя +публикация не влияет на результат уже завершённого действия. ## Журналы @@ -15,7 +295,12 @@ Crank сохраняет данные о тестовых запусках и в - краткий preview запроса и ответа; - категория ошибки, если вызов завершился ошибкой. -При обновлении опубликованного MCP-каталога отдельное событие `published agent catalog analyzed` содержит `tool_count`, `serialized_bytes`, `estimated_context_tokens`, `largest_tool_estimated_context_tokens`, `recommended_context_tokens`, `exceeds_recommended_budget` и число предупреждений качества. По этим полям можно заметить рост цены `tools/list` до того, как он ухудшит выбор инструментов моделью. +При обновлении опубликованного MCP-каталога событие `mcp.catalog.analyzed` +содержит `tool_count`, `serialized_bytes`, `estimated_context_tokens`, +`largest_tool_estimated_context_tokens`, `recommended_context_tokens`, +`exceeds_recommended_budget` и число предупреждений качества. По этим полям +можно заметить рост цены `tools/list` до того, как он ухудшит выбор +инструментов моделью. ## Использование diff --git a/docs/production-checklist.md b/docs/production-checklist.md index f55d6e7..c4c2537 100644 --- a/docs/production-checklist.md +++ b/docs/production-checklist.md @@ -9,6 +9,9 @@ - Сгенерирован сильный `CRANK_PASSWORD_PEPPER`. - Задан надежный `CRANK_BOOTSTRAP_ADMIN_PASSWORD`. - `CRANK_BASE_URL` указывает на публичный HTTPS URL. +- `CRANK_ENVIRONMENT=production`. +- Если используется внешний приёмник критических ошибок, задан корректный + `CRANK_SENTRY_DSN`; иначе значение оставлено пустым. - PostgreSQL доступен с host, где запускаются контейнеры. - Для PostgreSQL настроены бэкапы. - Reverse proxy проксирует `/`, `/api/admin/` и `/mcp/`. @@ -19,11 +22,16 @@ - `curl /health` для `admin-api` возвращает `ok`. - `curl /health` для `mcp-server` возвращает `ok`. +- `curl /ready` для `admin-api` и `mcp-server` возвращает `ready`. - UI открывается по публичному домену. - Вход под bootstrap admin работает. - Demo seed создал Frankfurter-пример, если `CRANK_DEMO_SEED=true`. - Тест операции `frankfurter_latest_rate` проходит. - MCP-клиент видит инструмент через агента `currency-rates`. +- Каждая непустая строка stdout `admin-api` и `mcp-server` является + корректным JSON и содержит `service`, `environment` и `event`. +- При включённом канале контрольная критическая ошибка появляется у приёмника + без исходного текста ошибки, request, user, payload и секретов. ## Безопасность @@ -39,7 +47,7 @@ - Включите мониторинг контейнеров. - Следите за свободным местом на диске. - Проверяйте размер PostgreSQL и директории artifact storage. -- Храните бэкапы PostgreSQL отдельно от application host. +- Храните полные комплекты PostgreSQL + artifact storage отдельно от application host. +- Регулярно выполняйте контрольное восстановление через `scripts/restore-community.sh` на чистом стенде. - Перед обновлением фиксируйте текущие image tags. - После обновления проверяйте UI, Admin API и MCP endpoint. - diff --git a/docs/runtime-config.md b/docs/runtime-config.md index 2a5287a..299bb80 100644 --- a/docs/runtime-config.md +++ b/docs/runtime-config.md @@ -131,14 +131,103 @@ CRANK_CACHE_DEFAULT_TTL_MS=60000 ## Логи -- `CRANK_LOG_LEVEL` - уровень логирования, например `info`, `debug`, `warn`. +- `CRANK_ENVIRONMENT` - короткая метка окружения. При локальном запуске по + умолчанию используется `development`, Community deployment явно передаёт + `production`. +- `CRANK_LOG_LEVEL` - фильтр `tracing`, например `info`, `debug`, `warn` или + `admin_api=debug,tower_http=info`. -Поля с паролями, токенами, ключами и заголовками авторизации удаляются из снимков -запросов и ответов. Один снимок ограничен 16 КиБ; более крупное значение хранится в -усечённом виде с исходным размером. +`admin-api` и `mcp-server` пишут в stdout по одному JSON-объекту на строку. +Поля с паролями, токенами, ключами, cookie, authorization, query, полным телом +или результатом очищаются до сериализации. Пример: ```env +CRANK_ENVIRONMENT=development CRANK_LOG_LEVEL=info ``` + +Внешний сборщик журналов не обязателен. Его отсутствие не влияет на `/health` +и `/ready`. + +## Критические ошибки + +- `CRANK_SENTRY_DSN` — DSN внешнего Sentry-совместимого приёмника. + +Пустое или отсутствующее значение отключает канал. Неверное непустое значение +останавливает запуск безопасной ошибкой без вывода DSN. Оба сервиса используют +одно значение, но передают собственные `service`, `release` и `environment`. + +```env +CRANK_SENTRY_DSN= +``` + +Community не разворачивает GlitchTip или другой приёмник. Подробный состав +события и правила очистки приведены в +[документе о наблюдаемости](observability.md). + +## Prometheus + +- `CRANK_METRICS_ENABLED` — включает отдельные listener-ы, по умолчанию + `true`; +- `CRANK_ADMIN_METRICS_BIND` — адрес показателей `admin-api`, по умолчанию + `127.0.0.1:9464`; +- `CRANK_MCP_METRICS_BIND` — адрес показателей `mcp-server`, по умолчанию + `127.0.0.1:9465`; +- `CRANK_METRICS_BEARER_TOKEN` — отдельный токен, обязательный для любого + non-loopback bind. + +Значения проверяются до запуска рабочих listener-ов. Для отключения +поверхности: + +```env +CRANK_METRICS_ENABLED=false +``` + +Подробная модель доступа, список показателей и пример настройки сборщика +приведены в [документе о наблюдаемости](observability.md). + +## Распределённые трассы OTLP + +Crank экспортирует через OTLP только трассы. Метрики остаются в Prometheus, +а эксплуатационные журналы — в stdout. Если оба endpoint пусты, tracer +provider и фоновый экспортёр не создаются. + +- `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` — полный HTTP endpoint трасс; имеет + приоритет; +- `OTEL_EXPORTER_OTLP_ENDPOINT` — общий endpoint, к которому Crank добавляет + `/v1/traces`; +- `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` и резервный + `OTEL_EXPORTER_OTLP_PROTOCOL` — только `http/protobuf`; +- `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` и резервный + `OTEL_EXPORTER_OTLP_TIMEOUT` — предел HTTP-запроса экспорта в миллисекундах; +- `OTEL_EXPORTER_OTLP_TRACES_HEADERS` — заголовки только для трасс; +- `OTEL_EXPORTER_OTLP_HEADERS` — резервные общие заголовки; +- `OTEL_BSP_MAX_QUEUE_SIZE` — конечная очередь spans, по умолчанию `2048`; +- `OTEL_BSP_MAX_EXPORT_BATCH_SIZE` — пакет, по умолчанию `512`, не больше + очереди; +- `OTEL_BSP_SCHEDULE_DELAY` — период отправки в миллисекундах, по умолчанию + `5000`; +- `OTEL_BSP_EXPORT_TIMEOUT` — совместимый предел пакетного экспортёра в + миллисекундах, по умолчанию `30000`. + +Фактический HTTP-запрос экспорта ограничивается более строгим из +`OTEL_EXPORTER_OTLP_TRACES_TIMEOUT`/`OTEL_EXPORTER_OTLP_TIMEOUT` и +`OTEL_BSP_EXPORT_TIMEOUT`. Это сохраняет оба верхних предела при работе +стабильного потокового `BatchSpanProcessor`. + +Допустимы только HTTP/HTTPS URL без учётных данных, query и fragment. +Некорректная явно заданная конфигурация останавливает запуск безопасной +типизированной ошибкой, не содержащей значений окружения. + +```env +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://otel.example.com/v1/traces +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_TIMEOUT=10000 +``` + +Заголовки обычно содержат токен приёмника. В production их следует хранить в +OpenBao как `OTEL_EXPORTER_OTLP_TRACES_HEADERS`; CD передаёт значение в +runtime `.env`, но не выводит его в журнал. Не записывайте токен в +репозиторий или Compose. diff --git a/docs/rust-code-health.md b/docs/rust-code-health.md index 45a89c5..d793d2e 100644 --- a/docs/rust-code-health.md +++ b/docs/rust-code-health.md @@ -23,14 +23,25 @@ - `scripts/check-rust-code-health.sh` — локальный repo-level gate для размера Rust-файлов. - `scripts/check-rust-boundaries.sh` — crate-level dependency direction и module-level import rules. - `scripts/check-rust-module-boundaries.sh` — запрет на опасные imports внутри слоев. +- `cargo deny --locked check advisories bans licenses sources` — обязательная проверка лицензий, известных уязвимостей и источников зависимостей. +- `scripts/check-dependencies.sh` — единая локальная проверка Rust- и npm-зависимостей. Что можно добавить позже: -- `cargo-deny` для лицензий, security advisories и duplicate dependencies. - `cargo-machete` для поиска неиспользуемых зависимостей. - `cargo-udeps` для более строгой проверки зависимостей, если nightly допустим в отдельной job. - `cargo-modules` или `cargo-guppy` для анализа графа модулей и зависимостей между crate-ами. +## Политика лицензий зависимостей + +По умолчанию разрешены `MIT`, `BSD-2-Clause`, `BSD-3-Clause`, `Apache-2.0`, `ISC` и `0BSD`. + +`MPL-2.0`, `LGPL`, `EPL` и `CDDL` требуют отдельного технического и юридического рассмотрения до добавления зависимости. + +`GPL`, `AGPL`, `SSPL`, `Commons Clause`, `BUSL` и зависимости с неизвестной лицензией запрещены без юридического согласования. Лицензия самого Community-продукта не считается внешней зависимостью и проверяется отдельно. + +Точечные исключения для обязательных транзитивных зависимостей фиксируются в `deny.toml` по имени пакета и конкретной лицензии. Безымянные или глобальные исключения запрещены. + ## Правила размера Новые Rust-файлы не должны быть больше `1000` строк. diff --git a/justfile b/justfile index 03c8641..d7ef3dd 100644 --- a/justfile +++ b/justfile @@ -16,6 +16,9 @@ rust-boundaries: rust-code-health: scripts/check-rust-code-health.sh +dependencies: + scripts/check-dependencies.sh + check: cargo check --workspace @@ -36,6 +39,7 @@ verify: just community-scope-check just rust-boundaries just rust-code-health + just dependencies just fmt-check just clippy just test diff --git a/scripts/check-dependencies.sh b/scripts/check-dependencies.sh new file mode 100755 index 0000000..0428ef9 --- /dev/null +++ b/scripts/check-dependencies.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +command -v cargo-deny >/dev/null 2>&1 || { + echo "cargo-deny 0.20.2 is required" >&2 + exit 1 +} + +cargo deny --locked check advisories bans licenses sources +(cd apps/ui && npm audit --audit-level=high) diff --git a/scripts/check-rust-boundaries.py b/scripts/check-rust-boundaries.py index 309da0a..d2fa1e0 100755 --- a/scripts/check-rust-boundaries.py +++ b/scripts/check-rust-boundaries.py @@ -49,6 +49,8 @@ def package_category(name: str, manifest_path: Path, workspace_root: Path) -> st return "app" if name == "crank-core": return "core" + if name == "crank-observability": + return "observability" if name == "crank-registry": return "registry" if name == "crank-runtime": @@ -99,6 +101,15 @@ def boundary_reason(source: Package, dependency: Package) -> str | None: if dependency.category == "app": return "workspace crates must not depend on apps" + if source.category == "observability": + return "crank-observability must not depend on other workspace crates" + + if ( + source.category in {"core", "registry", "runtime"} + and dependency.category == "observability" + ): + return "domain and runtime crates must not depend on crank-observability" + if source.category == "core" and dependency.category in {"runtime", "registry", "adapter"}: return "crank-core must stay below runtime, registry and adapters" diff --git a/scripts/deploy-community.sh b/scripts/deploy-community.sh new file mode 100755 index 0000000..7025237 --- /dev/null +++ b/scripts/deploy-community.sh @@ -0,0 +1,136 @@ +#!/bin/sh +set -eu + +cd "${1:-.}" + +env_value_from() { + env_file="$1" + key="$2" + default_value="${3:-}" + value="$(grep -E "^${key}=" "$env_file" | tail -n 1 | cut -d= -f2- || true)" + if [ -n "$value" ]; then + printf '%s' "$value" + else + printf '%s' "$default_value" + fi +} + +env_value() { + key="$1" + default_value="${2:-}" + env_value_from .env "$key" "$default_value" +} + +cache_backend="$(env_value CRANK_CACHE_BACKEND memory)" +compose_profiles="" +if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then + compose_profiles="--profile cache" +fi + +compose() { + # Intentional word splitting: compose_profiles is either empty or two arguments. + # shellcheck disable=SC2086 + docker compose $compose_profiles "$@" +} + +wait_for_stack() { + readiness_path="$1" + attempt=1 + while [ "$attempt" -le 45 ]; do + if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \ + && curl --fail --silent "http://127.0.0.1:3001/${readiness_path}" >/dev/null \ + && curl --fail --silent "http://127.0.0.1:3002/${readiness_path}" >/dev/null; then + return 0 + fi + sleep 2 + attempt=$((attempt + 1)) + done + return 1 +} + +create_backup() { + timestamp="$(date -u +%Y%m%dT%H%M%SZ)" + backup_dir="$(pwd)/backups/${timestamp}" + mkdir -p "$backup_dir" + chmod 700 "$backup_dir" + + backup_env_file=.env + previous_deployment=false + if [ -f .env.previous ]; then + backup_env_file=.env.previous + previous_deployment=true + cp "$backup_env_file" "$backup_dir/runtime.env" + else + cp .env "$backup_dir/runtime.env" + fi + if [ -f docker-compose.previous.yml ]; then + cp docker-compose.previous.yml "$backup_dir/docker-compose.yml" + else + cp docker-compose.yml "$backup_dir/docker-compose.yml" + fi + + postgres_host="$(env_value_from "$backup_env_file" POSTGRES_HOST)" + postgres_port="$(env_value_from "$backup_env_file" POSTGRES_PORT 5432)" + postgres_db="$(env_value_from "$backup_env_file" POSTGRES_DB crank)" + postgres_user="$(env_value_from "$backup_env_file" POSTGRES_USER crank)" + postgres_password="$(env_value_from "$backup_env_file" POSTGRES_PASSWORD)" + if [ -z "$postgres_host" ] || [ -z "$postgres_password" ]; then + echo "PostgreSQL credentials are required for the pre-update backup" >&2 + return 1 + fi + docker run --rm --network host \ + -e PGPASSWORD="$postgres_password" \ + -v "$backup_dir:/backup" \ + postgres:16-alpine \ + pg_dump --host "$postgres_host" --port "$postgres_port" \ + --username "$postgres_user" --dbname "$postgres_db" \ + --format custom --file /backup/postgres.dump + + admin_container="$(compose ps -q admin-api 2>/dev/null || true)" + if [ -n "$admin_container" ]; then + storage_root="$(env_value_from "$backup_env_file" CRANK_STORAGE_ROOT /var/lib/crank/storage)" + docker run --rm --volumes-from "$admin_container" \ + -v "$backup_dir:/backup" alpine:3.21 \ + tar -C "$storage_root" -czf /backup/artifacts.tar.gz . + elif [ "$previous_deployment" = true ]; then + echo "Existing deployment container is unavailable; artifact backup cannot be verified" >&2 + return 1 + else + tar -czf "$backup_dir/artifacts.tar.gz" --files-from /dev/null + fi + + (cd "$backup_dir" && sha256sum postgres.dump artifacts.tar.gz runtime.env docker-compose.yml > SHA256SUMS) + find backups -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \ + | sort -nr | awk 'NR > 5 { print $2 }' | xargs -r rm -rf +} + +rollback() { + echo "New release failed readiness; restoring previous deployment" >&2 + if [ ! -f .env.previous ] || [ ! -f docker-compose.previous.yml ]; then + echo "Previous deployment metadata is unavailable" >&2 + return 1 + fi + cp .env .env.failed + cp docker-compose.yml docker-compose.failed.yml + cp .env.previous .env + cp docker-compose.previous.yml docker-compose.yml + cache_backend="$(env_value CRANK_CACHE_BACKEND memory)" + compose_profiles="" + if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then + compose_profiles="--profile cache" + fi + compose up -d --remove-orphans + wait_for_stack health +} + +compose config -q +create_backup +compose pull + +if ! compose up -d --remove-orphans || ! wait_for_stack ready; then + compose ps >&2 || true + rollback + exit 1 +fi + +compose ps diff --git a/scripts/restore-community.sh b/scripts/restore-community.sh new file mode 100755 index 0000000..c4b30e7 --- /dev/null +++ b/scripts/restore-community.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +deploy_dir="${1:-.}" +backup_dir="${2:-}" +if [ -z "$backup_dir" ] || [ ! -d "$backup_dir" ]; then + echo "usage: CRANK_RESTORE_CONFIRM=restore scripts/restore-community.sh DEPLOY_DIR BACKUP_DIR" >&2 + exit 2 +fi +if [ "${CRANK_RESTORE_CONFIRM:-}" != "restore" ]; then + echo "set CRANK_RESTORE_CONFIRM=restore to confirm destructive restore" >&2 + exit 2 +fi + +cd "$deploy_dir" +backup_dir="$(cd "$backup_dir" && pwd)" +(cd "$backup_dir" && sha256sum --check SHA256SUMS) + +env_value() { + key="$1" + default_value="${2:-}" + value="$(grep -E "^${key}=" .env | tail -n 1 | cut -d= -f2- || true)" + if [ -n "$value" ]; then printf '%s' "$value"; else printf '%s' "$default_value"; fi +} + +cache_backend="$(env_value CRANK_CACHE_BACKEND memory)" +compose_profiles="" +if [ "$cache_backend" = "valkey" ] || [ "$cache_backend" = "redis" ]; then + compose_profiles="--profile cache" +fi +compose() { + # shellcheck disable=SC2086 + docker compose $compose_profiles "$@" +} + +compose stop admin-api mcp-server ui + +postgres_host="$(env_value POSTGRES_HOST)" +postgres_port="$(env_value POSTGRES_PORT 5432)" +postgres_db="$(env_value POSTGRES_DB crank)" +postgres_user="$(env_value POSTGRES_USER crank)" +postgres_password="$(env_value POSTGRES_PASSWORD)" +docker run --rm --network host \ + -e PGPASSWORD="$postgres_password" \ + -v "$backup_dir:/backup:ro" \ + postgres:16-alpine \ + pg_restore --host "$postgres_host" --port "$postgres_port" \ + --username "$postgres_user" --dbname "$postgres_db" \ + --clean --if-exists --no-owner --no-privileges /backup/postgres.dump + +admin_container="$(compose ps -aq admin-api)" +storage_root="$(env_value CRANK_STORAGE_ROOT /var/lib/crank/storage)" +docker run --rm --volumes-from "$admin_container" \ + -v "$backup_dir:/backup:ro" alpine:3.21 sh -eu -c \ + "find '$storage_root' -mindepth 1 -delete; tar -C '$storage_root' -xzf /backup/artifacts.tar.gz" + +compose up -d --remove-orphans +attempt=1 +while [ "$attempt" -le 45 ]; do + if curl --fail --silent http://127.0.0.1:3000/ >/dev/null \ + && curl --fail --silent http://127.0.0.1:3001/ready >/dev/null \ + && curl --fail --silent http://127.0.0.1:3002/ready >/dev/null; then + echo "Community state restored from $backup_dir" + exit 0 + fi + sleep 2 + attempt=$((attempt + 1)) +done + +compose ps >&2 +echo "restored stack failed readiness" >&2 +exit 1 diff --git a/scripts/scan-images.sh b/scripts/scan-images.sh new file mode 100755 index 0000000..63fc963 --- /dev/null +++ b/scripts/scan-images.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +if [ "$#" -eq 0 ]; then + echo "usage: scripts/scan-images.sh IMAGE [IMAGE ...]" >&2 + exit 2 +fi + +trivy_version="0.70.0" +archive="trivy_${trivy_version}_Linux-64bit.tar.gz" +expected_sha256="8b4376d5d6befe5c24d503f10ff136d9e0c49f9127a4279fd110b727929a5aa9" +temp_dir="$(mktemp -d)" +trap 'rm -rf "$temp_dir"' EXIT HUP INT TERM + +curl --fail --silent --show-error --location \ + "https://github.com/aquasecurity/trivy/releases/download/v${trivy_version}/${archive}" \ + --output "$temp_dir/$archive" +printf '%s %s\n' "$expected_sha256" "$temp_dir/$archive" | sha256sum --check --status +tar -C "$temp_dir" -xzf "$temp_dir/$archive" trivy + +for image in "$@"; do + "$temp_dir/trivy" image \ + --exit-code 1 \ + --ignore-unfixed \ + --severity HIGH,CRITICAL \ + --scanners vuln \ + --no-progress \ + "$image" +done diff --git a/tests/unit/test_check_rust_boundaries.py b/tests/unit/test_check_rust_boundaries.py index 283686b..a477617 100644 --- a/tests/unit/test_check_rust_boundaries.py +++ b/tests/unit/test_check_rust_boundaries.py @@ -87,6 +87,65 @@ class RustBoundaryCheckTests(unittest.TestCase): self.assertEqual(violations[0].source, "crank-registry") self.assertEqual(violations[0].dependency, "crank-adapter-rest") + def test_allows_apps_to_depend_on_observability(self) -> None: + packages = [ + package( + self.root, + "admin-api", + "apps/admin-api", + ["crank-observability"], + ), + package( + self.root, + "crank-observability", + "crates/crank-observability", + ), + ] + + violations = self.checker.find_violations(metadata(packages, self.root)) + + self.assertEqual(violations, []) + + def test_rejects_observability_dependency_on_workspace_crates(self) -> None: + for dependency in ("crank-core", "crank-registry", "crank-runtime"): + packages = [ + package( + self.root, + "crank-observability", + "crates/crank-observability", + [dependency], + ), + package(self.root, dependency, f"crates/{dependency}"), + ] + + violations = self.checker.find_violations(metadata(packages, self.root)) + + self.assertEqual(len(violations), 1, dependency) + self.assertEqual(violations[0].source, "crank-observability") + self.assertEqual(violations[0].dependency, dependency) + + def test_rejects_domain_and_runtime_dependencies_on_observability(self) -> None: + for source in ("crank-core", "crank-registry", "crank-runtime"): + packages = [ + package( + self.root, + source, + f"crates/{source}", + ["crank-observability"], + ), + package( + self.root, + "crank-observability", + "crates/crank-observability", + ), + ] + + violations = self.checker.find_violations(metadata(packages, self.root)) + + self.assertEqual(len(violations), 1, source) + self.assertEqual(violations[0].source, source) + self.assertEqual(violations[0].dependency, "crank-observability") + if __name__ == "__main__": unittest.main()