Compare commits

..
Author SHA1 Message Date
Hermes DevOps eecd5bad2f ci: skip unrelated UI work using the checked event base 2026-10-25 10:41:32 +00:00
Hermes DevOps a6a2ac9f34 ci: reuse the runner persistent dependency volumes
CI / UI Checks (pull_request) Successful in 32s
CI / Rust Checks (pull_request) Failing after 8m0s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 3s
CI / Deploy (pull_request) Skipped
2026-10-25 10:15:52 +00:00
Hermes DevOps 3b35702023 fix(test): avoid a false secret-marker match in a diagnostic comment
CI / UI Checks (pull_request) Failing after 1s
CI / Rust Checks (pull_request) Failing after 1s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
2026-10-25 10:08:30 +00:00
Hermes DevOps 87081a168b fix(ci): use the verified full checkout action revision
CI / Rust Feature Validation (pull_request) Skipped
CI / Rust Checks (pull_request) Failing after 19s
CI / UI Checks (pull_request) Failing after 19s
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
2026-10-25 10:06:46 +00:00
Hermes DevOps fd1f37537e ci: separate story ATDD from feature validation and enforce delivery checks
CI / UI Checks (pull_request) Failing after 27s
CI / Rust Checks (pull_request) Failing after 39s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 2s
CI / Deploy (pull_request) Skipped
2026-10-25 10:02:40 +00:00
Hermes DevOps 75fe5a123c [skip ci] wip: preserve Hermes crank development checkpoint
Owner-authorized recovery of accumulated source changes.
Includes accepted work and in-progress changes; NOT a release or acceptance.
Parent baseline 1b6db09bd1. Runtime data, build caches and local credentials excluded.
2026-10-25 10:00:22 +00:00
190 changed files with 1343 additions and 29554 deletions

No files matched your search

+47 -31
View File
@@ -11,12 +11,6 @@ on:
options: [story, feature]
default: story
required: true
deploy_target:
description: Deployment target for this run
type: choice
options: [none, production]
default: none
required: true
push:
branches:
- main
@@ -48,13 +42,9 @@ jobs:
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
exit 1
fi
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
fi
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
rustup default "$toolchain"
host="$(rustc -vV | sed -n 's/^host: //p')"
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
toolchain_bin="$toolchain_dir/bin"
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
@@ -91,9 +81,6 @@ jobs:
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
python3 scripts/check-metrics-boundaries.py --root .
- name: Run tooling unit tests
run: python3 -m unittest discover -s tests/unit
- name: Check Capability Inventory
run: |
required_args=""
@@ -158,15 +145,50 @@ jobs:
docker --version
- name: Detect UI changes
id: ui-diff
env:
FEATURE_VALIDATION: ${{ inputs.validation_level == 'feature' || contains(gitea.event.pull_request.labels.*.name, 'feature-ready') }}
BASE_SHA: ${{ gitea.event.pull_request.base.sha || gitea.event.before }}
run: |
run_ui=true
reason="no reliable base revision; checking UI conservatively"
if [ "$FEATURE_VALIDATION" = "true" ]; then
reason="feature validation requested"
elif [[ ! "$BASE_SHA" =~ ^([0-9a-fA-F]{40}|[0-9a-fA-F]{64})$ ]] || [[ "$BASE_SHA" =~ ^0+$ ]]; then
: # Keep the conservative default when the event has no usable base.
else
if ! git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null; then
git fetch --no-tags --depth=1 origin "$BASE_SHA" >/dev/null 2>&1 || true
fi
if git cat-file -e "$BASE_SHA^{commit}" 2>/dev/null && git diff --quiet "$BASE_SHA" HEAD -- apps/ui/ crank-community.png; then
run_ui=false
reason="UI inputs are unchanged from the event base"
else
reason="apps/ui changed or the base comparison was unavailable"
fi
fi
echo "run_ui=$run_ui" >> "$GITHUB_OUTPUT"
if [ "$run_ui" = "false" ]; then
echo "UI checks not applicable: $reason."
else
echo "Running UI checks: $reason."
fi
- name: Install UI dependencies
if: steps.ui-diff.outputs.run_ui == 'true'
working-directory: apps/ui
run: npm ci
- name: Audit UI dependencies
if: steps.ui-diff.outputs.run_ui == 'true'
working-directory: apps/ui
run: npm audit --audit-level=high
- name: Build UI bundle
if: steps.ui-diff.outputs.run_ui == 'true'
working-directory: apps/ui
run: npm run build
@@ -191,13 +213,9 @@ jobs:
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
exit 1
fi
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
fi
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
rustup default "$toolchain"
host="$(rustc -vV | sed -n 's/^host: //p')"
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
toolchain_bin="$toolchain_dir/bin"
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
@@ -220,6 +238,8 @@ jobs:
docker info
- name: Install dependency policy tool
run: cargo install cargo-deny --version 0.20.2 --locked
- name: Run tooling unit tests
run: python3 -m unittest discover -s tests/unit
- name: Check dependency licenses and advisories
run: cargo deny --locked check advisories bans licenses sources
- name: Run offline OpenAPI support corpus gate
@@ -274,13 +294,9 @@ jobs:
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
exit 1
fi
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
fi
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
rustup default "$toolchain"
host="$(rustc -vV | sed -n 's/^host: //p')"
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
toolchain_bin="$toolchain_dir/bin"
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
@@ -517,7 +533,7 @@ jobs:
runs-on: ubuntu-latest
needs:
- required-checks
if: ${{ gitea.event_name == 'workflow_dispatch' && gitea.ref == 'refs/heads/main' && inputs.validation_level == 'feature' && inputs.deploy_target == 'production' }}
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
env:
REGISTRY: git.itexp.me
IMAGE_TAG: ${{ gitea.sha }}
+9 -77
View File
@@ -12,20 +12,10 @@ on:
options: [story, feature]
default: story
required: true
resume_tag:
description: Existing draft release tag to resume with image-only steps
type: string
default: ""
required: false
resume_source_sha:
description: Exact accepted source commit SHA for the resume tag
type: string
default: ""
required: false
env:
REGISTRY: git.itexp.me
IMAGE_TAG: ${{ inputs.resume_tag || gitea.ref_name }}
IMAGE_TAG: ${{ gitea.ref_name }}
ADMIN_API_IMAGE: git.itexp.me/bsodfather/crank-community-admin-api
MCP_SERVER_IMAGE: git.itexp.me/bsodfather/crank-community-mcp-server
UI_IMAGE: git.itexp.me/bsodfather/crank-community-ui
@@ -41,47 +31,12 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0
- name: Verify existing release before resume
id: resume
if: inputs.resume_tag != ''
env:
GITEA_SERVER_URL: https://git.itexp.me
GITEA_REPOSITORY: ${{ gitea.repository }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
RELEASE_TAG: ${{ inputs.resume_tag }}
RESUME_SOURCE_SHA: ${{ inputs.resume_source_sha }}
run: python3 scripts/ci-release-resume.py
- name: Preserve updated image scanner for resume
id: pin_resume_scanner
if: inputs.resume_tag != ''
run: |
set -eu
scanner_dir="$(mktemp -d "${RUNNER_TEMP:?RUNNER_TEMP is unavailable}/crank-release-scanner.XXXXXX")"
cp scripts/scan-images.sh "$scanner_dir/scan-images.sh"
cp scripts/check-image-scan.py "$scanner_dir/check-image-scan.py"
chmod +x "$scanner_dir/scan-images.sh"
printf 'path=%s\n' "$scanner_dir/scan-images.sh" >> "$GITHUB_OUTPUT"
- name: Checkout validated release source
if: inputs.resume_tag != ''
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: ${{ steps.resume.outputs.release_tag }}
fetch-depth: 0
- name: Assert exact release source checkout
if: inputs.resume_tag != ''
env:
EXPECTED_SOURCE_SHA: ${{ steps.resume.outputs.source_sha }}
run: test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"
- name: Clear optional release evidence
if: inputs.resume_tag == ''
run: rm -f .tmp/openapi-playwright.json .tmp/openapi-ui-evidence.json
- name: Use preinstalled Rust toolchain
if: inputs.resume_tag == ''
run: |
set -eu
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/1.96.1-x86_64-unknown-linux-gnu"
@@ -99,7 +54,6 @@ jobs:
"$toolchain_bin/cargo-clippy" --version
- name: Verify runner toolchain
if: inputs.resume_tag == ''
run: |
python3 --version
rustc --version
@@ -111,23 +65,19 @@ jobs:
bao version
- name: Install dependency policy tool
if: inputs.resume_tag == ''
run: cargo install cargo-deny --version 0.20.2 --locked
- name: Check formatting
if: inputs.resume_tag == ''
run: cargo fmt --all --check
- name: Audit Rust dependencies
if: inputs.resume_tag == ''
run: cargo deny --locked check advisories bans licenses sources
- name: Run tooling unit tests
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
if: inputs.validation_level == 'feature'
run: python3 -m unittest discover -s tests/unit
- name: Check typed runtime configuration contract
if: inputs.resume_tag == ''
run: |
cargo run -p crank-config --bin crank-config-contract -- --check
python3 scripts/check-runtime-config.py --root .
@@ -135,17 +85,14 @@ jobs:
scripts/check-rust-boundaries.sh
- name: Check canonical migration contract
if: inputs.resume_tag == ''
run: cargo run -p admin-api --bin crank-migrate -- plan --check
- name: Check typed metrics contract
if: inputs.resume_tag == ''
run: |
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
python3 scripts/check-metrics-boundaries.py --root .
- name: Check Capability Inventory
if: inputs.resume_tag == ''
run: |
required_args=""
for number in $(seq 1 54); do
@@ -158,7 +105,6 @@ jobs:
$required_args
- name: Check Capability Baseline
if: inputs.resume_tag == ''
run: |
python3 scripts/validate-capability-baseline.py \
--root . \
@@ -166,46 +112,40 @@ jobs:
--schema docs/schemas/capability-baseline.schema.json
- name: Check Community scope
if: inputs.resume_tag == ''
run: scripts/check-community-scope.sh
- name: Run Crank story ATDD
if: inputs.resume_tag == ''
run: scripts/ci-story-checks.sh
- name: Run release quality gates
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
if: inputs.validation_level == 'feature'
run: |
cargo clippy --workspace --all-targets --all-features -- -D warnings
cargo test --workspace --all-targets -- --test-threads=1
- name: Build release binaries
if: inputs.resume_tag == ''
run: cargo build --release -p admin-api -p mcp-server
- name: Install UI dependencies
if: inputs.resume_tag == ''
working-directory: apps/ui
run: npm ci
- name: Audit UI dependencies
if: inputs.resume_tag == ''
working-directory: apps/ui
run: npm audit --audit-level=high
- name: Build UI dist
if: inputs.resume_tag == ''
working-directory: apps/ui
run: npm run build
- name: Install Playwright browser
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
if: inputs.validation_level == 'feature'
working-directory: apps/ui
run: npx playwright install --with-deps chromium
- name: Run release end-to-end tests
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
if: inputs.validation_level == 'feature'
working-directory: apps/ui
run: |
mkdir -p ../../.tmp
@@ -213,7 +153,7 @@ jobs:
PLAYWRIGHT_JSON_OUTPUT=../../.tmp/openapi-playwright.json npm run e2e
- name: Collect sanitized OpenAPI UI release evidence
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
if: inputs.validation_level == 'feature'
working-directory: apps/ui
run: |
trap 'rm -f ../../.tmp/openapi-playwright.json' EXIT
@@ -235,14 +175,12 @@ jobs:
--require-accepted
- name: Validate deployment manifests
if: inputs.resume_tag == ''
run: |
docker compose -f docker-compose.yml --env-file .env.example config -q
docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
docker compose -f deploy/community/docker-compose.images.yml --env-file deploy/community/.env.images.example --profile local-db config -q
- name: Package release artifacts
if: inputs.resume_tag == ''
run: |
set -euo pipefail
mkdir -p dist/release
@@ -266,18 +204,15 @@ jobs:
sha256sum "${files[@]}" > "dist/crank-community-${IMAGE_TAG}-checksums.txt"
- name: Generate SBOM
if: inputs.resume_tag == ''
run: |
docker run --rm -v "$PWD:/workspace" anchore/syft:latest \
dir:/workspace -o spdx-json \
> dist/crank-community-${IMAGE_TAG}-sbom.spdx.json
- name: Publish release bundles, checksums, evidence, and SBOM to Gitea
if: inputs.resume_tag == '' && startsWith(gitea.ref, 'refs/tags/v')
if: startsWith(gitea.ref, 'refs/tags/v')
env:
# gitea.server_url inside the job container is the internal http://gitea:3000 origin;
# the publisher only forwards its token to the public HTTPS origin.
GITEA_SERVER_URL: https://git.itexp.me
GITEA_SERVER_URL: ${{ gitea.server_url }}
GITEA_REPOSITORY: ${{ gitea.repository }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
RELEASE_TAG: ${{ gitea.ref_name }}
@@ -312,8 +247,6 @@ jobs:
docker login '${{ env.REGISTRY }}' -u "$DEPLOY_REGISTRY_USER" --password-stdin
- name: Build and push release images
env:
RELEASE_IMAGE_SCANNER: ${{ steps.pin_resume_scanner.outputs.path || 'scripts/scan-images.sh' }}
run: |
docker build -f apps/admin-api/Dockerfile \
-t '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
@@ -350,7 +283,7 @@ jobs:
--env-file .tmp/release-migration-smoke.env --profile local-db up -d --wait
docker compose -f deploy/community/docker-compose.images.yml \
--env-file .tmp/release-migration-smoke.env --profile local-db logs migrate | grep '"status":"applied"'
"$RELEASE_IMAGE_SCANNER" \
scripts/scan-images.sh \
'${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
'${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
@@ -362,6 +295,5 @@ jobs:
docker push '${{ env.UI_IMAGE }}:latest'
- name: Show release bundle summary
if: inputs.resume_tag == ''
run: |
ls -lh dist
Generated
+62 -205
View File
@@ -25,7 +25,7 @@ dependencies = [
"async-trait",
"axum",
"axum-extra",
"base64 0.22.1",
"base64",
"crank-artifacts",
"crank-community-auth",
"crank-community-mcp",
@@ -186,7 +186,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -197,7 +197,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -349,12 +349,6 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "base64ct"
version = "1.8.3"
@@ -404,7 +398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4"
dependencies = [
"async-stream",
"base64 0.22.1",
"base64",
"bitflags",
"bollard-buildkit-proto",
"bollard-stubs",
@@ -461,7 +455,7 @@ version = "1.52.1-rc.29.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066"
dependencies = [
"base64 0.22.1",
"base64",
"bollard-buildkit-proto",
"bytes",
"prost",
@@ -659,7 +653,7 @@ version = "0.3.1"
dependencies = [
"async-trait",
"axum",
"base64 0.22.1",
"base64",
"crank-core",
"crank-metrics",
"crank-trace",
@@ -694,7 +688,7 @@ dependencies = [
"argon2",
"async-trait",
"axum-extra",
"base64 0.22.1",
"base64",
"crank-core",
"crank-registry",
"rand 0.10.2",
@@ -711,7 +705,7 @@ version = "0.3.1"
dependencies = [
"async-trait",
"axum",
"base64 0.22.1",
"base64",
"crank-adapter-rest",
"crank-artifacts",
"crank-core",
@@ -728,7 +722,6 @@ dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"reqwest 0.12.28",
"rmcp",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -833,7 +826,6 @@ dependencies = [
name = "crank-registry"
version = "0.3.1"
dependencies = [
"async-trait",
"crank-artifacts",
"crank-core",
"crank-mapping",
@@ -857,7 +849,7 @@ dependencies = [
"aes-gcm",
"async-trait",
"axum",
"base64 0.22.1",
"base64",
"crank-adapter-rest",
"crank-core",
"crank-import",
@@ -999,18 +991,8 @@ version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
dependencies = [
"darling_core 0.23.0",
"darling_macro 0.23.0",
]
[[package]]
name = "darling"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec"
dependencies = [
"darling_core 0.24.1",
"darling_macro 0.24.1",
"darling_core",
"darling_macro",
]
[[package]]
@@ -1023,20 +1005,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
"syn 2.0.118",
]
[[package]]
name = "darling_core"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff"
dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 3.0.9",
"syn",
]
[[package]]
@@ -1045,20 +1014,9 @@ version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core 0.23.0",
"darling_core",
"quote",
"syn 2.0.118",
]
[[package]]
name = "darling_macro"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
dependencies = [
"darling_core 0.24.1",
"quote",
"syn 3.0.9",
"syn",
]
[[package]]
@@ -1110,7 +1068,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -1119,7 +1077,7 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d"
dependencies = [
"base64 0.22.1",
"base64",
"serde",
"serde_json",
]
@@ -1188,7 +1146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -1350,7 +1308,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -1702,7 +1660,7 @@ version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
"base64 0.22.1",
"base64",
"bytes",
"futures-channel",
"futures-util",
@@ -1713,7 +1671,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.4",
"socket2 0.5.10",
"tokio",
"tower-service",
"tracing",
@@ -1961,7 +1919,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -2097,7 +2055,7 @@ version = "0.3.1"
dependencies = [
"async-trait",
"axum",
"base64 0.22.1",
"base64",
"crank-artifacts",
"crank-community-mcp",
"crank-config",
@@ -2162,7 +2120,7 @@ version = "0.18.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
dependencies = [
"base64 0.22.1",
"base64",
"evmap",
"indexmap 2.14.0",
"metrics",
@@ -2498,7 +2456,7 @@ dependencies = [
"regex",
"regex-syntax",
"structmeta",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -2512,12 +2470,6 @@ dependencies = [
"subtle",
]
[[package]]
name = "pastey"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -2541,7 +2493,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -2627,7 +2579,7 @@ dependencies = [
"itertools",
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -2683,7 +2635,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls",
"socket2 0.6.4",
"socket2 0.5.10",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -2722,9 +2674,9 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.4",
"socket2 0.5.10",
"tracing",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -2899,7 +2851,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -2937,7 +2889,7 @@ version = "0.12.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64 0.22.1",
"base64",
"bytes",
"cookie",
"cookie_store",
@@ -2981,7 +2933,7 @@ version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64 0.22.1",
"base64",
"bytes",
"futures-channel",
"futures-core",
@@ -3028,52 +2980,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rmcp"
version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ece64e19631fc38758181e637c15509a2c2709ee53bfde1e1d58f9dc991f919e"
dependencies = [
"async-trait",
"base64 0.23.1",
"bytes",
"chrono",
"futures",
"http",
"http-body",
"http-body-util",
"indexmap 2.14.0",
"libc",
"pastey",
"pin-project-lite",
"rand 0.10.2",
"rmcp-macros",
"schemars 1.2.1",
"serde",
"serde_json",
"sse-stream",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tokio-util",
"tower-service",
"tracing",
"uuid",
]
[[package]]
name = "rmcp-macros"
version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3140f24a766f80f9ec1607bde85680959a69b83d46fa9ce87dadab7e67ab029"
dependencies = [
"darling 0.24.1",
"proc-macro2",
"quote",
"serde_json",
"syn 3.0.9",
]
[[package]]
name = "rustc-demangle"
version = "0.1.28"
@@ -3109,7 +3015,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -3168,7 +3074,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -3237,26 +3143,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
dependencies = [
"chrono",
"dyn-clone",
"ref-cast",
"schemars_derive",
"serde",
"serde_json",
]
[[package]]
name = "schemars_derive"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"syn 2.0.118",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
@@ -3400,18 +3292,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
]
[[package]]
name = "serde_derive_internals"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3446,7 +3327,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3467,7 +3348,7 @@ version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
dependencies = [
"base64 0.22.1",
"base64",
"bs58",
"chrono",
"hex",
@@ -3487,10 +3368,10 @@ version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
dependencies = [
"darling 0.23.0",
"darling",
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3528,7 +3409,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3662,7 +3543,7 @@ dependencies = [
name = "sqlx-core"
version = "0.9.0"
dependencies = [
"base64 0.22.1",
"base64",
"bytes",
"cfg-if",
"crc",
@@ -3704,7 +3585,7 @@ dependencies = [
"quote",
"sqlx-core",
"sqlx-macros-core",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3727,7 +3608,7 @@ dependencies = [
"sqlx-mysql",
"sqlx-postgres",
"sqlx-sqlite",
"syn 2.0.118",
"syn",
"thiserror 2.0.18",
"tokio",
"url",
@@ -3768,7 +3649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e"
dependencies = [
"atoi",
"base64 0.22.1",
"base64",
"bitflags",
"byteorder",
"crc",
@@ -3824,19 +3705,6 @@ dependencies = [
"uuid",
]
[[package]]
name = "sse-stream"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4"
dependencies = [
"bytes",
"futures-util",
"http-body",
"http-body-util",
"pin-project-lite",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
@@ -3869,7 +3737,7 @@ dependencies = [
"proc-macro2",
"quote",
"structmeta-derive",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3880,7 +3748,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3900,17 +3768,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
@@ -3928,7 +3785,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -3997,7 +3854,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -4008,7 +3865,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -4099,7 +3956,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -4144,7 +4001,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
dependencies = [
"async-trait",
"axum",
"base64 0.22.1",
"base64",
"bytes",
"h2",
"http",
@@ -4245,7 +4102,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -4374,7 +4231,7 @@ version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
dependencies = [
"base64 0.22.1",
"base64",
"log",
"percent-encoding",
"rustls",
@@ -4390,7 +4247,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
dependencies = [
"base64 0.22.1",
"base64",
"http",
"httparse",
"log",
@@ -4527,7 +4384,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
"wasm-bindgen-shared",
]
@@ -4619,7 +4476,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -4649,7 +4506,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -4660,7 +4517,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -4876,7 +4733,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
"synstructure",
]
@@ -4897,7 +4754,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
@@ -4917,7 +4774,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
"synstructure",
]
@@ -4957,7 +4814,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.118",
"syn",
]
[[package]]
+87 -99
View File
@@ -2,10 +2,6 @@
mod admin_auth_command;
#[path = "crank_migrate/db_connect.rs"]
mod db_connect;
#[path = "crank_migrate/reporting.rs"]
mod reporting;
use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json};
use crank_config::{ConfigSource, parse_migrator};
use crank_registry::{
@@ -15,7 +11,7 @@ use crank_registry::{
};
use crank_runtime::SecretCrypto;
use serde_json::json;
use std::process::ExitCode;
use std::{path::Path, process::ExitCode};
use time::OffsetDateTime;
const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000;
#[tokio::main]
@@ -769,59 +765,6 @@ async fn process_rotation_batch(
processed += 1;
}
}
let mut after_task_id: Option<String> = None;
loop {
let tasks = registry
.list_tasks_for_master_key_epoch_page(
rotation.source_epoch,
after_task_id.as_deref(),
MASTER_KEY_ROTATION_PAGE_SIZE,
)
.await
.map_err(CliError::from_registry)?;
if tasks.is_empty() {
break;
}
for task in tasks {
after_task_id = Some(task.task_id.clone());
if task.target_epoch == Some(rotation.target_epoch) {
continue;
}
if max_versions.is_some_and(|limit| processed as usize >= limit) {
return Ok(processed);
}
let plaintext = current_crypto
.decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext)
.map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let now = OffsetDateTime::now_utc();
registry
.stage_master_key_rotation_task_ciphertext(
&rotation.id,
&task.task_id,
rotation.source_epoch,
&target_ciphertext,
target_crypto.key_version(),
rotation.target_epoch,
&now,
)
.await
.map_err(CliError::from_registry)?;
processed += 1;
}
}
Ok(processed)
}
@@ -875,47 +818,6 @@ async fn verify_staged_targets(
verified += 1;
}
}
let mut after_task_id: Option<String> = None;
loop {
let tasks = registry
.list_target_tasks_for_master_key_rotation_page(
rotation.target_epoch,
after_task_id.as_deref(),
MASTER_KEY_ROTATION_PAGE_SIZE,
)
.await
.map_err(CliError::from_registry)?;
if tasks.is_empty() {
break;
}
for task in tasks {
after_task_id = Some(task.task_id.clone());
let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
let key_version = task.target_key_version.as_deref().ok_or_else(|| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
target_crypto
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
.map_err(|_| {
CliError::new(
"master_key_rotation_verification_failed",
"master_key.rotation",
"rerun_rotation_or_abort",
)
})?;
verified += 1;
}
}
if verified != rotation.total_secret_versions {
return Err(CliError::new(
"master_key_rotation_verification_failed",
@@ -975,3 +877,89 @@ fn decrypt_current(
)
})
}
async fn active_rotation(
registry: &PostgresRegistry,
allowed_states: &[&str],
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
let status = registry
.master_key_rotation_status()
.await
.map_err(CliError::from_registry)?;
Ok(status
.rotations
.into_iter()
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
}
fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
json!({
"id": rotation.id,
"source_epoch": rotation.source_epoch,
"target_epoch": rotation.target_epoch,
"state": rotation.state,
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
"checkpoint_secret_id": rotation.checkpoint_secret_id,
"total_secret_versions": rotation.total_secret_versions,
"processed_secret_versions": rotation.processed_secret_versions,
"verified_secret_versions": rotation.verified_secret_versions,
"failure_code": rotation.failure_code,
})
}
fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
let path = path.ok_or_else(|| {
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
})?;
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if !metadata.is_file() || metadata.len() > 16_384 {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if value.trim().is_empty() {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
Ok(value)
}
fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
if options.current_key_file.is_some()
|| options.target_key_file.is_some()
|| options.backup_ref.is_some()
|| options.rotation_id.is_some()
|| options.max_versions.is_some()
{
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_master_key_status",
));
}
Ok(())
}
@@ -1,98 +0,0 @@
//! Leaf/reporting helpers of the migrator CLI.
//!
//! Extracted from `crank-migrate.rs` without behaviour changes so the binary entry stays within
//! the code-health line limit.
use std::path::Path;
use crank_registry::{MasterKeyRotationRecord, PostgresRegistry};
use serde_json::json;
use super::{CliError, MasterKeyOptions};
pub(super) async fn active_rotation(
registry: &PostgresRegistry,
allowed_states: &[&str],
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
let status = registry
.master_key_rotation_status()
.await
.map_err(CliError::from_registry)?;
Ok(status
.rotations
.into_iter()
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
}
pub(super) fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
json!({
"id": rotation.id,
"source_epoch": rotation.source_epoch,
"target_epoch": rotation.target_epoch,
"state": rotation.state,
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
"checkpoint_secret_id": rotation.checkpoint_secret_id,
"total_secret_versions": rotation.total_secret_versions,
"processed_secret_versions": rotation.processed_secret_versions,
"verified_secret_versions": rotation.verified_secret_versions,
"failure_code": rotation.failure_code,
})
}
pub(super) fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
let path = path.ok_or_else(|| {
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
})?;
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if !metadata.is_file() || metadata.len() > 16_384 {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
)
})?;
if value.trim().is_empty() {
return Err(CliError::new(
"master_key_input_invalid",
"master_key.input",
"verify_operator_input",
));
}
Ok(value)
}
pub(super) fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
if options.current_key_file.is_some()
|| options.target_key_file.is_some()
|| options.backup_ref.is_some()
|| options.rotation_id.is_some()
|| options.max_versions.is_some()
{
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_master_key_status",
));
}
Ok(())
}
-1
View File
@@ -181,7 +181,6 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+1 -18
View File
@@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
use admin_api::{
app::build_app,
auth::{AuthSettings, BootstrapAdminConfig},
pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics},
pool_metrics::spawn_postgres_pool_metrics,
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
service::AdminServiceBuilder,
state::AppState,
@@ -169,7 +169,6 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let base_url = config
.runtime
@@ -215,16 +214,6 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(std::sync::Arc::new(
crank_registry::PostgresExecutionAdmission::new(
registry.pool().clone(),
crank_registry::AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
},
crank_core::AdmissionSchedulerConfig::default(),
)?,
))
.build();
let identity_provider =
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
@@ -332,12 +321,6 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
-14
View File
@@ -11,17 +11,3 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
+2 -27
View File
@@ -37,22 +37,13 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
} else {
"not_ready"
};
let execution_admission = if checks.execution_admission {
"ready"
} else {
"not_ready"
};
if checks.is_ready() {
(
StatusCode::OK,
Json(json!({
"service": "admin-api",
"status": "ready",
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
})),
)
} else {
@@ -61,11 +52,7 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
Json(json!({
"service": "admin-api",
"status": "not_ready",
"checks": {
"postgres": postgres,
"artifact_storage": artifact_storage,
"execution_admission": execution_admission
}
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
})),
)
}
@@ -80,7 +67,6 @@ mod tests {
let checks = ReadinessChecks {
postgres: false,
artifact_storage: true,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -106,7 +92,6 @@ mod tests {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: false,
execution_admission: true,
};
assert!(!checks.is_ready());
assert_eq!(
@@ -126,14 +111,4 @@ mod tests {
"not_ready"
);
}
#[test]
fn readiness_checks_identify_an_admission_state_failure() {
let checks = ReadinessChecks {
postgres: true,
artifact_storage: true,
execution_admission: false,
};
assert!(!checks.is_ready());
}
}
+3 -9
View File
@@ -55,8 +55,8 @@ use crate::{
storage::LocalArtifactStorage,
};
use operation_validation::{
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy,
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_protocol_target, validate_response_cache_policy,
};
#[derive(Clone)]
@@ -104,12 +104,11 @@ pub use crate::dto::*;
pub struct ReadinessChecks {
pub postgres: bool,
pub artifact_storage: bool,
pub execution_admission: bool,
}
impl ReadinessChecks {
pub const fn is_ready(self) -> bool {
self.postgres && self.artifact_storage && self.execution_admission
self.postgres && self.artifact_storage
}
}
@@ -151,8 +150,6 @@ pub(super) struct CredentialAuditRecord<'a> {
impl AdminService {
pub async fn readiness(&self) -> ReadinessChecks {
let postgres = self.registry.ping().await.is_ok();
// Story 4.1: the mandatory admission state is a readiness dependency of every execution.
let execution_admission = self.registry.execution_admission_health().await.is_ok();
let store = Arc::clone(&self.artifact_store);
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
.await
@@ -160,7 +157,6 @@ impl AdminService {
ReadinessChecks {
postgres,
artifact_storage,
execution_admission,
}
}
@@ -729,7 +725,6 @@ impl AdminService {
validate_response_cache_policy(&payload.target, &payload.execution_config)?;
validate_idempotency_policy(&payload.target, &payload.execution_config)?;
validate_approval_policy(&payload.execution_config)?;
validate_execution_policy(&payload.execution_config)?;
payload.input_mapping.validate_paths()?;
payload.output_mapping.validate_paths()?;
Ok(())
@@ -743,7 +738,6 @@ impl AdminService {
validate_response_cache_policy(&operation.target, &operation.execution_config)?;
validate_idempotency_policy(&operation.target, &operation.execution_config)?;
validate_approval_policy(&operation.execution_config)?;
validate_execution_policy(&operation.execution_config)?;
operation.input_mapping.validate_paths()?;
operation.output_mapping.validate_paths()?;
Ok(())
-1
View File
@@ -441,7 +441,6 @@ fn demo_rest_operation_payload() -> OperationPayload {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
-1
View File
@@ -635,7 +635,6 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -610,7 +610,6 @@ impl AdminService {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref,
headers: BTreeMap::new(),
},
@@ -160,23 +160,6 @@ pub(super) fn validate_approval_policy(
Ok(())
}
pub(super) fn validate_execution_policy(
execution_config: &crank_core::ExecutionConfig,
) -> Result<(), ApiError> {
let Some(policy) = execution_config.execution_policy.as_ref() else {
return Ok(());
};
if let Err(error) = policy.validate() {
return Err(ApiError::validation_with_context(
error.to_string(),
json!({
"field": "execution_config.execution_policy.deadline_ms",
}),
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
@@ -188,8 +171,8 @@ mod tests {
};
use super::{
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
validate_idempotency_policy, validate_response_cache_policy,
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
validate_response_cache_policy,
};
fn cacheable_execution_config() -> ExecutionConfig {
@@ -200,7 +183,6 @@ mod tests {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -219,7 +201,6 @@ mod tests {
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
@@ -384,36 +365,4 @@ mod tests {
"approval ttl must be between 1 and 300 seconds"
);
}
#[test]
fn accepts_published_execution_policy_and_absent_legacy_policy() {
let mut config = cacheable_execution_config();
validate_execution_policy(&config).unwrap();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Adaptive,
deadline_ms: 120_000,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
});
validate_execution_policy(&config).unwrap();
}
#[test]
fn rejects_zero_and_overflowing_execution_policy_deadline() {
let mut config = cacheable_execution_config();
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Synchronous,
deadline_ms: 0,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
let error = validate_execution_policy(&config).unwrap_err();
assert!(matches!(error, crate::error::ApiError::Validation { .. }));
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Background,
deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
});
assert!(validate_execution_policy(&config).is_err());
}
}
@@ -495,7 +495,6 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+1 -2
View File
@@ -29,7 +29,7 @@ sha2.workspace = true
sqlx.workspace = true
thiserror.workspace = true
time.workspace = true
tokio = { workspace = true, features = ["sync", "signal"] }
tokio = { workspace = true, features = ["sync"] }
tracing.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
@@ -45,6 +45,5 @@ opentelemetry_sdk.workspace = true
prost.workspace = true
reqwest.workspace = true
tower.workspace = true
tokio = { workspace = true, features = ["net", "time"] }
tracing-opentelemetry.workspace = true
metrics-util = "0.20.4"
+11 -193
View File
@@ -1,7 +1,7 @@
use std::{io, process::ExitCode, time::Duration};
use crank_community_mcp::{
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts_and_tasks,
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts,
session::PostgresTransportSessionStore,
};
use crank_config::{
@@ -14,14 +14,13 @@ use crank_observability::{
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
};
use crank_registry::{
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, PostgresTaskStore,
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics};
use mcp_server::pool_metrics::spawn_postgres_pool_metrics;
use sqlx::postgres::PgConnectOptions;
use tokio::net::TcpListener;
use tracing::info;
@@ -176,7 +175,6 @@ async fn run(
};
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
spawn_admission_metrics(registry.pool().clone());
}
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
let secret_crypto =
@@ -192,55 +190,12 @@ async fn run(
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.with_admission(execution_admission(
registry.pool().clone(),
runtime_limits,
)?)
.build();
// Published static Resources are read from the same immutable content-addressed root the
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
// Story 4.2/4.3: the durable Task application entry shares the registry pool, the published
// catalog refresh interval and the exact admission limits of the runtime; the public wire stays
// synchronous/unsupported while a fenced in-process worker executes accepted Tasks.
let task_store = std::sync::Arc::new(PostgresTaskStore::new(
registry.pool().clone(),
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?);
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
registry.clone(),
Duration::from_millis(config.refresh_ms),
cache_stores.coordination.clone(),
crank_runtime::TaskService::with_secret_crypto(
task_store.clone(),
std::sync::Arc::new(secret_crypto.clone()),
),
));
// Story 4.3: the durable Task worker runs in this process over the same PostgreSQL store,
// published catalog, executor and encrypted-input authority as the accepting application.
let task_handler =
std::sync::Arc::new(crank_community_mcp::TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime.clone(),
secret_crypto.clone(),
));
let task_worker = std::sync::Arc::new(crank_runtime::TaskWorker::new(
task_store,
task_handler,
crank_runtime::TaskWorkerConfig {
owner: format!("mcp-server-{}", std::process::id()),
..crank_runtime::TaskWorkerConfig::default()
},
)?);
let (task_shutdown_tx, task_shutdown_rx) = tokio::sync::watch::channel(false);
let task_worker_handle = tokio::spawn({
let worker = task_worker.clone();
async move { worker.run_until_shutdown(task_shutdown_rx).await }
});
let app = build_app_with_artifacts_and_tasks(
let app = build_app_with_artifacts(
registry,
Duration::from_millis(config.refresh_ms),
config.runtime.base_url.clone(),
@@ -256,7 +211,6 @@ async fn run(
std::sync::Arc::new(CommunityMachineCredentialVerifier),
runtime_limits.max_concurrent_sessions,
artifact_store,
tasks,
);
let listener = TcpListener::bind(config.bind_addr).await?;
@@ -277,77 +231,15 @@ async fn run(
info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening");
*startup_completed = true;
let serve_result: Result<(), Box<dyn std::error::Error>> = async {
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) => result?,
result = metrics_server.serve() => result?,
}
} else {
serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone())
.await?;
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = axum::serve(listener, app) => result?,
result = metrics_server.serve() => result?,
}
Ok(())
}
.await;
// Stop new claims; active attempts checkpoint/drain or are cancelled inside the bounded worker
// shutdown timeout (<=30s), and queued PostgreSQL work is left for recovery.
task_shutdown_tx.send(true).ok();
if tokio::time::timeout(Duration::from_secs(30), task_worker_handle)
.await
.is_err()
{
tracing::warn!(
name: "mcp.task_worker.shutdown_timeout",
"durable Task worker did not stop within the story shutdown budget"
);
}
serve_result
}
/// Gracefully stop the durable Task worker as soon as shutdown is requested, while still
/// draining in-flight HTTP connections before returning.
async fn serve_with_worker_shutdown<S>(
app: axum::Router,
listener: tokio::net::TcpListener,
signal: S,
worker_stop: tokio::sync::watch::Sender<bool>,
) -> std::io::Result<()>
where
S: std::future::Future<Output = ()> + Send + 'static,
{
let worker_stop_graceful = worker_stop.clone();
let graceful = async move {
signal.await;
let _ = worker_stop_graceful.send(true);
};
let result = axum::serve(listener, app)
.with_graceful_shutdown(graceful)
.await;
let _ = worker_stop.send(true);
result
}
/// Process shutdown trigger for the HTTP surface and the durable Task worker.
async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let terminate = async {
if let Ok(mut signal) =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
{
signal.recv().await;
}
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {}
_ = terminate => {}
} else {
axum::serve(listener, app).await?;
}
Ok(())
}
fn init_observability(
@@ -395,25 +287,6 @@ fn init_observability(
)?)
}
fn admission_limits(runtime_limits: RuntimeLimits) -> AdmissionLimits {
AdmissionLimits {
global_limit: runtime_limits.max_concurrent_unary,
..AdmissionLimits::default()
}
}
fn execution_admission(
pool: sqlx::PgPool,
runtime_limits: RuntimeLimits,
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
let store = PostgresExecutionAdmission::new(
pool,
admission_limits(runtime_limits),
crank_core::AdmissionSchedulerConfig::default(),
)?;
Ok(std::sync::Arc::new(store))
}
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
database_options(&config.database).map_err(|_| invalid("database.source"))?;
@@ -434,12 +307,6 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
crank_registry::AdmissionLimits {
global_limit: config.runtime.max_concurrent_unary,
..crank_registry::AdmissionLimits::default()
}
.validate()
.map_err(|_| invalid("runtime.admission"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
@@ -593,52 +460,3 @@ fn runtime_cache_config(
.map(|value| value.expose_secret().to_owned()),
)
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{Router, routing::get};
use std::time::Duration;
use tokio::sync::{oneshot, watch};
async fn slow_handler() -> &'static str {
tokio::time::sleep(Duration::from_millis(600)).await;
"ok"
}
#[tokio::test]
async fn worker_stops_before_http_drain_completes() {
let app = Router::new().route("/slow", get(slow_handler));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (stop_tx, mut stop_rx) = watch::channel(false);
let (signal_tx, signal_rx) = oneshot::channel::<()>();
let handle = tokio::spawn(serve_with_worker_shutdown(
app,
listener,
async {
let _ = signal_rx.await;
},
stop_tx,
));
let client = reqwest::Client::new();
let request = client.get(format!("http://{}/slow", addr));
let in_flight = tokio::spawn(async move { request.send().await });
tokio::time::sleep(Duration::from_millis(150)).await;
let _ = signal_tx.send(());
tokio::time::timeout(Duration::from_millis(250), stop_rx.changed())
.await
.expect("worker stop signal should arrive before HTTP drain")
.expect("watch changed");
assert!(*stop_rx.borrow());
let response = in_flight.await.unwrap().unwrap();
assert!(response.status().is_success());
let serve_result = handle.await.unwrap();
assert!(serve_result.is_ok());
}
}
-14
View File
@@ -11,17 +11,3 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
}
})
}
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
/// to zero) without waiting for a new request.
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
}
}
})
}
-874
View File
@@ -1,874 +0,0 @@
//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission.
//!
//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a
//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported
//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal.
//!
//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this
//! app already wires both and this is the real deployment shape.
use std::{
collections::BTreeMap,
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
use crank_core::{
AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence,
ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation,
OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget,
RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy,
WorkspaceId,
};
use crank_mapping::MappingSet;
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits};
use crank_schema::{Schema, SchemaKind};
use serde_json::json;
use sqlx::PgPool;
use time::OffsetDateTime;
struct CountingAdapter {
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
#[async_trait]
impl ProtocolAdapter for CountingAdapter {
fn protocol(&self) -> Protocol {
Protocol::Rest
}
fn supports_mode(&self, mode: ExecutionMode) -> bool {
mode == ExecutionMode::Unary
}
async fn invoke_unary(
&self,
_target: &Target,
_prepared: &crank_core::PreparedRequest,
context: &RuntimeRequestContext,
) -> Result<AdapterResponse, crank_core::ProtocolAdapterError> {
context.mark_dispatch_started();
self.calls.fetch_add(1, Ordering::SeqCst);
let current = self.current.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(current, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(150)).await;
self.current.fetch_sub(1, Ordering::SeqCst);
Ok(AdapterResponse {
status_code: 200,
headers: BTreeMap::new(),
body: json!({"ok": true}),
data: json!({"ok": true}),
})
}
}
fn schema(kind: SchemaKind) -> Schema {
Schema {
kind,
format: None,
description: None,
required: false,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
}
}
fn operation(policy: Option<ToolExecutionPolicy>) -> Operation<Schema, MappingSet> {
Operation {
id: OperationId::new("op_admission_budget"),
name: "admission_budget".to_owned(),
display_name: "Admission budget".to_owned(),
category: "general".to_owned(),
protocol: Protocol::Rest,
security_level: OperationSecurityLevel::Standard,
status: OperationStatus::Published,
version: 7,
target: Target::Rest(RestTarget {
base_url: "https://api.example.com".to_owned(),
method: HttpMethod::Get,
path_template: "/v1/items".to_owned(),
static_headers: BTreeMap::new(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: BTreeMap::new(),
}),
input_schema: schema(SchemaKind::Object),
output_schema: schema(SchemaKind::Object),
input_mapping: MappingSet::default(),
output_mapping: MappingSet::default(),
execution_config: ExecutionConfig {
timeout_ms: 5_000,
execution_policy: policy,
..ExecutionConfig::default()
},
tool_description: ToolDescription {
title: "Admission".to_owned(),
description: "Admission budget probe".to_owned(),
tags: Vec::new(),
examples: Vec::new(),
},
samples: None,
generated_draft: None,
config_export: None,
wizard_state: None,
created_at: OffsetDateTime::now_utc(),
updated_at: OffsetDateTime::now_utc(),
published_at: None,
}
}
async fn execute(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = WorkspaceId::new("ws_test");
let context = crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun);
execute_in(executor, operation, input, &context).await
}
async fn execute_in(
executor: &RuntimeExecutor,
operation: &crank_runtime::RuntimeOperation,
input: &serde_json::Value,
context: &crank_runtime::RuntimeRequestContext,
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
let workspace = context
.metering_context()
.map(|metering| metering.workspace_id.clone())
.unwrap_or_else(|| WorkspaceId::new("ws_test"));
let request = crank_runtime::RuntimeExecutionRequest::try_new(
&workspace,
ExecutionOrigin::AdminDraft,
None,
operation,
input,
crank_runtime::ExecutionAuthorization::Authorized,
None,
context,
Instant::now() + Duration::from_secs(30),
)
.map_err(|_| {
crank_core::ExecutionFailure::new(
ExecutionErrorCode::RuntimeInternal,
crank_core::CorrelationContext::new(
context.request_id.clone(),
context.trace_context.clone(),
),
)
})?;
executor
.execute_outcome(request)
.await
.map(|success| success.output)
}
async fn schema_pool(schema_url: &str) -> PgPool {
let pool = PgPool::connect(schema_url)
.await
.expect("admission schema pool");
crank_registry::MigrationAuthority::apply(&pool)
.await
.expect("migration authority applies the admission schema");
pool
}
fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc<PostgresExecutionAdmission> {
Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: global,
agent_limit: 8,
tool_limit: 8,
queue_limit: queue,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)
}
fn executor(
store: Arc<PostgresExecutionAdmission>,
calls: Arc<AtomicUsize>,
current: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
) -> RuntimeExecutor {
RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current,
peak,
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store)
.build()
}
#[tokio::test]
async fn two_executors_share_one_global_budget() {
let schema_url = crank_test_support::postgres_schema_url("admission_budget").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 8);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let first = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let second = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let mut tasks = Vec::new();
for index in 0..4 {
let executor = if index % 2 == 0 {
first.clone()
} else {
second.clone()
};
let operation = operation.clone();
tasks.push(tokio::spawn(async move {
execute(&executor, &operation, &json!({})).await
}));
}
let mut completed = 0;
for task in tasks {
match task.await.expect("execution task") {
Ok(_) => completed += 1,
Err(failure) => panic!("admission must grant every request: {failure:?}"),
}
}
assert_eq!(completed, 4);
assert_eq!(calls.load(Ordering::SeqCst), 4);
assert!(
peak.load(Ordering::SeqCst) <= 2,
"peak concurrency {} exceeded the shared global limit",
peak.load(Ordering::SeqCst)
);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn saturation_returns_typed_overload_without_an_outbound_call() {
let schema_url = crank_test_support::postgres_schema_url("admission_overload").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 1, 1);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// Another process already holds the only global permit.
let held = match store
.admit(AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_test".to_owned(),
agent_key: "admin".to_owned(),
operation_id: "op_admission_budget".to_owned(),
operation_version: 7,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_holder".to_owned(),
expires_at_unix_ms: crank_core::current_unix_ms() + 10_000,
})
.await
.expect("admission store is reachable")
{
crank_core::AdmissionOutcome::Permitted(permit) => permit,
other => panic!("expected a permit, got {other:?}"),
};
let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 250,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let waiting_runtime = runtime.clone();
let first_waiting_operation = waiting_operation.clone();
let waiting = tokio::spawn(async move {
execute(&waiting_runtime, &first_waiting_operation, &json!({})).await
});
// Let the first request occupy the bounded queue before the next one arrives.
while store.snapshot().await.unwrap().waiting == 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
let overloaded = execute(&runtime, &waiting_operation, &json!({})).await;
let failure = overloaded.expect_err("a full queue must refuse the request");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionOverloaded
);
assert_eq!(failure.stage(), ExecutionStage::Admission);
assert_eq!(failure.retry_after_ms(), Some(250));
// The queued request hits its absolute deadline instead of dispatching. That refusal happened
// inside admission (nothing was ever dispatched), so the client must see an admission-stage
// overload with retry guidance — not an `upstream_timeout`.
let expired = waiting
.await
.expect("waiting task")
.expect_err("deadline must refuse");
assert_eq!(
expired.error_code(),
ExecutionErrorCode::ExecutionOverloaded,
"queue waiting counts against the published deadline"
);
assert_eq!(expired.stage(), ExecutionStage::Admission);
assert_eq!(expired.retry_after_ms(), Some(250));
assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain);
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"no outbound call on refusal"
);
assert!(store.release(&held).await.unwrap());
}
#[tokio::test]
async fn background_policy_without_a_task_transport_is_typed_unsupported() {
let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
}))
.into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("no Task transport exists in Story 4.1");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::ExecutionModeUnsupported
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let snapshot = store.snapshot().await.unwrap();
assert_eq!(snapshot.waiting, 0);
assert_eq!(snapshot.executing, 0);
}
#[tokio::test]
async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() {
let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool, 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: 2_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}))
.into();
let output = execute(&runtime, &operation, &json!({}))
.await
.expect("adaptive resolves through the published synchronous fallback");
// The response shape is unchanged by admission: an operation without output mapping still
// returns the same mapped output object it returned before Story 4.1.
assert_eq!(output, json!({}));
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(store.snapshot().await.unwrap().executing, 0);
}
#[tokio::test]
async fn admission_store_outage_fails_closed_before_dispatch() {
let schema_url = crank_test_support::postgres_schema_url("admission_outage").await;
let pool = schema_pool(&schema_url).await;
let store = admission_store(pool.clone(), 2, 4);
let calls = Arc::new(AtomicUsize::new(0));
let current = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let runtime = executor(
Arc::clone(&store),
Arc::clone(&calls),
Arc::clone(&current),
Arc::clone(&peak),
);
// The mandatory state disappears after the store was built: every execution must fail closed.
sqlx::query("drop table execution_admission_entries")
.execute(&pool)
.await
.expect("drop the mandatory admission state");
let operation: crank_runtime::RuntimeOperation = operation(None).into();
let failure = execute(&runtime, &operation, &json!({}))
.await
.expect_err("a missing mandatory store must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(store.health().await.is_err());
}
/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2).
struct RecordingAdmission {
inner: Arc<PostgresExecutionAdmission>,
requests: Mutex<Vec<AdmissionRequest>>,
permits: AtomicUsize,
releases: AtomicUsize,
}
impl RecordingAdmission {
fn new(inner: Arc<PostgresExecutionAdmission>) -> Self {
Self {
inner,
requests: Mutex::new(Vec::new()),
permits: AtomicUsize::new(0),
releases: AtomicUsize::new(0),
}
}
fn requests(&self) -> Vec<AdmissionRequest> {
self.requests.lock().expect("admission log").clone()
}
fn permits(&self) -> usize {
self.permits.load(Ordering::SeqCst)
}
fn releases(&self) -> usize {
self.releases.load(Ordering::SeqCst)
}
}
#[async_trait]
impl ExecutionAdmission for RecordingAdmission {
async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> {
self.inner.health().await
}
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<crank_core::AdmissionOutcome, crank_core::AdmissionStoreError> {
self.requests
.lock()
.expect("admission log")
.push(request.clone());
let outcome = self.inner.admit(request).await?;
if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) {
self.permits.fetch_add(1, Ordering::SeqCst);
}
Ok(outcome)
}
async fn release(
&self,
permit: &crank_core::AdmissionPermit,
) -> Result<bool, crank_core::AdmissionStoreError> {
let released = self.inner.release(permit).await?;
if released {
self.releases.fetch_add(1, Ordering::SeqCst);
}
Ok(released)
}
async fn cancel(
&self,
ticket_id: u64,
owner_token: &str,
) -> Result<bool, crank_core::AdmissionStoreError> {
self.inner.cancel(ticket_id, owner_token).await
}
async fn snapshot(
&self,
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
self.inner.snapshot().await
}
}
fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation {
let mut source = operation(None);
source.id = OperationId::new("op_admission_confirmation");
source.name = "admission_confirmation".to_owned();
source.version = version;
let Target::Rest(target) = &mut source.target;
target.method = HttpMethod::Delete;
source.execution_config.safety = Some(OperationSafetyPolicy {
class: OperationSafetyClass::Destructive,
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
});
source.into()
}
fn confirmation_runtime(
admission: Arc<RecordingAdmission>,
coordination: Option<Arc<dyn crank_core::CoordinationStateStore>>,
calls: Arc<AtomicUsize>,
) -> RuntimeExecutor {
let store: Arc<dyn ExecutionAdmission> = admission;
let builder = RuntimeExecutorBuilder::new()
.register_adapter(Arc::new(CountingAdapter {
calls,
current: Arc::new(AtomicUsize::new(0)),
peak: Arc::new(AtomicUsize::new(0)),
}))
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
.with_admission(store);
match coordination {
Some(coordination) => builder.with_coordination_store(coordination).build(),
None => builder.build(),
}
}
fn confirmation_context() -> crank_runtime::RuntimeRequestContext {
crank_runtime::RuntimeRequestContext::from_correlation(
&crank_core::CorrelationContext::generate(),
)
.with_response_cache_scope("ws_test", "agent_a")
.with_metering_context(
WorkspaceId::new("ws_test"),
None,
InvocationSource::AdminTestRun,
)
}
#[tokio::test]
async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let coordination: Arc<dyn crank_core::CoordinationStateStore> =
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default());
let calls = Arc::new(AtomicUsize::new(0));
let runtime = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::clone(&coordination)),
Arc::clone(&calls),
);
let operation = confirmation_operation(3);
let context = confirmation_context();
let input = json!({ "reason": "remove the stale item" });
// 1. No approval yet: the existing challenge is returned and no execution capacity is taken.
let challenge = execute_in(&runtime, &operation, &input, &context)
.await
.expect_err("an unapproved destructive call must only stage confirmation");
assert_eq!(
challenge.error_code(),
ExecutionErrorCode::ConfirmationRequired
);
let token = challenge
.confirmation()
.expect("expected the confirmation challenge")
.token()
.to_owned();
assert_eq!(
recording.permits(),
0,
"an unapproved request must not take an execution permit"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let requests = recording.requests();
assert_eq!(
requests.len(),
1,
"the one admission snapshot is consulted once"
);
assert_eq!(
requests[0].approval,
ApprovalEvidence::NotGranted,
"admission must see the real approval state, not a fixed NotRequired"
);
assert_eq!(requests[0].scope.operation_version, 3);
let snapshot = recording.snapshot().await.expect("admission snapshot");
assert_eq!((snapshot.executing, snapshot.waiting), (0, 0));
// 2. An approval issued for another published version never opens this execution.
let other_version = confirmation_operation(4);
let foreign = execute_in(
&runtime,
&other_version,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a foreign approval must not authorise this version");
assert_eq!(
foreign.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(
recording.requests().len(),
1,
"an unusable approval is refused before any capacity request"
);
// 3. An approval is bound to its exact input as well.
let mismatched_input = json!({ "reason": "remove another item" });
let mismatched = execute_in(
&runtime,
&operation,
&mismatched_input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect_err("a confirmation token must not authorise different input");
assert_eq!(
mismatched.error_code(),
ExecutionErrorCode::ConfirmationInvalid
);
assert_eq!(recording.permits(), 0);
assert_eq!(recording.requests().len(), 1);
// 4. The matching approval passes admission with truthful evidence and executes exactly once.
let executed = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token.clone()),
)
.await
.expect("a verified approval executes");
assert!(executed.is_object());
assert_eq!(calls.load(Ordering::SeqCst), 1);
let requests = recording.requests();
assert_eq!(requests.len(), 2);
assert_eq!(requests[1].approval, ApprovalEvidence::Granted);
assert_eq!(requests[1].scope.operation_version, 3);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.releases(),
1,
"the permit is released after execution"
);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
// 5. Replay of the consumed confirmation is refused before capacity and without a side effect.
let replay = execute_in(
&runtime,
&operation,
&input,
&context.clone().with_confirmation_token(token),
)
.await
.expect_err("a confirmation token is single-use");
assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(recording.permits(), 1);
assert_eq!(
recording.requests().len(),
2,
"a replayed approval never reaches admission"
);
// 6. The trusted approved-recovery path still passes the same gate with real evidence.
let approved = execute_in(
&runtime,
&operation,
&json!({ "reason": "approved by an operator" }),
&context.with_approval_granted(),
)
.await;
assert!(approved.is_ok());
assert_eq!(calls.load(Ordering::SeqCst), 2);
let requests = recording.requests();
assert_eq!(requests.len(), 3);
assert_eq!(requests[2].approval, ApprovalEvidence::Granted);
assert_eq!(recording.permits(), 2);
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
}
#[tokio::test]
async fn mandatory_safety_store_outage_fails_closed_before_admission() {
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await;
let pool = schema_pool(&schema_url).await;
let recording = Arc::new(RecordingAdmission::new(Arc::new(
PostgresExecutionAdmission::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("valid admission limits"),
)));
let calls = Arc::new(AtomicUsize::new(0));
// Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail
// closed instead of reserving capacity and hoping for an in-memory fallback.
let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls));
let operation = confirmation_operation(3);
let failure = execute_in(
&runtime,
&operation,
&json!({ "reason": "no store" }),
&confirmation_context(),
)
.await
.expect_err("a mandatory safety store outage must fail closed");
assert_eq!(
failure.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(recording.permits(), 0);
let after_missing_store = recording.requests().len();
// The same Tool with a claimed approval and a *failing* safety store never reaches the capacity
// store at all: the approval claim stays unverified, so no permit and no outbound call exist.
let failing = confirmation_runtime(
Arc::clone(&recording),
Some(Arc::new(UnavailableSafetyStore)),
Arc::clone(&calls),
);
let outage = execute_in(
&failing,
&operation,
&json!({ "reason": "failing store" }),
&confirmation_context().with_confirmation_token("ct_unavailable"),
)
.await
.expect_err("an unreachable safety store must refuse the claimed approval");
assert_eq!(
outage.error_code(),
ExecutionErrorCode::SafetyStoreUnavailable
);
assert_eq!(
recording.requests().len(),
after_missing_store,
"an unverified approval never becomes a capacity request"
);
assert_eq!(recording.permits(), 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
/// Coordination store whose mandatory backend is down: every operation fails closed.
struct UnavailableSafetyStore;
impl UnavailableSafetyStore {
fn error() -> crank_core::CacheStoreError {
crank_core::CacheStoreError::Unavailable {
message: "test backend unavailable".to_owned(),
}
}
}
#[async_trait]
impl crank_core::CoordinationStateStore for UnavailableSafetyStore {
async fn get_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn put_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn delete_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<(), crank_core::CacheStoreError> {
Err(Self::error())
}
async fn take_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn reserve_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<crank_core::CoordinationStateReservation, crank_core::CacheStoreError> {
Err(Self::error())
}
async fn compare_and_set_value(
&self,
_scope: crank_core::CacheScope,
_key: &str,
_expected: &crank_core::CoordinationStateValue,
_value: crank_core::CoordinationStateValue,
_ttl: Duration,
) -> Result<bool, crank_core::CacheStoreError> {
Err(Self::error())
}
}
-3
View File
@@ -1,7 +1,6 @@
mod integration {
mod catalog_access;
mod common;
mod durable_tasks;
mod era_isolation;
mod execution_certainty;
mod execution_stages;
@@ -10,11 +9,9 @@ mod integration {
mod mcp_completion;
mod mcp_logging;
mod mcp_prompts;
mod mcp_signals;
mod request_context;
mod revision_matrix;
mod static_resources;
mod tool_search;
mod tools_lifecycle;
mod transport_protocol;
}
@@ -60,31 +60,6 @@ impl SharedLogWriter {
}
}
/// Story 3.4: a capturing writer plus the subscriber that records into it, so a test can prove that a
/// request-scoped progress token never reaches the log projection. Install it with
/// `tracing::subscriber::set_default` around the request on the `current_thread` runtime.
pub(super) fn capturing_subscriber() -> (
CapturedLogs,
impl tracing::Subscriber + Send + Sync + 'static,
) {
let writer = SharedLogWriter::default();
let subscriber = tracing_subscriber::fmt()
.with_writer(writer.clone())
.with_max_level(LevelFilter::TRACE)
.with_ansi(false)
.finish();
(CapturedLogs(writer), subscriber)
}
/// The captured log text of one scoped subscriber.
pub(super) struct CapturedLogs(SharedLogWriter);
impl CapturedLogs {
pub(super) fn output(&self) -> String {
self.0.output()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
@@ -621,7 +596,6 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation<Schema, Ma
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
@@ -1,555 +0,0 @@
//! Story 4.4 ATDD: no outbound send exists without a committed durable attempt, and an elapsed
//! accepted deadline fails before dispatch.
//!
//! Both oracles drive the production composition (application entry → PG store → runtime executor →
//! real local HTTP upstream); only the upstream and the checkpoint failure are test-controlled.
use super::*;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{
AgentOperationBinding, PlatformApiKeyScope, TaskState, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use time::OffsetDateTime;
async fn composition(
registry: &PostgresRegistry,
upstream: &str,
policy: ToolExecutionPolicy,
caller_name: &str,
scoped_key: &str,
) -> (
Arc<PostgresTaskStore>,
crank_community_mcp::TaskApplication,
SecretCrypto,
crank_community_mcp::TaskPrincipal,
) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
registry,
caller_name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let _ = scoped_key;
(store, tasks, crypto, caller)
}
fn worker_for(
registry: &PostgresRegistry,
store: &Arc<PostgresTaskStore>,
tasks: &TaskApplication,
crypto: &SecretCrypto,
owner: &str,
) -> TaskWorker {
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
tasks,
registry.clone(),
runtime,
crypto.clone(),
));
let config = TaskWorkerConfig {
owner: owner.to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
TaskWorker::new(store.clone(), handler, config).expect("valid worker")
}
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
/// AC1: a dispatch checkpoint that cannot commit produces zero outbound calls and no attempt row.
#[tokio::test]
async fn checkpoint_commit_failure_never_reaches_upstream() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let (store, tasks, crypto, caller) = composition(
&registry,
&upstream,
background_policy(),
"attempt-guards-checkpoint",
"guards-checkpoint-key",
)
.await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-checkpoint-key",
&json!({"email": "user@example.com"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
// The checkpoint is the only statement that commits an attempt; it is refused here.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create function refuse_task_checkpoint() returns trigger language plpgsql as $$
begin
if new.current_attempt_id is not null and old.current_attempt_id is null then
raise exception 'durable checkpoint refused for test';
end if;
return new;
end $$;
create trigger refuse_task_checkpoint before update on tasks
for each row execute function refuse_task_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint refusal trigger");
let worker = worker_for(
&registry,
&store,
&tasks,
&crypto,
"worker-guards-checkpoint",
);
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::sleep(Duration::from_secs(2)).await;
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an uncommitted durable checkpoint must never produce an outbound call"
);
let attempts: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
attempts, 0,
"no attempt history without a committed checkpoint"
);
assert_ne!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("completed"),
"the refused attempt cannot complete the Task"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(
"drop trigger refuse_task_checkpoint on tasks; drop function refuse_task_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint refusal trigger removed");
server.abort();
}
/// AC6: an accepted absolute deadline that already elapsed fails the Task before any dispatch.
#[tokio::test]
async fn elapsed_deadline_fails_before_dispatch_without_upstream_call() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let short_deadline = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 300,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
};
let (store, tasks, crypto, caller) = composition(
&registry,
&upstream,
short_deadline,
"attempt-guards-deadline",
"guards-deadline-key",
)
.await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-deadline-key",
&json!({"email": "user@example.com"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
// The accepted deadline elapses before any worker claims the Task.
tokio::time::sleep(Duration::from_millis(500)).await;
let worker = worker_for(&registry, &store, &tasks, &crypto, "worker-guards-deadline");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("failed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state.as_deref(), Some("failed"));
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an elapsed deadline must fail before dispatch"
);
let attempts: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts, 0, "no attempt is checkpointed after the deadline");
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("task_deadline_elapsed"));
assert_eq!(
sqlx::query_scalar::<_, String>("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap(),
TaskState::Failed.as_str()
);
server.abort();
}
/// AC2: a write whose upstream idempotency is backed by server-owned evidence is retried with the
/// same stable key, in a bounded number of distinct durable attempts, and applies exactly one effect.
#[tokio::test]
async fn verified_idempotent_write_retries_with_stable_key_and_one_effect() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
use std::collections::HashSet;
use std::sync::Mutex;
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let effects = Arc::new(AtomicUsize::new(0));
let keys = Arc::new(Mutex::new(Vec::<String>::new()));
let seen = Arc::new(Mutex::new(HashSet::<String>::new()));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
let effects = Arc::clone(&effects);
let keys = Arc::clone(&keys);
let seen = Arc::clone(&seen);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
let effects = Arc::clone(&effects);
let keys = Arc::clone(&keys);
let seen = Arc::clone(&seen);
async move {
let key = headers
.get("idempotency-key")
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
keys.lock().expect("keys").push(key.clone());
let first_application = seen.lock().expect("seen").insert(key);
if first_application {
effects.fetch_add(1, Ordering::SeqCst);
}
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// The first application is stored but its acknowledgement is lost.
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error": "acknowledgement lost"})),
);
}
(
StatusCode::OK,
axum::Json(json!({"id": "lead-1", "deduped": !first_application})),
)
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"attempt-guards-verified",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-verified-key",
&json!({"email": "user@example.com", "idempotency_key": "k-1"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&tasks, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
let config = TaskWorkerConfig {
owner: "worker-guards-verified".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("completed"));
assert_eq!(
requests.load(Ordering::SeqCst),
2,
"exactly one retry of the write"
);
assert_eq!(
effects.load(Ordering::SeqCst),
1,
"the upstream applied the write exactly once"
);
assert_eq!(
keys.lock().expect("keys").as_slice(),
["k-1".to_owned(), "k-1".to_owned()],
"the same stable key is delivered on both attempts"
);
let attempts = store
.attempts(
&crank_core::TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: crank_core::TaskOwnerKind::Agent,
owner_id: "agent_attempt_guards".to_owned(),
authorization_scope: String::new(),
},
acceptance.task_id,
)
.await;
// Owner scope of this principal is the agent of the published binding; read the history through
// the application entry instead of guessing the authorization scope string.
assert!(attempts.is_err(), "a foreign scope never reads the history");
let attempts = tasks
.attempts(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("owner-scoped attempt history");
assert_eq!(attempts.len(), 2, "two distinct durable attempts");
assert_eq!(attempts[0].ordinal, 1);
assert_eq!(attempts[1].ordinal, 2);
assert!(attempts.iter().all(|attempt| attempt.retry_allowed));
assert_eq!(
attempts[0].outcome,
Some(crank_core::AttemptOutcomeClass::RetryScheduled)
);
assert_eq!(
attempts[1].outcome,
Some(crank_core::AttemptOutcomeClass::Completed)
);
assert_eq!(
attempts[0].upstream_key_digest, attempts[1].upstream_key_digest,
"the same stable key digest is recorded on both attempts"
);
assert!(attempts[0].upstream_key_digest.is_some());
let record = tasks
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("status");
assert_eq!(record.pins.retry.max_attempts, 2);
}
@@ -1,432 +0,0 @@
//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1).
//!
//! End-to-end: a real durable Task worker dispatches a retrying GET through a real local
//! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is
//! persisted in `tasks.current_attempt_id` on both attempts.
use super::super::common;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::get;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{
AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy,
};
use crank_runtime::{
OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy,
};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use time::OffsetDateTime;
use tracing::{Id, Subscriber, field::Visit};
use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "attempt_trace";
const KEY: &str = "attempt-trace-key-1";
async fn publish(
registry: &crank_registry::PostgresRegistry,
operation_id: &crank_core::OperationId,
version: u32,
) {
registry
.publish_operation(crank_registry::PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id,
version,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
}
async fn application_with_store(
registry: &crank_registry::PostgresRegistry,
) -> (
crank_community_mcp::TaskApplication,
std::sync::Arc<crank_registry::PostgresTaskStore>,
SecretCrypto,
) {
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = std::sync::Arc::new(
crank_registry::PostgresTaskStore::new(
registry.pool().clone(),
crank_registry::AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = crank_community_mcp::TaskApplication::new(
registry.clone(),
std::time::Duration::ZERO,
std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()),
crank_runtime::TaskService::with_secret_crypto(
store.clone(),
std::sync::Arc::new(crypto.clone()),
),
);
(tasks, store, crypto)
}
async fn principal(
registry: &crank_registry::PostgresRegistry,
name: &str,
scopes: &[crank_core::PlatformApiKeyScope],
) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("an active platform key resolves");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
#[tokio::test]
async fn adapter_attempt_span_carries_durable_attempt_id() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
get({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
let n = calls.fetch_add(1, Ordering::SeqCst);
if n == 0 {
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(serde_json::json!({"error":"boom"})),
);
}
(
StatusCode::OK,
axum::Json(serde_json::json!({"id":"lead-1"})),
)
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
let Target::Rest(target) = &mut operation.target;
target.method = crank_core::HttpMethod::Get;
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: crank_core::ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"attempt-trace",
&[
crank_core::PlatformApiKeyScope::Read,
crank_core::PlatformApiKeyScope::Write,
],
)
.await;
let (tasks, store, crypto) = application_with_store(&registry).await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&serde_json::json!({"email": "user@example.com"}),
&CorrelationContext::generate(),
)
.await
.expect("durable acceptance");
let runtime =
community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let config = TaskWorkerConfig {
owner: "runtime-worker-attempt-trace".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: std::time::Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: std::time::Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let capture = TraceCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
let _guard = tracing::subscriber::set_default(subscriber);
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(std::time::Duration::from_secs(10), worker_task)
.await
.expect("worker stops inside the budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
server.abort();
let durable_attempt_id: Option<String> = sqlx::query_scalar::<_, Option<String>>(
"select current_attempt_id from tasks where task_id = $1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(state.as_deref(), Some("completed"));
// Story 4.4: every outbound send is its own durable attempt, so the two retries of one lease carry
// two distinct attempt identities, both recorded in the immutable attempt history.
let attempt_rows: Vec<(String, i32)> = sqlx::query_as(
"select attempt_id, ordinal from task_attempts where task_id = $1 order by ordinal",
)
.bind(acceptance.task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap();
assert_eq!(attempt_rows.len(), 2, "two durable attempts were committed");
assert_eq!(attempt_rows[0].1, 1);
assert_eq!(attempt_rows[1].1, 2);
assert_eq!(
Some(attempt_rows[1].0.as_str()),
durable_attempt_id.as_deref(),
"the Task points at its latest durable attempt"
);
let spans = capture.snapshot();
let attempt_spans: Vec<_> = spans
.iter()
.filter(|s| s.name == "adapter_attempt")
.collect();
assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans");
for span in &attempt_spans {
let aid = span.fields.get("attempt_id").cloned().unwrap_or_default();
assert!(!aid.is_empty(), "attempt_id must not be empty");
assert!(
!aid.contains("traceparent"),
"attempt_id must not contain traceparent"
);
assert!(
attempt_rows.iter().any(|(id, _)| id == &aid),
"every attempt span names an attempt that was durably committed"
);
}
let ids: Vec<String> = attempt_spans
.iter()
.map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default())
.collect();
assert_ne!(ids[0], ids[1], "each retry has its own durable attempt id");
assert_eq!(
spans
.iter()
.filter(|span| span.name == "durable_task_attempt")
.count(),
1,
"one durable attempt span per dispatch"
);
let runtime_span = spans
.iter()
.find(|span| span.name == "runtime.execute")
.expect("runtime.execute span");
assert_eq!(
runtime_span.parent_name,
Some("durable_task_attempt"),
"the execution continues the durable attempt trace"
);
for span in &attempt_spans {
assert_eq!(
span.parent_name,
Some("runtime.execute"),
"each retry is a child attempt span under the durable execution"
);
}
let attempts: Vec<String> = attempt_spans
.iter()
.map(|s| s.fields.get("attempt").cloned().unwrap_or_default())
.collect();
assert!(attempts.contains(&"1".to_string()));
assert!(attempts.contains(&"2".to_string()));
for span in &attempt_spans {
assert_eq!(
span.fields.get("max_attempts").cloned().unwrap_or_default(),
"2"
);
}
}
// TraceCapture minimal port from crates/crank-runtime/tests/stages.rs
#[derive(Clone, Default)]
struct TraceCapture {
spans: Arc<Mutex<Vec<CapturedSpan>>>,
}
impl TraceCapture {
fn snapshot(&self) -> Vec<CapturedSpan> {
self.spans.lock().expect("span lock").clone()
}
}
#[derive(Clone, Debug)]
struct CapturedSpan {
name: &'static str,
parent_name: Option<&'static str>,
fields: std::collections::BTreeMap<String, String>,
}
impl<S> Layer<S> for TraceCapture
where
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
{
fn on_new_span(
&self,
attributes: &tracing::span::Attributes<'_>,
id: &Id,
context: tracing_subscriber::layer::Context<'_, S>,
) {
let parent = attributes
.parent()
.and_then(|parent| context.span(parent))
.or_else(|| {
attributes
.is_contextual()
.then(|| context.lookup_current())
.flatten()
});
let mut visitor = FieldVisitor::default();
attributes.record(&mut visitor);
let mut spans = self.spans.lock().expect("span lock");
let index = spans.len();
spans.push(CapturedSpan {
name: attributes.metadata().name(),
parent_name: parent.map(|span| span.metadata().name()),
fields: visitor.fields,
});
context
.span(id)
.expect("span exists")
.extensions_mut()
.insert(index);
}
fn on_record(
&self,
id: &Id,
values: &tracing::span::Record<'_>,
context: tracing_subscriber::layer::Context<'_, S>,
) {
let mut visitor = FieldVisitor::default();
values.record(&mut visitor);
let span = context.span(id).expect("span exists");
let index = *span.extensions().get::<usize>().expect("capture index");
self.spans.lock().expect("span lock")[index]
.fields
.extend(visitor.fields);
}
}
#[derive(Default)]
struct FieldVisitor {
fields: std::collections::BTreeMap<String, String>,
}
impl Visit for FieldVisitor {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.fields
.insert(field.name().to_owned(), value.to_owned());
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.fields
.insert(field.name().to_owned(), format!("{value:?}"));
}
}
@@ -1,846 +0,0 @@
//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream.
use super::common;
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
use crank_core::{
AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState,
TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential,
};
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
};
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "report_gen";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
}
async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(
registry,
AGENT_SLUG,
name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("active key");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store, Arc::new(crypto)),
)
}
#[tokio::test]
async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
let signal = Arc::new(std::sync::Mutex::new(Some(tx)));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
move || {
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
async move {
calls.fetch_add(1, Ordering::SeqCst);
if let Some(tx) = signal.lock().unwrap().take() {
let _ = tx.send(());
}
tokio::time::sleep(Duration::from_millis(1500)).await;
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s1").await;
let foreign = principal(&registry, "cancel-s1-foreign").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s1-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&app,
registry.clone(),
runtime,
crypto.clone(),
));
let worker = TaskWorker::new(
store,
handler,
TaskWorkerConfig {
owner: "worker-s1".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
renewal_interval: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(5),
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let _ = tokio::time::timeout(Duration::from_secs(5), rx).await;
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, TaskState::OutcomeUnknown);
}
other => panic!("expected OutcomeUnknown got {other:?}"),
}
let foreign_cancel = app
.cancel(&foreign, TOOL_NAME, acceptance.task_id, None)
.await;
assert!(matches!(
foreign_cancel,
Err(crank_community_mcp::TaskApplicationError::NotFound)
));
tokio::time::sleep(Duration::from_millis(1600)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1);
let cancel2 = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel2");
assert!(matches!(
cancel2,
crank_core::TaskCancelOutcome::OutcomeUnknown(_)
));
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
}
#[tokio::test]
async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s3").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s3-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::Cancelled(record) => {
assert_eq!(record.state, TaskState::Cancelled)
}
other => panic!("expected Cancelled got {other:?}"),
}
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("cancelled")
);
let attempts = app
.attempts(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("attempts");
assert!(attempts.is_empty());
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s2").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s2-key",
&json!({"email":"u","idempotency_key":"k-2"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
// Reproduce deterministically the durable state the executor commits after an ambiguous attempt
// whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and
// `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a
// running executor against the cancel would not be a repeatable oracle.
let lease = store
.claim_batch("worker-s2-prep", 1, 10_000)
.await
.expect("claim")
.pop()
.expect("one claimed lease");
let attempt = store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: acceptance.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("durable checkpoint");
let resolution = store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Uncertain {
error: crank_core::TaskEnvelope::try_redacted(
&json!({"code": "upstream_ack_lost", "phase": "dispatched"}),
)
.expect("bounded attempt error"),
repeatable: true,
phase: crank_core::AttemptPhase::Dispatched,
},
)
.await
.expect("attempt resolution");
assert!(
matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }),
"an ambiguous but safely repeatable attempt arms the pinned retry"
);
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown)
}
other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"),
}
// A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not
// claimable and no second terminal write is possible.
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s2".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::sleep(Duration::from_secs(1)).await;
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
requests.load(Ordering::SeqCst),
0,
"no outbound send after the cancel"
);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
}
#[tokio::test]
async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s4").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s4-key",
&json!({"email":"u","idempotency_key":"k-4"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s4".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
// Revoke the key the Task was accepted with while the first outbound write is provably in flight:
// that attempt still resolves as an uncertain write, and the next attempt is refused by the
// authority before its checkpoint.
for _ in 0..200 {
if requests.load(Ordering::SeqCst) >= 1 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the first attempt is dispatched before the refusal"
);
registry
.revoke_platform_api_key_for_agent(
&common::test_workspace_id(),
&common::test_agent_id(AGENT_SLUG),
&PlatformApiKeyId::new("pk_cancel-s4".to_owned()),
&OffsetDateTime::now_utc(),
)
.await
.expect("revoke");
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("outcome_unknown") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
state.as_deref(),
Some("outcome_unknown"),
"a refused attempt after a lost acknowledgement must stay uncertain"
);
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the refused attempt is never dispatched"
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let phase: Option<String> = sqlx::query_scalar(
"select phase from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(phase.as_deref(), Some("dispatched"));
let error: Option<(Option<String>, Option<String>)> = sqlx::query_as(
"select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
let (code, error_phase) = error.expect("the refused Task carries its bounded error");
assert_eq!(code.as_deref(), Some("task_authority_rejected"));
assert_eq!(
error_phase.as_deref(),
Some("dispatched"),
"the error keeps the phase of the attempt that may have reached the upstream"
);
}
@@ -1,965 +0,0 @@
//! ATDD Story 4.5 (Beam): durable input continuation through TaskApplication -> PG -> worker -> upstream.
use super::common;
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
use crank_core::{
NoTasksFallback, PlatformApiKeyScope, TaskId, TaskState, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
};
use serde_json::json;
use sqlx::Row;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
const AGENT_SLUG: &str = "reports";
pub(crate) const TOOL_NAME: &str = "report_gen";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
pub(crate) async fn state_of(registry: &PostgresRegistry, task_id: TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
pub(crate) async fn wait_for_state(
registry: &PostgresRegistry,
task_id: TaskId,
desired: &str,
limit: Duration,
) -> bool {
let deadline = tokio::time::Instant::now() + limit;
while tokio::time::Instant::now() < deadline {
if state_of(registry, task_id).await.as_deref() == Some(desired) {
return true;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
state_of(registry, task_id).await.as_deref() == Some(desired)
}
pub(crate) async fn seed_tool(
registry: &PostgresRegistry,
upstream: &str,
policy: ToolExecutionPolicy,
) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![crank_core::AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
}
pub(crate) async fn principal(
registry: &PostgresRegistry,
name: &str,
) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(
registry,
AGENT_SLUG,
name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("active key");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
pub(crate) async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store, Arc::new(crypto)),
)
}
pub(crate) fn spawn_worker(
registry: &PostgresRegistry,
app: &TaskApplication,
) -> (
Arc<std::sync::atomic::AtomicBool>,
tokio::task::JoinHandle<()>,
) {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let pool = registry.pool().clone();
let store = Arc::new(
PostgresTaskStore::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
app,
registry.clone(),
runtime,
crypto,
));
let worker = TaskWorker::new(
store,
handler,
TaskWorkerConfig {
owner: "worker-input".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
renewal_interval: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(5),
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let jh = tokio::spawn(async move {
let _ = worker.run().await;
});
(stop_flag, jh)
}
#[tokio::test]
async fn input_continuation_happy_path_accept_update_resume_replay() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "input-happy").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-1",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
assert_eq!(acceptance.state, TaskState::Queued);
let task_id = acceptance.task_id;
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(
&registry,
task_id,
"input_required",
Duration::from_secs(10)
)
.await,
"worker must suspend for missing email"
);
// positive control: before update no outbound request
assert_eq!(calls.load(Ordering::SeqCst), 0);
// persisted request fields
let row = sqlx::query(
"select wait_id, expected_version, schema_snapshot, projected_schema, prompt_code, expires_at from task_input_requests where task_id = $1",
)
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
let wait_id: String = row.get("wait_id");
let expected_version: i32 = row.get("expected_version");
let schema_snapshot: serde_json::Value = row.get("schema_snapshot");
let projected_schema: serde_json::Value = row.get("projected_schema");
let prompt_code: String = row.get("prompt_code");
let _expires_at: OffsetDateTime = row.get("expires_at");
assert!(!wait_id.is_empty());
assert!(expected_version > 0);
assert!(schema_snapshot.is_object());
assert!(projected_schema.is_object());
assert_eq!(prompt_code, "required_input_missing");
let audit_count_before: i64 = sqlx::query_scalar(
"select count(*)::bigint from task_input_audit where task_id = $1 and event = 'input_required'",
)
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(audit_count_before >= 1);
// stop worker before update to avoid races
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let outcome = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version as u32,
"update-key-1",
&json!({"email": "user@example.com"}),
)
.await
.expect("valid update");
assert_eq!(outcome.state, TaskState::Working);
assert!(!outcome.replayed);
assert_eq!(outcome.task_id, task_id);
assert_eq!(outcome.wait_id, wait_id);
let (stop2, worker_task2) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"worker must complete after update"
);
stop2.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
let attempts = app
.attempts(&caller, TOOL_NAME, task_id)
.await
.expect("attempts");
assert_eq!(attempts.len(), 1);
let effective_fp: String =
sqlx::query_scalar("select effective_input_fingerprint from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts[0].input_fingerprint, effective_fp);
let input_fp: String =
sqlx::query_scalar("select input_fingerprint from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_ne!(effective_fp, input_fp);
// An identical answer replay still returns the committed outcome after the Task completed:
// no second transition, audit row or outbound call.
let completed_replay = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version as u32,
"update-key-1",
&json!({"email": "user@example.com"}),
)
.await
.expect("identical replay after completion");
assert!(completed_replay.replayed);
assert_eq!(completed_replay.version, outcome.version);
assert_eq!(calls.load(Ordering::SeqCst), 1);
// replay with original scoped key/partial input: same Task, no fingerprint conflict
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-1",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("replay accepted");
assert_eq!(replay.task_id, task_id);
server.abort();
}
pub(crate) async fn start_and_suspend(
registry: &PostgresRegistry,
app: &TaskApplication,
caller: &crank_community_mcp::TaskPrincipal,
scoped_key: &str,
) -> (TaskId, String, u32) {
let acceptance = app
.accept_background(
caller,
TOOL_NAME,
None,
scoped_key,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
let (stop, worker_task) = spawn_worker(registry, app);
assert!(
wait_for_state(
registry,
acceptance.task_id,
"input_required",
Duration::from_secs(10)
)
.await,
"worker must suspend for missing email"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let (wait_id, expected_version) = wait_row(registry, acceptance.task_id).await;
(acceptance.task_id, wait_id, expected_version)
}
async fn wait_row(registry: &PostgresRegistry, task_id: TaskId) -> (String, u32) {
let row =
sqlx::query("select wait_id, expected_version from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
(
row.get::<String, _>("wait_id"),
u32::try_from(row.get::<i32, _>("expected_version")).unwrap_or_default(),
)
}
pub(crate) async fn audit_events(
registry: &PostgresRegistry,
task_id: TaskId,
) -> Vec<(String, Option<String>)> {
sqlx::query("select event, reason_code from task_input_audit where task_id = $1 order by id")
.bind(task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap()
.into_iter()
.map(|row| (row.get("event"), row.get("reason_code")))
.collect()
}
pub(crate) async fn admission_entries(
registry: &PostgresRegistry,
task_id: TaskId,
) -> Vec<(String, i64)> {
sqlx::query(
"select state, count(*)::bigint as count from execution_admission_entries
where task_id = $1 group by state order by state",
)
.bind(task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap()
.into_iter()
.map(|row| (row.get("state"), row.get("count")))
.collect()
}
fn key_id(principal: &crank_community_mcp::TaskPrincipal) -> crank_core::PlatformApiKeyId {
match principal {
crank_community_mcp::TaskPrincipal::Agent { credential, .. } => credential
.platform_api_key_id
.clone()
.expect("agent principal carries a key id"),
crank_community_mcp::TaskPrincipal::Admin { .. } => panic!("agent principal expected"),
}
}
#[tokio::test]
async fn invalid_schema_foreign_scope_and_revoked_key_never_transition_or_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-scope").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-scope").await;
// Invalid answer: server-side schema validation refuses it, nothing transitions and nothing leaks.
let invalid = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-invalid",
&json!({}),
)
.await
.expect_err("the pinned schema requires email");
assert!(
matches!(
invalid,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::Invalid { .. })
),
"unexpected error: {invalid:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert!(
audit_events(&registry, task_id)
.await
.iter()
.all(|(event, _)| event == "input_required"),
"a rejected answer must not be audited as accepted"
);
let stored_snapshot: serde_json::Value =
sqlx::query_scalar("select schema_snapshot from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(!stored_snapshot.to_string().contains("user@example.com"));
// A different key of the same agent carries a different authorization scope: opaque not-found.
let foreign = principal(&registry, "input-foreign").await;
let foreign_result = app
.update_input(
&foreign,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-foreign",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a foreign scope cannot continue the Task");
assert_eq!(
foreign_result,
crank_community_mcp::TaskApplicationError::NotFound
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
// Positive control: the accepting principal (same key/scope) completes the transition.
let accepted = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-valid",
&json!({"email": "user@example.com"}),
)
.await
.expect("valid answer for the accepting scope");
assert_eq!(accepted.state, TaskState::Working);
assert!(!accepted.replayed);
// Finish the positive-control Task through a real worker so the revocation case starts from a
// known dispatch count instead of leaving resumable work behind.
let (finish_stop, finish_worker) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"the positive-control Task completes"
);
finish_stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), finish_worker).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
// Revocation race: the accepting key is revoked after suspension. The same waiting scope must
// fail closed inside the update transaction and never revive or dispatch the Task.
let revocable = principal(&registry, "input-revoked").await;
let (revoked_task, revoked_wait, revoked_version) =
start_and_suspend(&registry, &app, &revocable, "input-key-revoked").await;
registry
.revoke_platform_api_key(
&common::test_workspace_id(),
&key_id(&revocable),
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
let revoked_result = app
.update_input(
&revocable,
TOOL_NAME,
revoked_task,
&revoked_wait,
revoked_version,
"update-revoked",
&json!({"email": "revoked@example.com"}),
)
.await
.expect_err("a revoked key cannot continue the Task");
assert_eq!(
revoked_result,
crank_community_mcp::TaskApplicationError::NotFound
);
assert_eq!(
state_of(&registry, revoked_task).await.as_deref(),
Some("input_required")
);
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"the revoked answer must not add an outbound call"
);
server.abort();
}
#[tokio::test]
async fn replay_conflict_and_stale_answers_have_one_typed_outcome() {
let registry = common::test_registry().await;
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, "http://127.0.0.1:9", policy).await;
let caller = principal(&registry, "input-replay").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-replay").await;
let first = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "user@example.com"}),
)
.await
.expect("first valid answer");
assert!(!first.replayed);
assert_eq!(first.state, TaskState::Working);
// Identical replay returns the stored outcome without a second CAS/audit.
let replay = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "user@example.com"}),
)
.await
.expect("identical replay");
assert!(replay.replayed);
assert_eq!(replay.version, first.version);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("working")
);
let accepted_audit = audit_events(&registry, task_id)
.await
.into_iter()
.filter(|(event, _)| event == "input_accepted")
.count();
assert_eq!(accepted_audit, 1);
// Same key with another payload is a fingerprint conflict.
let conflict = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-key",
&json!({"email": "other@example.com"}),
)
.await
.expect_err("same key with another payload conflicts");
assert_eq!(
conflict,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::FingerprintConflict)
);
// Another key after a committed answer, and a foreign wait id, are state conflicts.
let other_key = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
first.version,
"update-key-2",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("another key after commit conflicts");
assert_eq!(
other_key,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
let foreign_wait = app
.update_input(
&caller,
TOOL_NAME,
task_id,
"wait-does-not-exist",
first.version,
"update-key-3",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a foreign wait id conflicts");
assert_eq!(
foreign_wait,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
// A stale generation on a fresh wait is refused; the valid generation still works.
let (stale_task, stale_wait, stale_version) =
start_and_suspend(&registry, &app, &caller, "input-key-stale").await;
let stale = app
.update_input(
&caller,
TOOL_NAME,
stale_task,
&stale_wait,
stale_version.saturating_sub(1).max(1),
"update-stale",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("a stale generation cannot be continued");
assert_eq!(
stale,
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
);
let repaired = app
.update_input(
&caller,
TOOL_NAME,
stale_task,
&stale_wait,
stale_version,
"update-stale",
&json!({"email": "user@example.com"}),
)
.await
.expect("the current generation still accepts the answer");
assert!(!repaired.replayed);
}
#[tokio::test]
async fn ttl_expiry_fails_with_audit_and_releases_capacity_without_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
// The wait TTL is the remaining pinned absolute execution deadline.
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 1_200,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-ttl").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"input-key-ttl",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("acceptance");
let task_id = acceptance.task_id;
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "input_required", Duration::from_secs(5)).await,
"worker must suspend"
);
assert!(
wait_for_state(&registry, task_id, "failed", Duration::from_secs(6)).await,
"the per-poll sweeper must fail the expired wait"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
let (wait_id, expected_version) = wait_row(&registry, task_id).await;
let events = audit_events(&registry, task_id).await;
assert!(
events.iter().any(|(event, reason)| event == "input_expired"
&& reason.as_deref() == Some("input_ttl_expired")),
"expiry reason must be audited: {events:?}"
);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
assert_eq!(calls.load(Ordering::SeqCst), 0);
// A late answer cannot revive the failed Task.
let late = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-late",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("an expired wait never revives");
assert!(
matches!(
late,
crank_community_mcp::TaskApplicationError::Task(
crank_core::TaskError::InputExpired | crank_core::TaskError::StateConflict
)
),
"unexpected late answer error: {late:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("failed")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn competing_identical_updates_commit_once_and_keep_one_permit() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "input-race").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-race").await;
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)],
"the waiting Task keeps exactly one executing permit"
);
let payload = json!({"email": "user@example.com"});
let first = app.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-race",
&payload,
);
let second = app.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-race",
&payload,
);
let (first, second) = tokio::join!(first, second);
let mut outcomes = [
first.expect("competing answer"),
second.expect("competing answer"),
];
outcomes.sort_by_key(|outcome| outcome.replayed);
assert!(!outcomes[0].replayed);
assert!(outcomes[1].replayed);
assert_eq!(outcomes[0].version, outcomes[1].version);
assert_eq!(
audit_events(&registry, task_id)
.await
.into_iter()
.filter(|(event, _)| event == "input_accepted")
.count(),
1
);
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)],
"the update must not grant a second permit"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_id, "completed", Duration::from_secs(10)).await,
"the resumed Task completes"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
server.abort();
}
@@ -1,457 +0,0 @@
//! ATDD Story 4.5 QA1 guards (F1/F3): one pinned deadline for several waiting Tasks, cancellation
//! while the Task waits for input and refusal of a stale lease/fence around the continuation seam.
use super::common;
use super::input_continuation::{
TOOL_NAME, admission_entries, application, audit_events, principal, seed_tool, spawn_worker,
start_and_suspend, state_of, wait_for_state,
};
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_core::{
BeginAttempt, NoTasksFallback, TASK_INPUT_SCHEMA_FORMAT_VERSION, TaskInputRequest,
TaskInputStore, TaskState, TaskStore, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{AdmissionLimits, PostgresTaskStore};
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
#[tokio::test]
async fn two_waits_with_one_pinned_deadline_and_cancelled_leftover_do_not_block_suspension() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "deadline-cancel").await;
let app = application(&registry).await;
let acc_a = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-a",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept a");
let acc_b = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-b",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept b");
let task_a = acc_a.task_id;
let task_b = acc_b.task_id;
let pinned_ms = (OffsetDateTime::now_utc().unix_timestamp() as f64 * 1000.0) + 60_000.0;
for tid in [task_a, task_b] {
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
.bind(pinned_ms)
.bind(tid.as_uuid())
.execute(registry.pool())
.await
.unwrap();
}
let (stop, worker_task) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_a, "input_required", Duration::from_secs(10)).await,
"a must suspend"
);
assert!(
wait_for_state(&registry, task_b, "input_required", Duration::from_secs(10)).await,
"b must suspend"
);
stop.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(calls.load(Ordering::SeqCst), 0);
// Both active waits carry the same absolute TTL value: the per-Task uniqueness above is the only
// uniqueness, so this must not be a UNIQUE index on expires_at.
let same_deadline_wait: i64 = sqlx::query_scalar(
"select count(*)::bigint from task_input_requests a
join task_input_requests b on b.task_id = $2
where a.task_id = $1 and a.accepted_at is null and b.accepted_at is null
and a.expires_at = b.expires_at",
)
.bind(task_a.as_uuid())
.bind(task_b.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
same_deadline_wait, 1,
"both waits share the pinned deadline"
);
let outcome = app
.cancel(&caller, TOOL_NAME, task_a, None)
.await
.expect("cancel a");
assert!(matches!(
outcome,
crank_core::TaskCancelOutcome::Cancelled(_)
));
assert_eq!(
state_of(&registry, task_a).await.as_deref(),
Some("cancelled")
);
assert_eq!(admission_entries(&registry, task_a).await, Vec::new());
// accept C with same pinned deadline
let acc_c = app
.accept_background(
&caller,
TOOL_NAME,
None,
"deadline-key-c",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept c");
let task_c = acc_c.task_id;
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
.bind(pinned_ms)
.bind(task_c.as_uuid())
.execute(registry.pool())
.await
.unwrap();
let (stop2, worker_task2) = spawn_worker(&registry, &app);
assert!(
wait_for_state(&registry, task_c, "input_required", Duration::from_secs(10)).await,
"c must suspend despite leftover a"
);
stop2.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
assert_eq!(
state_of(&registry, task_b).await.as_deref(),
Some("input_required")
);
let c_deadline_ms: i64 = sqlx::query_scalar(
"select (extract(epoch from expires_at) * 1000)::bigint
from task_input_requests where task_id = $1",
)
.bind(task_c.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
c_deadline_ms, pinned_ms as i64,
"the third wait reuses the same pinned deadline value"
);
let a_accepted: Option<OffsetDateTime> =
sqlx::query_scalar("select accepted_at from task_input_requests where task_id = $1")
.bind(task_a.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(
a_accepted.is_none(),
"cancellation leaves the wait row without an accepted receipt"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn cancel_while_input_required_never_revives_or_dispatches() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "cancel-wait").await;
let app = application(&registry).await;
let (task_id, wait_id, expected_version) =
start_and_suspend(&registry, &app, &caller, "input-key-cancel").await;
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)]
);
let outcome = app
.cancel(&caller, TOOL_NAME, task_id, None)
.await
.expect("cancel");
assert!(matches!(
outcome,
crank_core::TaskCancelOutcome::Cancelled(_)
));
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("cancelled")
);
assert_eq!(admission_entries(&registry, task_id).await, Vec::new());
let late = app
.update_input(
&caller,
TOOL_NAME,
task_id,
&wait_id,
expected_version,
"update-late",
&json!({"email": "user@example.com"}),
)
.await
.expect_err("late answer must not revive");
assert!(
matches!(
late,
crank_community_mcp::TaskApplicationError::Task(
crank_core::TaskError::StateConflict | crank_core::TaskError::InputExpired
)
),
"unexpected: {late:?}"
);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("cancelled")
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
let events = audit_events(&registry, task_id).await;
assert!(events.iter().any(|(e, _)| e == "input_required"));
assert!(!events.iter().any(|(e, _)| e == "input_accepted"));
let attempts = app
.attempts(&caller, TOOL_NAME, task_id)
.await
.expect("attempts");
assert!(attempts.is_empty());
server.abort();
}
#[tokio::test]
async fn stale_lease_fence_cannot_suspend_or_dispatch() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &format!("http://{address}"), policy).await;
let caller = principal(&registry, "stale-fence").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"stale-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let task_id = acceptance.task_id;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let mut old_claims = store
.claim_batch("worker-old", 1, 50)
.await
.expect("old claim");
assert!(!old_claims.is_empty());
let old = old_claims.remove(0);
tokio::time::sleep(Duration::from_millis(120)).await;
let mut new_claims = store
.claim_batch("worker-new", 1, 10_000)
.await
.expect("new claim");
assert!(!new_claims.is_empty());
let new = new_claims.remove(0);
assert!(new.fence > old.fence);
assert_eq!(new.record.task_id, task_id);
let request = TaskInputRequest {
wait_id: "stale-wait".to_string(),
expected_version: old.record.version + 1,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({}),
projected_schema: json!({}),
prompt_code: "required_input_missing".to_string(),
missing_fields: vec!["$.email".to_string()],
expires_at_unix_ms: OffsetDateTime::now_utc().unix_timestamp() * 1000 + 30_000,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
};
let suspended =
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &old, request).await;
assert!(
suspended.is_err(),
"stale fence must not suspend: {suspended:?}"
);
let attempt_id = "attempt-stale".to_string();
let checkpoint = store
.checkpoint_dispatch(BeginAttempt {
task_id,
owner: old.owner.clone(),
fence: old.fence,
expected_version: old.record.version,
attempt_id,
input_fingerprint: old.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(
checkpoint.is_err(),
"stale fence must not checkpoint: {checkpoint:?}"
);
assert_ne!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
let attempt_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempt_count, 0);
let req_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_input_requests where task_id = $1")
.bind(task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(req_count, 0);
assert_eq!(calls.load(Ordering::SeqCst), 0);
// Positive control: the same request under the current lease suspends the Task exactly once, so
// the refusal above is the stale fence and not an invalid request shape.
let request = TaskInputRequest {
wait_id: "current-wait".to_string(),
expected_version: new.record.version + 1,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({}),
projected_schema: json!({}),
prompt_code: "required_input_missing".to_string(),
missing_fields: vec!["$.email".to_string()],
expires_at_unix_ms: new.record.pins.execution_deadline_unix_ms,
};
let suspended =
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &new, request)
.await
.expect("the current lease suspends");
assert_eq!(suspended.state, TaskState::InputRequired);
assert_eq!(
state_of(&registry, task_id).await.as_deref(),
Some("input_required")
);
assert_eq!(
admission_entries(&registry, task_id).await,
vec![("executing".to_owned(), 1)]
);
// The stale fence still cannot checkpoint a dispatch of this generation.
let checkpoint = store
.checkpoint_dispatch(BeginAttempt {
task_id,
owner: old.owner.clone(),
fence: old.fence,
expected_version: old.record.version,
attempt_id: "attempt-stale-after-suspend".to_string(),
input_fingerprint: old.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(
checkpoint.is_err(),
"stale fence must not checkpoint: {checkpoint:?}"
);
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
@@ -1,659 +0,0 @@
//! ATDD Story 4.2 (FR-22/FR-23): the durable Task application entry over a real published catalog
//! and a real PostgreSQL schema.
//!
//! These oracles drive the production application entry (`TaskApplication`) that the mcp-server
//! composition root constructs: a verified machine credential is checked against the published Tool,
//! the Task is accepted durably, a freshly constructed application instance reads the same identity
//! and pins, replay/version conflicts never rebind, foreign callers receive no metadata, and a missing
//! Task store fails closed and reports readiness as not ready. The public `tools/call` wire stays
//! synchronous/unsupported: this suite never calls it.
use super::common;
use std::time::Duration;
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
use crank_core::{
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId,
TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_registry::{
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
};
use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService};
use serde_json::json;
use sqlx::{PgPool, Row};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "report_gen";
const KEY: &str = "retry-key-1";
fn operation(policy: ToolExecutionPolicy) -> crank_registry::RegistryOperation {
let mut operation = common::test_operation("https://api.example.com", TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
operation
}
fn background_policy() -> ToolExecutionPolicy {
ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 60_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
}
}
async fn seed_tool(registry: &PostgresRegistry, policy: ToolExecutionPolicy) {
let operation = operation(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(registry, &operation.id, 1).await;
// A second, explicitly synchronous Tool of the same Agent: the durable entry must refuse it.
let synchronous = common::test_operation("https://api.example.com", "report_sync");
registry
.create_operation(&common::test_workspace_id(), &synchronous, Some("alice"))
.await
.unwrap();
publish(registry, &synchronous.id, 1).await;
let binding = AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
};
let synchronous_binding = AgentOperationBinding {
operation_id: synchronous.id.clone(),
operation_version: 1,
tool_name: synchronous.name.clone(),
tool_title: synchronous.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
};
common::publish_agent_with_bindings(registry, AGENT_SLUG, vec![binding, synchronous_binding])
.await;
}
async fn publish(
registry: &PostgresRegistry,
operation_id: &crank_core::OperationId,
version: u32,
) {
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id,
version,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
}
async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
let mut next = operation(background_policy());
next.version = version;
next.status = crank_core::OperationStatus::Draft;
next.published_at = None;
registry
.create_version(CreateVersionRequest {
workspace_id: &common::test_workspace_id(),
snapshot: &next,
change_note: Some("next publication"),
created_by: Some("alice"),
})
.await
.unwrap();
publish(registry, &next.id, version).await;
// The agent catalog must bind the published Tool version for the new publication to be addressed.
let agent_id = common::test_agent_id(AGENT_SLUG);
let draft = crank_core::AgentVersion {
agent_id: agent_id.clone(),
version: 2,
status: crank_core::AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: Default::default(),
created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
};
registry
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
workspace_id: &common::test_workspace_id(),
agent_id: &agent_id,
version: &draft,
bindings: &[AgentOperationBinding {
operation_id: next.id.clone(),
operation_version: version,
tool_name: next.name.clone(),
tool_title: next.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
})
.await
.unwrap();
registry
.publish_agent(crank_registry::PublishAgentRequest {
workspace_id: &common::test_workspace_id(),
agent_id: &agent_id,
version: 2,
published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
}
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService {
let store = PostgresTaskStore::new(
pool,
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration");
TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto))
}
/// The application entry under the same encrypted-input authority the production process uses.
async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key");
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
TaskApplication::new(
registry.clone(),
Duration::from_millis(0),
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
task_service(registry.pool().clone(), crypto),
)
}
async fn principal(
registry: &PostgresRegistry,
name: &str,
scopes: &[PlatformApiKeyScope],
) -> TaskPrincipal {
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
// The production machine-access path resolves the presented secret against the active platform
// key of the addressed Agent and derives the scope identity from that verified record.
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("an active platform key resolves");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: crank_core::VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
fn unsampled_correlation() -> CorrelationContext {
CorrelationContext::new(
crank_core::RequestId::generate(),
crank_core::TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
)
}
async fn counts(pool: &PgPool) -> (i64, i64) {
let row = sqlx::query(
"select
(select count(*)::bigint from tasks) as tasks,
(select count(*)::bigint from execution_admission_entries where task_id is not null)
as intents",
)
.fetch_one(pool)
.await
.unwrap();
(row.get("tasks"), row.get("intents"))
}
#[tokio::test]
async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-accept",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&correlation,
)
.await
.expect("acceptance over the real publication");
assert_eq!(acceptance.state, TaskState::Queued);
assert!(!acceptance.replayed);
assert_eq!(acceptance.operation_version, 1);
assert_eq!(acceptance.support.request_id, *correlation.request_id());
assert_eq!(acceptance.support.trace_id, *correlation.trace_id());
assert!(!acceptance.support.trace_sampled);
assert_eq!(
uuid::Uuid::parse_str(&acceptance.task_id.to_string())
.unwrap()
.get_version_num(),
7,
"the durable Task id is UUIDv7"
);
// Task, dispatch intent and queue slot committed together.
assert_eq!(counts(registry.pool()).await, (1, 1));
// A freshly constructed application + service reads the same durable identity and pins.
let reader = application(&registry).await;
let record = reader
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
.unwrap();
assert_eq!(record.state, TaskState::Queued);
assert_eq!(record.pins.operation_version, 1);
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
assert_eq!(record.support.request_id, acceptance.support.request_id);
assert_eq!(record.support.traceparent, acceptance.support.traceparent);
assert!(!record.support.trace_sampled);
let outcome = reader
.result(&caller, TOOL_NAME, acceptance.task_id)
.await
.unwrap();
assert!(outcome.result.is_none() && outcome.error.is_none());
// Replay of the same scoped key and input returns the accepted Task instead of a new one.
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(replay.task_id, acceptance.task_id);
assert!(replay.replayed);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn a_new_publication_never_rebinds_an_accepted_task() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-rebind",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry).await;
let accepted = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
let before = app
.status(&caller, TOOL_NAME, accepted.task_id)
.await
.unwrap();
publish_tool_version(&registry, 2).await;
// The same retry keeps the accepted Task and its v1 pin; it never becomes a v2 Task.
let replay = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(replay.task_id, accepted.task_id);
assert_eq!(replay.operation_version, 1);
let after = app
.status(&caller, TOOL_NAME, accepted.task_id)
.await
.unwrap();
assert_eq!(after.pins.operation_version, 1);
assert_eq!(after.support.request_id, before.support.request_id);
assert_eq!(counts(registry.pool()).await, (1, 1));
// An explicit other exact version is a typed conflict, not a rebind.
let explicit = app
.accept_background(
&caller,
TOOL_NAME,
Some(2),
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("a different explicit pin cannot rebind the accepted Task");
assert_eq!(explicit, TaskApplicationError::VersionUnavailable);
// The original explicit pin still resolves to the accepted Task.
let pinned = app
.accept_background(
&caller,
TOOL_NAME,
Some(1),
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
assert_eq!(pinned.task_id, accepted.task_id);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let owner = principal(
&registry,
"task-owner",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry).await;
let accepted = app
.accept_background(
&owner,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap();
// Positive control: the owner reads its Task.
assert!(
app.status(&owner, TOOL_NAME, accepted.task_id)
.await
.is_ok()
);
// A second verified key of the same Agent is another authorization scope: no payload, no metadata.
let foreign_scope =
principal(&registry, "task-other-scope", &[PlatformApiKeyScope::Read]).await;
match app
.status(&foreign_scope, TOOL_NAME, accepted.task_id)
.await
{
Err(TaskApplicationError::NotFound) => {}
other => panic!("foreign scope must be not-found without metadata, got {other:?}"),
}
assert!(matches!(
app.result(&foreign_scope, TOOL_NAME, accepted.task_id)
.await,
Err(TaskApplicationError::NotFound)
));
// A synchronous publication is not accepted as background work at all.
let synchronous_app = application(&registry).await;
let refused = synchronous_app
.accept_background(
&owner,
"report_sync",
None,
"sync-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("a synchronous policy has no durable acceptance");
assert_eq!(refused, TaskApplicationError::NotBackground);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
let registry = common::test_registry().await;
seed_tool(&registry, background_policy()).await;
let app = application(&registry).await;
let admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_1".to_owned(),
};
let accepted = app
.accept_background(
&admin,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect("an authenticated administrator accepts durable work");
assert_eq!(accepted.state, TaskState::Queued);
// Positive control: the same administrator reads its own Task.
assert!(
app.status(&admin, TOOL_NAME, accepted.task_id)
.await
.is_ok()
);
// An administrator is never a wildcard owner: another admin user and the owning Agent of the
// same workspace both see nothing.
let other_admin = TaskPrincipal::Admin {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_2".to_owned(),
};
assert_eq!(
app.status(&other_admin, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
assert_eq!(
app.result(&other_admin, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
let agent = principal(&registry, "task-admin-cross", &[PlatformApiKeyScope::Read]).await;
assert_eq!(
app.status(&agent, TOOL_NAME, accepted.task_id)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
// An admin identity addressed to another workspace cannot resolve the publication at all.
let foreign_workspace = TaskPrincipal::Admin {
workspace_slug: "other-workspace".to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
user_id: "user_admin_1".to_owned(),
};
assert_eq!(
app.accept_background(
&foreign_workspace,
TOOL_NAME,
None,
"foreign-key",
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.unwrap_err(),
TaskApplicationError::NotFound
);
assert_eq!(counts(registry.pool()).await, (1, 1));
}
#[tokio::test]
async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
let database_url = crank_test_support::postgres_schema_url("durable_tasks_outage").await;
let pool = PgPool::connect(&database_url).await.unwrap();
// No migration: the durable Task state does not exist.
let outage_registry = PostgresRegistry::connect(&database_url).await;
assert!(
outage_registry.is_err(),
"an unmigrated registry cannot be used"
);
crank_registry::MigrationAuthority::apply(&pool)
.await
.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
seed_tool(&registry, background_policy()).await;
let caller = principal(
&registry,
"task-outage",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = application(&registry).await;
assert!(app.health().await.is_ok());
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
sqlx::query("drop table tasks cascade")
.execute(&pool)
.await
.unwrap();
assert!(app.health().await.is_err());
let error = app
.accept_background(
&caller,
TOOL_NAME,
None,
KEY,
&json!({"report": "q3"}),
&unsampled_correlation(),
)
.await
.expect_err("no durable Task store, no acceptance");
assert!(matches!(
error,
TaskApplicationError::Task(crank_core::TaskError::Unavailable { .. })
));
// Readiness of the app that wires the same store reports the durable Task dependency as not ready.
let router = common::build_test_app(registry.clone(), Duration::from_millis(0), None);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
});
let response = reqwest::get(format!("http://{address}/ready"))
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE);
let body: serde_json::Value = response.json().await.unwrap();
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
server.abort();
}
/// Commits and resolves exactly one durable attempt of the current lease through the real store.
///
/// Story 4.4 moved the dispatch checkpoint and the attempt resolution into the runtime seam, so a
/// worker-level test double persists exactly one attempt instead of inventing a second retry loop.
pub(crate) async fn commit_attempt(
context: &crank_runtime::TaskAttemptContext,
outcome: crank_core::AttemptOutcome,
) -> crank_runtime::TaskAttemptOutcome {
use crank_core::{AttemptResolution, BeginAttempt, TaskState};
let lease = context.lease();
let begin = BeginAttempt {
task_id: lease.record.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: lease.record.pins.retry.max_attempts > 1,
};
match context.store().checkpoint_dispatch(begin).await {
Ok(attempt) => match context.store().resolve_attempt(&attempt, outcome).await {
Ok(AttemptResolution::Finished(task)) if task.state == TaskState::Completed => {
crank_runtime::TaskAttemptOutcome::Completed
}
Ok(AttemptResolution::Finished(_)) => crank_runtime::TaskAttemptOutcome::Failed {
code: "test_failure".to_owned(),
},
Ok(AttemptResolution::Unknown(_)) => crank_runtime::TaskAttemptOutcome::OutcomeUnknown,
Ok(AttemptResolution::RetryAt { .. }) | Err(_) => {
crank_runtime::TaskAttemptOutcome::Stopped
}
},
Err(_) => crank_runtime::TaskAttemptOutcome::Stopped,
}
}
/// Bounded success outcome used by the worker-level test doubles.
pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::AttemptOutcome {
crank_core::AttemptOutcome::Completed(
crank_core::TaskEnvelope::try_redacted(&value).expect("test result is bounded"),
)
}
mod attempt_guards;
mod attempt_trace;
mod cancel_chain;
mod input_continuation;
mod input_continuation_guards;
mod shutdown_window;
mod task_worker;
@@ -1,220 +0,0 @@
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
//!
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
use super::*;
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
/// does not consume work that is still queued.
///
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
#[tokio::test]
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
// statement keeps the window deterministic without competing for a second connection.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
begin
if new.current_attempt_id is not null and old.current_attempt_id is null then
perform pg_sleep(1.5);
end if;
return new;
end $$;
create trigger delay_task_dispatch_checkpoint before update on tasks
for each row execute function delay_task_dispatch_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint delay trigger");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let mut accepted = Vec::new();
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: scoped_key.repeat(64),
input_fingerprint: scoped_key.repeat(64),
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
accepted.push(store.accept(request).await.expect("PG accepts"));
}
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
let executed = Arc::new(AtomicUsize::new(0));
struct GatedHandler {
entered: Arc<tokio::sync::Semaphore>,
release: Arc<tokio::sync::Semaphore>,
executed: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for GatedHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
self.entered.add_permits(1);
let _ = self.release.acquire().await;
Ok(())
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
// The trigger delays the durable checkpoint; a stop that lands while it is in flight must
// finalize failed(shutdown_before_dispatch) without an outbound send.
let begin = crank_core::BeginAttempt {
task_id: context.lease().record.task_id,
owner: context.lease().owner.clone(),
fence: context.lease().fence,
expected_version: context.lease().record.version,
attempt_id: "attempt-stop-window".to_owned(),
input_fingerprint: context.lease().record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
};
let Ok(attempt) = context.store().checkpoint_dispatch(begin).await else {
return TaskAttemptOutcome::Stopped;
};
if context.is_stopping() {
let _ = context
.store()
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Failed {
error: crank_core::TaskEnvelope::try_redacted(&json!({
"code": "shutdown_before_dispatch",
"phase": "before_dispatch",
}))
.expect("bounded error envelope"),
retryable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await;
return TaskAttemptOutcome::Failed {
code: "shutdown_before_dispatch".to_owned(),
};
}
self.executed.fetch_add(1, Ordering::SeqCst);
commit_attempt(&context, completed_outcome(json!({ "ok": true }))).await
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
entered: revalidate_entered.clone(),
release: revalidate_release.clone(),
executed: executed.clone(),
});
let config = TaskWorkerConfig {
owner: "runtime-worker-stop-window".to_owned(),
batch_limit: 1,
lease_ms: 30_000,
poll_interval: Duration::from_millis(50),
renewal_interval: Duration::from_secs(20),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(20),
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
.await
.expect("worker must claim the first Task and enter revalidation")
.expect("revalidation gate is open")
.forget();
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
revalidate_release.add_permits(1);
tokio::time::sleep(Duration::from_millis(300)).await;
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
// finalize the Task, but no permit exists yet, so no outbound call can start.
let stop_requested = std::time::Instant::now();
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(Duration::from_secs(30), worker_task)
.await
.expect("worker stops inside the 30s budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
assert!(
stop_requested.elapsed() >= Duration::from_millis(500),
"the checkpoint must have still been in flight when the stop arrived"
);
assert_eq!(
executed.load(Ordering::SeqCst),
0,
"a stop between checkpoint and permit must never reach the upstream adapter"
);
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_stop_window".to_owned(),
};
let record = store.read(&access, accepted[0].task_id).await.unwrap();
assert_eq!(record.state, TaskState::Failed);
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
assert!(record.current_attempt_id.is_some());
assert_eq!(
record
.error
.as_ref()
.and_then(|error| error.value().get("code"))
.and_then(|code| code.as_str()),
Some("shutdown_before_dispatch")
);
// Work that was still queued is not consumed by the stopped worker.
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
assert_eq!(queued.state, TaskState::Queued);
let recovered = store
.claim_batch("worker_after_stop", 4, 10_000)
.await
.expect("queued work stays claimable");
assert_eq!(recovered.len(), 1);
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
}
@@ -1,821 +0,0 @@
//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch.
//!
//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit.
use super::*;
/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a
/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan,
/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented.
#[tokio::test]
async fn task_worker_store_lease_recovery_contract() {
let registry = common::test_registry().await;
let store = PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration");
// Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and
// leases durable work, preserving the same admission-backed intent.
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
// This store-level control exercises lease/fence behavior, not the encrypted acceptance path.
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
let leases = store
.claim_batch("worker_1", 4, 10_000)
.await
.expect("claim");
assert_eq!(leases.len(), 1);
assert_eq!(leases[0].record.task_id, accepted.task_id);
let lease = leases[0].clone();
// A stale or competing owner cannot win CAS with the issued fencing token.
assert!(
store
.renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000)
.await
.is_err()
);
assert!(
store
.write_progress(
"worker_1",
lease.record.task_id,
lease.fence + 1,
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
)
.await
.is_err()
);
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
assert!(
store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence + 1,
expected_version: lease.record.version,
attempt_id: "attempt-stale".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.is_err()
);
// The valid owner still commits the dispatch checkpoint before any outbound work.
let committed = store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-1".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.unwrap();
assert_eq!(committed.attempt.attempt_id, "attempt-1");
assert_eq!(
committed.task.current_attempt_id.as_deref(),
Some("attempt-1")
);
}
#[tokio::test]
async fn task_worker_runtime_dispatch_and_recovery_contract() {
use async_trait::async_trait;
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let pool = registry.pool().clone();
let store = Arc::new(
PostgresTaskStore::new(
pool.clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let observed = Arc::new(AtomicUsize::new(0));
// Positive control: PostgreSQL alone accepts the durable Task.
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_runtime".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "r".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
let task_id = accepted.task_id;
struct RuntimeHandler {
observed: Arc<AtomicUsize>,
}
#[async_trait]
impl TaskHandler for RuntimeHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
observed: observed.clone(),
});
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
let config = TaskWorkerConfig {
owner: "runtime-worker-1".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(100),
renewal_interval: Duration::from_millis(1_000),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_runtime".to_owned(),
};
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(100)).await;
if let Ok(record) = store.read(&access, task_id).await {
state = Some(record.state);
if record.state == TaskState::Completed {
break;
}
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state, Some(TaskState::Completed));
assert_eq!(
observed.load(Ordering::SeqCst),
1,
"exactly one outbound attempt"
);
// Terminal state is not duplicated.
let record = store.read(&access, task_id).await.expect("terminal read");
assert_eq!(record.state, TaskState::Completed);
// Bounded shutdown with no fresh work returns promptly.
let idle_config = TaskWorkerConfig {
owner: "runtime-worker-idle".to_owned(),
poll_interval: Duration::from_millis(50),
shutdown_timeout: Duration::from_secs(2),
..TaskWorkerConfig::default()
};
let idle_worker = TaskWorker::new(
store.clone(),
handler_for_no_work(store.clone()),
idle_config,
)
.expect("valid idle worker");
idle_worker.stop();
let started = std::time::Instant::now();
idle_worker.run().await.expect("idle worker stops");
assert!(started.elapsed() <= Duration::from_secs(30));
}
/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call.
#[tokio::test]
async fn task_worker_revalidation_denial_never_dispatches() {
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_denied".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "d".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
struct DeniedHandler {
observed: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for DeniedHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Err(TaskWorkerError::Store(TaskError::invalid(
"publication or credential revoked",
)))
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
}
}
let observed = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
observed: observed.clone(),
});
let config = TaskWorkerConfig {
owner: "runtime-worker-denied".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_denied".to_owned(),
};
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(50)).await;
if let Ok(record) = store.read(&access, accepted.task_id).await {
state = Some(record.state);
if record.state == TaskState::Failed {
break;
}
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state, Some(TaskState::Failed));
assert_eq!(
observed.load(Ordering::SeqCst),
0,
"invalidated work never reaches the upstream"
);
}
fn handler_for_no_work(
_store: std::sync::Arc<PostgresTaskStore>,
) -> std::sync::Arc<dyn crank_runtime::TaskHandler> {
struct NoWorkHandler;
#[async_trait::async_trait]
impl crank_runtime::TaskHandler for NoWorkHandler {
async fn revalidate(
&self,
_lease: &crank_core::TaskLease,
) -> Result<(), crank_runtime::TaskWorkerError> {
Ok(())
}
async fn execute(
&self,
context: crank_runtime::TaskAttemptContext,
) -> crank_runtime::TaskAttemptOutcome {
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
}
}
std::sync::Arc::new(NoWorkHandler)
}
/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a
/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the
/// accepted parent trace is continued on the wire.
#[tokio::test]
async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
use axum::Router;
use axum::http::{HeaderMap, StatusCode};
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let seen_traceparent = Arc::new(Mutex::new(None::<String>));
let app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
let seen = Arc::clone(&seen_traceparent);
move |headers: HeaderMap| {
let calls = Arc::clone(&calls);
let seen = Arc::clone(&seen);
async move {
calls.fetch_add(1, Ordering::SeqCst);
if let Some(value) = headers
.get("traceparent")
.and_then(|value| value.to_str().ok())
{
*seen.lock().unwrap() = Some(value.to_owned());
}
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"task-worker-e2e",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"worker-e2e-key",
&json!({"email": "user@example.com"}),
&correlation,
)
.await
.expect("durable acceptance with executable input");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let config = TaskWorkerConfig {
owner: "runtime-worker-e2e".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("completed"));
assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call");
assert_eq!(
seen_traceparent.lock().unwrap().as_deref(),
Some(correlation.trace_context().traceparent()),
"the durable attempt continues the accepted parent trace"
);
}
/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as
/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically.
#[tokio::test]
async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::Arc;
use std::sync::atomic::Ordering;
let registry = common::test_registry().await;
let upstream_app = Router::new().route(
"/crm/leads",
post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"task-worker-uncertain",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let correlation = unsampled_correlation();
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"worker-uncertain-key",
&json!({"email": "user@example.com"}),
&correlation,
)
.await
.expect("durable acceptance with executable input");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&tasks,
registry.clone(),
runtime,
crypto,
));
let config = TaskWorkerConfig {
owner: "runtime-worker-uncertain".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
let current: Option<String> =
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
state = current;
if state.as_deref() == Some("outcome_unknown") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("outcome_unknown"));
// No automatic retry: another authoritative scan never redispaches the uncertain attempt.
assert!(
store
.claim_batch("worker_after", 4, 10_000)
.await
.unwrap()
.is_empty()
);
}
/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced
/// terminal commit within the 30s budget.
#[tokio::test]
async fn task_worker_graceful_stop_drains_active_attempt() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let registry = common::test_registry().await;
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let request = TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_drain".to_owned(),
platform_api_key_id: None,
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 1,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "s".repeat(64),
input_fingerprint: "d".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&crank_core::TraceContext::parse(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
)
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
};
let accepted = store.accept(request).await.expect("PG accepts");
struct SlowHandler {
started: Arc<AtomicUsize>,
}
#[async_trait::async_trait]
impl TaskHandler for SlowHandler {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.started.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(500)).await;
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
}
}
let started = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
started: started.clone(),
});
let config = TaskWorkerConfig {
owner: "runtime-worker-drain".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(20),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
for _ in 0..100 {
if started.load(Ordering::SeqCst) > 0 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started");
let stop_requested = std::time::Instant::now();
stop_flag.store(true, Ordering::SeqCst);
tokio::time::timeout(Duration::from_secs(30), worker_task)
.await
.expect("worker stops inside the 30s budget")
.expect("worker exits cleanly")
.expect("worker reports no error");
assert!(stop_requested.elapsed() <= Duration::from_secs(30));
let access = TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_drain".to_owned(),
};
let record = store.read(&access, accepted.task_id).await.unwrap();
assert_eq!(record.state, TaskState::Completed);
}
@@ -241,11 +241,6 @@ async fn inventory_projection_sidecar_is_provenance_only_and_names_the_projectio
assert_eq!(
triples,
vec![
(
"server".to_owned(),
"capability".to_owned(),
"logging".to_owned()
),
(
"server".to_owned(),
"capability".to_owned(),
@@ -267,7 +262,7 @@ async fn inventory_projection_sidecar_is_provenance_only_and_names_the_projectio
"tools".to_owned()
)
],
"the advertised server capabilities of the current scope are `logging`, `completions`, `prompts`, `resources` and `tools`"
"the advertised server capabilities of the current scope are `completions`, `prompts`, `resources` and `tools`"
);
// The sidecar is not the payload: byte comparison is against the frozen file only.
@@ -454,18 +454,6 @@ async fn a_request_value_can_never_reach_a_notification() {
"the positive control must reflect the canary in the error message, got {message:?}"
);
// Positive control on delivery: the event is really delivered and its body is exactly the
// closed one for a rejected request, so the canary assertions below cannot pass vacuously.
assert_eq!(
reply.notifications,
vec![expected_notification(
"warning",
"mcp.request.rejected",
"invalid_params"
)],
"a rejected request must deliver exactly one safe warning event"
);
// The delivered notification is exactly the closed safe body: no canary, no argument value.
for notification in &reply.notifications {
assert_eq!(
@@ -736,45 +724,6 @@ async fn the_modern_era_has_no_set_level_method() {
assert!(after.notifications.is_empty());
}
/// AT-3.18-01/03: a method this era does not implement is a pinned client error, so the safe event
/// of an opted-in client describes a client-side rejection, never an internal server failure.
#[tokio::test]
async fn an_unknown_method_is_a_rejection_not_an_internal_failure() {
let endpoint = logging_endpoint().await;
let session = endpoint.legacy_session(AGENT_SLUG, &endpoint.api_key).await;
endpoint.set_level(&session, json!("info")).await;
// `server/discover` is a modern-era method, so the legacy adapter answers it with -32601.
let reply = endpoint
.legacy_post(
AGENT_SLUG,
&endpoint.api_key,
&session,
json!({
"jsonrpc": "2.0",
"id": 71,
"method": "server/discover",
"params": {}
}),
)
.await;
assert_eq!(
reply.error().0,
-32601,
"an unimplemented method is a pinned client error: {}",
reply.response
);
assert_eq!(
reply.notifications,
vec![expected_notification(
"warning",
"mcp.request.rejected",
"invalid_params"
)],
"a pinned client error must not be reported as an internal failure"
);
}
/// AT-3.18-01/03 (independence): a level belongs to one client context, so a second session and a
/// second Agent keep their own (or no) level.
#[tokio::test]
@@ -23,7 +23,7 @@ async fn advertisement_declares_prompts_on_both_revisions() {
let discover = endpoint.modern("server/discover", json!({}), None).await;
assert_eq!(
discover["result"]["capabilities"]["prompts"],
json!({"listChanged": true}),
json!({"listChanged": false}),
"the modern advertisement must declare the prompts capability: {discover}"
);
@@ -50,7 +50,7 @@ async fn advertisement_declares_prompts_on_both_revisions() {
let initialize = json_body(initialize_response).await;
assert_eq!(
initialize["result"]["capabilities"]["prompts"],
json!({"listChanged": true}),
json!({"listChanged": false}),
"the legacy advertisement must declare the prompts capability: {initialize}"
);
}
@@ -1,928 +0,0 @@
//! Story 3.19 acceptance tests (AT-3.19-01..03, FR-19): committed Resource/Prompt catalog changes
//! are announced to a client that asked for the signal of its revision, and the ordinary list
//! methods stay authoritative when a notification is lost.
//!
//! Everything here is the production path: the real loopback MCP endpoint, the real PostgreSQL
//! registry, the real immutable artifact store and real commits through the registry API. No stubbed
//! route, no faked notification, no seeded notification table.
#![allow(dead_code)]
mod scenarios;
use std::{
fs,
path::PathBuf,
sync::atomic::{AtomicU64, Ordering},
time::Duration,
};
use axum::http::header;
use crank_artifacts::ArtifactStore;
use crank_core::{
Agent, AgentId, AgentStatus, AgentVersion, PlatformApiKeyScope, ToolAccessMode, ToolGroup,
ToolSearchSettings, ToolSelectionPolicy,
ids::{PromptId, ResourceId},
prompt::{PromptDefinition, PromptSensitivity},
};
use crank_registry::model::{
AgentPromptBinding, CreatePromptDraftRequest, PublishPromptRequest,
SaveAgentCatalogConfigRequest,
resource::{CreateResourceDraftRequest, PublishResourceVersionRequest},
};
use crank_registry::{
AgentResourceBinding, CreateAgentDraftVersionRequest, CreateAgentRequest, PostgresRegistry,
PublishAgentRequest, PublishRequest,
};
use serde_json::{Value, json};
use time::OffsetDateTime;
use super::common::*;
const MODERN_REVISION: &str = "2026-07-28";
const LEGACY_REVISION: &str = "2025-11-25";
const META_PROTOCOL_VERSION: &str = "io.modelcontextprotocol/protocolVersion";
const META_CLIENT_CAPABILITIES: &str = "io.modelcontextprotocol/clientCapabilities";
const META_CLIENT_INFO: &str = "io.modelcontextprotocol/clientInfo";
const RESOURCE_LIST_CHANGED: &str = "notifications/resources/list_changed";
const PROMPT_LIST_CHANGED: &str = "notifications/prompts/list_changed";
const TOOL_LIST_CHANGED: &str = "notifications/tools/list_changed";
const SUBSCRIPTIONS_ACKNOWLEDGED: &str = "notifications/subscriptions/acknowledged";
const SUBSCRIPTION_ID_KEY: &str = "io.modelcontextprotocol/subscriptionId";
const AGENT_SLUG: &str = "sales-signals";
/// A second published Agent of the same workspace: its catalog must never reach another Agent.
const OTHER_AGENT_SLUG: &str = "sales-other";
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
/// Test-owned artifact root (0700), removed on drop.
struct TestRoot(PathBuf);
impl TestRoot {
fn new() -> Self {
let path = std::env::temp_dir().join(format!(
"crank-mcp-signals-{}-{}",
std::process::id(),
NEXT_ROOT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir_all(&path).expect("artifact root must be creatable");
let mut permissions = fs::metadata(&path).unwrap().permissions();
std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o700);
fs::set_permissions(&path, permissions).expect("artifact root must be private");
Self(path)
}
}
impl Drop for TestRoot {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
/// The published fixture: one Tool, two bound Prompts, one published but initially unbound Resource.
struct SignalsEndpoint {
client: reqwest::Client,
base_url: String,
mcp_url: String,
api_key: String,
registry: PostgresRegistry,
artifact_store: ArtifactStore,
root: TestRoot,
agent_id: AgentId,
operation: crank_core::Operation<crank_schema::Schema, crank_mapping::MappingSet>,
second_operation: crank_core::Operation<crank_schema::Schema, crank_mapping::MappingSet>,
greeting: PromptId,
review: PromptId,
extra_prompt: PromptId,
notice: ResourceId,
}
impl SignalsEndpoint {
fn modern_meta() -> Value {
json!({
META_PROTOCOL_VERSION: MODERN_REVISION,
META_CLIENT_CAPABILITIES: {},
META_CLIENT_INFO: { "name": "crank-signals-atdd", "version": "0.1.0" }
})
}
async fn modern_post(&self, payload: Value) -> reqwest::Response {
self.client
.post(&self.mcp_url)
.header(header::ACCEPT, "application/json, text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {}", self.api_key))
.json(&payload)
.send()
.await
.unwrap()
}
/// One modern request whose result is a JSON body (never a stream).
async fn modern(&self, method: &str, params: Value) -> Value {
let mut params = params;
params["_meta"] = Self::modern_meta();
let response = self
.client
.post(&self.mcp_url)
.header(header::ACCEPT, "application/json")
.header(header::AUTHORIZATION, format!("Bearer {}", self.api_key))
.header("MCP-Protocol-Version", MODERN_REVISION)
.header("Mcp-Method", method)
.json(&json!({"jsonrpc": "2.0", "id": 5, "method": method, "params": params}))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
serde_json::from_str(&body).unwrap_or_else(|error| {
panic!("expected JSON for {method}, {status} {body:?}: {error}")
})
}
/// One modern request that offers both response modes and is expected to answer a JSON body
/// (an admission or validation rejection, never a stream).
async fn modern_offering_stream(&self, method: &str, params: Value) -> Value {
let mut params = params;
params["_meta"] = Self::modern_meta();
let response = self
.client
.post(&self.mcp_url)
.header(header::ACCEPT, "application/json, text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {}", self.api_key))
.header("MCP-Protocol-Version", MODERN_REVISION)
.header("Mcp-Method", method)
.json(&json!({"jsonrpc": "2.0", "id": 9, "method": method, "params": params}))
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
let body = response.text().await.unwrap();
serde_json::from_str(&body).unwrap_or_else(|error| {
panic!("expected a JSON rejection for {method}: {body:?} {error}")
})
}
/// One legacy request inside the initialized session of this Agent.
async fn legacy(&self, session: &str, method: &str, params: Value) -> Value {
let response = self
.client
.post(&self.mcp_url)
.header(header::ACCEPT, "application/json, text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {}", self.api_key))
.header("MCP-Protocol-Version", LEGACY_REVISION)
.header("MCP-Session-Id", session)
.json(&json!({"jsonrpc": "2.0", "id": 6, "method": method, "params": params}))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
serde_json::from_str(&body).unwrap_or_else(|error| {
panic!("expected JSON for {method}, {status} {body:?}: {error}")
})
}
async fn legacy_session(&self) -> String {
initialize_session(&self.client, &self.mcp_url, &self.api_key).await
}
/// The legacy notification stream (`GET`), consumed as an SSE response.
async fn legacy_stream(&self, session: &str) -> SseResponse {
let response = self
.client
.get(&self.mcp_url)
.header(header::ACCEPT, "text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {}", self.api_key))
.header("MCP-Protocol-Version", LEGACY_REVISION)
.header("MCP-Session-Id", session)
.send()
.await
.unwrap();
SseResponse::assert_open(response).await
}
/// Open the modern `subscriptions/listen` stream with an explicit filter.
async fn listen(&self, request_id: Value, filter: Value) -> SseResponse {
let params = json!({ "notifications": filter, "_meta": Self::modern_meta() });
let response = self
.client
.post(&self.mcp_url)
.header(header::ACCEPT, "text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {}", self.api_key))
.header("MCP-Protocol-Version", MODERN_REVISION)
.header("Mcp-Method", "subscriptions/listen")
.json(&json!({
"jsonrpc": "2.0",
"id": request_id,
"method": "subscriptions/listen",
"params": params
}))
.send()
.await
.unwrap();
SseResponse::assert_open(response).await
}
fn agent_version(&self, version: u32) -> AgentVersion {
AgentVersion {
agent_id: self.agent_id.clone(),
version,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: OffsetDateTime::now_utc(),
}
}
async fn open_draft(&self, version: u32) {
self.registry
.create_agent_draft_version(CreateAgentDraftVersionRequest {
workspace_id: &test_workspace_id(),
agent_id: &self.agent_id,
version: &self.agent_version(version),
bindings: &[],
updated_at: &OffsetDateTime::now_utc(),
})
.await
.expect("a new Agent draft version opens");
}
/// Write the exact catalog config of one draft version: Tool bindings, Resource bindings and
/// Prompt bindings in the single accepted transition.
async fn configure_draft(
&self,
version: u32,
tool_bindings: &[crank_core::AgentOperationBinding],
resource_bindings: &[AgentResourceBinding],
prompt_bindings: &[AgentPromptBinding],
) {
self.configure_draft_with_policy(
version,
tool_bindings,
&ToolSelectionPolicy::default(),
resource_bindings,
prompt_bindings,
)
.await;
}
/// The same transition with an explicit Tool selection policy (Story 3.5: the policy is part of
/// the served `tools/list` projection).
async fn configure_draft_with_policy(
&self,
version: u32,
tool_bindings: &[crank_core::AgentOperationBinding],
tool_selection_policy: &ToolSelectionPolicy,
resource_bindings: &[AgentResourceBinding],
prompt_bindings: &[AgentPromptBinding],
) {
self.registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &test_workspace_id(),
agent_id: &self.agent_id,
agent_version: version,
bindings: tool_bindings,
tool_selection_policy,
resource_bindings,
expected_state: None,
},
prompt_bindings,
)
.await
.expect("the exact catalog config of the draft version saves");
}
async fn publish_version(&self, version: u32) {
self.registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &self.agent_id,
version,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("alice"),
expected_state: None,
})
.await
.expect("the draft Agent version publishes");
}
fn tool_bindings(&self) -> Vec<crank_core::AgentOperationBinding> {
vec![binding_for_operation(&self.operation)]
}
fn both_tool_bindings(&self) -> Vec<crank_core::AgentOperationBinding> {
vec![
binding_for_operation(&self.operation),
binding_for_operation(&self.second_operation),
]
}
fn base_prompts(&self) -> Vec<AgentPromptBinding> {
vec![
AgentPromptBinding {
prompt_id: self.greeting.clone(),
prompt_version: 1,
},
AgentPromptBinding {
prompt_id: self.review.clone(),
prompt_version: 1,
},
]
}
fn extended_prompts(&self) -> Vec<AgentPromptBinding> {
let mut prompts = self.base_prompts();
prompts.push(AgentPromptBinding {
prompt_id: self.extra_prompt.clone(),
prompt_version: 1,
});
prompts
}
fn notice_binding(&self) -> AgentResourceBinding {
AgentResourceBinding {
resource_id: self.notice.clone(),
resource_version: 1,
}
}
/// The draft version this Agent would open next, so the commit helpers do not depend on the
/// order the test calls them in.
async fn next_draft_version(&self) -> u32 {
self.registry
.get_agent_summary(&test_workspace_id(), &self.agent_id)
.await
.expect("the Agent summary reads")
.expect("the fixture Agent exists")
.current_draft_version
+ 1
}
/// The committed catalog change of AT-3.19-01: one Resource binding and one Prompt binding join
/// the published Agent snapshot in a single publication.
async fn commit_resource_and_prompt_change(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft(
version,
&self.tool_bindings(),
&[self.notice_binding()],
&self.extended_prompts(),
)
.await;
self.publish_version(version).await;
}
/// A commit that only changes the Tool catalog: the visible Resource/Prompt projection is
/// identical, so it must not produce a Resource or Prompt signal.
async fn commit_tool_only_change(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft(
version,
&self.both_tool_bindings(),
&[self.notice_binding()],
&self.extended_prompts(),
)
.await;
self.publish_version(version).await;
}
/// A commit that only changes the Resource catalog: the visible Prompt projection is identical,
/// so it must produce a Resource signal only.
async fn commit_resource_only_change(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft(
version,
&self.tool_bindings(),
&[self.notice_binding()],
&self.base_prompts(),
)
.await;
self.publish_version(version).await;
}
/// A commit that only changes the Prompt catalog: the visible Resource projection is identical,
/// so it must produce a Prompt signal only.
async fn commit_prompt_only_change(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft(
version,
&self.tool_bindings(),
&[self.notice_binding()],
&self.extended_prompts(),
)
.await;
self.publish_version(version).await;
}
/// A draft-only edit: nothing is published, so no committed catalog state changed.
async fn edit_draft_without_publishing(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft(
version,
&self.both_tool_bindings(),
&[self.notice_binding()],
&self.base_prompts(),
)
.await;
}
/// The reverse Tool-only commit: the second Tool binding leaves the published snapshot while the
/// visible Resource/Prompt projection stays identical.
async fn commit_tool_only_revert(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft(
version,
&self.tool_bindings(),
&[self.notice_binding()],
&self.extended_prompts(),
)
.await;
self.publish_version(version).await;
}
/// The search-mode Tool selection policy of this Agent: one group over the bound Tool.
fn search_policy(&self) -> ToolSelectionPolicy {
ToolSelectionPolicy {
mode: ToolAccessMode::Search,
groups: vec![ToolGroup {
id: "crm".to_owned(),
name: "CRM".to_owned(),
description: "CRM operations".to_owned(),
tool_names: vec![self.operation.name.clone()],
}],
search: ToolSearchSettings::default(),
}
}
/// A commit that changes only the Tool selection policy: same Tool, Resource and Prompt
/// bindings, direct mode -> search mode. The served `tools/list` changes from the bound Tool to
/// `search_tools`/`call_tool`, so this is a visible Tool catalog change.
async fn commit_policy_only_change(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft_with_policy(
version,
&self.tool_bindings(),
&self.search_policy(),
&[],
&self.base_prompts(),
)
.await;
self.publish_version(version).await;
}
/// A search-mode commit that only adds a Tool binding: `tools/list` still serves exactly
/// `search_tools`/`call_tool`, so it must not announce a Tools change.
async fn commit_search_mode_binding_change(&self) {
let version = self.next_draft_version().await;
self.open_draft(version).await;
self.configure_draft_with_policy(
version,
&self.both_tool_bindings(),
&self.search_policy(),
&[],
&self.base_prompts(),
)
.await;
self.publish_version(version).await;
}
/// Publish an independent Agent of the same workspace, bound to the first fixture operation, and
/// return its read-only MCP client key.
async fn publish_other_agent(&self, slug: &str) -> String {
let agent_id = test_agent_id(slug);
let now = OffsetDateTime::now_utc();
self.registry
.create_agent(CreateAgentRequest {
agent: &Agent {
id: agent_id.clone(),
workspace_id: test_workspace_id(),
slug: slug.to_owned(),
display_name: slug.to_owned(),
description: "Story 3.5 isolation fixture".to_owned(),
status: AgentStatus::Draft,
current_draft_version: 1,
latest_published_version: None,
created_at: now,
updated_at: now,
published_at: None,
},
version: &AgentVersion {
agent_id: agent_id.clone(),
version: 1,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: now,
},
bindings: &[binding_for_operation(&self.operation)],
})
.await
.expect("the second Agent is created");
self.registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
version: 1,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("alice"),
expected_state: None,
})
.await
.expect("the second Agent publishes");
create_platform_api_key(
&self.registry,
slug,
"mcp-signals-other",
&[PlatformApiKeyScope::Read],
)
.await
}
/// A committed Tool-only change of the independent Agent: it binds the second fixture Tool.
async fn commit_other_agent_tool_change(&self, slug: &str) {
let agent_id = test_agent_id(slug);
let version = AgentVersion {
agent_id: agent_id.clone(),
version: 2,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: OffsetDateTime::now_utc(),
};
self.registry
.create_agent_draft_version(CreateAgentDraftVersionRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
version: &version,
bindings: &[],
updated_at: &OffsetDateTime::now_utc(),
})
.await
.expect("a second draft version of the other Agent opens");
self.registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
agent_version: 2,
bindings: &self.both_tool_bindings(),
tool_selection_policy: &ToolSelectionPolicy::default(),
resource_bindings: &[],
expected_state: None,
},
&[],
)
.await
.expect("the other Agent's second catalog config saves");
self.registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
version: 2,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("alice"),
expected_state: None,
})
.await
.expect("the other Agent's second version publishes");
}
}
/// A live SSE response, read event by event.
struct SseResponse {
response: reqwest::Response,
buffer: String,
}
impl SseResponse {
async fn assert_open(response: reqwest::Response) -> Self {
assert_eq!(
response.status(),
reqwest::StatusCode::OK,
"the notification stream must be served"
);
let content_type = response
.headers()
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
assert!(
content_type.starts_with("text/event-stream"),
"the notification stream must be an SSE response, got {content_type}"
);
Self {
response,
buffer: String::new(),
}
}
/// The payload of the next `data:` frame, or `None` when nothing arrived within `timeout`.
async fn next_data(&mut self, timeout: Duration) -> Option<Value> {
loop {
if let Some(index) = self.buffer.find("\n\n") {
let block: String = self.buffer.drain(..index + 2).collect();
for line in block.lines() {
if let Some(data) = line.strip_prefix("data:") {
return Some(
serde_json::from_str(data.trim_start())
.unwrap_or_else(|error| panic!("SSE frame must be JSON: {error}")),
);
}
}
continue;
}
let chunk = tokio::time::timeout(timeout, self.response.chunk())
.await
.ok()?
.ok()??;
self.buffer.push_str(&String::from_utf8_lossy(&chunk));
}
}
/// The next frame within the watcher's bound.
async fn expect_frame(&mut self, what: &str) -> Value {
self.next_data(Duration::from_secs(6))
.await
.unwrap_or_else(|| panic!("expected {what} on the notification stream"))
}
/// Exactly one safe negative window: no frame within `wait`.
async fn expect_silence(&mut self, wait: Duration, what: &str) {
if let Some(frame) = self.next_data(wait).await {
panic!("expected no {what}, got {frame}");
}
}
}
async fn publish_prompt(registry: &PostgresRegistry, prompt_id: &PromptId, name: &str) -> PromptId {
let definition = PromptDefinition::parse(&json!({
"name": name,
"description": format!("{name} fixture"),
"arguments": [],
"messages": [{"role": "user", "content": format!("Use {name}.")}]
}))
.expect("the fixture Prompt definition is valid");
let created = registry
.create_prompt_draft(CreatePromptDraftRequest {
workspace_id: test_workspace_id(),
prompt_id: prompt_id.clone(),
sensitivity: PromptSensitivity::Internal,
definition,
created_at: OffsetDateTime::now_utc(),
})
.await
.expect("the Prompt Draft is created");
registry
.publish_prompt_draft(PublishPromptRequest {
workspace_id: test_workspace_id(),
prompt_id: prompt_id.clone(),
expected_updated_at: created.updated_at,
published_at: OffsetDateTime::now_utc(),
})
.await
.expect("the Prompt version publishes");
prompt_id.clone()
}
/// Publish one static Resource through the production 3.6/3.7 registry transition.
async fn publish_resource(
registry: &PostgresRegistry,
store: &ArtifactStore,
resource_id: &ResourceId,
bytes: &[u8],
) {
let artifact = store
.put_registered(bytes)
.expect("the fixture bytes are registered durably");
registry
.create_resource_draft(CreateResourceDraftRequest {
workspace_id: test_workspace_id(),
resource_id: resource_id.clone(),
uri: format!("crank://resources/{}", resource_id.as_str()),
name: resource_id.as_str().to_owned(),
description: "Story 3.19 signal fixture".to_owned(),
kind: "text".to_owned(),
mime_type: "text/plain".to_owned(),
sensitivity: "internal".to_owned(),
artifact,
created_at: OffsetDateTime::now_utc(),
})
.await
.expect("the Resource Draft is created");
let current = registry
.get_resource_draft(test_workspace_id().as_str(), resource_id)
.await
.expect("the Resource Draft reads");
registry
.publish_resource_version(PublishResourceVersionRequest {
workspace_id: test_workspace_id(),
resource_id: resource_id.clone(),
expected_updated_at: current.resource.updated_at,
expected_version: current.version.version,
expected_digest: current.version.digest.clone(),
expected_size_bytes: current.version.size_bytes,
artifact: store
.put_registered(bytes)
.expect("the published bytes are registered durably"),
published_at: OffsetDateTime::now_utc(),
})
.await
.expect("the Resource version publishes");
}
async fn signals_endpoint() -> SignalsEndpoint {
let root = TestRoot::new();
let artifact_store = ArtifactStore::open(root.0.clone()).expect("the artifact store opens");
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_create_lead");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("alice"),
})
.await
.unwrap();
let second_operation = test_operation(&upstream_base_url, "crm_create_second");
registry
.create_operation(&test_workspace_id(), &second_operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &second_operation.id,
version: 1,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("alice"),
})
.await
.unwrap();
let greeting = publish_prompt(&registry, &PromptId::new("prm_a_greeting"), "greeting").await;
let review = publish_prompt(&registry, &PromptId::new("prm_b_review"), "review").await;
let extra_prompt = publish_prompt(&registry, &PromptId::new("prm_c_extra"), "extra").await;
let notice = ResourceId::new("res_a_notice");
publish_resource(&registry, &artifact_store, &notice, b"first notice").await;
let agent_id = AgentId::new(format!("agent_{AGENT_SLUG}"));
let now = OffsetDateTime::now_utc();
registry
.create_agent(CreateAgentRequest {
agent: &Agent {
id: agent_id.clone(),
workspace_id: test_workspace_id(),
slug: AGENT_SLUG.to_owned(),
display_name: AGENT_SLUG.to_owned(),
description: "Story 3.19 signal fixture".to_owned(),
status: AgentStatus::Draft,
current_draft_version: 1,
latest_published_version: None,
created_at: now,
updated_at: now,
published_at: None,
},
version: &AgentVersion {
agent_id: agent_id.clone(),
version: 1,
status: AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: ToolSelectionPolicy::default(),
created_at: now,
},
bindings: &[binding_for_operation(&operation)],
})
.await
.expect("the draft Agent is created");
registry
.save_agent_catalog_config_editable_with_prompts(
SaveAgentCatalogConfigRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
agent_version: 1,
bindings: &[binding_for_operation(&operation)],
tool_selection_policy: &ToolSelectionPolicy::default(),
resource_bindings: &[],
expected_state: None,
},
&[
AgentPromptBinding {
prompt_id: greeting.clone(),
prompt_version: 1,
},
AgentPromptBinding {
prompt_id: review.clone(),
prompt_version: 1,
},
],
)
.await
.expect("the exact bound Prompt versions are accepted");
registry
.publish_agent(PublishAgentRequest {
workspace_id: &test_workspace_id(),
agent_id: &agent_id,
version: 1,
published_at: &OffsetDateTime::now_utc(),
published_by: Some("alice"),
expected_state: None,
})
.await
.expect("the bound Agent publishes");
let api_key = create_platform_api_key(
&registry,
AGENT_SLUG,
"mcp-signals",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let base_url = spawn_mcp_server(build_test_app_with_artifacts(
registry.clone(),
Duration::from_millis(0),
None,
artifact_store.clone(),
))
.await;
SignalsEndpoint {
client: reqwest::Client::new(),
mcp_url: agent_mcp_url(&base_url, AGENT_SLUG),
base_url,
api_key,
registry,
artifact_store,
root,
agent_id,
operation,
second_operation,
greeting,
review,
extra_prompt,
notice,
}
}
fn prompt_names(list: &Value) -> Vec<String> {
list["result"]["prompts"]
.as_array()
.unwrap_or_else(|| panic!("prompts/list must return a page: {list}"))
.iter()
.map(|prompt| prompt["name"].as_str().unwrap().to_owned())
.collect()
}
fn resource_uris(list: &Value) -> Vec<String> {
list["result"]["resources"]
.as_array()
.unwrap_or_else(|| panic!("resources/list must return a page: {list}"))
.iter()
.map(|resource| resource["uri"].as_str().unwrap().to_owned())
.collect()
}
fn tool_names(list: &Value) -> Vec<String> {
let mut names: Vec<String> = list["result"]["tools"]
.as_array()
.unwrap_or_else(|| panic!("tools/list must return a page: {list}"))
.iter()
.map(|tool| tool["name"].as_str().unwrap().to_owned())
.collect();
names.sort();
names
}
/// Write a stand file that only the current user may read (credential files live here).
fn write_private(path: &PathBuf, contents: &str) {
use std::io::Write as _;
use std::os::unix::fs::OpenOptionsExt as _;
let mut file = fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.mode(0o600)
.open(path)
.unwrap_or_else(|error| panic!("stand file {path:?} must be writable: {error}"));
file.write_all(contents.as_bytes())
.expect("the stand file must be written");
}
@@ -1,791 +0,0 @@
//! Story 3.19 acceptance scenarios (AT-3.19-01..03, FR-19) on the harness in the parent module.
//!
//! Moved out of `mcp_signals.rs` verbatim (code-health split, 1000-line limit); no scenario,
//! assertion or fixture changed.
use super::*;
/// Loopback disposable stand for the AC-3 representative-client evidence. This is deliberately not
/// part of the automated ATDD: it publishes the fixture catalog, serves the real MCP endpoint on
/// `127.0.0.1`, and then performs one commit per command file so an external MCP client can observe
/// the signals on its own stream.
///
/// Run it explicitly (the credential file is `0600` and never printed):
///
/// ```text
/// MCP_SIGNALS_STAND_DIR=/tmp/crank-signals-stand \
/// cargo test -p mcp-server --test integration mcp_signals::scenarios::representative_client_stand \
/// -- --ignored --exact --nocapture
/// ```
///
/// Commands (create the file, wait for `<command>.done`): `commit-both`, `commit-prompt`,
/// `commit-resource`, `commit-tool`, `draft`, `stop`.
#[tokio::test]
#[ignore = "manual loopback stand for the representative MCP Inspector client evidence"]
async fn representative_client_stand() {
let directory = PathBuf::from(
std::env::var("MCP_SIGNALS_STAND_DIR")
.expect("MCP_SIGNALS_STAND_DIR names the disposable stand directory"),
);
fs::create_dir_all(&directory).expect("the stand directory is creatable");
let endpoint = signals_endpoint().await;
write_private(
&directory.join("stand.json"),
&json!({ "url": endpoint.mcp_url, "token": endpoint.api_key }).to_string(),
);
println!("stand ready: {}", directory.join("stand.json").display());
let deadline = tokio::time::Instant::now() + Duration::from_secs(900);
while tokio::time::Instant::now() < deadline {
for command in [
"commit-both",
"commit-prompt",
"commit-resource",
"commit-tool",
"draft",
"stop",
] {
let marker = directory.join(command);
if !marker.exists() {
continue;
}
fs::remove_file(&marker).expect("the command file is consumed once");
if command == "stop" {
println!("stand stopped: {command}");
return;
}
match command {
"commit-both" => endpoint.commit_resource_and_prompt_change().await,
"commit-prompt" => endpoint.commit_prompt_only_change().await,
"commit-resource" => endpoint.commit_resource_only_change().await,
"commit-tool" => endpoint.commit_tool_only_change().await,
_ => endpoint.edit_draft_without_publishing().await,
}
write_private(&directory.join(format!("{command}.done")), "done");
println!("stand committed: {command}");
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
/// AT-3.19-01 (legacy): an open notification stream receives the Resource and Prompt list-change
/// notifications only after the catalog commit, and `resources/list` / `prompts/list` serve the new
/// state afterwards.
#[tokio::test]
async fn legacy_stream_announces_a_committed_catalog_change_after_the_commit() {
let endpoint = signals_endpoint().await;
let session = endpoint.legacy_session().await;
let mut stream = endpoint.legacy_stream(&session).await;
// The watcher takes its baseline before it can report anything; let it settle before the commit.
tokio::time::sleep(Duration::from_millis(1_500)).await;
stream
.expect_silence(Duration::from_millis(1_200), "signal before any commit")
.await;
// A draft-only edit is not a commit: nothing is announced.
endpoint.edit_draft_without_publishing().await;
stream
.expect_silence(Duration::from_millis(2_500), "signal for a draft-only edit")
.await;
endpoint.commit_resource_and_prompt_change().await;
let mut methods = Vec::new();
for _ in 0..2 {
let frame = stream
.expect_frame("the committed list-change signals")
.await;
assert_eq!(frame["jsonrpc"], json!("2.0"));
assert!(
frame.get("params").is_none(),
"a legacy list-change notification carries no parameters: {frame}"
);
methods.push(frame["method"].as_str().unwrap().to_owned());
}
methods.sort();
assert_eq!(
methods,
vec![
PROMPT_LIST_CHANGED.to_owned(),
RESOURCE_LIST_CHANGED.to_owned()
],
"one signal per changed catalog type"
);
// The ordinary list methods are authoritative for the same committed state.
let prompts = endpoint.legacy(&session, "prompts/list", json!({})).await;
assert!(
prompt_names(&prompts).contains(&"extra".to_owned()),
"prompts/list must serve the newly bound Prompt: {prompts}"
);
let resources = endpoint.legacy(&session, "resources/list", json!({})).await;
assert!(
resource_uris(&resources)
.iter()
.any(|uri| uri == &format!("crank://resources/{}", endpoint.notice.as_str())),
"resources/list must serve the newly bound Resource: {resources}"
);
// A Tool-only commit keeps the Resource/Prompt projection: only a Tools frame follows (Story 3.5).
endpoint.commit_tool_only_change().await;
let frame = stream
.expect_frame("the committed Tools list-change signal")
.await;
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
assert!(
frame.get("params").is_none(),
"a legacy list-change notification carries no parameters: {frame}"
);
stream
.expect_silence(
Duration::from_millis(2_500),
"a Resource or Prompt signal for a Tool-only commit",
)
.await;
}
/// AT-3.19-01/03 (modern): `subscriptions/listen` acknowledges the honoured filter with the request
/// id, then delivers coalesced list-change notifications carrying the subscription id.
#[tokio::test]
async fn modern_listen_stream_acknowledges_then_announces_coalesced_changes() {
let endpoint = signals_endpoint().await;
let mut stream = endpoint
.listen(
json!(41),
json!({
"resourcesListChanged": true,
"promptsListChanged": true,
"toolsListChanged": true,
"resourceSubscriptions": ["crank://resources/res_a_notice"]
}),
)
.await;
let acknowledgement = stream.expect_frame("the acknowledgment").await;
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
assert_eq!(
acknowledgement["params"]["_meta"][SUBSCRIPTION_ID_KEY],
json!(41)
);
assert_eq!(
acknowledgement["params"]["notifications"],
json!({
"promptsListChanged": true,
"resourcesListChanged": true,
"toolsListChanged": true
}),
"only the implemented list-change types are acknowledged: {acknowledgement}"
);
tokio::time::sleep(Duration::from_millis(1_500)).await;
// Two commits in quick succession coalesce into one signal per changed type.
endpoint.commit_resource_and_prompt_change().await;
endpoint.commit_tool_only_change().await;
let mut methods = Vec::new();
for _ in 0..3 {
let frame = stream
.expect_frame("the coalesced list-change signal")
.await;
assert_eq!(frame["params"]["_meta"][SUBSCRIPTION_ID_KEY], json!(41));
methods.push(frame["method"].as_str().unwrap().to_owned());
}
methods.sort();
assert_eq!(
methods,
vec![
PROMPT_LIST_CHANGED.to_owned(),
RESOURCE_LIST_CHANGED.to_owned(),
TOOL_LIST_CHANGED.to_owned()
]
);
stream
.expect_silence(
Duration::from_millis(2_500),
"a duplicate or unrequested signal",
)
.await;
// A client that missed every notification still recovers the authoritative catalog.
let prompts = endpoint.modern("prompts/list", json!({})).await;
assert!(
prompt_names(&prompts).contains(&"extra".to_owned()),
"{prompts}"
);
}
/// AT-3.19-01/03 (modern): the acknowledged filter is the delivery contract. A prompts-only stream
/// stays silent for a Resource-only commit, delivers exactly one Prompt frame with the subscription
/// id for a Prompt commit, and never delivers an unrequested type.
#[tokio::test]
async fn modern_listen_delivers_only_the_negotiated_signal_types() {
let endpoint = signals_endpoint().await;
let mut stream = endpoint
.listen(
json!("prompts-only"),
json!({
"promptsListChanged": true,
"toolsListChanged": true,
"resourceSubscriptions": ["crank://resources/res_a_notice"]
}),
)
.await;
let acknowledgement = stream.expect_frame("the acknowledgment").await;
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
assert_eq!(
acknowledgement["params"]["_meta"][SUBSCRIPTION_ID_KEY],
json!("prompts-only")
);
assert_eq!(
acknowledgement["params"]["notifications"],
json!({ "promptsListChanged": true, "toolsListChanged": true }),
"only the implemented and requested types are acknowledged: {acknowledgement}"
);
// The watcher takes its baseline before it can report anything.
tokio::time::sleep(Duration::from_millis(1_500)).await;
// A Resource-only commit is outside the negotiated filter: a server that ignored the filter would
// emit a Resource frame here.
endpoint.commit_resource_only_change().await;
stream
.expect_silence(
Duration::from_millis(2_500),
"signal for a Resource-only commit on a prompts-only stream",
)
.await;
// A Prompt-only commit is inside it: exactly one frame, correlated by the subscription id.
endpoint.commit_prompt_only_change().await;
let frame = stream
.expect_frame("the negotiated Prompt list-change signal")
.await;
assert_eq!(frame["method"], PROMPT_LIST_CHANGED);
assert_eq!(
frame["params"]["_meta"][SUBSCRIPTION_ID_KEY],
json!("prompts-only")
);
// No Tool commit happened, so no Tools frame follows; `resourceSubscriptions` was not honoured.
stream
.expect_silence(
Duration::from_millis(2_500),
"a duplicate, Tools or URI-content signal",
)
.await;
}
/// AT-3.19-03: the unsupported URI-content subscription surface is never acknowledged, and the
/// legacy revision has no `subscriptions/listen` request at all.
#[tokio::test]
async fn unsupported_subscription_surfaces_are_refused_rather_than_faked() {
let endpoint = signals_endpoint().await;
for method in ["resources/subscribe", "resources/unsubscribe"] {
let reply = endpoint
.modern(method, json!({ "uri": "crank://resources/res_a_notice" }))
.await;
assert_eq!(
reply["error"]["code"],
json!(-32601),
"{method} must stay unimplemented: {reply}"
);
}
// The legacy revision opens its notification stream with GET only.
let session = endpoint.legacy_session().await;
let legacy_listen = endpoint
.legacy(
&session,
"subscriptions/listen",
json!({ "notifications": {} }),
)
.await;
assert_eq!(
legacy_listen["error"]["code"],
json!(-32601),
"subscriptions/listen is a modern-only request: {legacy_listen}"
);
// A malformed filter is a bounded invalid-params answer with no echoed value.
let malformed = endpoint
.modern_offering_stream(
"subscriptions/listen",
json!({ "notifications": ["promptsListChanged"] }),
)
.await;
assert_eq!(malformed["error"]["code"], json!(-32602));
assert_eq!(
malformed["error"]["message"],
json!("invalid subscriptions/listen notification filter")
);
// Without an SSE-capable client the stream cannot be served at all.
let without_sse = endpoint
.client
.post(&endpoint.mcp_url)
.header(header::ACCEPT, "application/json")
.header(
header::AUTHORIZATION,
format!("Bearer {}", endpoint.api_key),
)
.header("MCP-Protocol-Version", MODERN_REVISION)
.header("Mcp-Method", "subscriptions/listen")
.json(&json!({
"jsonrpc": "2.0",
"id": 42,
"method": "subscriptions/listen",
"params": {
"notifications": { "promptsListChanged": true },
"_meta": SignalsEndpoint::modern_meta()
}
}))
.send()
.await
.unwrap();
assert_eq!(without_sse.status(), reqwest::StatusCode::NOT_ACCEPTABLE);
}
/// AT-3.19-02: a missed notification never loses committed state — a legacy client re-initializes
/// (the retired session is a safe `404`) and reads the authoritative catalog, while the modern
/// stateless client recovers through `server/discover` and the list methods.
#[tokio::test]
async fn a_lost_notification_is_recovered_through_the_authoritative_methods() {
let endpoint = signals_endpoint().await;
// Modern recovery: the commit happens with no stream open at all.
endpoint.commit_resource_and_prompt_change().await;
let discover = endpoint.modern("server/discover", json!({})).await;
assert_eq!(
discover["result"]["capabilities"]["prompts"]["listChanged"],
json!(true)
);
assert_eq!(
discover["result"]["capabilities"]["resources"]["listChanged"],
json!(true)
);
assert_eq!(
discover["result"]["capabilities"]["tools"]["listChanged"],
json!(true),
"the Tools list-change capability is advertised: {discover}"
);
let resources = endpoint.modern("resources/list", json!({})).await;
assert!(
resource_uris(&resources)
.iter()
.any(|uri| uri == &format!("crank://resources/{}", endpoint.notice.as_str())),
"the committed Resource is served to a client that missed the signal: {resources}"
);
// Legacy recovery: the negotiated session can be dropped, and the revision-correct answer is a
// safe `404` that makes the client re-initialize instead of resuming a stale stream.
let session = endpoint.legacy_session().await;
let mut stream = endpoint.legacy_stream(&session).await;
tokio::time::sleep(Duration::from_millis(1_500)).await;
let deleted = endpoint
.client
.delete(&endpoint.mcp_url)
.header(
header::AUTHORIZATION,
format!("Bearer {}", endpoint.api_key),
)
.header("MCP-Protocol-Version", LEGACY_REVISION)
.header("MCP-Session-Id", &session)
.send()
.await
.unwrap();
assert_eq!(deleted.status(), reqwest::StatusCode::NO_CONTENT);
stream
.expect_silence(
Duration::from_millis(2_500),
"a signal on the retired session",
)
.await;
let stale_get = endpoint
.client
.get(&endpoint.mcp_url)
.header(header::ACCEPT, "text/event-stream")
.header(
header::AUTHORIZATION,
format!("Bearer {}", endpoint.api_key),
)
.header("MCP-Protocol-Version", LEGACY_REVISION)
.header("MCP-Session-Id", &session)
.send()
.await
.unwrap();
assert_eq!(
stale_get.status(),
reqwest::StatusCode::NOT_FOUND,
"a retired session is never resumed"
);
let fresh = endpoint.legacy_session().await;
let prompts = endpoint.legacy(&fresh, "prompts/list", json!({})).await;
assert!(
prompt_names(&prompts).contains(&"extra".to_owned()),
"re-initialization serves the authoritative catalog: {prompts}"
);
}
// ------------------------------------------------------------------------------------------------
// Story 3.5 (FR-14): Tool catalog change signals. A client that negotiated the Tools list-changed
// notification learns about a *committed* Tool catalog change, and `tools/list` stays the
// authoritative source for a client without the notification or after a reconnect.
// ------------------------------------------------------------------------------------------------
/// AT-3.5-01/02 (modern): the Tools signal is delivered only to the stream that negotiated it, once
/// per committed change, and `tools/list` serves the same committed snapshot.
#[tokio::test]
async fn modern_tools_list_changed_is_delivered_only_to_a_client_that_asks_for_it() {
let endpoint = signals_endpoint().await;
let mut tools = endpoint
.listen(json!(51), json!({ "toolsListChanged": true }))
.await;
let mut prompts = endpoint
.listen(json!(52), json!({ "promptsListChanged": true }))
.await;
let acknowledgement = tools.expect_frame("the Tools acknowledgment").await;
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
assert_eq!(
acknowledgement["params"]["notifications"],
json!({ "toolsListChanged": true }),
"the Tools list-change type must be honoured and acknowledged: {acknowledgement}"
);
let other = prompts.expect_frame("the Prompts acknowledgment").await;
assert_eq!(
other["params"]["notifications"],
json!({ "promptsListChanged": true }),
"a stream that did not ask for Tools is acknowledged without it: {other}"
);
// The watchers take their baseline before they can report anything.
tokio::time::sleep(Duration::from_millis(1_500)).await;
// A draft-only edit is not a commit: nothing is announced.
endpoint.edit_draft_without_publishing().await;
tools
.expect_silence(
Duration::from_millis(2_500),
"Tools signal for a draft-only edit",
)
.await;
// A committed Resource/Prompt change is not a Tools change: the Tools stream stays silent while
// the Prompts stream receives exactly its own type.
endpoint.commit_resource_and_prompt_change().await;
let prompt_frame = prompts
.expect_frame("the committed Prompt list-change signal")
.await;
assert_eq!(prompt_frame["method"], PROMPT_LIST_CHANGED);
tools
.expect_silence(
Duration::from_millis(2_500),
"Tools signal for a Resource/Prompt commit",
)
.await;
// One committed Tool catalog change is announced exactly once.
endpoint.commit_tool_only_change().await;
let frame = tools
.expect_frame("the committed Tools list-change signal")
.await;
assert_eq!(frame["jsonrpc"], json!("2.0"));
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
assert_eq!(frame["params"]["_meta"][SUBSCRIPTION_ID_KEY], json!(51));
tools
.expect_silence(Duration::from_millis(2_500), "a repeated Tools signal")
.await;
// The prompts-only stream was never told about Tools, even though the commit happened after it
// opened.
prompts
.expect_silence(Duration::from_millis(1_200), "unrequested Tools signal")
.await;
// `tools/list` is authoritative for the same committed state, notification or not.
let list = endpoint.modern("tools/list", json!({})).await;
assert_eq!(
tool_names(&list),
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()]
);
}
/// AT-3.5-01/02 (legacy): the legacy `GET` stream carries the Tools signal after the commit; a
/// Resource/Prompt commit produces no Tools frame; a committed change of another Agent is never
/// disclosed on this stream, and `tools/list` stays authoritative.
#[tokio::test]
async fn legacy_stream_announces_a_committed_tools_change_and_never_another_agents() {
let endpoint = signals_endpoint().await;
let other_key = endpoint.publish_other_agent(OTHER_AGENT_SLUG).await;
let session = endpoint.legacy_session().await;
let mut stream = endpoint.legacy_stream(&session).await;
tokio::time::sleep(Duration::from_millis(1_500)).await;
stream
.expect_silence(Duration::from_millis(1_200), "signal before any commit")
.await;
// A committed Resource/Prompt change keeps the Tool binding set: no Tools frame.
endpoint.commit_resource_and_prompt_change().await;
let mut methods = Vec::new();
for _ in 0..2 {
let frame = stream.expect_frame("the committed catalog signals").await;
assert!(
frame.get("params").is_none(),
"a legacy list-change notification carries no parameters: {frame}"
);
methods.push(frame["method"].as_str().unwrap().to_owned());
}
methods.sort();
assert_eq!(
methods,
vec![
PROMPT_LIST_CHANGED.to_owned(),
RESOURCE_LIST_CHANGED.to_owned()
],
"a Resource/Prompt commit must not announce Tools"
);
// A committed Tool change of another Agent of the same workspace never reaches this stream or
// this Agent's `tools/list`.
endpoint
.commit_other_agent_tool_change(OTHER_AGENT_SLUG)
.await;
stream
.expect_silence(
Duration::from_millis(2_500),
"Tools signal for another Agent's commit",
)
.await;
let isolated = endpoint.legacy(&session, "tools/list", json!({})).await;
assert_eq!(
tool_names(&isolated),
vec!["crm_create_lead".to_owned()],
"another Agent's commit must not change this Agent's catalog: {isolated}"
);
// The commit really happened: the other Agent serves its own two-Tool catalog.
let other_url = agent_mcp_url(&endpoint.base_url, OTHER_AGENT_SLUG);
let other_session = initialize_session(&endpoint.client, &other_url, &other_key).await;
let other_list = endpoint
.client
.post(&other_url)
.header(header::ACCEPT, "application/json, text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {other_key}"))
.header("MCP-Protocol-Version", LEGACY_REVISION)
.header("MCP-Session-Id", &other_session)
.json(&json!({
"jsonrpc": "2.0",
"id": 7,
"method": "tools/list",
"params": {}
}))
.send()
.await
.unwrap();
let other_list: Value = serde_json::from_str(&other_list.text().await.unwrap()).unwrap();
assert_eq!(
tool_names(&other_list),
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
"the other Agent's committed catalog changed: {other_list}"
);
// This Agent's committed Tool change is announced after the commit, exactly once.
endpoint.commit_tool_only_change().await;
let frame = stream
.expect_frame("the committed Tools list-change signal")
.await;
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
assert!(
frame.get("params").is_none(),
"a legacy list-change notification carries no parameters: {frame}"
);
stream
.expect_silence(
Duration::from_millis(2_500),
"a repeated or unrequested signal",
)
.await;
let list = endpoint.legacy(&session, "tools/list", json!({})).await;
assert_eq!(
tool_names(&list),
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
"tools/list must serve the committed catalog without the notification: {list}"
);
}
/// AT-3.5-04: the Tools signal follows the same projection `tools/list` serves, so a committed
/// publication that only changes the Tool selection policy is announced on both revisions, while a
/// committed binding change that leaves `tools/list` unchanged is silent.
#[tokio::test]
async fn a_policy_only_commit_is_announced_on_both_revisions_and_an_invisible_change_is_silent() {
let endpoint = signals_endpoint().await;
let direct_projection = vec!["crm_create_lead".to_owned()];
// `tool_names` sorts the served page.
let search_projection = vec!["call_tool".to_owned(), "search_tools".to_owned()];
// Both revisions negotiate the Tools signal and take their baseline before anything can arrive.
let mut modern = endpoint
.listen(json!(61), json!({ "toolsListChanged": true }))
.await;
let acknowledgement = modern.expect_frame("the Tools acknowledgment").await;
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
assert_eq!(
acknowledgement["params"]["notifications"],
json!({ "toolsListChanged": true })
);
let session = endpoint.legacy_session().await;
let mut legacy = endpoint.legacy_stream(&session).await;
tokio::time::sleep(Duration::from_millis(1_500)).await;
// The served projection before the commit is the direct-mode binding itself.
assert_eq!(
tool_names(&endpoint.modern("tools/list", json!({})).await),
direct_projection
);
// A policy-only commit: identical Tool/Resource/Prompt bindings, direct -> search mode. The
// served list changes, so each negotiated stream receives exactly one Tools frame.
endpoint.commit_policy_only_change().await;
let modern_frame = modern
.expect_frame("the committed Tools list-change signal (modern)")
.await;
assert_eq!(modern_frame["method"], TOOL_LIST_CHANGED);
assert_eq!(
modern_frame["params"]["_meta"][SUBSCRIPTION_ID_KEY],
json!(61)
);
let legacy_frame = legacy
.expect_frame("the committed Tools list-change signal (legacy)")
.await;
assert_eq!(legacy_frame["method"], TOOL_LIST_CHANGED);
assert!(
legacy_frame.get("params").is_none(),
"a legacy list-change notification carries no parameters: {legacy_frame}"
);
modern
.expect_silence(
Duration::from_millis(2_500),
"a repeated modern Tools signal",
)
.await;
legacy
.expect_silence(
Duration::from_millis(2_500),
"a repeated legacy Tools signal",
)
.await;
// `tools/list` is authoritative for the new projection on both revisions.
assert_eq!(
tool_names(&endpoint.modern("tools/list", json!({})).await),
search_projection
);
assert_eq!(
tool_names(&endpoint.legacy(&session, "tools/list", json!({})).await),
search_projection
);
// The inverse direction: in search mode a binding-only commit does not change `tools/list`, so
// neither stream may claim a visible Tool change.
endpoint.commit_search_mode_binding_change().await;
modern
.expect_silence(
Duration::from_millis(3_500),
"a Tools signal for an unchanged modern tools/list",
)
.await;
legacy
.expect_silence(
Duration::from_millis(2_500),
"a Tools signal for an unchanged legacy tools/list",
)
.await;
assert_eq!(
tool_names(&endpoint.modern("tools/list", json!({})).await),
search_projection
);
}
/// AT-3.5-02/03: a client that missed the notification recovers Tool discovery from the authoritative
/// state — the modern stateless client after a reconnect, and the legacy client after re-initializing
/// a retired session — and a reconnected stream never replays the missed change.
#[tokio::test]
async fn a_reconnected_client_recovers_tools_discovery_from_the_authoritative_state() {
let endpoint = signals_endpoint().await;
// Modern: the commit happens while no stream is open at all, then a client reconnects.
endpoint.commit_tool_only_change().await;
let mut reconnected = endpoint
.listen(
json!("after-reconnect"),
json!({ "toolsListChanged": true }),
)
.await;
let acknowledgement = reconnected.expect_frame("the Tools acknowledgment").await;
assert_eq!(
acknowledgement["params"]["notifications"],
json!({ "toolsListChanged": true })
);
// The baseline of a new stream is the committed state: the missed change is not replayed.
reconnected
.expect_silence(
Duration::from_millis(2_500),
"a replayed Tools signal after reconnect",
)
.await;
let list = endpoint.modern("tools/list", json!({})).await;
assert_eq!(
tool_names(&list),
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
"a client without the notification recovers the committed catalog: {list}"
);
// Legacy: one more committed Tool change is announced on the live stream, then the session is
// retired and the client re-initializes.
let session = endpoint.legacy_session().await;
let mut stale = endpoint.legacy_stream(&session).await;
tokio::time::sleep(Duration::from_millis(1_500)).await;
endpoint.commit_tool_only_revert().await;
let frame = stale
.expect_frame("the committed Tools list-change signal")
.await;
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
let deleted = endpoint
.client
.delete(&endpoint.mcp_url)
.header(
header::AUTHORIZATION,
format!("Bearer {}", endpoint.api_key),
)
.header("MCP-Protocol-Version", LEGACY_REVISION)
.header("MCP-Session-Id", &session)
.send()
.await
.unwrap();
assert_eq!(deleted.status(), reqwest::StatusCode::NO_CONTENT);
let fresh = endpoint.legacy_session().await;
let list = endpoint.legacy(&fresh, "tools/list", json!({})).await;
assert_eq!(
tool_names(&list),
vec!["crm_create_lead".to_owned()],
"re-initialization serves the committed catalog: {list}"
);
// A stream reopened after the reconnect starts from the committed baseline, not from a replay.
let mut reopened = endpoint.legacy_stream(&fresh).await;
reopened
.expect_silence(
Duration::from_millis(2_500),
"a replayed Tools signal on a reconnected stream",
)
.await;
}
@@ -114,8 +114,8 @@ fn now_rfc3339() -> String {
.unwrap_or_else(|error| format!("unformatted-utc ({error})"))
}
/// A single timed `resources/list` call, built directly on `reqwest` so transport/timeout/HTTP
/// failures are recorded as outcomes instead of aborting the run.
/// A single timed `resources/list`, built directly on `reqwest` so transport/timeout/HTTP failures are
/// recorded as outcomes instead of aborting the run.
async fn timed_list(
endpoint: &ResourceEndpoint,
revision: &'static str,
@@ -1,56 +1,18 @@
use super::*;
use std::{
env,
io::{Read, Write},
net::{TcpListener, TcpStream, ToSocketAddrs},
process::Command,
thread,
time::{Duration, Instant},
};
use std::process::Command;
pub(super) struct TestValkey(String);
impl TestValkey {
pub(super) fn start() -> (Self, u16) {
let in_docker_job = if std::path::Path::new("/.dockerenv").exists() {
let hostname = env::var("HOSTNAME").unwrap_or_default();
assert!(
matches!(hostname.len(), 12 | 64)
&& hostname.bytes().all(|byte| byte.is_ascii_hexdigit()),
"Docker job hostname must be a full 12- or 64-character container ID"
);
Some(hostname)
} else {
None
};
let port = if in_docker_job.is_some() {
let listener = TcpListener::bind(("127.0.0.1", 0))
.expect("test-owned Valkey loopback port must be available");
let port = listener.local_addr().unwrap().port();
drop(listener);
Some(port)
} else {
None
};
let mut docker = Command::new("docker");
docker.args([
"run",
"--detach",
"--label",
"crank.test.fixture=static-resources-valkey",
]);
if let (Some(hostname), Some(port)) = (&in_docker_job, port) {
docker
.arg("--network")
.arg(format!("container:{hostname}"))
.arg("valkey/valkey:8-alpine")
.args(["valkey-server", "--port", &port.to_string()]);
} else {
docker.args(["--publish", "127.0.0.1::6379", "valkey/valkey:8-alpine"]);
}
let output = docker
let output = Command::new("docker")
.args([
"run",
"--detach",
"--publish",
"127.0.0.1::6379",
"valkey/valkey:8-alpine",
])
.output()
.expect("Docker CLI must start the test-owned Valkey container");
assert!(
@@ -63,29 +25,22 @@ impl TestValkey {
.trim()
.to_owned();
let valkey = Self(id);
let port = match port {
Some(port) => port,
None => {
let port_output = Command::new("docker")
.args(["port", &valkey.0, "6379/tcp"])
.output()
.expect("Docker CLI must report the test-owned port");
assert!(
port_output.status.success(),
"test-owned Valkey port must be available"
);
let address =
String::from_utf8(port_output.stdout).expect("Docker port must be UTF-8");
address
.trim()
.rsplit(':')
.next()
.unwrap()
.parse()
.expect("Docker mapped port must be numeric")
}
};
wait_for_valkey(port);
let port_output = Command::new("docker")
.args(["port", &valkey.0, "6379/tcp"])
.output()
.expect("Docker CLI must report the test-owned port");
assert!(
port_output.status.success(),
"test-owned Valkey port must be available"
);
let address = String::from_utf8(port_output.stdout).expect("Docker port must be UTF-8");
let port = address
.trim()
.rsplit(':')
.next()
.unwrap()
.parse()
.expect("Docker mapped port must be numeric");
(valkey, port)
}
@@ -98,30 +53,6 @@ impl TestValkey {
}
}
fn wait_for_valkey(port: u16) {
let address = ("127.0.0.1", port)
.to_socket_addrs()
.expect("test-owned Valkey loopback address must resolve")
.next()
.unwrap();
let deadline = Instant::now() + Duration::from_secs(10);
while Instant::now() < deadline {
if let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(250)) {
let _ = stream.set_read_timeout(Some(Duration::from_millis(250)));
let _ = stream.set_write_timeout(Some(Duration::from_millis(250)));
let mut reply = [0_u8; 7];
if stream.write_all(b"*1\r\n$4\r\nPING\r\n").is_ok()
&& stream.read_exact(&mut reply).is_ok()
&& &reply == b"+PONG\r\n"
{
return;
}
}
thread::sleep(Duration::from_millis(100));
}
panic!("test-owned Valkey did not answer PING on 127.0.0.1:{port} within 10 seconds");
}
impl Drop for TestValkey {
fn drop(&mut self) {
let _ = Command::new("docker").args(["rm", "-f", &self.0]).output();
@@ -1,868 +0,0 @@
//! Story 3.4 (FR-14) ATDD: progress and cancellation of one synchronous `tools/call`.
//!
//! Every case drives the real HTTP endpoint of the real application against a real (counted, slow)
//! upstream and real published catalog/registry state; nothing here is a fake client of the server
//! internals. The oracles are the pinned MCP shapes: `notifications/progress` (monotonic `progress`,
//! the request's own `progressToken`, no invented `total`) and the terminal `tools/call` result.
//!
//! Cancellation is revision-correct: `2025-11-25` cancels with `notifications/cancelled`, and
//! `2026-07-28` cancels by closing the request's own response stream. Both are proven to stop the
//! work, to release the call, and to never produce a `completed` record.
use std::{
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{Json, Router, routing::post};
use crank_core::{InvocationStatus, OperationId, PlatformApiKeyScope};
use crank_registry::{ListInvocationLogsQuery, PostgresRegistry, PublishRequest};
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::net::TcpListener;
use super::common::{
agent_mcp_url, build_test_app, capturing_subscriber, create_platform_api_key,
initialize_session, post_jsonrpc, post_jsonrpc_with_headers, publish_agent_for_operation,
spawn_mcp_server, test_operation, test_registry, test_workspace_id,
};
const LEGACY: &str = "2025-11-25";
const MODERN: &str = "2026-07-28";
const META_PROTOCOL_VERSION: &str = "io.modelcontextprotocol/protocolVersion";
const META_CLIENT_CAPABILITIES: &str = "io.modelcontextprotocol/clientCapabilities";
const META_CLIENT_INFO: &str = "io.modelcontextprotocol/clientInfo";
const TOOL: &str = "crm_lifecycle_lead";
/// A call that is long enough for the client to observe real progress, short enough for a suite.
const HOLD_MS: u64 = 250;
/// A call that is still running when the client cancels it.
const LONG_HOLD_MS: u64 = 1_500;
/// The grace period after a cancellation. It must exceed [`LONG_HOLD_MS`]: a call that was *not*
/// stopped completes its upstream request and records the outcome inside this window, so the checks
/// after it prove that the cancellation stopped the call rather than only that it had not finished
/// yet.
const CANCEL_GRACE_MS: u64 = 2_500;
/// The suite depends on that order: the constant must not silently drop back below the hold.
const _: () = assert!(CANCEL_GRACE_MS > LONG_HOLD_MS);
#[derive(Clone, Copy, Debug)]
enum Revision {
Legacy,
Modern,
}
impl Revision {
const fn label(self) -> &'static str {
match self {
Self::Legacy => LEGACY,
Self::Modern => MODERN,
}
}
}
struct Upstream {
base_url: String,
started: Arc<AtomicUsize>,
completed: Arc<AtomicUsize>,
}
/// A real upstream that records when a request starts and when it finishes. A cancelled call must
/// increment `started` exactly once and never `completed`: the request really was dispatched (so the
/// outcome is not `certain`-not-sent) and its response really was abandoned (so it is not retried).
async fn spawn_counted_slow_upstream(hold_ms: u64) -> Upstream {
let started = Arc::new(AtomicUsize::new(0));
let completed = Arc::new(AtomicUsize::new(0));
let handler_started = Arc::clone(&started);
let handler_completed = Arc::clone(&completed);
let app = Router::new().route(
"/crm/lifecycle-leads",
post(move |Json(payload): Json<Value>| {
let started = Arc::clone(&handler_started);
let completed = Arc::clone(&handler_completed);
async move {
started.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(hold_ms)).await;
completed.fetch_add(1, Ordering::SeqCst);
Json(json!({ "id": "lead_123", "email": payload["email"] }))
}
}),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
Upstream {
base_url: format!("http://{address}"),
started,
completed,
}
}
struct Fixture {
client: reqwest::Client,
mcp_url: String,
api_key: String,
other_api_key: String,
registry: PostgresRegistry,
operation_id: OperationId,
upstream: Upstream,
}
async fn fixture(agent_slug: &str, hold_ms: u64, timeout_ms: u64) -> Fixture {
let upstream = spawn_counted_slow_upstream(hold_ms).await;
let registry = test_registry().await;
let mut operation = test_operation(&upstream.base_url, TOOL);
let crank_core::Target::Rest(target) = &mut operation.target;
target.path_template = "/crm/lifecycle-leads".to_owned();
operation.execution_config.timeout_ms = timeout_ms;
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, agent_slug).await;
let scopes = [PlatformApiKeyScope::Read, PlatformApiKeyScope::Write];
let api_key = create_platform_api_key(&registry, agent_slug, "mcp-lifecycle", &scopes).await;
let other_api_key =
create_platform_api_key(&registry, agent_slug, "mcp-lifecycle-other", &scopes).await;
let base_url = spawn_mcp_server(build_test_app(
registry.clone(),
Duration::ZERO,
Some("https://crank.example.com".to_owned()),
))
.await;
Fixture {
client: reqwest::Client::new(),
mcp_url: agent_mcp_url(&base_url, agent_slug),
api_key,
other_api_key,
registry,
operation_id: operation.id,
upstream,
}
}
/// One parsed answer: the HTTP status, the negotiated media type and every JSON-RPC frame in order.
struct Reply {
status: reqwest::StatusCode,
content_type: String,
frames: Vec<Value>,
raw: String,
}
impl Reply {
fn parse(status: reqwest::StatusCode, content_type: String, body: String) -> Self {
let mut frames = Vec::new();
for block in body.split("\n\n") {
for line in block.lines() {
let Some(data) = line.strip_prefix("data:") else {
continue;
};
let data = data.trim_start();
if data.is_empty() {
continue;
}
frames.push(
serde_json::from_str(data)
.unwrap_or_else(|error| panic!("every frame is JSON: {data:?} {error}")),
);
}
}
if frames.is_empty() && !body.trim().is_empty() {
frames.push(
serde_json::from_str(body.trim()).unwrap_or_else(|error| {
panic!("expected a JSON-RPC body, got {body:?}: {error}")
}),
);
}
Self {
status,
content_type,
frames,
raw: body,
}
}
fn assert_stream(&self) {
assert_eq!(self.status, reqwest::StatusCode::OK);
assert!(
self.content_type.starts_with("text/event-stream"),
"a progress answer must be the request's SSE stream, got {:?}: {}",
self.content_type,
self.raw
);
}
fn progress_frames(&self) -> Vec<&Value> {
self.frames
.iter()
.filter(|frame| frame["method"] == json!("notifications/progress"))
.collect()
}
fn terminal(&self) -> &Value {
let terminal = self
.frames
.last()
.unwrap_or_else(|| panic!("the stream must carry a terminal frame: {}", self.raw));
assert!(
terminal.get("method").is_none(),
"the terminal frame must be the JSON-RPC response, got {terminal}"
);
terminal
}
}
impl Fixture {
fn call_params(&self, token: Option<Value>, revision: Revision) -> Value {
let mut params = json!({
"name": TOOL,
"arguments": { "email": "lifecycle@example.com" },
});
let mut meta = match revision {
Revision::Modern => json!({
META_PROTOCOL_VERSION: MODERN,
META_CLIENT_CAPABILITIES: {},
META_CLIENT_INFO: { "name": "crank-3.4-atdd", "version": "0.1.0" },
}),
Revision::Legacy => json!({}),
};
if let Some(token) = token {
meta["progressToken"] = token;
}
if meta.as_object().is_some_and(|object| !object.is_empty()) {
params["_meta"] = meta;
}
params
}
fn headers(&self, revision: Revision, session: Option<&str>) -> Vec<(&'static str, String)> {
let mut headers = vec![("MCP-Protocol-Version", revision.label().to_owned())];
match revision {
Revision::Legacy => {
if let Some(session) = session {
headers.push(("MCP-Session-Id", session.to_owned()));
}
}
Revision::Modern => {
headers.push(("Mcp-Method", "tools/call".to_owned()));
headers.push(("Mcp-Name", TOOL.to_owned()));
}
}
headers
}
fn request(
&self,
accept: &'static str,
key: &str,
headers: &[(&'static str, String)],
payload: Value,
) -> reqwest::RequestBuilder {
let mut request = self
.client
.post(&self.mcp_url)
.header(reqwest::header::ACCEPT, accept)
.header(reqwest::header::AUTHORIZATION, format!("Bearer {key}"));
for (name, value) in headers {
request = request.header(*name, value);
}
request.json(&payload)
}
/// One `tools/call` whose answer is negotiated as the request's own SSE stream.
async fn call_stream(
&self,
revision: Revision,
id: u64,
params: Value,
session: Option<&str>,
request_id: &str,
) -> Reply {
self.call_stream_with_key(revision, &self.api_key, id, params, session, request_id)
.await
}
async fn call_stream_with_key(
&self,
revision: Revision,
key: &str,
id: u64,
params: Value,
session: Option<&str>,
request_id: &str,
) -> Reply {
self.send(
"text/event-stream",
key,
revision,
id,
params,
session,
Some(request_id),
)
.await
}
/// One `tools/call` whose answer is negotiated as a single JSON body.
async fn call_json(
&self,
revision: Revision,
id: u64,
params: Value,
session: Option<&str>,
) -> Reply {
self.send(
"application/json",
&self.api_key,
revision,
id,
params,
session,
None,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn send(
&self,
accept: &'static str,
key: &str,
revision: Revision,
id: u64,
params: Value,
session: Option<&str>,
request_id: Option<&str>,
) -> Reply {
let headers = self.headers(revision, session);
let payload =
json!({ "jsonrpc": "2.0", "id": id, "method": "tools/call", "params": params });
let mut request = self.request(accept, key, &headers, payload);
if let Some(request_id) = request_id {
request = request.header("x-request-id", request_id);
}
let response = request.send().await.expect("the MCP endpoint answers");
let status = response.status();
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
let body = response.text().await.expect("the body is readable");
Reply::parse(status, content_type, body)
}
async fn cancel(&self, key: &str, session: &str, request_id: Value) -> reqwest::StatusCode {
let response = self
.request(
"application/json",
key,
&[
("MCP-Protocol-Version", LEGACY.to_owned()),
("MCP-Session-Id", session.to_owned()),
],
json!({
"jsonrpc": "2.0",
"method": "notifications/cancelled",
"params": { "requestId": request_id, "reason": "user requested cancellation" }
}),
)
.send()
.await
.expect("the cancellation is acknowledged");
response.status()
}
/// Every Invocation History row of this operation and request id, oldest first.
async fn history(&self, request_id: &str) -> Vec<crank_core::InvocationLog> {
self.registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
status: None,
outcome_group: None,
search_text: None,
source: Some(crank_core::InvocationSource::AgentToolCall),
operation_id: Some(&self.operation_id),
agent_id: None,
created_after: None,
created_before: None,
cursor_created_at: None,
cursor_id: None,
limit: 50,
})
.await
.unwrap()
.into_iter()
.map(|record| record.log)
.filter(|log| log.request_id.as_deref() == Some(request_id))
.collect()
}
/// Wait for the counted upstream to be entered, so a cancellation happens while it really runs.
async fn wait_until_dispatched(&self) {
for _ in 0..200 {
if self.upstream.started.load(Ordering::SeqCst) > 0 {
return;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
panic!("the upstream was never dispatched");
}
}
fn progress_values(frames: &[&Value]) -> Vec<u64> {
frames
.iter()
.map(|frame| {
frame["params"]["progress"]
.as_u64()
.unwrap_or_else(|| panic!("progress must be a number, got {frame}"))
})
.collect()
}
// ---------------------------------------------------------------------------------------------
// AT-3.4-01: a negotiated stream carries bounded monotonic progress of the real execution, and a
// missing token (or a missing carrier) never changes the execution result.
// ---------------------------------------------------------------------------------------------
#[tokio::test]
async fn at_3_4_01_legacy_progress_stream_is_monotonic_and_ends_with_the_real_result() {
let fixture = fixture("lifecycle-legacy-progress", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let reply = fixture
.call_stream(
Revision::Legacy,
501,
fixture.call_params(Some(json!("tok-legacy-progress")), Revision::Legacy),
Some(&session),
"req-legacy-progress",
)
.await;
reply.assert_stream();
let progress = reply.progress_frames();
assert!(
!progress.is_empty(),
"a token with a stream carrier must produce progress, got {}",
reply.raw
);
for frame in &progress {
assert_eq!(frame["jsonrpc"], json!("2.0"));
assert_eq!(
frame["params"]["progressToken"],
json!("tok-legacy-progress")
);
assert!(
frame["params"].get("total").is_none(),
"this server knows no total and must not invent one: {frame}"
);
assert!(
frame["params"]["message"].is_string(),
"every real step names itself: {frame}"
);
}
let values = progress_values(&progress);
assert!(
values.windows(2).all(|pair| pair[0] < pair[1]),
"progress must be strictly monotonic, got {values:?}"
);
// One event per real execution step this server actually crosses: resolved, authorized,
// dispatched. Never a timer-driven percentage and never an unbounded stream.
assert_eq!(values, vec![1, 2, 3], "got {values:?}");
let terminal = reply.terminal();
assert_eq!(terminal["id"], json!(501));
assert_eq!(terminal["error"], Value::Null);
assert_eq!(terminal["result"]["isError"], json!(false));
assert_eq!(
terminal["result"]["structuredContent"],
json!({ "id": "lead_123" })
);
assert_eq!(fixture.upstream.started.load(Ordering::SeqCst), 1);
assert_eq!(fixture.upstream.completed.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn at_3_4_01_modern_progress_stream_is_monotonic_and_ends_with_the_real_result() {
let fixture = fixture("lifecycle-modern-progress", HOLD_MS, 5_000).await;
let reply = fixture
.call_stream(
Revision::Modern,
502,
fixture.call_params(Some(json!(77)), Revision::Modern),
None,
"req-modern-progress",
)
.await;
reply.assert_stream();
let progress = reply.progress_frames();
assert!(!progress.is_empty(), "got {}", reply.raw);
for frame in &progress {
assert_eq!(frame["params"]["progressToken"], json!(77));
}
let values = progress_values(&progress);
assert!(
values.windows(2).all(|pair| pair[0] < pair[1]),
"{values:?}"
);
let terminal = reply.terminal();
assert_eq!(terminal["result"]["isError"], json!(false));
assert_eq!(
terminal["result"]["structuredContent"],
json!({ "id": "lead_123" })
);
}
// ---------------------------------------------------------------------------------------------
// AT-3.4-01 (TL correction): the official SDK sends the dual `Accept` JSON-first. A `tools/call`
// that carries a validated token and offers `text/event-stream` in that same header must still be
// answered on its own stream, wherever the media types are ordered.
// ---------------------------------------------------------------------------------------------
#[tokio::test]
async fn at_3_4_01_json_first_dual_accept_keeps_progress_for_a_valid_token() {
/// Exactly what the official MCP SDK sends when it can read a stream.
const SDK_ACCEPT: &str = "application/json, text/event-stream";
let fixture = fixture("lifecycle-json-first", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
for revision in [Revision::Legacy, Revision::Modern] {
let carrier = match revision {
Revision::Legacy => Some(session.as_str()),
Revision::Modern => None,
};
let token = json!("tok-json-first");
let reply = fixture
.send(
SDK_ACCEPT,
&fixture.api_key,
revision,
550,
fixture.call_params(Some(token.clone()), revision),
carrier,
Some("req-json-first"),
)
.await;
reply.assert_stream();
let progress = reply.progress_frames();
assert!(
!progress.is_empty(),
"{revision:?}: a valid token with an acceptable stream must produce progress, got {}",
reply.raw
);
for frame in &progress {
assert_eq!(frame["params"]["progressToken"], token, "{frame}");
assert!(
frame["params"].get("total").is_none(),
"this server knows no total: {frame}"
);
}
let values = progress_values(&progress);
assert!(
values.windows(2).all(|pair| pair[0] < pair[1]),
"{revision:?}: progress must be strictly monotonic, got {values:?}"
);
assert_eq!(values, vec![1, 2, 3], "{revision:?}: got {values:?}");
let terminal = reply.terminal();
assert_eq!(terminal["id"], json!(550), "{revision:?}");
assert_eq!(terminal["result"]["isError"], json!(false), "{revision:?}");
assert_eq!(
terminal["result"]["structuredContent"],
json!({ "id": "lead_123" }),
"{revision:?}"
);
}
// The correction is scoped to a call that can actually carry progress: the same JSON-first
// header without a token, and with an unusable token, keeps the negotiated single JSON body.
for token in [None, Some(json!("x".repeat(4096)))] {
let reply = fixture
.send(
SDK_ACCEPT,
&fixture.api_key,
Revision::Legacy,
551,
fixture.call_params(token.clone(), Revision::Legacy),
Some(&session),
None,
)
.await;
assert_eq!(reply.status, reqwest::StatusCode::OK);
assert!(
reply.content_type.starts_with("application/json"),
"token {token:?} must keep the negotiated JSON body, got {:?}: {}",
reply.content_type,
reply.raw
);
assert!(reply.progress_frames().is_empty(), "{token:?}");
assert_eq!(
reply.terminal()["result"]["structuredContent"],
json!({ "id": "lead_123" }),
"token {token:?}"
);
}
}
#[tokio::test]
async fn at_3_4_01_no_token_and_no_carrier_keep_the_result_identical() {
let fixture = fixture("lifecycle-no-token", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
// Without a token the very same call is one plain JSON body: no notification exists.
let without_token = fixture
.call_json(
Revision::Legacy,
510,
fixture.call_params(None, Revision::Legacy),
Some(&session),
)
.await;
assert_eq!(without_token.status, reqwest::StatusCode::OK);
assert!(without_token.progress_frames().is_empty());
let plain = without_token.terminal()["result"].clone();
assert_eq!(plain["isError"], json!(false));
// With a token but only the `application/json` carrier the result is the same and progress is
// simply not sent (the receiver is never obligated to send it).
let token_without_carrier = fixture
.call_json(
Revision::Legacy,
511,
fixture.call_params(Some(json!("tok-no-carrier")), Revision::Legacy),
Some(&session),
)
.await;
assert_eq!(token_without_carrier.status, reqwest::StatusCode::OK);
assert!(token_without_carrier.progress_frames().is_empty());
assert_eq!(token_without_carrier.terminal()["result"], plain);
// In the stream mode without a token the only frame is the terminal response itself.
let stream_without_token = fixture
.call_stream(
Revision::Legacy,
512,
fixture.call_params(None, Revision::Legacy),
Some(&session),
"req-no-token-stream",
)
.await;
assert!(stream_without_token.progress_frames().is_empty());
assert_eq!(stream_without_token.frames.len(), 1);
assert_eq!(stream_without_token.terminal()["result"], plain);
}
/// The streamed answer keeps the Story 3.18 projection: a client that opted in to logging gets the
/// same single closed `notifications/message` of this call before the terminal frame, next to the
/// progress events of Story 3.4.
#[tokio::test]
async fn at_3_4_01_a_progress_stream_keeps_the_logging_projection() {
let fixture = fixture("lifecycle-progress-logging", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let set_level = fixture
.request(
"application/json",
&fixture.api_key,
&[
("MCP-Protocol-Version", LEGACY.to_owned()),
("MCP-Session-Id", session.clone()),
],
json!({
"jsonrpc": "2.0", "id": 900, "method": "logging/setLevel",
"params": { "level": "debug" }
}),
)
.send()
.await
.expect("the level is accepted");
assert_eq!(set_level.status(), reqwest::StatusCode::OK);
let reply = fixture
.call_stream(
Revision::Legacy,
901,
fixture.call_params(Some(json!("tok-logging")), Revision::Legacy),
Some(&session),
"req-progress-logging",
)
.await;
reply.assert_stream();
assert!(!reply.progress_frames().is_empty(), "got {}", reply.raw);
let logged: Vec<&Value> = reply
.frames
.iter()
.filter(|frame| frame["method"] == json!("notifications/message"))
.collect();
assert_eq!(
logged.len(),
1,
"exactly one closed log event per response, got {}",
reply.raw
);
assert_eq!(
logged[0]["params"]["data"]["event"],
json!("mcp.request.served")
);
assert_eq!(logged[0]["params"]["data"]["outcome"], json!("ok"));
assert_eq!(reply.terminal()["result"]["isError"], json!(false));
}
// ---------------------------------------------------------------------------------------------
// AT-3.4-02: a malformed, reused-in-another-scope or oversized token disables progress, and the
// token never becomes a metric label, a log field or an authorization scope.
// ---------------------------------------------------------------------------------------------
#[tokio::test]
async fn at_3_4_02_malformed_and_oversized_tokens_disable_progress_without_changing_the_result() {
let fixture = fixture("lifecycle-bad-token", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
// Both supported revisions read the token from the same pinned `RequestMetaObject` field, so an
// unusable token must behave identically in each of them: no progress, unchanged result.
for revision in [Revision::Legacy, Revision::Modern] {
let carrier = match revision {
Revision::Legacy => Some(session.as_str()),
Revision::Modern => None,
};
let tokens = [
json!(""),
json!({ "not": "a token" }),
json!([1, 2]),
json!(true),
json!("x".repeat(4096)),
];
let mut observed = Vec::new();
for token in tokens {
let reply = fixture
.call_stream(
revision,
520,
fixture.call_params(Some(token.clone()), revision),
carrier,
"req-bad-token",
)
.await;
observed.push((token, reply));
}
for (token, reply) in observed {
assert!(
reply.progress_frames().is_empty(),
"{revision:?}: token {token} must not produce progress, got {}",
reply.raw
);
let terminal = reply.terminal();
assert_eq!(
terminal["result"]["structuredContent"],
json!({ "id": "lead_123" }),
"{revision:?}: token {token} must not change the execution result, got {terminal}"
);
assert_eq!(terminal["result"]["isError"], json!(false));
assert!(terminal["error"].is_null() || terminal.get("error").is_none());
}
}
}
#[tokio::test]
async fn at_3_4_02_a_reused_token_is_scoped_to_its_own_request() {
let fixture = fixture("lifecycle-reused-token", HOLD_MS, 5_000).await;
let first = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let second = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let token = json!("tok-shared");
let (first_reply, second_reply) = tokio::join!(
fixture.call_stream(
Revision::Legacy,
530,
fixture.call_params(Some(token.clone()), Revision::Legacy),
Some(&first),
"req-first",
),
fixture.call_stream(
Revision::Legacy,
531,
fixture.call_params(Some(token.clone()), Revision::Legacy),
Some(&second),
"req-second",
)
);
for (reply, id) in [(&first_reply, 530), (&second_reply, 531)] {
reply.assert_stream();
let values = progress_values(&reply.progress_frames());
// Each request counts its own real steps: a reused token never inherits another call's count.
assert_eq!(values, vec![1, 2, 3], "request {id} got {values:?}");
assert_eq!(reply.terminal()["id"], json!(id));
}
}
#[tokio::test]
async fn at_3_4_02_a_progress_token_never_reaches_logs_or_invocation_history() {
const CANARY: &str = "crank-progress-canary-4f1c2d";
let fixture = fixture("lifecycle-token-leak", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let (writer, subscriber) = capturing_subscriber();
let reply = {
let _guard = tracing::subscriber::set_default(subscriber);
fixture
.call_stream(
Revision::Legacy,
540,
fixture.call_params(Some(json!(CANARY)), Revision::Legacy),
Some(&session),
"req-token-leak",
)
.await
};
reply.assert_stream();
let logs = writer.output();
assert!(
logs.contains("mcp request received"),
"the capture must really see the endpoint's own log projection, got {logs}"
);
assert!(
!logs.contains(CANARY),
"the progress token is client data and must never be logged, got {logs}"
);
let history = fixture.history("req-token-leak").await;
assert_eq!(history.len(), 1, "got {history:?}");
let row = &history[0];
let serialized = serde_json::to_string(row).unwrap();
assert!(
!serialized.contains(CANARY),
"the progress token must never reach Invocation History, got {serialized}"
);
assert_eq!(row.status, InvocationStatus::Ok);
}
// AT-3.4-03: a cancellation stops a call that can still be stopped, releases it and never yields a
// `completed` result or record.
mod cancellation;
// AT-3.4-05: the client-visible smoke of both supported revisions.
mod smoke;
@@ -1,294 +0,0 @@
//! AT-3.4-03: a cancellation stops a call that can still be stopped, releases it, and never yields a
//! `completed` result or history row. The harness of the suite lives in `super`.
use super::*;
// ---------------------------------------------------------------------------------------------
// AT-3.4-03: a cancellation stops a call that can still be stopped, releases it, and never yields a
// `completed` result or history row.
// ---------------------------------------------------------------------------------------------
#[tokio::test]
async fn at_3_4_03_legacy_cancellation_stops_the_call_and_never_completes_it() {
let fixture = fixture("lifecycle-legacy-cancel", LONG_HOLD_MS, 30_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let call_session = session.clone();
let call = tokio::spawn({
let fixture_client = fixture.client.clone();
let mcp_url = fixture.mcp_url.clone();
let api_key = fixture.api_key.clone();
let params = fixture.call_params(Some(json!("tok-cancel")), Revision::Legacy);
async move {
let response = fixture_client
.post(mcp_url)
.header(reqwest::header::ACCEPT, "text/event-stream")
.header(reqwest::header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("MCP-Protocol-Version", LEGACY)
.header("MCP-Session-Id", call_session)
.header("x-request-id", "req-legacy-cancel")
.json(&json!({
"jsonrpc": "2.0",
"id": 601,
"method": "tools/call",
"params": params,
}))
.send()
.await
.unwrap();
(response.status(), response.text().await.unwrap())
}
});
fixture.wait_until_dispatched().await;
assert_eq!(
fixture.cancel(&fixture.api_key, &session, json!(601)).await,
reqwest::StatusCode::ACCEPTED,
"an accepted cancellation is acknowledged without a body"
);
let (status, body) = tokio::time::timeout(Duration::from_secs(10), call)
.await
.expect("the cancelled call is released, it does not hang")
.expect("the request task joins");
assert_eq!(status, reqwest::StatusCode::OK);
let reply = Reply::parse(status, String::new(), body);
assert!(
reply
.frames
.iter()
.all(|frame| frame.get("method").is_some()),
"a cancelled request gets no terminal message, only the progress it already saw: {}",
reply.raw
);
assert_eq!(
reply.progress_frames().len(),
3,
"the client really saw the real steps before the stop: {}",
reply.raw
);
tokio::time::sleep(Duration::from_millis(CANCEL_GRACE_MS)).await;
assert_eq!(
fixture.upstream.completed.load(Ordering::SeqCst),
0,
"the abandoned upstream response must never complete the call"
);
assert_eq!(
fixture.upstream.started.load(Ordering::SeqCst),
1,
"the write really was dispatched and must not be retried automatically"
);
let history = fixture.history("req-legacy-cancel").await;
assert!(
history.iter().all(|row| row.status != InvocationStatus::Ok),
"a cancelled call must never be recorded as completed, got {history:?}"
);
// The record contract admits a non-`ok` row only with a closed execution error code, and a
// cancellation is not one of them: the honest terminal state of this call is "no record" instead
// of an invented error code. Either way nothing claims that the call completed.
assert!(
history.is_empty(),
"a stopped call has no terminal outcome to record, got {history:?}"
);
}
#[tokio::test]
async fn at_3_4_03_a_foreign_or_unknown_cancellation_never_touches_the_active_call() {
let fixture = fixture("lifecycle-foreign-cancel", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let foreign_session =
initialize_session(&fixture.client, &fixture.mcp_url, &fixture.other_api_key).await;
let call = tokio::spawn({
let client = fixture.client.clone();
let mcp_url = fixture.mcp_url.clone();
let api_key = fixture.api_key.clone();
let call_session = session.clone();
let params = fixture.call_params(Some(json!("tok-owner")), Revision::Legacy);
async move {
let response = client
.post(mcp_url)
.header(reqwest::header::ACCEPT, "text/event-stream")
.header(reqwest::header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("MCP-Protocol-Version", LEGACY)
.header("MCP-Session-Id", call_session)
.header("x-request-id", "req-owner")
.json(&json!({
"jsonrpc": "2.0", "id": 602, "method": "tools/call", "params": params,
}))
.send()
.await
.unwrap();
response.text().await.unwrap()
}
});
fixture.wait_until_dispatched().await;
// A foreign credential of the same agent, and an unknown request id: neither may cancel this call.
assert_eq!(
fixture
.cancel(&fixture.other_api_key, &session, json!(602))
.await,
reqwest::StatusCode::ACCEPTED
);
assert_eq!(
fixture
.cancel(&fixture.api_key, &session, json!(999_999))
.await,
reqwest::StatusCode::ACCEPTED
);
// A foreign session of the same credential cannot reach this call either.
assert_eq!(
fixture
.cancel(&fixture.api_key, &foreign_session, json!(602))
.await,
reqwest::StatusCode::ACCEPTED
);
let body = tokio::time::timeout(Duration::from_secs(10), call)
.await
.expect("the untouched call completes")
.expect("the request task joins");
let reply = Reply::parse(reqwest::StatusCode::OK, String::new(), body);
assert_eq!(
reply.terminal()["result"]["structuredContent"],
json!({ "id": "lead_123" }),
"a foreign cancellation must not stop the owner's call, got {}",
reply.raw
);
let history = fixture.history("req-owner").await;
assert_eq!(history.len(), 1, "got {history:?}");
assert_eq!(history[0].status, InvocationStatus::Ok);
}
#[tokio::test]
async fn at_3_4_03_a_late_cancellation_after_the_result_is_ignored() {
let fixture = fixture("lifecycle-late-cancel", HOLD_MS, 5_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let reply = fixture
.call_stream(
Revision::Legacy,
603,
fixture.call_params(Some(json!("tok-late")), Revision::Legacy),
Some(&session),
"req-late-cancel",
)
.await;
reply.assert_stream();
assert_eq!(reply.terminal()["result"]["isError"], json!(false));
assert_eq!(
fixture.cancel(&fixture.api_key, &session, json!(603)).await,
reqwest::StatusCode::ACCEPTED
);
tokio::time::sleep(Duration::from_millis(200)).await;
let history = fixture.history("req-late-cancel").await;
assert_eq!(
history.len(),
1,
"a cancellation of a finished call must not add a second record, got {history:?}"
);
assert_eq!(history[0].status, InvocationStatus::Ok);
}
#[tokio::test]
async fn at_3_4_03_a_plain_json_call_is_cancellable_too() {
let fixture = fixture("lifecycle-json-cancel", LONG_HOLD_MS, 30_000).await;
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let call = tokio::spawn({
let client = fixture.client.clone();
let mcp_url = fixture.mcp_url.clone();
let api_key = fixture.api_key.clone();
let call_session = session.clone();
let params = fixture.call_params(None, Revision::Legacy);
async move {
let response = client
.post(mcp_url)
.header(reqwest::header::ACCEPT, "application/json")
.header(reqwest::header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("MCP-Protocol-Version", LEGACY)
.header("MCP-Session-Id", call_session)
.header("x-request-id", "req-json-cancel")
.json(&json!({
"jsonrpc": "2.0", "id": 605, "method": "tools/call", "params": params,
}))
.send()
.await
.unwrap();
let status = response.status();
(status, response.text().await.unwrap())
}
});
fixture.wait_until_dispatched().await;
assert_eq!(
fixture.cancel(&fixture.api_key, &session, json!(605)).await,
reqwest::StatusCode::ACCEPTED
);
let (status, body) = tokio::time::timeout(Duration::from_secs(10), call)
.await
.expect("the cancelled call is released")
.expect("the request task joins");
assert_eq!(status, reqwest::StatusCode::OK);
let reply = Reply::parse(status, String::new(), body);
assert!(
reply.frames.is_empty(),
"a cancelled request gets no terminal message, got {}",
reply.raw
);
tokio::time::sleep(Duration::from_millis(CANCEL_GRACE_MS)).await;
assert_eq!(fixture.upstream.completed.load(Ordering::SeqCst), 0);
let history = fixture.history("req-json-cancel").await;
assert!(
history.iter().all(|row| row.status != InvocationStatus::Ok),
"a cancelled call must never be recorded as completed, got {history:?}"
);
}
#[tokio::test]
async fn at_3_4_03_closing_the_modern_response_stream_cancels_the_call() {
let fixture = fixture("lifecycle-modern-disconnect", LONG_HOLD_MS, 30_000).await;
let response = fixture
.request(
"text/event-stream",
&fixture.api_key,
&fixture.headers(Revision::Modern, None),
json!({
"jsonrpc": "2.0",
"id": 604,
"method": "tools/call",
"params": fixture.call_params(Some(json!("tok-disconnect")), Revision::Modern),
}),
)
.header("x-request-id", "req-modern-disconnect")
.send()
.await
.expect("the stream opens before the call finishes");
assert_eq!(response.status(), reqwest::StatusCode::OK);
fixture.wait_until_dispatched().await;
// The pinned `2026-07-28` cancellation signal: closing the request's own response stream.
drop(response);
tokio::time::sleep(Duration::from_millis(CANCEL_GRACE_MS)).await;
assert_eq!(
fixture.upstream.completed.load(Ordering::SeqCst),
0,
"the closed stream must stop the dispatched call instead of letting it finish"
);
assert_eq!(fixture.upstream.started.load(Ordering::SeqCst), 1);
let history = fixture.history("req-modern-disconnect").await;
assert!(
history.iter().all(|row| row.status != InvocationStatus::Ok),
"a cancelled call must never be recorded as completed, got {history:?}"
);
}
@@ -1,79 +0,0 @@
//! AT-3.4-05: the client-visible smoke of both supported revisions.
//!
//! A representative client of each revision performs the whole flow a real MCP client performs —
//! discovery, then a `tools/call` that asks for progress and reads the request's own stream to the
//! terminal response — against the real endpoint. The harness of the suite lives in `super`.
use super::*;
#[tokio::test]
async fn at_3_4_05_representative_client_smoke_of_both_revisions() {
let fixture = fixture("lifecycle-client-smoke", HOLD_MS, 5_000).await;
// Legacy client: initialize → tools/list → tools/call with progress.
let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await;
let tools = post_jsonrpc(
&fixture.client,
&fixture.mcp_url,
&fixture.api_key,
Some(&session),
json!({ "jsonrpc": "2.0", "id": 700, "method": "tools/list", "params": {} }),
)
.await;
assert_eq!(tools["result"]["tools"][0]["name"], json!(TOOL));
let legacy_call = fixture
.call_stream(
Revision::Legacy,
701,
fixture.call_params(Some(json!("tok-smoke-legacy")), Revision::Legacy),
Some(&session),
"req-smoke-legacy",
)
.await;
legacy_call.assert_stream();
assert_eq!(
legacy_call.terminal()["result"]["structuredContent"],
json!({ "id": "lead_123" })
);
// Modern client: `tools/list` and `tools/call` of the stateless revision, progress on the stream.
let modern_list = post_jsonrpc_with_headers(
&fixture.client,
&fixture.mcp_url,
&fixture.api_key,
&[
("MCP-Protocol-Version", MODERN),
("Mcp-Method", "tools/list"),
],
json!({
"jsonrpc": "2.0",
"id": 710,
"method": "tools/list",
"params": { "_meta": {
META_PROTOCOL_VERSION: MODERN,
META_CLIENT_CAPABILITIES: {},
META_CLIENT_INFO: { "name": "crank-3.4-smoke", "version": "0.1.0" },
}}
}),
)
.await;
let modern_list = modern_list.json::<Value>().await.unwrap();
assert_eq!(modern_list["result"]["tools"][0]["name"], json!(TOOL));
let modern_call = fixture
.call_stream(
Revision::Modern,
711,
fixture.call_params(Some(json!("tok-smoke-modern")), Revision::Modern),
None,
"req-smoke-modern",
)
.await;
modern_call.assert_stream();
assert!(!modern_call.progress_frames().is_empty());
assert_eq!(
modern_call.terminal()["result"]["structuredContent"],
json!({ "id": "lead_123" })
);
}
+1 -6
View File
@@ -1,10 +1,5 @@
FROM node:22-alpine AS build
# The CI host uplink drops TLS 1.3 ClientHello carrying the hybrid
# X25519MLKEM768 key share; pin the npm stage to TLS 1.2 until that network
# defect is fixed. Certificates, SRI and the lockfile are unaffected.
ENV NODE_OPTIONS=--tls-max-v1.2
WORKDIR /app
COPY apps/ui/package.json apps/ui/package-lock.json ./
@@ -15,7 +10,7 @@ COPY crank-community.png ./crank-community.png
RUN npm run build
FROM nginx:1.30.6-alpine3.24-slim
FROM nginx:1.30.4-alpine3.24-slim
RUN apk add --no-cache --upgrade \
'pcre2>=10.49-r0' \
-2
View File
@@ -1036,7 +1036,6 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Secret name is required.',
'wizard.error.secret_value_required': 'Secret value is required.',
'wizard.error.save_upstream_first': 'Save the API host after creating a new authorization profile before continuing.',
'wizard.error.execution_policy_invalid': 'Execution policy is invalid: mode, deadline from 1 to 3600000 ms and no-Tasks fallback are required.',
'wizard.toast.auth_profile_created_title': 'Authorization profile created',
'wizard.toast.auth_profile_created_body': 'New authorization profile is ready and selected.',
'wizard.toast.auth_profile_error_title': 'Authorization profile creation failed',
@@ -2354,7 +2353,6 @@ var TRANSLATIONS = {
'wizard.error.secret_name_required': 'Нужно указать имя секрета.',
'wizard.error.secret_value_required': 'Нужно указать значение секрета.',
'wizard.error.save_upstream_first': 'Сначала сохраните API-хост после создания нового профиля авторизации.',
'wizard.error.execution_policy_invalid': 'Политика выполнения некорректна: нужны режим, deadline от 1 до 3600000 мс и поведение без Tasks.',
'wizard.toast.auth_profile_created_title': 'Профиль авторизации создан',
'wizard.toast.auth_profile_created_body': 'Новый профиль авторизации готов и уже выбран.',
'wizard.toast.auth_profile_error_title': 'Не удалось создать профиль авторизации',
-30
View File
@@ -214,33 +214,10 @@ function parseExecutionConfig(text) {
auth_profile_ref: authProfileRef,
headers: headers,
};
var executionPolicy = parseExecutionPolicy(value.execution_policy);
if (executionPolicy) {
config.execution_policy = executionPolicy;
}
return config;
}
// Story 4.1: the published execution policy must survive a wizard save. The editor is a free-form
// document, so unknown/legacy shapes stay untouched: only a fully valid policy is projected, and an
// explicit invalid value is reported instead of being silently replaced by defaults.
function parseExecutionPolicy(raw) {
if (raw === undefined || raw === null) return null;
if (typeof raw !== 'object' || Array.isArray(raw)) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
var mode = raw.mode === undefined ? 'synchronous' : raw.mode;
var fallback = raw.no_tasks_fallback === undefined ? 'synchronous' : raw.no_tasks_fallback;
var deadline = Number(raw.deadline_ms);
if (['synchronous', 'background', 'adaptive'].indexOf(mode) < 0
|| ['synchronous', 'unsupported'].indexOf(fallback) < 0
|| !Number.isInteger(deadline) || deadline < 1 || deadline > 3600000) {
throw new Error(tKey('wizard.error.execution_policy_invalid'));
}
return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback };
}
function currentUpstream() {
var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; });
if (selected) return selected;
@@ -438,13 +415,6 @@ function executionConfigToEditorValue(config) {
var value = {
timeout_ms: config.timeout_ms,
};
if (config.execution_policy) {
value.execution_policy = {
mode: config.execution_policy.mode,
deadline_ms: config.execution_policy.deadline_ms,
no_tasks_fallback: config.execution_policy.no_tasks_fallback,
};
}
if (config.retry_policy) {
value.retry = { max_attempts: config.retry_policy.max_attempts };
}
@@ -1,139 +0,0 @@
const { test, expect } = require('@playwright/test');
const { getCurrentWorkspace, login } = require('./helpers');
// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a
// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit
// invalid value must be reported instead of being silently replaced by defaults.
function draftSnapshot(operationId) {
return {
id: operationId,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
target: {
kind: 'rest',
base_url: 'https://api.example.test',
method: 'GET',
path_template: '/rates',
static_headers: {},
},
input_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
output_schema: {
type: 'object',
description: null,
required: false,
nullable: false,
default_value: null,
fields: {},
items: null,
enum_values: [],
variants: [],
},
input_mapping: { rules: [] },
output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] },
execution_config: {
timeout_ms: 10000,
retry_policy: null,
auth_profile_ref: null,
headers: {},
approval_policy: null,
execution_policy: {
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
},
},
tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] },
wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] },
};
}
test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => {
await login(page);
const workspace = await getCurrentWorkspace(page);
const operationId = 'policy_rates_41';
const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`;
const savedPayloads = [];
await page.route(operationPath, async (route) => {
if (route.request().method() === 'PATCH') {
savedPayloads.push(route.request().postDataJSON());
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }),
});
return;
}
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
id: operationId,
workspace_id: workspace.id,
name: operationId,
display_name: 'Policy rates',
category: 'general',
protocol: 'rest',
status: 'draft',
current_draft_version: 1,
draft_version_ref: { version: 1, status: 'draft' },
published_version_ref: null,
agent_refs: [],
}),
});
});
await page.route(`${operationPath}/versions/*`, async (route) => {
await route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
operation_id: operationId,
workspace_id: workspace.id,
version: 1,
status: 'draft',
snapshot: draftSnapshot(operationId),
}),
});
});
await page.route('**/operations/analyze-quality', async (route) => {
await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) });
});
page.on('dialog', (dialog) => dialog.accept());
await page.goto(`/wizard/?mode=edit&operationId=${operationId}`);
await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates');
const editorValue = await page.locator('#tool-exec-config').inputValue();
expect(editorValue).toContain('execution_policy');
expect(editorValue).toContain('adaptive');
expect(editorValue).toContain('45000');
expect(editorValue).toContain('unsupported');
// A real edit still saves the pinned policy unchanged.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(4));
await page.locator('#tool-display-name').fill('Policy rates edited');
await page.locator('.btn-save-draft').click();
await expect.poll(() => savedPayloads.length).toBe(1);
expect(savedPayloads[0].execution_config.execution_policy).toEqual({
mode: 'adaptive',
deadline_ms: 45000,
no_tasks_fallback: 'unsupported',
});
// An explicit invalid policy is refused instead of being silently rewritten to defaults.
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
await expect(page.locator('#tool-exec-config')).toBeVisible();
await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n');
await page.locator('.btn-save-draft').click();
await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/);
expect(savedPayloads.length).toBe(1);
});
-2
View File
@@ -21,7 +21,6 @@ crank-schema = { path = "../crank-schema" }
crank-trace = { path = "../crank-trace" }
futures-util = "0.3"
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
@@ -33,7 +32,6 @@ tracing.workspace = true
uuid.workspace = true
[dev-dependencies]
rmcp = { version = "=3.7.0", features = ["transport-streamable-http-server"] }
crank-mapping = { path = "../crank-mapping" }
crank-test-support = { path = "../crank-test-support" }
metrics-util = "0.20.4"
+1 -4
View File
@@ -272,10 +272,7 @@ fn record_unknown_approval_access_denial(reason: &'static str) {
);
}
pub(super) fn allows_scope(
scopes: &[PlatformApiKeyScope],
required_scope: PlatformApiKeyScope,
) -> bool {
fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool {
match required_scope {
PlatformApiKeyScope::Read => scopes.iter().any(|scope| {
matches!(
+16 -128
View File
@@ -30,22 +30,9 @@ mod approval_policy;
#[cfg(test)]
pub(crate) mod dispatch_observer;
mod invocation_history;
pub(crate) mod lifecycle;
mod metrics;
mod request;
mod response;
#[cfg(test)]
mod rmcp_http;
#[cfg(test)]
#[path = "rmcp_http_tests.rs"]
mod rmcp_http_tests;
#[cfg(test)]
mod rmcp_session_tests;
#[cfg(test)]
mod rmcp_tools;
#[cfg(test)]
#[path = "rmcp_tools_tests.rs"]
mod rmcp_tools_tests;
mod stages;
mod tool_resolution;
pub(crate) use self::approval_policy::{
@@ -62,16 +49,13 @@ mod approvals;
mod auth_resolution;
mod dispatch;
mod endpoint;
mod legacy_dispatch;
mod task_worker;
mod tasks;
use self::approvals::{
approve_request, deny_request, get_approval_request, list_pending_approvals,
};
pub(super) use self::auth_resolution::resolve_operation_auth;
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
pub use self::task_worker::{TaskWorkerHandler, VerifiedIdempotentWrite};
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
// through `super`, and the plain lib build does not use them (same pattern as
// `#[cfg(test)] use invocation_history::observe_invocation_history_outcome;` above).
@@ -101,14 +85,6 @@ pub(super) struct AppState {
/// Story 3.18: the bounded in-memory `logging/setLevel` opt-in of the legacy sessions. It is a
/// client-scoped level, never a process-global one, and it needs no schema migration.
pub(super) logging_levels: Arc<crate::protocol::LogLevelStore>,
/// Story 3.19: the bounded table of the legacy session that currently owns the notification
/// stream, so one session never broadcasts the same list-change signal into two streams.
pub(super) signal_streams: Arc<crate::protocol::StreamRegistry>,
/// Story 3.4: the bounded table of the `tools/call` requests a legacy client can still cancel.
pub(super) in_flight: Arc<self::lifecycle::InFlightCalls>,
/// Story 4.2: durable background Tasks over the same registry pool. The public wire stays
/// synchronous/unsupported; readiness depends on this mandatory store.
tasks: Arc<TaskApplication>,
}
#[derive(Clone)]
@@ -154,7 +130,6 @@ pub fn build_app(
DEFAULT_MAX_CONCURRENT_SESSIONS,
false,
None,
None,
)
}
@@ -210,7 +185,6 @@ pub fn build_app_with_background_workers_and_limits(
max_concurrent_sessions,
true,
None,
None,
)
}
@@ -246,41 +220,6 @@ pub fn build_app_with_artifacts(
max_concurrent_sessions,
true,
Some(artifact_store),
None,
)
}
/// Production launcher entry point with the durable Task application entry constructed by the
/// process composition root from the same registry pool and admission limits as the runtime.
#[allow(clippy::too_many_arguments)]
pub fn build_app_with_artifacts_and_tasks(
registry: PostgresRegistry,
refresh_interval: Duration,
public_base_url: Option<String>,
secret_crypto: SecretCrypto,
runtime: RuntimeExecutor,
api_rate_limiter: RequestRateLimiter,
coordination_store: Arc<dyn CoordinationStateStore>,
sessions: SharedSessionStore,
credential_verifier: SharedMachineCredentialVerifier,
max_concurrent_sessions: usize,
artifact_store: crank_artifacts::ArtifactStore,
tasks: Arc<TaskApplication>,
) -> Router {
build_app_inner(
registry,
refresh_interval,
public_base_url,
secret_crypto,
runtime,
api_rate_limiter,
coordination_store,
sessions,
credential_verifier,
max_concurrent_sessions,
true,
Some(artifact_store),
Some(tasks),
)
}
@@ -298,23 +237,8 @@ fn build_app_inner(
max_concurrent_sessions: usize,
start_background_workers: bool,
artifact_store: Option<crank_artifacts::ArtifactStore>,
tasks: Option<Arc<TaskApplication>>,
) -> Router {
let session_metrics = ActiveSessionMetrics::start(Arc::clone(&sessions));
let tasks = tasks.unwrap_or_else(|| {
let store = crank_registry::PostgresTaskStore::new(
registry.pool().clone(),
crank_registry::AdmissionLimits::default(),
crank_core::AdmissionSchedulerConfig::default(),
)
.expect("default admission limits and scheduler weights are valid");
Arc::new(TaskApplication::new(
registry.clone(),
refresh_interval,
Arc::clone(&coordination_store),
crank_runtime::TaskService::new(Arc::new(store)),
))
});
let state = Arc::new(AppState {
registry: registry.clone(),
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
@@ -328,11 +252,6 @@ fn build_app_inner(
credential_verifier,
allowed_origins: AllowedOrigins::new(public_base_url),
logging_levels: Arc::new(crate::protocol::LogLevelStore::new()),
signal_streams: Arc::new(crate::protocol::StreamRegistry::new()),
in_flight: Arc::new(lifecycle::InFlightCalls::new(
lifecycle::MAX_IN_FLIGHT_CALLS,
)),
tasks,
});
if start_background_workers {
spawn_approval_recovery(Arc::clone(&state));
@@ -381,55 +300,24 @@ async fn health() -> Json<Value> {
}
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
let postgres_ready = state.registry.ping().await.is_ok();
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
// reflect it rather than reporting ready on a bare `select 1`.
let admission_ready = state.registry.execution_admission_health().await.is_ok();
// Story 4.2: the durable Task store is mandatory for a durable acceptance acknowledgement.
let durable_tasks_ready = state.tasks.health().await.is_ok();
if postgres_ready && admission_ready && durable_tasks_ready {
return Json(json!({
match state.registry.ping().await {
Ok(()) => Json(json!({
"service": "mcp-server",
"status": "ready",
"checks": {
"postgres": "ready",
"execution_admission": "ready",
"durable_tasks": "ready"
}
"checks": { "postgres": "ready" }
}))
.into_response();
.into_response(),
Err(_) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": "database is unavailable"
})),
)
.into_response(),
}
let mut checks = serde_json::Map::new();
checks.insert(
"postgres".to_owned(),
json!(if postgres_ready { "ready" } else { "not_ready" }),
);
checks.insert(
"execution_admission".to_owned(),
json!(if admission_ready {
"ready"
} else {
"not_ready"
}),
);
checks.insert(
"durable_tasks".to_owned(),
json!(if durable_tasks_ready {
"ready"
} else {
"not_ready"
}),
);
(
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "mcp-server",
"status": "not_ready",
"checks": checks,
"error": "a mandatory dependency is unavailable"
})),
)
.into_response()
}
#[cfg(test)]
@@ -19,31 +19,18 @@ pub(crate) async fn resolve_operation_auth(
workspace_id: &crank_core::WorkspaceId,
execution_config: &crank_core::ExecutionConfig,
) -> Result<Option<ResolvedAuth>, RuntimeError> {
resolve_operation_auth_with(
if execution_config.auth_profile_ref.is_none() {
return Ok(None);
}
let span = Stage::AuthResolve.span();
let result = resolve_runtime_auth_for_task(
&state.registry,
&state.secret_crypto,
workspace_id,
execution_config,
)
.await
}
/// Same resolution for callers that hold the authoritative registry/secret-crypto pair without the
/// HTTP application state (the durable Task worker).
pub(crate) async fn resolve_operation_auth_with(
registry: &PostgresRegistry,
secret_crypto: &SecretCrypto,
workspace_id: &crank_core::WorkspaceId,
execution_config: &crank_core::ExecutionConfig,
) -> Result<Option<ResolvedAuth>, RuntimeError> {
if execution_config.auth_profile_ref.is_none() {
return Ok(None);
}
let span = Stage::AuthResolve.span();
let result =
resolve_runtime_auth_for_task(registry, secret_crypto, workspace_id, execution_config)
.instrument(span.clone())
.await;
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
+20 -102
View File
@@ -23,9 +23,7 @@ use crate::{
},
session::SessionState,
tool_error::execution_locale,
transport::{
ResponseMode, json_response, session_id_from_headers, sse_response, transport_response,
},
transport::{ResponseMode, json_response, session_id_from_headers, transport_response},
};
use axum::{
http::{HeaderMap, StatusCode},
@@ -39,7 +37,8 @@ use time::OffsetDateTime;
use super::response::{
completion_result, internal_jsonrpc_error, prompt_arguments_error_response, prompt_get_result,
prompts_list_result, resource_error_response, resource_read_result,
resource_templates_list_result, resources_list_result, tools_list_result,
resource_templates_list_result, resources_list_result, tool_command_response,
tools_list_result,
};
use super::{AgentRoutePath, AppState, TRANSPORT_SESSION_TTL_MS};
use crate::protocol::{ResourceTemplatesListError, list_resource_templates};
@@ -155,8 +154,6 @@ pub(super) async fn handle_modern_request(
locale: execution_locale(message),
},
ToolsCommand::List(list_command),
// A list command has no progress or cancellation lifecycle (Story 3.4).
None,
)
.await;
match outcome {
@@ -221,35 +218,30 @@ pub(super) async fn handle_modern_request(
// session-less, so its calls cannot advertise elicitation support.
let mut arguments = tool_call_params.arguments.unwrap_or_else(|| json!({}));
let confirmation_token = take_confirmation_token(&mut arguments);
// Story 3.4: the client's own `_meta.progressToken` of this request, validated and bounded.
let progress = crate::app::lifecycle::token_from_meta(tool_call_params.meta.as_ref());
super::lifecycle::run_tool_call(
Arc::clone(state),
crate::app::lifecycle::ToolCallRequest {
workspace_slug: path.workspace_slug.clone(),
agent_slug: path.agent_slug.clone(),
credential: credential.clone(),
correlation: transport_correlation.clone(),
let outcome = handle_tools_command(
state,
ServiceContext {
workspace_slug: &path.workspace_slug,
agent_slug: &path.agent_slug,
credential,
correlation: transport_correlation,
continuation: ContinuationSupport::ElicitationUnsupported,
locale: execution_locale(message),
},
crate::app::lifecycle::ToolCallWire {
message: message.clone(),
protocol_version: version.to_owned(),
response_mode,
progress,
session_id: None,
// The modern revision cancels by closing this request's response stream, so no
// session-scoped cancellation table is involved.
in_flight: None,
},
ToolCallCommand {
ToolsCommand::Call(ToolCallCommand {
tool_name: tool_call_params.name,
arguments,
confirmation_token,
},
}),
)
.await;
tool_command_response(
message,
response_mode,
version,
transport_correlation,
outcome,
)
.await
}
DispatchDecision::Allowed {
handler: MethodHandler::ResourceList,
@@ -666,80 +658,6 @@ pub(super) async fn handle_modern_request(
}
}
}
// Story 3.19: the modern long-lived notification stream. The response is itself the SSE
// stream that stays open, so it is built here instead of going through the one-shot
// response helpers: the acknowledgment is its first frame, then the coalesced list-change
// signals, and closing the HTTP stream ends the producer.
DispatchDecision::Allowed {
handler: MethodHandler::SubscriptionsListen,
..
} => {
// A notification stream is an SSE body by construction: a client that did not offer
// `text/event-stream` could not receive it.
if !crate::transport::accepts_event_stream(headers) {
return StatusCode::NOT_ACCEPTABLE.into_response();
}
let filter: crate::protocol::ListenParams =
match serde_json::from_value(params(message)) {
Ok(filter) => filter,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
crate::protocol::INVALID_LISTEN_FILTER,
),
response_mode,
None,
Some(version),
);
}
};
let accepted = match filter.notifications.accepted() {
crate::protocol::FilterOutcome::Accepted(accepted) => accepted,
crate::protocol::FilterOutcome::Invalid(detail) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(version),
);
}
};
// The subscription id is the JSON-RPC id of this request; a value whose encoding is not
// bounded cannot be a frame identity carrier and is refused before the stream opens.
let Some(subscription_id) = crate::protocol::subscription_id(&request_id(message))
else {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32600, "invalid JSON-RPC message"),
response_mode,
None,
Some(version),
);
};
let Ok(permit) = super::metrics::ActiveStreamGuard::try_acquire(&state.session_slots)
else {
return StatusCode::TOO_MANY_REQUESTS.into_response();
};
let acknowledged = crate::protocol::acknowledged_frame(&subscription_id, accepted);
let receiver = crate::protocol::spawn_signal_watch(
Arc::clone(state),
crate::protocol::Watch {
workspace_slug: path.workspace_slug.clone(),
agent_slug: path.agent_slug.clone(),
signals: accepted,
wire: crate::protocol::Wire::Modern(subscription_id),
legacy_stream: None,
deadline: None,
},
);
let stream = crate::protocol::frames_stream(vec![acknowledged], receiver, permit);
sse_response(StatusCode::OK, stream, None, Some(version))
}
// The standard Prompt requests are registered capabilities now; any other
// unregistered/wrong-era method goes straight to the era-correct unsupported answer.
_ => crate::protocol::modern_method_not_found_response(message),
@@ -144,7 +144,7 @@ fn increment(counter: impl Fn(&CounterSet) -> &AtomicUsize) {
/// execution-boundary reason as the tests below).
#[cfg(test)]
#[path = "../dispatch_observer_support.rs"]
pub(crate) mod support;
mod support;
/// The behavioural section 7.2 harness: real app, real PostgreSQL, one counted loopback upstream and
/// one request-scoped observer per request. Its file lives outside `src/app/` only because the
+708 -65
View File
@@ -3,22 +3,32 @@
//!
//! Moved verbatim out of `app.rs` (Story 3.6, kanban t_987bf309); `mcp_post_response` is reached from
//! `build_app_inner` via `mcp_post`, hence `pub(super)`. No behaviour changed.
//!
//! The legacy-era dispatch itself lives in `super::legacy_dispatch` (Story 3.18 code-health, kanban
//! t_d1b45ddc) to keep this transport entry file inside the 1000-line limit.
use super::metrics::{ActiveStreamGuard, McpRequestMetrics};
use super::request::ToolCallParams;
use super::response::take_confirmation_token;
use super::stages::{enforce_traced_rate_limit, require_traced_machine_access};
use crate::{
jsonrpc::{jsonrpc_error, method_name, request_id},
jsonrpc::{
LEGACY_PROTOCOL_VERSION, is_notification, is_request, jsonrpc_error, jsonrpc_result,
method_name, params, request_id,
},
protocol::{
EnvelopeKind, RequestEra, ambiguous_session_carrier_response, resolve_request_identity,
CompletionCommand, CompletionOutcome, ContinuationSupport, DispatchDecision,
EVENT_LEVEL_APPLIED, EnvelopeKind, INVALID_LEVEL, Kind, LoggingOutcome, MethodHandler,
PromptGetCommand, PromptsCommand, PromptsListCommand, PromptsOutcome, RequestEra,
ResourceReadCommand, ResourcesCommand, ResourcesListCommand, ResourcesOutcome,
ServiceContext, SetLevelCommand, ToolCallCommand, ToolsCommand, ToolsListCommand,
ToolsOutcome, ambiguous_session_carrier_response, handle_completion_command,
handle_logging_command, handle_prompts_command, handle_resources_command,
handle_tools_command, resolve_request_identity,
},
rate_limit::{rate_limited_jsonrpc_response, rate_limited_status_response},
request_context::RequestContext,
tool_error::execution_locale,
transport::{
ResponseMode, accepts_event_stream, negotiate_post_response_mode, session_id_from_headers,
sse_response, transport_response, validate_get_accept_header, validate_origin,
accepts_event_stream, negotiate_post_response_mode, session_id_from_headers, sse_response,
transport_response, validate_get_accept_header, validate_origin,
validate_session_protocol_version, with_request_id_header,
},
};
@@ -26,14 +36,24 @@ use axum::{
Json,
extract::{Extension, Path, State, rejection::JsonRejection},
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
response::{IntoResponse, Response, sse::Event},
};
use crank_core::{CorrelationContext, PlatformApiKeyScope};
use serde_json::Value;
use std::{sync::Arc, time::Duration, time::Instant};
use futures_util::stream;
use serde_json::{Value, json};
use std::{convert::Infallible, sync::Arc, time::Duration};
use tracing::info;
use super::dispatch::handle_modern_request;
use super::dispatch::{
handle_initialize, handle_initialized_notification, handle_modern_request,
require_initialized_session,
};
use super::response::{
completion_result, internal_jsonrpc_error, prompt_arguments_error_response, prompt_get_result,
prompts_list_result, resource_error_response, resource_read_result,
resource_templates_list_result, resources_list_result, tool_command_response,
tools_list_result,
};
use super::{AgentRoutePath, AppState, TRANSPORT_SESSION_TTL_MS};
pub(super) async fn mcp_get(
Path(path): Path<AgentRoutePath>,
@@ -102,32 +122,11 @@ pub(super) async fn mcp_get(
return StatusCode::TOO_MANY_REQUESTS.into_response();
};
// Story 3.19: the pinned legacy transport forbids broadcasting one JSON-RPC message across the
// SSE streams of a session, so exactly one admitted `GET` receives the Resource/Prompt/Tool
// list-change signals. A newer `GET` takes them over and the older stream simply ends; a
// `DELETE` retires the producer, so a stale generation never writes into a reconnect.
let Some(generation) = state.signal_streams.admit(&session_id) else {
// The bounded owner table is contended: refuse this admission instead of waiting.
return StatusCode::TOO_MANY_REQUESTS.into_response();
};
let receiver = crate::protocol::spawn_signal_watch(
Arc::clone(&state),
crate::protocol::Watch {
workspace_slug: path.workspace_slug.clone(),
agent_slug: path.agent_slug.clone(),
signals: crate::protocol::Signals::ALL,
wire: crate::protocol::Wire::Legacy,
legacy_stream: Some((
Arc::clone(&state.signal_streams),
session_id.clone(),
generation,
)),
deadline: Some(Instant::now() + Duration::from_millis(TRANSPORT_SESSION_TTL_MS)),
},
);
// The stream permit lives in the response stream: a client that closes the SSE connection ends
// the producer and releases the slot, and no signal is queued for a reader that is gone.
let stream = crate::protocol::frames_stream(Vec::new(), receiver, permit);
let stream = stream::unfold(permit, |permit| async move {
tokio::time::sleep(Duration::from_millis(TRANSPORT_SESSION_TTL_MS)).await;
drop(permit);
None::<(Result<Event, Infallible>, _)>
});
sse_response(
StatusCode::OK,
@@ -180,7 +179,6 @@ pub(super) async fn mcp_delete(
match state.sessions.delete(&session_id).await {
Ok(true) => {
state.signal_streams.retire(&session_id);
state.session_metrics.refresh();
StatusCode::NO_CONTENT.into_response()
}
@@ -256,7 +254,7 @@ async fn mcp_post_response(
return status.into_response();
}
let mut response_mode = match negotiate_post_response_mode(headers) {
let response_mode = match negotiate_post_response_mode(headers) {
Ok(mode) => mode,
Err(status) => return status.into_response(),
};
@@ -365,19 +363,6 @@ async fn mcp_post_response(
Err(error) => return error.into_response(),
};
// Story 3.4 correction: the official SDK lists the media types JSON-first. For a `tools/call`
// that carries a validated progress token and offered `text/event-stream`, the stream is the only
// carrier that can deliver the progress the client asked for, so the media-type order is not a
// preference for the JSON body here. Every other request — another method, no token, an unusable
// token, a JSON-only `Accept` — keeps the negotiated mode, and no admission check moved.
if matches!(response_mode, ResponseMode::Json)
&& method_name(message) == Some("tools/call")
&& accepts_event_stream(headers)
&& crate::app::lifecycle::token_from_meta(message.pointer("/params/_meta")).is_some()
{
response_mode = ResponseMode::Sse;
}
if matches!(identity.era, RequestEra::Modern) {
// ERA_MODERN_ENTER (spec section 7.2): this request entered the modern era-adapter.
#[cfg(test)]
@@ -399,17 +384,675 @@ async fn mcp_post_response(
#[cfg(test)]
crate::app::dispatch_observer::era_legacy_enter();
// R5/F5: the owning registration decides which handler serves this method in the request's
// revision; that legacy dispatch lives in `super::legacy_dispatch`.
super::legacy_dispatch::handle_legacy_request(
state,
path,
headers,
message,
response_mode,
&identity,
&credential,
transport_correlation,
)
.await
// R5/F5: the owning registration decides *which handler* serves this method in the request's
// revision. The adapter holds no method list of its own — the method-to-handler mapping lives in
// `protocol::registry` and is read through `dispatch_decision`, so a newly registered method
// really reaches its handler and removing a descriptor removes the dispatch.
let registration_revision = protocol_version
.as_deref()
.unwrap_or(LEGACY_PROTOCOL_VERSION);
let dispatch = crate::protocol::dispatch_decision(
method_name(message).unwrap_or_default(),
registration_revision,
);
match dispatch {
// G2: the *registered kind* of the matched method, not the envelope shape, decides which
// legacy lifecycle action runs. `initialize` is a request and `notifications/initialized` is a
// notification: the two share one handler but are not interchangeable, so an `initialize`
// notification may not initialize an existing session and a `notifications/initialized`
// request may not create one.
DispatchDecision::Allowed {
handler: MethodHandler::Lifecycle,
kind: Kind::Method,
} if is_request(message) => handle_initialize(state, path, message, response_mode).await,
DispatchDecision::Allowed {
handler: MethodHandler::Lifecycle,
kind: Kind::Notification,
} if is_notification(message) => {
handle_initialized_notification(state, path, headers).await
}
DispatchDecision::Allowed {
handler: MethodHandler::Ping,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
json!({ "protocolVersion": session.protocol_version }),
),
response_mode,
None,
Some(&session.protocol_version),
)
}
DispatchDecision::Allowed {
handler: MethodHandler::ToolList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
// The adapter parses the wire form and maps the neutral outcome back; the shared Tools
// service never sees the envelope, the response mode or the protocol revision.
let list_command: ToolsListCommand = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, "invalid tools/list parameters"),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_tools_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential: &credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
ToolsCommand::List(list_command),
)
.await;
match outcome {
ToolsOutcome::List { tools, next_cursor } => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), tools_list_result(tools, next_cursor)),
response_mode,
None,
Some(&session.protocol_version),
),
ToolsOutcome::InvalidParams(message_text) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, message_text),
response_mode,
None,
Some(&session.protocol_version),
),
ToolsOutcome::CatalogUnavailable => {
internal_jsonrpc_error(message, "published catalog is unavailable")
}
// A list command never produces a call outcome; it is answered with the adapter's
// internal error instead of being silently ignored.
ToolsOutcome::Call(_) => internal_jsonrpc_error(
message,
"unexpected shared tools outcome for a tools/list",
),
}
}
DispatchDecision::Allowed {
handler: MethodHandler::ToolCall,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let tool_call_params: ToolCallParams = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, "invalid tool call parameters"),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
// The adapter owns the wire form: it parses the call parameters, removes the reserved
// confirmation marker and the shared Tools service never sees the envelope, the
// response mode or the protocol revision (spec section 6.2).
let mut arguments = tool_call_params.arguments.unwrap_or_else(|| json!({}));
let confirmation_token = take_confirmation_token(&mut arguments);
let outcome = handle_tools_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential: &credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
ToolsCommand::Call(ToolCallCommand {
tool_name: tool_call_params.name,
arguments,
confirmation_token,
}),
)
.await;
tool_command_response(
message,
response_mode,
&session.protocol_version,
transport_correlation,
outcome,
)
}
DispatchDecision::Allowed {
handler: MethodHandler::ResourceList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let list_command: ResourcesListCommand = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/list parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_resources_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential: &credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
ResourcesCommand::List(list_command),
)
.await;
match outcome {
ResourcesOutcome::Listed {
resources,
next_cursor,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
resources_list_result(resources, next_cursor),
),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
_ => internal_jsonrpc_error(message, "published resources are unavailable"),
}
}
DispatchDecision::Allowed {
handler: MethodHandler::ResourceRead,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let read_command: ResourceReadCommand = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/read parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_resources_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential: &credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
ResourcesCommand::Read(read_command),
)
.await;
match outcome {
ResourcesOutcome::Read { contents } => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), resource_read_result(contents)),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::NotAvailable => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"the requested resource is not available",
),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::ExecutionError(contract) => resource_error_response(
message,
response_mode,
&session.protocol_version,
contract,
),
_ => internal_jsonrpc_error(message, "published resources are unavailable"),
}
}
// Story 3.11: the shared service returns exact bound Published templates; the legacy adapter
// retains its earlier result shape and still requires the initialized session.
DispatchDecision::Allowed {
handler: MethodHandler::ResourceTemplateList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: ResourcesListCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/templates/list parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let (templates, next_cursor) = match crate::protocol::list_resource_templates(
&state,
&path.workspace_slug,
&path.agent_slug,
&session.protocol_version,
command,
)
.await
{
Ok(page) => page,
Err(crate::protocol::ResourceTemplatesListError::InvalidCursor) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/templates/list cursor",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
Err(crate::protocol::ResourceTemplatesListError::Unavailable) => {
return internal_jsonrpc_error(message, "published templates are unavailable");
}
};
transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
resource_templates_list_result(templates, next_cursor),
),
response_mode,
None,
Some(&session.protocol_version),
)
}
// Story 3.14: `prompts/list` / `prompts/get` of the initialized legacy session, served from
// the same exact bound Published Prompt Versions as the modern era.
DispatchDecision::Allowed {
handler: MethodHandler::PromptList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: PromptsListCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid prompts/list parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_prompts_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential: &credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
PromptsCommand::List(command),
)
.await;
match outcome {
PromptsOutcome::Listed {
prompts,
next_cursor,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
prompts_list_result(prompts, next_cursor),
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
_ => internal_jsonrpc_error(message, "published prompts are unavailable"),
}
}
DispatchDecision::Allowed {
handler: MethodHandler::PromptGet,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: PromptGetCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid prompts/get parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_prompts_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential: &credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
PromptsCommand::Get(command),
)
.await;
match outcome {
PromptsOutcome::Got {
description,
messages,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
prompt_get_result(description, messages),
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::NotAvailable => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"the requested prompt is not available",
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::ArgumentsRejected { reason, argument } => {
prompt_arguments_error_response(
message,
response_mode,
&session.protocol_version,
reason,
argument,
)
}
PromptsOutcome::DependencyUnavailable { position } => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32603,
format!("the pinned dependency at dependencies[{position}] is unavailable"),
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
_ => internal_jsonrpc_error(message, "published prompts are unavailable"),
}
}
// Story 3.17: `completion/complete` of the initialized legacy session. The shared service is
// the same one the modern era uses; only the result envelope differs (no `resultType`).
DispatchDecision::Allowed {
handler: MethodHandler::Completion,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: CompletionCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid completion/complete parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_completion_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential: &credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
command,
)
.await;
match outcome {
CompletionOutcome::Completed {
values,
total,
has_more,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
completion_result(values, total, has_more),
),
response_mode,
None,
Some(&session.protocol_version),
),
CompletionOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
CompletionOutcome::NotAvailable => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"the completion reference is not available",
),
response_mode,
None,
Some(&session.protocol_version),
),
CompletionOutcome::CatalogUnavailable => {
internal_jsonrpc_error(message, "published completion sources are unavailable")
}
}
}
// Story 3.18: the pinned legacy `logging/setLevel`. The level belongs to the initialized
// session of this request, so setting it can never change another client's level. The request
// answers an empty result; the acknowledgment event is filtered by the level it just set.
DispatchDecision::Allowed {
handler: MethodHandler::LoggingSetLevel,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: SetLevelCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
// A non-string, missing or otherwise unparsable level shares the one safe detail of a
// value outside the pinned union: nothing the client sent is echoed back.
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, INVALID_LEVEL),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
match handle_logging_command(&state.logging_levels, &session.id, command) {
LoggingOutcome::Applied(level) => {
crate::protocol::set_active_level(Some(level));
crate::protocol::note(
crate::protocol::LogLevel::Notice,
EVENT_LEVEL_APPLIED,
"applied",
);
transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), json!({})),
response_mode,
None,
Some(&session.protocol_version),
)
}
LoggingOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
}
}
// A notification whose method the registration does not map to this adapter's handlers is
// acknowledged without being dispatched, exactly as before.
_ if is_notification(message) => StatusCode::ACCEPTED.into_response(),
// Every remaining decision (unregistered, wrong era, or a handler this adapter does not
// serve) is a method-level rejection: no handler, no service entry, no outbound call.
_ => match method_name(message) {
Some(method) => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32601,
format!("method {method} is not supported"),
),
response_mode,
None,
protocol_version.as_deref(),
),
None => transport_response(
StatusCode::BAD_REQUEST,
jsonrpc_error(Value::Null, -32600, "invalid JSON-RPC message"),
response_mode,
None,
protocol_version.as_deref(),
),
},
}
}
@@ -1,794 +0,0 @@
//! Legacy-era JSON-RPC dispatch: the registered method-to-handler mapping, the
//! initialized-session guard and every legacy envelope the transport endpoint answers.
//!
//! Moved verbatim out of `app/endpoint.rs` (Story 3.18 code-health, kanban t_d1b45ddc) so the
//! transport entry file stays inside the 1000-line limit; the entry point is `pub(super)`.
//! No behaviour changed.
use std::sync::Arc;
use axum::{
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
};
use crank_core::CorrelationContext;
use serde_json::{Value, json};
use super::dispatch::{
handle_initialize, handle_initialized_notification, require_initialized_session,
};
use super::request::ToolCallParams;
use super::response::{
completion_result, internal_jsonrpc_error, prompt_arguments_error_response, prompt_get_result,
prompts_list_result, resource_error_response, resource_read_result,
resource_templates_list_result, resources_list_result, take_confirmation_token,
tools_list_result,
};
use super::{AgentRoutePath, AppState};
use crate::{
auth::VerifiedMachineCredential,
jsonrpc::{
LEGACY_PROTOCOL_VERSION, is_notification, is_request, jsonrpc_error, jsonrpc_result,
method_name, params, request_id,
},
protocol::{
CompletionCommand, CompletionOutcome, ContinuationSupport, DispatchDecision,
EVENT_LEVEL_APPLIED, INVALID_LEVEL, Kind, LoggingOutcome, MethodHandler, PromptGetCommand,
PromptsCommand, PromptsListCommand, PromptsOutcome, RequestIdentity, ResourceReadCommand,
ResourcesCommand, ResourcesListCommand, ResourcesOutcome, ServiceContext, SetLevelCommand,
ToolCallCommand, ToolsCommand, ToolsListCommand, ToolsOutcome, handle_completion_command,
handle_logging_command, handle_prompts_command, handle_resources_command,
handle_tools_command,
},
tool_error::execution_locale,
transport::{ResponseMode, session_id_from_headers, transport_response},
};
// The transport invariants of one request travel together here; a parameter struct would only
// duplicate them (the same shape every other dispatch entry point in this crate uses).
#[allow(clippy::too_many_arguments)]
pub(super) async fn handle_legacy_request(
state: Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
message: &Value,
response_mode: ResponseMode,
identity: &RequestIdentity,
credential: &VerifiedMachineCredential,
transport_correlation: &CorrelationContext,
) -> Response {
let protocol_version = identity.version.as_deref();
// R5/F5: the owning registration decides *which handler* serves this method in the request's
// revision. The adapter holds no method list of its own — the method-to-handler mapping lives in
// `protocol::registry` and is read through `dispatch_decision`, so a newly registered method
// really reaches its handler and removing a descriptor removes the dispatch.
let registration_revision = protocol_version.unwrap_or(LEGACY_PROTOCOL_VERSION);
let dispatch = crate::protocol::dispatch_decision(
method_name(message).unwrap_or_default(),
registration_revision,
);
match dispatch {
// G2: the *registered kind* of the matched method, not the envelope shape, decides which
// legacy lifecycle action runs. `initialize` is a request and `notifications/initialized` is a
// notification: the two share one handler but are not interchangeable, so an `initialize`
// notification may not initialize an existing session and a `notifications/initialized`
// request may not create one.
DispatchDecision::Allowed {
handler: MethodHandler::Lifecycle,
kind: Kind::Method,
} if is_request(message) => handle_initialize(state, path, message, response_mode).await,
DispatchDecision::Allowed {
handler: MethodHandler::Lifecycle,
kind: Kind::Notification,
} if is_notification(message) => {
handle_initialized_notification(state, path, headers).await
}
DispatchDecision::Allowed {
handler: MethodHandler::Ping,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
json!({ "protocolVersion": session.protocol_version }),
),
response_mode,
None,
Some(&session.protocol_version),
)
}
DispatchDecision::Allowed {
handler: MethodHandler::ToolList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
// The adapter parses the wire form and maps the neutral outcome back; the shared Tools
// service never sees the envelope, the response mode or the protocol revision.
let list_command: ToolsListCommand = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, "invalid tools/list parameters"),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_tools_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
ToolsCommand::List(list_command),
// A list command has no progress or cancellation lifecycle (Story 3.4).
None,
)
.await;
match outcome {
ToolsOutcome::List { tools, next_cursor } => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), tools_list_result(tools, next_cursor)),
response_mode,
None,
Some(&session.protocol_version),
),
ToolsOutcome::InvalidParams(message_text) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, message_text),
response_mode,
None,
Some(&session.protocol_version),
),
ToolsOutcome::CatalogUnavailable => {
internal_jsonrpc_error(message, "published catalog is unavailable")
}
// A list command never produces a call outcome; it is answered with the adapter's
// internal error instead of being silently ignored.
ToolsOutcome::Call(_) => internal_jsonrpc_error(
message,
"unexpected shared tools outcome for a tools/list",
),
}
}
DispatchDecision::Allowed {
handler: MethodHandler::ToolCall,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let tool_call_params: ToolCallParams = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, "invalid tool call parameters"),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
// The adapter owns the wire form: it parses the call parameters, removes the reserved
// confirmation marker and the shared Tools service never sees the envelope, the
// response mode or the protocol revision (spec section 6.2).
let mut arguments = tool_call_params.arguments.unwrap_or_else(|| json!({}));
let confirmation_token = take_confirmation_token(&mut arguments);
// Story 3.4: the client's own `_meta.progressToken`, validated and bounded, and the
// session/credential ownership of the in-flight cancellation table.
let progress = crate::app::lifecycle::token_from_meta(tool_call_params.meta.as_ref());
super::lifecycle::run_tool_call(
Arc::clone(&state),
crate::app::lifecycle::ToolCallRequest {
workspace_slug: session.workspace_slug.clone(),
agent_slug: session.agent_slug.clone(),
credential: credential.clone(),
correlation: transport_correlation.clone(),
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
crate::app::lifecycle::ToolCallWire {
message: message.clone(),
protocol_version: session.protocol_version.clone(),
response_mode,
progress,
session_id: Some(session.id.clone()),
in_flight: Some(crate::app::lifecycle::InFlightScope {
session_id: session.id.clone(),
owner: in_flight_owner(credential),
}),
},
ToolCallCommand {
tool_name: tool_call_params.name,
arguments,
confirmation_token,
},
)
.await
}
DispatchDecision::Allowed {
handler: MethodHandler::Cancelled,
kind: Kind::Notification,
} if is_notification(message) => {
handle_cancelled_notification(&state, path, headers, message, credential).await
}
DispatchDecision::Allowed {
handler: MethodHandler::ResourceList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let list_command: ResourcesListCommand = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/list parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_resources_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
ResourcesCommand::List(list_command),
)
.await;
match outcome {
ResourcesOutcome::Listed {
resources,
next_cursor,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
resources_list_result(resources, next_cursor),
),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
_ => internal_jsonrpc_error(message, "published resources are unavailable"),
}
}
DispatchDecision::Allowed {
handler: MethodHandler::ResourceRead,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let read_command: ResourceReadCommand = match serde_json::from_value(params(message)) {
Ok(value) => value,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/read parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_resources_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
ResourcesCommand::Read(read_command),
)
.await;
match outcome {
ResourcesOutcome::Read { contents } => transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), resource_read_result(contents)),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::NotAvailable => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"the requested resource is not available",
),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
ResourcesOutcome::ExecutionError(contract) => resource_error_response(
message,
response_mode,
&session.protocol_version,
contract,
),
_ => internal_jsonrpc_error(message, "published resources are unavailable"),
}
}
// Story 3.11: the shared service returns exact bound Published templates; the legacy adapter
// retains its earlier result shape and still requires the initialized session.
DispatchDecision::Allowed {
handler: MethodHandler::ResourceTemplateList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: ResourcesListCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/templates/list parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let (templates, next_cursor) = match crate::protocol::list_resource_templates(
&state,
&path.workspace_slug,
&path.agent_slug,
&session.protocol_version,
command,
)
.await
{
Ok(page) => page,
Err(crate::protocol::ResourceTemplatesListError::InvalidCursor) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid resources/templates/list cursor",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
Err(crate::protocol::ResourceTemplatesListError::Unavailable) => {
return internal_jsonrpc_error(message, "published templates are unavailable");
}
};
transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
resource_templates_list_result(templates, next_cursor),
),
response_mode,
None,
Some(&session.protocol_version),
)
}
// Story 3.14: `prompts/list` / `prompts/get` of the initialized legacy session, served from
// the same exact bound Published Prompt Versions as the modern era.
DispatchDecision::Allowed {
handler: MethodHandler::PromptList,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: PromptsListCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid prompts/list parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_prompts_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
PromptsCommand::List(command),
)
.await;
match outcome {
PromptsOutcome::Listed {
prompts,
next_cursor,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
prompts_list_result(prompts, next_cursor),
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
_ => internal_jsonrpc_error(message, "published prompts are unavailable"),
}
}
DispatchDecision::Allowed {
handler: MethodHandler::PromptGet,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: PromptGetCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid prompts/get parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_prompts_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
PromptsCommand::Get(command),
)
.await;
match outcome {
PromptsOutcome::Got {
description,
messages,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
prompt_get_result(description, messages),
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::NotAvailable => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"the requested prompt is not available",
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::ArgumentsRejected { reason, argument } => {
prompt_arguments_error_response(
message,
response_mode,
&session.protocol_version,
reason,
argument,
)
}
PromptsOutcome::DependencyUnavailable { position } => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32603,
format!("the pinned dependency at dependencies[{position}] is unavailable"),
),
response_mode,
None,
Some(&session.protocol_version),
),
PromptsOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
_ => internal_jsonrpc_error(message, "published prompts are unavailable"),
}
}
// Story 3.17: `completion/complete` of the initialized legacy session. The shared service is
// the same one the modern era uses; only the result envelope differs (no `resultType`).
DispatchDecision::Allowed {
handler: MethodHandler::Completion,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: CompletionCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"invalid completion/complete parameters",
),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
let outcome = handle_completion_command(
&state,
ServiceContext {
workspace_slug: &session.workspace_slug,
agent_slug: &session.agent_slug,
credential,
correlation: transport_correlation,
continuation: ContinuationSupport::from_supported(session.supports_elicitation),
locale: execution_locale(message),
},
&session.protocol_version,
command,
)
.await;
match outcome {
CompletionOutcome::Completed {
values,
total,
has_more,
} => transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
completion_result(values, total, has_more),
),
response_mode,
None,
Some(&session.protocol_version),
),
CompletionOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
CompletionOutcome::NotAvailable => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32602,
"the completion reference is not available",
),
response_mode,
None,
Some(&session.protocol_version),
),
CompletionOutcome::CatalogUnavailable => {
internal_jsonrpc_error(message, "published completion sources are unavailable")
}
}
}
// Story 3.18: the pinned legacy `logging/setLevel`. The level belongs to the initialized
// session of this request, so setting it can never change another client's level. The request
// answers an empty result; the acknowledgment event is filtered by the level it just set.
DispatchDecision::Allowed {
handler: MethodHandler::LoggingSetLevel,
..
} if is_request(message) => {
let session = match require_initialized_session(&state, path, headers, message).await {
Ok(session) => session,
Err(response) => return response,
};
let command: SetLevelCommand = match serde_json::from_value(params(message)) {
Ok(command) => command,
// A non-string, missing or otherwise unparsable level shares the one safe detail of a
// value outside the pinned union: nothing the client sent is echoed back.
Err(_) => {
return transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, INVALID_LEVEL),
response_mode,
None,
Some(&session.protocol_version),
);
}
};
match handle_logging_command(&state.logging_levels, &session.id, command) {
LoggingOutcome::Applied(level) => {
crate::protocol::set_active_level(Some(level));
crate::protocol::note(
crate::protocol::LogLevel::Notice,
EVENT_LEVEL_APPLIED,
"applied",
);
transport_response(
StatusCode::OK,
jsonrpc_result(request_id(message), json!({})),
response_mode,
None,
Some(&session.protocol_version),
)
}
LoggingOutcome::InvalidParams(detail) => transport_response(
StatusCode::OK,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(&session.protocol_version),
),
}
}
// A notification whose method the registration does not map to this adapter's handlers is
// acknowledged without being dispatched, exactly as before.
_ if is_notification(message) => StatusCode::ACCEPTED.into_response(),
// Every remaining decision (unregistered, wrong era, or a handler this adapter does not
// serve) is a method-level rejection: no handler, no service entry, no outbound call.
_ => match method_name(message) {
Some(method) => transport_response(
StatusCode::OK,
jsonrpc_error(
request_id(message),
-32601,
format!("method {method} is not supported"),
),
response_mode,
None,
protocol_version,
),
None => transport_response(
StatusCode::BAD_REQUEST,
jsonrpc_error(Value::Null, -32600, "invalid JSON-RPC message"),
response_mode,
None,
protocol_version,
),
},
}
}
/// The credential half of the cancellation ownership of one call (Story 3.4). The progress token is
/// deliberately not part of it: a token never authorizes a cancellation.
fn in_flight_owner(credential: &VerifiedMachineCredential) -> String {
credential
.platform_api_key_id
.as_ref()
.map_or_else(String::new, |id| id.as_str().to_owned())
}
/// Route one legacy `notifications/cancelled` to the call it names, then acknowledge it.
///
/// The pinned rules of a cancelled request are all "ignore": an unknown or already finished request,
/// a foreign session or credential, a malformed `requestId` and an unknown session are acknowledged
/// without any effect. That is also why this handler never fails the transport — a client that
/// cancels must not learn anything about another client's calls.
async fn handle_cancelled_notification(
state: &Arc<AppState>,
path: &AgentRoutePath,
headers: &HeaderMap,
message: &Value,
credential: &VerifiedMachineCredential,
) -> Response {
let request_id = params(message).get("requestId").cloned();
let session_id = session_id_from_headers(headers).ok().flatten();
if let (Some(request_id), Some(session_id)) = (request_id, session_id) {
// Only a string or a number is a pinned `RequestId`; anything else cannot name a call.
if matches!(request_id, Value::String(_) | Value::Number(_)) {
let owned_by_session = match state.sessions.get(&session_id).await {
Ok(Some(session)) => {
session.workspace_slug == path.workspace_slug
&& session.agent_slug == path.agent_slug
}
_ => false,
};
if owned_by_session {
state.in_flight.cancel(
&crate::app::lifecycle::InFlightScope {
session_id,
owner: in_flight_owner(credential),
},
&request_id,
);
}
}
}
StatusCode::ACCEPTED.into_response()
}
@@ -1,566 +0,0 @@
//! Story 3.4 (FR-14): request-scoped progress and cancellation of one synchronous `tools/call`.
//!
//! The pinned `RequestMetaObject` of *both* supported revisions carries an optional opaque
//! `progressToken`; a request that has one asks for out-of-band `notifications/progress` messages
//! about that request. Such a message only has a carrier on the request's own SSE response stream, so
//! a token is honoured only when the client negotiated `text/event-stream`. Without a token — or
//! without the stream — the call runs exactly as before: the receiver of a progress token is never
//! obligated to send progress, so the execution result cannot change.
//!
//! Cancellation follows the revision-specific rule of the pinned specification:
//! * `2025-11-25`: the client POSTs `notifications/cancelled` with the id of the request it issued.
//! The bounded in-flight table routes it to that call. Ownership is the session *and* the platform
//! credential of the call, never the progress token: a foreign session or key cancels nothing.
//! * `2026-07-28`: closing the request's SSE response stream *is* the cancellation signal, so there is
//! no notification to route; dropping the response stream drops the call future.
//!
//! Either way a cancelled call never becomes a false `completed`: no further message is sent for it
//! (the pinned rule for a cancelled request), and the call never reaches its terminal outcome, so no
//! Invocation History row claims a completion. The record contract only admits a *closed* execution
//! error code for a non-`ok` row and a cancellation is not one of them, so the honest state of a
//! stopped call is "no terminal record" rather than an invented error code; a dedicated cancellation
//! code would be a schema/vocabulary decision of its own.
use std::{
collections::{HashMap, VecDeque},
future::Future,
pin::Pin,
sync::{Arc, Mutex},
};
use axum::{
http::StatusCode,
response::{IntoResponse, Response, sse::Event},
};
use crank_core::{CorrelationContext, ExecutionLocale, VerifiedMachineCredential};
use futures_util::stream;
use serde_json::{Number, Value, json};
use tokio::sync::{mpsc, watch};
use super::{
AppState,
response::{ToolCommandPayload, tool_command_payload, tool_command_response},
};
use crate::{
jsonrpc::request_id,
protocol::{
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsOutcome,
handle_tools_command,
},
transport::{ResponseMode, sse_response},
};
/// The bounded progress queue of one call. A client that reads its stream slowly can never block the
/// call: a full queue drops the event instead of waiting, and the remaining events stay monotonic.
pub(crate) const PROGRESS_CAPACITY: usize = 8;
/// The pinned `ProgressToken` bound this server accepts. A longer token is not rejected as an error:
/// progress is simply disabled for that request, exactly as for a malformed one.
pub(crate) const MAX_PROGRESS_TOKEN_LEN: usize = 128;
/// The bounded table of calls that can still be cancelled. A call that finds the table full runs
/// without cancellation instead of growing the table.
pub(crate) const MAX_IN_FLIGHT_CALLS: usize = 256;
/// A validated `RequestMetaObject.progressToken`: the pinned opaque token is a string or a number.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum ProgressToken {
Text(String),
Number(Number),
}
impl ProgressToken {
/// `None` means "disable progress", never "fail the call": the pinned schema declares the token
/// as an opaque client value the receiver may simply ignore.
fn parse(value: &Value) -> Option<Self> {
match value {
Value::String(text)
if !text.is_empty()
&& text.len() <= MAX_PROGRESS_TOKEN_LEN
&& !text.chars().any(char::is_control) =>
{
Some(Self::Text(text.clone()))
}
Value::Number(number) => Some(Self::Number(number.clone())),
_ => None,
}
}
fn as_value(&self) -> Value {
match self {
Self::Text(text) => Value::String(text.clone()),
Self::Number(number) => Value::Number(number.clone()),
}
}
}
/// The progress token of one request parameters object, already validated and bounded. The adapters
/// read the `_meta` object they already parsed ([`token_from_meta`]); this is the test seam of the
/// same rule.
#[cfg(test)]
pub(crate) fn progress_token(params: &Value) -> Option<ProgressToken> {
token_from_meta(params.get("_meta"))
}
/// The validated token of one `RequestMetaObject`. A non-object `_meta` has no token.
pub(crate) fn token_from_meta(meta: Option<&Value>) -> Option<ProgressToken> {
meta.and_then(|meta| meta.get("progressToken"))
.and_then(ProgressToken::parse)
}
/// One real, already reached execution step: the shared Tools service emits these at the stage
/// boundaries it actually crosses, so no percentage is invented and no timer drives them.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Phase {
Resolved,
Authorized,
Dispatching,
}
impl Phase {
const fn message(self) -> &'static str {
match self {
Self::Resolved => "tool_resolved",
Self::Authorized => "authorization_resolved",
Self::Dispatching => "execution_dispatched",
}
}
}
#[derive(Clone, Debug)]
pub(crate) struct ProgressEvent {
progress: u64,
message: &'static str,
}
/// The pinned `notifications/progress` frame. Both supported revisions declare the same parameters,
/// so the projection is shared; the token is echoed verbatim and never validated as a credential.
pub(super) fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
json!({
"jsonrpc": "2.0",
"method": "notifications/progress",
"params": {
"progressToken": token.as_value(),
"progress": event.progress,
"message": event.message,
}
})
}
#[derive(Clone)]
struct ProgressSink {
events: mpsc::Sender<ProgressEvent>,
produced: Arc<std::sync::atomic::AtomicU64>,
}
impl ProgressSink {
fn emit(&self, message: &'static str) {
let progress = self
.produced
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
+ 1;
let _ = self.events.try_send(ProgressEvent { progress, message });
}
}
/// A cooperative cancellation signal of one call. It is set at most once; a signal whose only sender
/// is gone simply never fires.
#[derive(Clone)]
pub(crate) struct CancelSignal {
receiver: watch::Receiver<bool>,
}
impl CancelSignal {
fn never() -> Self {
let (_sender, receiver) = watch::channel(false);
Self { receiver }
}
pub(crate) async fn cancelled(&mut self) {
if self
.receiver
.wait_for(|cancelled| *cancelled)
.await
.is_err()
{
// The owning call is gone, so nothing can cancel it any more.
std::future::pending::<()>().await;
}
}
}
/// The request-scoped lifecycle of one `tools/call`: its progress sink (when the client asked for
/// progress *and* offered the stream carrier) and its cancellation signal.
#[derive(Clone)]
pub(crate) struct ToolCallLifecycle {
progress: Option<ProgressSink>,
cancel: CancelSignal,
}
impl ToolCallLifecycle {
pub(crate) fn new(
token: Option<ProgressToken>,
) -> (Self, Option<mpsc::Receiver<ProgressEvent>>) {
match token {
Some(_) => {
let (sender, receiver) = mpsc::channel(PROGRESS_CAPACITY);
(
Self {
progress: Some(ProgressSink {
events: sender,
produced: Arc::new(std::sync::atomic::AtomicU64::new(0)),
}),
cancel: CancelSignal::never(),
},
Some(receiver),
)
}
None => (
Self {
progress: None,
cancel: CancelSignal::never(),
},
None,
),
}
}
pub(crate) fn with_cancel(mut self, cancel: CancelSignal) -> Self {
self.cancel = cancel;
self
}
/// Record one real execution step. Without a progress sink this is a no-op, so the shared service
/// does not branch on the presence of a token.
pub(crate) fn advance(&self, phase: Phase) {
if let Some(progress) = &self.progress {
progress.emit(phase.message());
}
}
pub(crate) fn cancelled(&mut self) -> impl Future<Output = ()> + '_ {
self.cancel.cancelled()
}
}
/// The owned identity of one call, so the call future can be driven by the response stream itself.
pub(crate) struct ToolCallRequest {
pub(crate) workspace_slug: String,
pub(crate) agent_slug: String,
pub(crate) credential: VerifiedMachineCredential,
pub(crate) correlation: CorrelationContext,
pub(crate) continuation: ContinuationSupport,
pub(crate) locale: ExecutionLocale,
}
/// The wire facts the adapter owns: the envelope, the negotiated revision and response mode, the
/// validated token and — for the legacy revision — the session-scoped in-flight ownership.
pub(crate) struct ToolCallWire {
pub(crate) message: Value,
pub(crate) protocol_version: String,
pub(crate) response_mode: ResponseMode,
pub(crate) progress: Option<ProgressToken>,
pub(crate) session_id: Option<String>,
pub(crate) in_flight: Option<InFlightScope>,
}
/// Who owns a cancellable call. The session and the platform credential are the ownership of the
/// legacy revision; the progress token is deliberately not part of it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct InFlightScope {
pub(crate) session_id: String,
pub(crate) owner: String,
}
/// The bounded table of the calls a legacy client can still cancel.
pub(crate) struct InFlightCalls {
entries: Mutex<HashMap<String, watch::Sender<bool>>>,
capacity: usize,
}
impl InFlightCalls {
pub(crate) fn new(capacity: usize) -> Self {
Self {
entries: Mutex::new(HashMap::new()),
capacity,
}
}
fn entries(&self) -> std::sync::MutexGuard<'_, HashMap<String, watch::Sender<bool>>> {
// A panic of another request must not make the cancellation table unusable.
self.entries
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn key(scope: &InFlightScope, request_id: &Value) -> String {
// The pinned `RequestId` is a string or a number; the two must not collapse into one key.
format!(
"{}\u{1f}{}\u{1f}{request_id}",
scope.session_id, scope.owner
)
}
/// Register one call as cancellable. `None` means the table is full (or the id is already live),
/// and the call then runs without cancellation instead of evicting someone else's entry.
pub(crate) fn register(
self: &Arc<Self>,
scope: &InFlightScope,
request_id: &Value,
) -> Option<InFlightEntry> {
let key = Self::key(scope, request_id);
let (sender, receiver) = watch::channel(false);
{
let mut entries = self.entries();
if entries.len() >= self.capacity || entries.contains_key(&key) {
return None;
}
entries.insert(key.clone(), sender);
}
Some(InFlightEntry {
registry: Arc::clone(self),
key,
cancel: CancelSignal { receiver },
})
}
/// Route one `notifications/cancelled` to the call it names. `false` means there is no such
/// in-flight call of this owner, which the specification requires to be ignored.
pub(crate) fn cancel(&self, scope: &InFlightScope, request_id: &Value) -> bool {
let entries = self.entries();
entries
.get(&Self::key(scope, request_id))
.is_some_and(|sender| sender.send(true).is_ok())
}
}
/// The live registration of one call: dropping it (terminal response, cancellation or a dropped
/// response stream) releases the table slot.
pub(crate) struct InFlightEntry {
registry: Arc<InFlightCalls>,
key: String,
cancel: CancelSignal,
}
impl InFlightEntry {
pub(crate) fn cancel_signal(&self) -> CancelSignal {
self.cancel.clone()
}
}
impl Drop for InFlightEntry {
fn drop(&mut self) {
self.registry.entries().remove(&self.key);
}
}
/// Run one `tools/call` through the shared Tools service with the lifecycle of this request.
///
/// Without a progress token — or without the SSE carrier the client negotiated — the response is the
/// unchanged single JSON (or single-event SSE) response. With both, the response is the request's own
/// bounded stream: the real progress events of this call, then the terminal response as the last
/// event. Dropping that stream (a `2026-07-28` cancellation) or setting the signal of a legacy
/// `notifications/cancelled` drops the call future, which is the cooperative stop.
pub(crate) async fn run_tool_call(
state: Arc<AppState>,
request: ToolCallRequest,
wire: ToolCallWire,
command: ToolCallCommand,
) -> Response {
// A token is only honoured when the client offered the only carrier a notification has: the
// request's own SSE stream. Otherwise the call runs unchanged (a receiver is never obligated).
let token = match (&wire.response_mode, &wire.progress) {
(ResponseMode::Sse, Some(token)) => Some(token.clone()),
_ => None,
};
let (mut lifecycle, progress_events) = ToolCallLifecycle::new(token.clone());
let in_flight = wire.in_flight.as_ref().and_then(|scope| {
let entry = state.in_flight.register(scope, &request_id(&wire.message));
if let Some(entry) = &entry {
lifecycle = lifecycle.clone().with_cancel(entry.cancel_signal());
}
entry
});
let message = wire.message.clone();
let correlation = request.correlation.clone();
let protocol_version = wire.protocol_version.clone();
let response_mode = wire.response_mode;
let mut call = Box::pin(call_future(
Arc::clone(&state),
request,
lifecycle.clone(),
command,
));
let Some(progress_events) = progress_events else {
// No notification carrier for this call: run it as before, still cancellable.
let outcome = tokio::select! {
biased;
outcome = &mut call => outcome,
_ = lifecycle.cancelled() => return cancelled_response(),
};
return tool_command_response(
&message,
response_mode,
&protocol_version,
&correlation,
outcome,
);
};
debug_assert!(matches!(response_mode, ResponseMode::Sse));
let token = token.expect("a progress receiver implies a validated token");
let stream = stream::unfold(
CallStream {
call,
events: progress_events,
cancel: lifecycle.cancel.clone(),
token,
message,
protocol_version: protocol_version.clone(),
correlation,
pending: VecDeque::new(),
finished: false,
logging: crate::protocol::carry_scope(),
_in_flight: in_flight,
},
next_event,
);
sse_response(
StatusCode::OK,
stream,
wire.session_id.as_deref(),
Some(&protocol_version),
)
}
async fn call_future(
state: Arc<AppState>,
request: ToolCallRequest,
lifecycle: ToolCallLifecycle,
command: ToolCallCommand,
) -> ToolsOutcome {
handle_tools_command(
&state,
ServiceContext {
workspace_slug: &request.workspace_slug,
agent_slug: &request.agent_slug,
credential: &request.credential,
correlation: &request.correlation,
continuation: request.continuation,
locale: request.locale,
},
ToolsCommand::Call(command),
Some(lifecycle),
)
.await
}
/// A cancelled call is not answered: the pinned rule for a cancelled request is that no further
/// message is sent for it, and the client that cancelled it ignores a late response anyway. Nothing
/// here claims that an outbound side effect did or did not happen — and nothing else claims the
/// opposite either: the call is stopped before its terminal outcome, so its Invocation History holds
/// no row for this call at all (see the module docs for why there is no invented cancellation code).
fn cancelled_response() -> Response {
StatusCode::OK.into_response()
}
struct CallStream {
call: Pin<Box<dyn Future<Output = ToolsOutcome> + Send>>,
events: mpsc::Receiver<ProgressEvent>,
cancel: CancelSignal,
token: ProgressToken,
message: Value,
protocol_version: String,
correlation: CorrelationContext,
pending: VecDeque<Event>,
finished: bool,
/// The request's logging context, so the terminal event of the stream is projected exactly as the
/// single response of the same call would be (Story 3.18 carried across the stream, Story 3.4).
logging: crate::protocol::CarriedScope,
/// Held for the lifetime of the stream: the slot is released when the call is over, however it ends.
_in_flight: Option<InFlightEntry>,
}
enum Step {
Progress(ProgressEvent),
Outcome(ToolsOutcome),
Cancelled,
}
/// One stream item: a queued frame, the next real progress event, the terminal response, or the end of
/// a cancelled stream. A cancelled call sends no terminal frame at all.
async fn next_event(
call: CallStream,
) -> Option<(Result<Event, std::convert::Infallible>, CallStream)> {
let scope = call.logging;
crate::protocol::in_carried_scope(scope, next_event_in_scope(call)).await
}
async fn next_event_in_scope(
mut call: CallStream,
) -> Option<(Result<Event, std::convert::Infallible>, CallStream)> {
if let Some(frame) = call.pending.pop_front() {
return Some((Ok(frame), call));
}
if call.finished {
return None;
}
let step = {
let CallStream {
call,
events,
cancel,
..
} = &mut call;
tokio::select! {
biased;
Some(event) = events.recv() => Step::Progress(event),
outcome = call => Step::Outcome(outcome),
_ = cancel.cancelled() => Step::Cancelled,
}
};
match step {
Step::Progress(event) => {
let frame = progress_frame(&call.token, &event);
Some((Ok(Event::default().data(frame.to_string())), call))
}
Step::Outcome(outcome) => {
call.finished = true;
let payload = match tool_command_payload(
&call.message,
&call.protocol_version,
&call.correlation,
outcome,
) {
ToolCommandPayload::Ok(payload) => payload,
ToolCommandPayload::Internal(_) => jsonrpc_internal_error(&call.message),
};
// The same projection point the single response uses, so a streamed answer delivers the
// same closed `notifications/message` (Story 3.18) before its terminal frame.
crate::transport::note_logging_outcome(&payload);
for notification in crate::protocol::take_notifications() {
call.pending
.push_back(Event::default().data(notification.to_string()));
}
call.pending
.push_back(Event::default().data(payload.to_string()));
Some((Ok(call.pending.pop_front().expect("queued frame")), call))
}
Step::Cancelled => {
call.finished = true;
call.pending.clear();
None
}
}
}
fn jsonrpc_internal_error(message: &Value) -> Value {
crate::jsonrpc::jsonrpc_error(request_id(message), -32603, "internal server error")
}
#[cfg(test)]
#[path = "lifecycle_tests.rs"]
mod tests;
@@ -1,170 +0,0 @@
//! Story 3.4 bounds that do not need the HTTP surface: token validation, the monotonic bounded
//! progress queue, the pinned frame shape and the ownership of the in-flight cancellation table.
use super::*;
fn token(value: Value) -> Option<ProgressToken> {
progress_token(&json!({ "_meta": { "progressToken": value } }))
}
#[test]
fn a_pinned_progress_token_is_a_bounded_string_or_a_number() {
assert_eq!(
token(json!("tok-1")),
Some(ProgressToken::Text("tok-1".to_owned()))
);
assert!(matches!(token(json!(7)), Some(ProgressToken::Number(_))));
assert!(matches!(token(json!(1.5)), Some(ProgressToken::Number(_))));
// Malformed and out-of-bound tokens disable progress instead of failing the call.
assert_eq!(token(json!("")), None);
assert_eq!(token(json!("x".repeat(MAX_PROGRESS_TOKEN_LEN + 1))), None);
assert_eq!(token(json!("line\nbreak")), None);
assert_eq!(token(json!(null)), None);
assert_eq!(token(json!(true)), None);
assert_eq!(token(json!({ "nested": 1 })), None);
assert_eq!(token(json!([1, 2])), None);
// A token at the bound is still accepted.
assert!(token(json!("x".repeat(MAX_PROGRESS_TOKEN_LEN))).is_some());
// Absent `_meta`, absent token and a non-object `_meta` are all "no progress".
assert_eq!(progress_token(&json!({})), None);
assert_eq!(progress_token(&json!({ "_meta": { "other": 1 } })), None);
assert_eq!(progress_token(&json!({ "_meta": "not-an-object" })), None);
}
#[tokio::test]
async fn real_steps_produce_strictly_increasing_bounded_progress_events() {
let (lifecycle, events) = ToolCallLifecycle::new(token(json!("tok-monotonic")));
let mut events = events.expect("a validated token opens the progress queue");
for phase in [Phase::Resolved, Phase::Authorized, Phase::Dispatching] {
lifecycle.advance(phase);
}
drop(lifecycle);
let mut observed = Vec::new();
while let Some(event) = events.recv().await {
observed.push((event.progress, event.message));
}
assert_eq!(
observed,
vec![
(1, "tool_resolved"),
(2, "authorization_resolved"),
(3, "execution_dispatched"),
]
);
assert!(
observed.windows(2).all(|pair| pair[0].0 < pair[1].0),
"progress must increase monotonically, got {observed:?}"
);
}
#[tokio::test]
async fn a_full_progress_queue_drops_events_instead_of_blocking_the_call() {
let (lifecycle, events) = ToolCallLifecycle::new(token(json!(9)));
let mut events = events.expect("a validated token opens the progress queue");
// Many more steps than the queue holds: the call must never wait for the reader.
for _ in 0..(PROGRESS_CAPACITY * 4) {
lifecycle.advance(Phase::Resolved);
}
let mut delivered = 0;
while events.try_recv().is_ok() {
delivered += 1;
}
assert_eq!(delivered, PROGRESS_CAPACITY);
}
#[tokio::test]
async fn without_a_token_no_progress_is_produced_at_all() {
let (lifecycle, events) = ToolCallLifecycle::new(None);
assert!(events.is_none());
lifecycle.advance(Phase::Resolved);
lifecycle.advance(Phase::Dispatching);
}
/// The pinned `notifications/progress` frame (schema `ProgressNotification`, both supported
/// revisions): `progressToken` echoed verbatim, `progress` increasing, `message` optional and
/// `total` absent because this server never invents one.
#[test]
fn the_progress_frame_matches_the_pinned_notification_shape() {
let event = ProgressEvent {
progress: 2,
message: Phase::Authorized.message(),
};
assert_eq!(
progress_frame(&ProgressToken::Text("tok-2".to_owned()), &event),
json!({
"jsonrpc": "2.0",
"method": "notifications/progress",
"params": {
"progressToken": "tok-2",
"progress": 2,
"message": "authorization_resolved"
}
})
);
assert_eq!(
progress_frame(&ProgressToken::Number(7.into()), &event)["params"]["progressToken"],
json!(7)
);
}
#[tokio::test]
async fn a_cancellation_only_reaches_the_call_of_the_same_session_and_credential() {
let registry = Arc::new(InFlightCalls::new(MAX_IN_FLIGHT_CALLS));
let scope = InFlightScope {
session_id: "sess-1".to_owned(),
owner: "pk_a".to_owned(),
};
let entry = registry
.register(&scope, &json!(42))
.expect("the table has room");
let mut signal = entry.cancel_signal();
// A foreign credential, a foreign session and a different request id cancel nothing.
for foreign in [
InFlightScope {
session_id: "sess-1".to_owned(),
owner: "pk_b".to_owned(),
},
InFlightScope {
session_id: "sess-2".to_owned(),
owner: "pk_a".to_owned(),
},
] {
assert!(!registry.cancel(&foreign, &json!(42)));
}
assert!(!registry.cancel(&scope, &json!(43)));
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), signal.cancelled())
.await
.is_err(),
"a foreign cancellation must not fire the signal"
);
assert!(registry.cancel(&scope, &json!(42)));
tokio::time::timeout(std::time::Duration::from_millis(50), signal.cancelled())
.await
.expect("the owner's cancellation fires the signal");
}
#[tokio::test]
async fn a_released_call_releases_its_slot_and_the_table_is_bounded() {
let registry = Arc::new(InFlightCalls::new(1));
let scope = InFlightScope {
session_id: "sess-1".to_owned(),
owner: "pk_a".to_owned(),
};
let entry = registry.register(&scope, &json!(1)).expect("first slot");
assert!(
registry.register(&scope, &json!(2)).is_none(),
"the table refuses a call beyond its bound instead of evicting the live one"
);
assert!(registry.cancel(&scope, &json!(1)));
drop(entry);
assert!(
!registry.cancel(&scope, &json!(1)),
"a terminal call is no longer cancellable"
);
assert!(registry.register(&scope, &json!(2)).is_some());
}
@@ -14,11 +14,6 @@ pub(super) struct ToolCallParams {
pub(super) name: String,
#[serde(default, deserialize_with = "deserialize_arguments_object")]
pub(super) arguments: Option<Value>,
/// Story 3.4: the pinned `RequestMetaObject` of this request. Only the validated
/// `progressToken` of it is read, and only to decide whether the client asked for out-of-band
/// progress; a malformed or foreign `_meta` never fails the call.
#[serde(default, rename = "_meta")]
pub(super) meta: Option<Value>,
}
/// The pinned `CallToolRequest.params.arguments` is an **optional object**. Omitting it stays valid
+81 -107
View File
@@ -23,72 +23,68 @@ pub(crate) fn take_confirmation_token(arguments: &mut Value) -> Option<String> {
.filter(|value| !value.trim().is_empty())
}
/// The era-neutral success payload of one `tools/call` (Story 3.4: the streamed response needs the
/// payload itself, because the terminal response is the last event of the request's stream).
pub(super) fn success_tool_payload(
message: &Value,
protocol_version: &str,
output: Value,
) -> Value {
jsonrpc_result(
request_id(message),
project_modern_result(
protocol_version,
json!({
"content": [
{
"type": "text",
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
}
],
"structuredContent": output,
"isError": false
}),
),
)
}
#[cfg(test)]
pub(crate) fn tool_error_response(
pub(crate) fn success_tool_response(
message: &Value,
response_mode: ResponseMode,
protocol_version: &str,
error: ToolErrorContract,
output: Value,
) -> Response {
transport_response(
StatusCode::OK,
tool_error_payload(message, protocol_version, error),
jsonrpc_result(
request_id(message),
project_modern_result(
protocol_version,
json!({
"content": [
{
"type": "text",
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
}
],
"structuredContent": output,
"isError": false
}),
),
),
response_mode,
None,
Some(protocol_version),
)
}
pub(super) fn tool_error_payload(
pub(crate) fn tool_error_response(
message: &Value,
response_mode: ResponseMode,
protocol_version: &str,
error: ToolErrorContract,
) -> Value {
) -> Response {
let error_message = tool_error_text(&error);
let error_value = tool_error_value(&error);
jsonrpc_result(
request_id(message),
project_modern_result(
protocol_version,
json!({
"content": [
{
"type": "text",
"text": error_message
}
],
"structuredContent": {
"error": error_value
},
"isError": true
}),
transport_response(
StatusCode::OK,
jsonrpc_result(
request_id(message),
project_modern_result(
protocol_version,
json!({
"content": [
{
"type": "text",
"text": error_message
}
],
"structuredContent": {
"error": error_value
},
"isError": true
}),
),
),
response_mode,
None,
Some(protocol_version),
)
}
@@ -197,66 +193,12 @@ pub(super) fn prompt_arguments_error_response(
)
}
/// One `tools/call` wire frame, or the adapter's own internal-error detail.
/// Map one shared Tools-command outcome back to the era-specific wire response (spec section 6.2).
///
/// Story 3.4: the same projection serves the single response and the terminal event of a streamed
/// response, so both forms of one outcome can never drift apart.
pub(super) enum ToolCommandPayload {
Ok(Value),
Internal(&'static str),
}
/// Map one shared Tools-command outcome back to its era-specific wire payload (spec section 6.2).
///
/// This adapter function is the only place that turns a neutral outcome into a wire value: the shared
/// This adapter function is the only place that turns a neutral outcome into a response: the shared
/// service never sees `ResponseMode`, the protocol revision, the request envelope or an
/// `axum::Response`. `ApprovalPending` is a success for the client (it still has to answer), while
/// an unavailable elicitation and a failing call keep their era-correct tool-error shape.
pub(super) fn tool_command_payload(
message: &Value,
protocol_version: &str,
correlation: &CorrelationContext,
outcome: ToolsOutcome,
) -> ToolCommandPayload {
match outcome {
ToolsOutcome::Call(ToolCallOutcome::Completed(output))
| ToolsOutcome::Call(ToolCallOutcome::ApprovalPending(output)) => {
ToolCommandPayload::Ok(success_tool_payload(message, protocol_version, output))
}
ToolsOutcome::Call(ToolCallOutcome::ApprovalUnavailable(control_error)) => {
ToolCommandPayload::Ok(tool_error_payload(
message,
protocol_version,
control_tool_error_contract(
control_error,
correlation.request_id().as_str(),
correlation.trace_id().as_str(),
execution_locale(message),
),
))
}
ToolsOutcome::Call(ToolCallOutcome::InvalidParams(detail)) => {
ToolCommandPayload::Ok(jsonrpc_error(request_id(message), -32602, detail))
}
ToolsOutcome::Call(ToolCallOutcome::Error(contract)) => {
ToolCommandPayload::Ok(tool_error_payload(message, protocol_version, contract))
}
// The pending approval could not be persisted: the adapter answers its internal error shape,
// as the previous wire path did.
ToolsOutcome::Call(ToolCallOutcome::InternalFailure) => {
ToolCommandPayload::Internal("the pending approval request could not be persisted")
}
// A call command never produces these variants; they are answered with the adapter's
// internal error instead of being silently ignored.
ToolsOutcome::List { .. }
| ToolsOutcome::InvalidParams(_)
| ToolsOutcome::CatalogUnavailable => {
ToolCommandPayload::Internal("unexpected shared tools outcome for a tools/call")
}
}
}
/// The single-response form of [`tool_command_payload`].
pub(super) fn tool_command_response(
message: &Value,
response_mode: ResponseMode,
@@ -264,15 +206,47 @@ pub(super) fn tool_command_response(
correlation: &CorrelationContext,
outcome: ToolsOutcome,
) -> Response {
match tool_command_payload(message, protocol_version, correlation, outcome) {
ToolCommandPayload::Ok(payload) => transport_response(
match outcome {
ToolsOutcome::Call(ToolCallOutcome::Completed(output))
| ToolsOutcome::Call(ToolCallOutcome::ApprovalPending(output)) => {
success_tool_response(message, response_mode, protocol_version, output)
}
ToolsOutcome::Call(ToolCallOutcome::ApprovalUnavailable(control_error)) => {
tool_error_response(
message,
response_mode,
protocol_version,
control_tool_error_contract(
control_error,
correlation.request_id().as_str(),
correlation.trace_id().as_str(),
execution_locale(message),
),
)
}
ToolsOutcome::Call(ToolCallOutcome::InvalidParams(detail)) => transport_response(
StatusCode::OK,
payload,
jsonrpc_error(request_id(message), -32602, detail),
response_mode,
None,
Some(protocol_version),
),
ToolCommandPayload::Internal(detail) => internal_jsonrpc_error(message, detail),
ToolsOutcome::Call(ToolCallOutcome::Error(contract)) => {
tool_error_response(message, response_mode, protocol_version, contract)
}
// The pending approval could not be persisted: the adapter answers its internal error shape,
// as the previous wire path did.
ToolsOutcome::Call(ToolCallOutcome::InternalFailure) => internal_jsonrpc_error(
message,
"the pending approval request could not be persisted",
),
// A call command never produces these variants; they are answered with the adapter's
// internal error instead of being silently ignored.
ToolsOutcome::List { .. }
| ToolsOutcome::InvalidParams(_)
| ToolsOutcome::CatalogUnavailable => {
internal_jsonrpc_error(message, "unexpected shared tools outcome for a tools/call")
}
}
}
@@ -1,197 +0,0 @@
//! Isolated single-Agent loopback experiment. Not installed in the production router.
//! Each instance owns one route and one SDK manager; no durable store is replaced.
use super::{
AgentRoutePath, AppState,
lifecycle::InFlightScope,
metrics::ActiveStreamGuard,
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
stages::{enforce_traced_rate_limit, require_traced_machine_access},
};
use crate::{
protocol::{self, ContinuationSupport, RequestEra},
transport,
};
use axum::{
Router,
body::{Body, to_bytes},
extract::Request,
http::{Method, StatusCode},
response::{IntoResponse, Response},
};
use crank_core::PlatformApiKeyScope;
use futures_util::StreamExt;
use rmcp::transport::streamable_http_server::{
StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager,
};
use serde_json::Value;
use std::sync::Arc;
const BODY_LIMIT: usize = 2 * 1024 * 1024;
pub(super) fn router(
state: Arc<AppState>,
path: AgentRoutePath,
active: Arc<std::sync::atomic::AtomicUsize>,
) -> Router {
let handler_state = state.clone();
let mut config = StreamableHttpServerConfig::default().with_json_response(true);
config.max_request_body_bytes = BODY_LIMIT;
let shutdown = Arc::new(config.cancellation_token.clone().drop_guard());
let mut manager = LocalSessionManager::default();
manager.session_config.channel_capacity = 16;
let service = StreamableHttpService::new(
move || {
Ok(RmcpTools {
state: handler_state.clone(),
active: active.clone(),
})
},
Arc::new(manager),
config,
);
let route = format!("/v1/{}/{}", path.workspace_slug, path.agent_slug);
Router::new()
.route(
&route,
axum::routing::any(move |request: Request| {
let state = state.clone();
let path = path.clone();
let service = service.clone();
let shutdown = shutdown.clone();
async move {
let _shutdown = shutdown;
let (mut parts, body) = request.into_parts();
if let Err(status) =
transport::validate_origin(&state.allowed_origins, &parts.headers)
{
return status.into_response();
}
let bytes = match to_bytes(body, BODY_LIMIT).await {
Ok(bytes) => bytes,
Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(),
};
let message: Value = if parts.method == Method::POST {
match serde_json::from_slice(&bytes) {
Ok(message) => message,
Err(_) => return StatusCode::BAD_REQUEST.into_response(),
}
} else {
Value::Null
};
let method = message["method"].as_str().unwrap_or("");
let required = if method == "tools/call" {
PlatformApiKeyScope::Write
} else {
PlatformApiKeyScope::Read
};
let credential = match require_traced_machine_access(
&state,
&path,
&parts.headers,
required,
)
.await
{
Ok(credential) => credential,
Err(error) => return error.into_response(),
};
let session = match transport::session_id_from_headers(&parts.headers) {
Ok(session) => session,
Err(_) => {
return protocol::ambiguous_session_carrier_response(
&parts.headers,
&message,
);
}
};
let era = if parts.method == Method::POST {
if let Err(response) = protocol::validate_jsonrpc_envelope(&message) {
return *response;
}
if let Err(status) = transport::negotiate_post_response_mode(&parts.headers)
{
return status.into_response();
}
match protocol::resolve_request_identity(&parts.headers, &message) {
Ok(identity) => identity.era,
Err(response) => return *response,
}
} else {
if let Err(response) = protocol::session_lifecycle_admission(&parts.headers)
{
return *response;
}
RequestEra::Legacy
};
if matches!(era, RequestEra::Modern) && session.is_some() {
return protocol::modern_carrier_conflict_response(&message);
}
if let Err(error) =
enforce_traced_rate_limit(&state, &path, &parts.headers).await
{
return crate::rate_limit::rate_limited_status_response(error);
}
// SDK cancellation is keyed only by request id. Never forward an unchecked cancel.
if method == "notifications/cancelled" {
if let Some(session_id) = session {
state.in_flight.cancel(
&InFlightScope {
session_id,
owner: credential
.platform_api_key_id
.as_ref()
.map(|id| id.as_str())
.unwrap_or("")
.to_owned(),
},
&message["params"]["requestId"],
);
}
return StatusCode::ACCEPTED.into_response();
}
let Some(context) = parts
.extensions
.get::<crate::request_context::RequestContext>()
else {
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
};
let correlation = context.correlation.clone();
parts.extensions.insert(RmcpRequestIdentity {
workspace_slug: path.workspace_slug,
agent_slug: path.agent_slug,
credential,
correlation,
continuation: ContinuationSupport::from_supported(false),
locale: crate::tool_error::execution_locale(&message),
session_id: session,
progress_meta: message.pointer("/params/_meta").cloned(),
sse_acceptable: transport::accepts_event_stream(&parts.headers),
});
// Legacy SDK calls outlive disconnected bodies: both lifetimes share one slot.
let permit = match ActiveStreamGuard::try_acquire(&state.session_slots) {
Ok(permit) => Arc::new(permit),
Err(_) => return StatusCode::TOO_MANY_REQUESTS.into_response(),
};
if method == "tools/call" {
parts.extensions.insert(permit.clone());
}
let response = service
.handle(Request::from_parts(parts, Body::from(bytes)))
.await;
let (parts, body) = response.into_parts();
let stream = futures_util::stream::unfold(
(Box::pin(Body::new(body).into_data_stream()), permit),
|(mut stream, permit)| async move {
stream.next().await.map(|chunk| (chunk, (stream, permit)))
},
);
Response::from_parts(parts, Body::from_stream(stream))
}
}),
)
.layer(axum::middleware::from_fn(
crate::request_context::apply_request_context,
))
}
@@ -1,43 +0,0 @@
//! Behavioural ATDD for rmcp 3.7.0 LocalSessionManager reconnect/restart semantics.
use rmcp::transport::streamable_http_server::session::{
SessionManager, local::LocalSessionManager,
};
#[tokio::test]
async fn rmcp_local_session_manager_preserves_access_to_created_session() {
let manager = LocalSessionManager::default();
let (session_id, _transport) = manager
.create_session()
.await
.expect("local manager must create a session");
assert!(
manager
.has_session(&session_id)
.await
.expect("live manager must answer has_session"),
"a live manager must keep access to a session it created"
);
}
#[tokio::test]
async fn rmcp_fresh_session_manager_without_external_store_rejects_known_session_id() {
let first_manager = LocalSessionManager::default();
let (session_id, _transport) = first_manager
.create_session()
.await
.expect("first local manager must create a session");
let second_manager = LocalSessionManager::default();
assert!(
!second_manager
.has_session(&session_id)
.await
.expect("fresh manager must answer has_session"),
"a fresh in-memory manager must not claim ownership of another manager's session"
);
}
@@ -1,258 +0,0 @@
//! Isolated rmcp 3.7.0 Tools adapter over Crank's existing typed Tools service.
//!
//! This module deliberately does not provide a second command engine. It owns only
//! the verified request identity (`ServiceContext` data) and delegates both
//! `tools/list` and `tools/call` to `handle_tools_command` via rmcp's
//! `ServerHandler` trait. HTTP admission must supply identity for each request.
use std::sync::Arc;
use crank_core::{CorrelationContext, ExecutionLocale};
use crate::app::{AppState, lifecycle::ToolCallLifecycle};
use crate::auth::VerifiedMachineCredential;
use crate::protocol::{
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsListCommand,
ToolsOutcome, handle_tools_command,
};
/// Long-lived handler: never stores credentials obtained on initialization.
#[derive(Clone)]
pub(super) struct RmcpTools {
pub(super) state: Arc<AppState>,
pub(super) active: Arc<std::sync::atomic::AtomicUsize>,
}
struct ActiveCall(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for ActiveCall {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
/// Server-created identity attached to the individual admitted HTTP request.
#[derive(Clone)]
pub(super) struct RmcpRequestIdentity {
pub(super) workspace_slug: String,
pub(super) agent_slug: String,
pub(super) credential: VerifiedMachineCredential,
pub(super) correlation: CorrelationContext,
pub(super) continuation: ContinuationSupport,
pub(super) locale: ExecutionLocale,
pub(super) session_id: Option<String>,
pub(super) progress_meta: Option<serde_json::Value>,
pub(super) sse_acceptable: bool,
}
impl RmcpRequestIdentity {
/// Build the neutral `ServiceContext` borrowed from this adapter's owned data.
pub(super) fn service_context(&self) -> ServiceContext<'_> {
ServiceContext {
workspace_slug: &self.workspace_slug,
agent_slug: &self.agent_slug,
credential: &self.credential,
correlation: &self.correlation,
continuation: self.continuation,
locale: self.locale,
}
}
}
impl RmcpTools {
/// Delegate one neutral Tools command to the existing shared service.
pub(super) async fn handle(
&self,
identity: &RmcpRequestIdentity,
command: ToolsCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
handle_tools_command(&self.state, identity.service_context(), command, lifecycle).await
}
pub(super) async fn list_tools(
&self,
identity: &RmcpRequestIdentity,
command: ToolsListCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
self.handle(identity, ToolsCommand::List(command), lifecycle)
.await
}
pub(super) async fn call_tool(
&self,
identity: &RmcpRequestIdentity,
command: ToolCallCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
self.handle(identity, ToolsCommand::Call(command), lifecycle)
.await
}
}
use rmcp::ErrorData as McpError;
use rmcp::model::{
CallToolRequestParams, CallToolResult, ContentBlock, ListToolsResult, PaginatedRequestParams,
Tool,
};
use rmcp::service::RequestContext;
use rmcp::{RoleServer, ServerHandler};
fn request_identity(context: &RequestContext<RoleServer>) -> Result<RmcpRequestIdentity, McpError> {
context
.extensions
.get::<axum::http::request::Parts>()
.and_then(|parts| parts.extensions.get::<RmcpRequestIdentity>())
.cloned()
.ok_or_else(|| McpError::internal_error("missing authenticated request context", None))
}
impl ServerHandler for RmcpTools {
fn supported_protocol_versions(
&self,
) -> std::borrow::Cow<'static, [rmcp::model::ProtocolVersion]> {
std::borrow::Cow::Borrowed(&[
rmcp::model::ProtocolVersion::V_2025_11_25,
rmcp::model::ProtocolVersion::V_2026_07_28,
])
}
fn get_info(&self) -> rmcp::model::ServerConfig {
let mut info = rmcp::model::ServerConfig::default();
info.capabilities = rmcp::model::ServerCapabilities::builder()
.enable_tools()
.build();
info
}
fn list_tools(
&self,
request: Option<PaginatedRequestParams>,
context: RequestContext<RoleServer>,
) -> impl std::future::Future<Output = Result<ListToolsResult, McpError>> + Send + '_ {
let request = request.unwrap_or_default();
async move {
let identity = request_identity(&context)?;
let cursor = request.cursor;
let command = crate::protocol::ToolsListCommand {
cursor,
limit: None,
};
match self.list_tools(&identity, command, None).await {
crate::protocol::ToolsOutcome::List { tools, next_cursor } => {
let tools = tools
.into_iter()
.map(serde_json::from_value::<Tool>)
.collect::<Result<Vec<_>, _>>()
.map_err(|_| {
McpError::internal_error("invalid tool catalog entry", None)
})?;
Ok(ListToolsResult {
tools,
next_cursor,
..Default::default()
})
}
crate::protocol::ToolsOutcome::InvalidParams(message) => {
Err(McpError::invalid_params(message, None))
}
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
}
}
}
async fn call_tool(
&self,
request: CallToolRequestParams,
context: RequestContext<RoleServer>,
) -> Result<rmcp::model::CallToolResponse, McpError> {
let identity = request_identity(&context)?;
let arguments = request
.arguments
.map(serde_json::Value::Object)
.unwrap_or_else(|| serde_json::json!({}));
let command = crate::protocol::ToolCallCommand {
tool_name: request.name.to_string(),
arguments,
confirmation_token: None,
};
let token = identity
.sse_acceptable
.then(|| super::lifecycle::token_from_meta(identity.progress_meta.as_ref()))
.flatten();
// Observe future cleanup directly in the isolated loopback oracle.
self.active
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let _active = ActiveCall(self.active.clone());
let (mut lifecycle, mut events) = ToolCallLifecycle::new(token.clone());
let entry = identity.session_id.as_ref().and_then(|session_id| {
self.state.in_flight.register(
&super::lifecycle::InFlightScope {
session_id: session_id.clone(),
owner: identity
.credential
.platform_api_key_id
.as_ref()
.map(|id| id.as_str())
.unwrap_or("")
.to_owned(),
},
&serde_json::to_value(&context.id).expect("request id serializes"),
)
});
if let Some(entry) = &entry {
lifecycle = lifecycle.with_cancel(entry.cancel_signal());
}
let call = self.call_tool(&identity, command, Some(lifecycle.clone()));
tokio::pin!(call);
let outcome = loop {
tokio::select! {
biased;
event = async { match events.as_mut() { Some(events) => events.recv().await, None => std::future::pending().await } } => {
if let Some(event) = event {
if let Some(token) = &token {
let params = serde_json::from_value(super::lifecycle::progress_frame(token, &event)["params"].clone()).map_err(|_| McpError::internal_error("invalid progress event", None))?;
let _ = context.peer.notify_progress(params).await;
}
} else { events = None; }
}
outcome = &mut call => break outcome,
_ = lifecycle.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
_ = context.ct.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
}
};
match outcome {
crate::protocol::ToolsOutcome::Call(outcome) => match outcome {
crate::protocol::ToolCallOutcome::Completed(value)
| crate::protocol::ToolCallOutcome::ApprovalPending(value) => {
let text = value.to_string();
let mut result = CallToolResult::default();
result.content = vec![ContentBlock::text(text)];
result.structured_content = if value.is_null() { None } else { Some(value) };
Ok(result.into())
}
crate::protocol::ToolCallOutcome::ApprovalUnavailable(_) => {
let mut result = CallToolResult::default();
result.is_error = Some(true);
result.content = vec![ContentBlock::text("approval elicitation not supported")];
Ok(result.into())
}
crate::protocol::ToolCallOutcome::InvalidParams(message) => {
Err(McpError::invalid_params(message, None))
}
crate::protocol::ToolCallOutcome::Error(contract) => {
let text = serde_json::to_string(&contract).unwrap_or_default();
let mut result = CallToolResult::default();
result.is_error = Some(true);
result.content = vec![ContentBlock::text(text)];
Ok(result.into())
}
crate::protocol::ToolCallOutcome::InternalFailure => {
Err(McpError::internal_error("tool execution failed", None))
}
},
crate::protocol::ToolsOutcome::InvalidParams(message) => {
Err(McpError::invalid_params(message, None))
}
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
}
}
}
@@ -1,381 +0,0 @@
//! Application-level durable Task handler and attempt authority (Story 4.3/4.4).
//!
//! The handler owns everything that touches product authority: the exact published pin, the typed
//! Platform API key, the approval policy, the encrypted executable input and the resolved upstream
//! auth. The runtime owns the single bounded retry loop and the durable attempt checkpoint/resolution;
//! this module only revalidates and prepares one attempt at a time.
use std::time::{Duration, Instant};
use async_trait::async_trait;
use crank_core::{
AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId,
current_unix_ms,
};
use crank_registry::PostgresRegistry;
use crank_registry::PublishedAgentTool;
use crank_runtime::{
ExecutionAuthorization, PrepareAttempt, RuntimeExecutor, RuntimeOperation, SecretCrypto,
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal, TaskHandler, TaskService, TaskWorkerError,
};
use sha2::{Digest, Sha256};
use super::TaskApplication;
use super::auth_resolution::resolve_operation_auth_with;
use super::tool_resolution::runtime_operation;
use crate::access::{allows_scope, credential_allows_security_level};
use crate::catalog::PublishedToolCatalog;
/// Server-owned evidence that one published operation deduplicates repeated sends of the same stable
/// idempotency key.
///
/// The descriptor is constructed by the application for an integration whose upstream contract was
/// actually qualified (key delivery, dedup scope, canonical input binding and a dedup window that
/// covers the pinned retry budget). It is never deserialized from Task input, so a caller key or
/// header alone can never make a write retryable. Production wiring passes an empty set until such a
/// contract exists.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct VerifiedIdempotentWrite {
pub operation_id: String,
pub operation_version: u32,
/// Header through which the stable key is delivered to this exact upstream.
pub key_header: String,
/// Dedup window of the upstream contract, which must cover the pinned retry budget.
pub validity_ms: i64,
}
pub struct TaskWorkerHandler {
registry: PostgresRegistry,
catalog: PublishedToolCatalog,
runtime: RuntimeExecutor,
secret_crypto: SecretCrypto,
service: TaskService,
verified_writes: Vec<VerifiedIdempotentWrite>,
}
impl TaskWorkerHandler {
/// Production construction from the same application entry, registry and runtime the MCP
/// process serves requests with. The worker needs no HTTP state, only these authoritative
/// dependencies.
pub fn from_application(
tasks: &TaskApplication,
registry: PostgresRegistry,
runtime: RuntimeExecutor,
secret_crypto: SecretCrypto,
) -> Self {
Self {
registry,
catalog: tasks.catalog(),
runtime,
secret_crypto,
service: tasks.task_service(),
verified_writes: Vec::new(),
}
}
/// Registers server-owned verified-idempotency evidence of qualified integrations.
pub fn with_verified_writes(mut self, verified_writes: Vec<VerifiedIdempotentWrite>) -> Self {
self.verified_writes = verified_writes;
self
}
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
///
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
/// policy and revalidates the typed persisted Platform API key identity against the registry:
/// the key must still be active, unexpired and bound to this workspace/agent, and its exact
/// scopes/security level must still authorize the operation. Any denial is a typed error, so the
/// worker finishes the attempt without dispatch (fail closed before any network use).
///
/// An administrator Task has no authoritative Task revalidation API in the registry, so it stays
/// fail-closed here instead of inventing authority.
async fn authorize(&self, lease: &TaskLease) -> Result<PublishedAgentTool, TaskError> {
if lease.record.owner_kind != TaskOwnerKind::Agent {
return Err(TaskError::invalid(
"admin task authority revalidation is not implemented",
));
}
let Some(key_id) = lease.record.platform_api_key_id.as_ref() else {
return Err(TaskError::invalid("task authority reference is missing"));
};
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
let workspace = self
.registry
.get_workspace(&workspace_id)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?
.ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?;
let agents = self
.registry
.list_agents(&workspace_id)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
let agent = agents
.into_iter()
.find(|agent| agent.id.as_str() == lease.record.owner_id.as_str())
.ok_or_else(|| TaskError::invalid("task agent no longer exists"))?;
let catalog = self
.catalog
.get_catalog(&workspace.workspace.slug, &agent.slug)
.await
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
let tool = catalog
.tools
.into_iter()
.find(|tool| {
tool.operation.id.as_str() == lease.record.operation_id.as_str()
&& tool.operation.version == lease.record.pins.operation_version
})
.ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?;
if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() {
return Err(TaskError::invalid("task workspace binding mismatch"));
}
if tool.agent_id.as_str() != lease.record.owner_id.as_str() {
return Err(TaskError::invalid("task agent binding mismatch"));
}
if tool
.operation
.execution_config
.approval_policy
.as_ref()
.is_some_and(|policy| policy.required)
{
return Err(TaskError::invalid(
"missing approval proof for task operation",
));
}
// Authoritative revalidation of the exact immutable key identity. A revoked, expired,
// foreign-workspace or foreign-agent key is absent here and never reaches the network.
let key = self
.registry
.revalidate_platform_api_key(&workspace_id, &agent_id, key_id)
.await
.map_err(|_| TaskError::unavailable("task authority store unavailable"))?
.ok_or_else(|| TaskError::invalid("task authority is no longer active"))?;
let credential = VerifiedMachineCredential {
machine_access_mode: MachineAccessMode::StaticAgentKey,
max_security_level: OperationSecurityLevel::Standard,
scopes: key.api_key.scopes.clone(),
platform_api_key_id: Some(key.api_key.id.clone()),
};
if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) {
return Err(TaskError::invalid("task authority scope is insufficient"));
}
if !credential_allows_security_level(&credential, tool.operation.security_level) {
return Err(TaskError::invalid(
"task authority security level is insufficient",
));
}
Ok(tool)
}
/// Safe-retry classification of one attempt and the stable upstream key digest.
///
/// A read is always safe to repeat. A write is only safe to repeat when this process holds
/// server-owned evidence for the exact published operation and that evidence's dedup window still
/// covers the accepted retry budget; the presence of a client key or an idempotency header alone
/// never qualifies.
fn retry_classification(
&self,
operation: &RuntimeOperation,
input: &serde_json::Value,
retry_window_ms: i64,
) -> (bool, Option<String>) {
if matches!(
&operation.target,
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
) {
return (true, None);
}
let Some(policy) = operation.execution_config.idempotency.as_ref() else {
return (false, None);
};
let Some(field) = policy
.input_field
.as_deref()
.filter(|field| !field.trim().is_empty())
else {
return (false, None);
};
let Some(header) = policy
.header_name
.as_deref()
.filter(|value| !value.is_empty())
else {
return (false, None);
};
let key = match input.get(field) {
Some(serde_json::Value::String(value)) if !value.trim().is_empty() => value.clone(),
Some(serde_json::Value::Number(value)) => value.to_string(),
_ => return (false, None),
};
let evidence = self.verified_writes.iter().find(|evidence| {
evidence.operation_id == operation.operation_id.as_str()
&& evidence.operation_version == operation.operation_version
&& evidence.key_header.eq_ignore_ascii_case(header)
&& evidence.validity_ms >= retry_window_ms
});
let Some(evidence) = evidence else {
return (false, None);
};
// Only a digest of the stable key is persisted; the key itself stays in the encrypted input.
let mut digest = Sha256::new();
digest.update(evidence.key_header.as_bytes());
digest.update([0u8]);
digest.update(key.as_bytes());
(true, Some(format!("{:x}", digest.finalize())))
}
/// Story 4.5 producer: the pinned operation still lacks a required input item, so the Task must be
/// suspended before any dispatch. The schema snapshot and its presentation projection are frozen
/// here; the TTL never exceeds the accepted absolute execution deadline.
fn input_request(
&self,
tool: &PublishedAgentTool,
lease: &TaskLease,
missing_path: &str,
expires_at_unix_ms: i64,
) -> Result<crank_core::TaskInputRequest, TaskAttemptRefusal> {
let schema_snapshot = serde_json::to_value(&tool.operation.input_schema).map_err(|_| {
TaskAttemptRefusal::Failed {
code: "task_input_schema_unavailable".to_owned(),
}
})?;
let request = crank_core::TaskInputRequest {
wait_id: format!("wait-{}", uuid::Uuid::now_v7()),
expected_version: lease.record.version.saturating_add(1),
schema_revision: tool.operation.version,
schema_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot,
projected_schema: crate::manifest::schema_to_json_schema(&tool.operation.input_schema),
prompt_code: "required_input_missing".to_owned(),
missing_fields: vec![missing_path.to_owned()],
expires_at_unix_ms,
fingerprint_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
};
request.validate().map_err(|_| TaskAttemptRefusal::Failed {
code: "task_input_request_invalid".to_owned(),
})?;
Ok(request)
}
/// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store
/// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal.
fn refusal(error: TaskError) -> TaskAttemptRefusal {
match error {
TaskError::Unavailable { .. } => TaskAttemptRefusal::Stopped,
_ => TaskAttemptRefusal::Failed {
code: "task_authority_rejected".to_owned(),
},
}
}
}
#[async_trait]
impl TaskHandler for TaskWorkerHandler {
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> {
self.authorize(lease)
.await
.map_err(TaskWorkerError::Store)?;
Ok(())
}
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome {
self.runtime.execute_task_outcome(&context, self).await
}
}
#[async_trait]
impl TaskAttemptAuthority for TaskWorkerHandler {
async fn prepare_attempt(
&self,
lease: &TaskLease,
) -> Result<PrepareAttempt, TaskAttemptRefusal> {
let tool = self.authorize(lease).await.map_err(Self::refusal)?;
let input = self
.service
.executable_input(lease)
.await
.map_err(Self::refusal)?;
let operation = runtime_operation(&tool);
let now_ms = current_unix_ms();
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
if deadline_ms <= now_ms {
// The accepted absolute deadline already elapsed: no further attempt may be dispatched,
// and any previous uncertainty was already recorded as such.
return Err(TaskAttemptRefusal::Failed {
code: "task_deadline_elapsed".to_owned(),
});
}
// Only a missing required item of the pinned schema becomes a typed input request. Any other
// schema violation keeps the existing pre-dispatch attempt validation, and a generation that
// already accepted a continuation can never open a second wait for the same Task.
if let Some(missing_path) =
crank_runtime::missing_required_input(&tool.operation.input_schema, &input)
{
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
return Err(TaskAttemptRefusal::Failed {
code: "task_input_still_incomplete".to_owned(),
});
}
return Ok(PrepareAttempt::InputRequired(self.input_request(
&tool,
lease,
&missing_path,
deadline_ms,
)?));
}
let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64);
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
let resolved_auth = resolve_operation_auth_with(
&self.registry,
&self.secret_crypto,
&workspace_id,
&operation.execution_config,
)
.await
.map_err(|_| TaskAttemptRefusal::Failed {
code: "task_auth_resolution_failed".to_owned(),
})?;
// Continue the parent trace persisted with the accepted Task; every attempt adds its own
// child `adapter_attempt` span carrying the committed durable attempt ID.
let trace_context =
TraceContext::parse(&lease.record.support.traceparent).map_err(|_| {
TaskAttemptRefusal::Failed {
code: "task_trace_parent_invalid".to_owned(),
}
})?;
let request_context = crank_runtime::RuntimeRequestContext::new(
lease.record.support.request_id.clone(),
trace_context,
)
.with_metering_context(
workspace_id.clone(),
Some(agent_id.clone()),
crank_core::InvocationSource::AgentToolCall,
);
let (retry_allowed, upstream_key_digest) =
self.retry_classification(&operation, &input, deadline_ms - now_ms);
Ok(PrepareAttempt::Ready(Box::new(TaskAttemptPlan {
workspace_id,
origin: ExecutionOrigin::AgentSnapshot,
agent_id: Some(agent_id),
operation,
input,
authorization: ExecutionAuthorization::Authorized,
resolved_auth,
request_context,
deadline,
retry_allowed,
upstream_key_digest,
})))
}
}
-414
View File
@@ -1,414 +0,0 @@
//! Application entry of durable background Tasks (Story 4.2, FR-22/FR-23).
//!
//! The entry takes an already authenticated principal (a verified machine credential, or an
//! administrator identity verified by the calling admin surface), resolves the exact published Tool,
//! repeats the caller's authorization, pins the Published Version/policy/deadline and delegates the
//! durable acceptance to the runtime Task service. The repository owns the transaction, so the
//! returned Task ID exists only after Task + dispatch intent + queue slot committed.
//!
//! Deliberately not wired: the public MCP `tools/call`/Tasks wire (Story 4.7) and the worker/leases
//! (Story 4.3). The existing synchronous/unsupported `tools/call` path is unchanged, and this entry
//! never spawns an ephemeral execution future.
use std::sync::Arc;
use std::time::{Duration, Instant};
use crank_core::{
CorrelationContext, PlatformApiKeyId, TaskAcceptance, TaskAccess, TaskError, TaskId,
TaskOwnerKind, TaskRecord, ToolExecutionMode,
};
use crank_registry::{PostgresRegistry, PublishedAgentTool};
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
use serde_json::Value;
use crate::access::credential_allows_security_level;
use crate::catalog::PublishedToolCatalog;
/// Authenticated principal of one Task operation. The authorization scope identity is derived here
/// from the verified credential/admin identity, never from a caller-supplied request string.
#[derive(Clone, Debug)]
pub enum TaskPrincipal {
Agent {
workspace_slug: String,
agent_slug: String,
credential: crank_core::VerifiedMachineCredential,
},
/// Administrator identity that the admin surface already authenticated. It is never a wildcard:
/// owner, workspace and authorization scope all participate in the stored predicate.
Admin {
workspace_slug: String,
agent_slug: String,
user_id: String,
},
}
impl TaskPrincipal {
fn workspace_slug(&self) -> &str {
match self {
Self::Agent { workspace_slug, .. } | Self::Admin { workspace_slug, .. } => {
workspace_slug
}
}
}
fn agent_slug(&self) -> &str {
match self {
Self::Agent { agent_slug, .. } | Self::Admin { agent_slug, .. } => agent_slug,
}
}
fn owner(&self, tool: &PublishedAgentTool) -> (TaskOwnerKind, String) {
match self {
Self::Agent { .. } => (TaskOwnerKind::Agent, tool.agent_id.as_str().to_owned()),
Self::Admin { user_id, .. } => (TaskOwnerKind::Admin, user_id.clone()),
}
}
/// Typed immutable Platform API key identity of this principal. A verified agent credential
/// without a key ID cannot be revalidated by a worker, so it is refused instead of accepted;
/// an administrator has no machine key and is stored as `None` (the worker fails closed).
fn platform_api_key_id(&self) -> Result<Option<PlatformApiKeyId>, TaskApplicationError> {
match self {
Self::Agent { credential, .. } => credential
.platform_api_key_id
.clone()
.map(Some)
.ok_or(TaskApplicationError::Denied),
Self::Admin { .. } => Ok(None),
}
}
/// Bounded server-derived authorization scope identity of this principal.
fn authorization_scope(&self) -> String {
let identity = match self {
Self::Agent { credential, .. } => {
let mut scopes = credential
.scopes
.iter()
.map(|scope| format!("{scope:?}"))
.collect::<Vec<_>>();
scopes.sort();
format!(
"key:{}|mode:{:?}|level:{:?}|scopes:{}",
credential
.platform_api_key_id
.as_ref()
.map(|id| id.as_str().to_owned())
.unwrap_or_else(|| "-".to_owned()),
credential.machine_access_mode,
credential.max_security_level,
scopes.join(",")
)
}
Self::Admin { user_id, .. } => format!("admin:{user_id}"),
};
identity.chars().take(160).collect()
}
}
/// Typed refusal of the application entry. `NotFound`/`Denied` never reveal whether a foreign Task
/// exists, and no variant carries Task payload or fingerprint.
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum TaskApplicationError {
#[error("task request was not found")]
NotFound,
#[error("task request is not authorized")]
Denied,
#[error("the published Tool policy does not accept durable background work")]
NotBackground,
#[error("the requested published Tool version is not available")]
VersionUnavailable,
#[error("this Tool requires a verified approval before acceptance")]
ApprovalRequired,
#[error(transparent)]
Task(#[from] TaskError),
}
#[derive(Clone)]
pub struct TaskApplication {
catalog: PublishedToolCatalog,
service: TaskService,
}
impl TaskApplication {
/// Production construction of the application entry over the process registry, the published
/// catalog and the durable Task service.
pub fn new(
registry: PostgresRegistry,
refresh_interval: Duration,
coordination_store: Arc<dyn crank_core::CoordinationStateStore>,
service: TaskService,
) -> Self {
Self {
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
service,
}
}
/// Runtime task service carrying encrypted executable input.
pub fn task_service(&self) -> TaskService {
self.service.clone()
}
/// Published catalog of the same application entry, for the durable Task worker's pre-dispatch
/// revalidation.
pub(super) fn catalog(&self) -> PublishedToolCatalog {
self.catalog.clone()
}
/// Readiness of the durable Task store this entry depends on.
pub async fn health(&self) -> Result<(), TaskError> {
self.service.health().await
}
/// Accepts one background request for an already authenticated principal.
///
/// `expected_version` is the caller's explicit Published Version pin. Without it the current
/// publication is pinned for new work and a retry of the same scoped key returns the stored Task
/// even after a newer publication. An explicit pin may only retry the Task accepted with that
/// exact version: a different pin is a typed conflict, never a rebind.
pub async fn accept_background(
&self,
principal: &TaskPrincipal,
tool_name: &str,
expected_version: Option<u32>,
scoped_key: &str,
input: &Value,
correlation: &CorrelationContext,
) -> Result<TaskAcceptance, TaskApplicationError> {
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
let current_version = tool.operation.version;
let Some(policy) = tool.operation.execution_config.execution_policy else {
return Err(TaskApplicationError::NotBackground);
};
if policy.mode == ToolExecutionMode::Synchronous {
return Err(TaskApplicationError::NotBackground);
}
if tool
.operation
.execution_config
.approval_policy
.as_ref()
.is_some_and(|approval| approval.required)
{
// Without a verified, non-consuming approval reference this entry fails closed instead
// of accepting work a worker could never safely execute.
return Err(TaskApplicationError::ApprovalRequired);
}
let platform_api_key_id = principal.platform_api_key_id()?;
let draft = |operation_version: u32| {
let (owner_kind, owner_id) = principal.owner(&tool);
TaskAcceptDraft {
workspace_id: tool.workspace_id.as_str().to_owned(),
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
platform_api_key_id: platform_api_key_id.clone(),
operation_id: tool.operation.id.as_str().to_owned(),
operation_version,
policy_mode: policy.mode,
scoped_key: scoped_key.to_owned(),
input: input.clone(),
correlation: correlation.clone(),
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
retention_ms: TaskService::default_retention_ms(),
approval_reference: None,
retry: task_retry_policy(&tool),
}
};
let Some(requested) = expected_version else {
return Ok(self.service.accept(draft(current_version)).await?);
};
let draft = draft(requested);
match self.service.lookup(&draft).await? {
Some(record) => {
if record.input_fingerprint != TaskService::input_fingerprint(input)? {
return Err(TaskApplicationError::Task(TaskError::FingerprintConflict));
}
if record.pins.operation_version != requested {
return Err(TaskApplicationError::VersionUnavailable);
}
Ok(TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support,
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
})
}
None if requested == current_version => Ok(self.service.accept(draft).await?),
// A new Task can only be accepted against the currently published version.
None => Err(TaskApplicationError::VersionUnavailable),
}
}
/// Owner-scoped status read of one Task of the addressed published Tool.
pub async fn status(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<TaskRecord, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.status(&access, task_id).await {
Ok(record) => Ok(record),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped result projection of one Task of the addressed published Tool.
pub async fn result(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<TaskOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.result(&access, task_id).await {
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped cancellation of one Task of the addressed published Tool (Story 4.4).
///
/// Cancellation is decided by the durable store under a row lock: before the dispatch checkpoint
/// it is an honest cancellation with zero outbound calls, after it the Task becomes the
/// non-terminal uncertain outcome instead of a plain cancel or a retry.
pub async fn cancel(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
expected_version: Option<u32>,
) -> Result<crank_core::TaskCancelOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self
.service
.cancel(&access, task_id, expected_version, "user_cancelled")
.await
{
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Applies one additional validated input to a waiting Task (Story 4.5).
///
/// This is the server-internal application entry: the public Task MCP API stays Story 4.7. The
/// principal is already authenticated here, the owner scope is derived from the verified
/// credential and the durable transaction revalidates the same key before any receipt lookup.
/// An identical authorized replay returns the committed outcome; a conflicting or stale answer is
/// typed, and an elapsed TTL never revives the Task.
#[allow(clippy::too_many_arguments)]
pub async fn update_input(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
wait_id: &str,
expected_version: u32,
update_key: &str,
input: &Value,
) -> Result<crank_core::TaskInputUpdateOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self
.service
.update_input(
&access,
task_id,
wait_id,
expected_version,
update_key,
input,
)
.await
{
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped attempt history of one Task of the addressed published Tool.
pub async fn attempts(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<Vec<crank_core::TaskAttempt>, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.attempts(&access, task_id).await {
Ok(attempts) => Ok(attempts),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
async fn task_access(
&self,
principal: &TaskPrincipal,
tool_name: &str,
) -> Result<TaskAccess, TaskApplicationError> {
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
let (owner_kind, owner_id) = principal.owner(&tool);
Ok(TaskAccess {
workspace_id: tool.workspace_id.as_str().to_owned(),
owner_kind,
owner_id,
authorization_scope: principal.authorization_scope(),
})
}
/// Resolves the addressed Tool of the principal's published catalog and repeats its authorization
/// check. A foreign workspace, an unknown Tool and an insufficient security level are typed
/// refusals without metadata.
async fn resolve_authorized_tool(
&self,
principal: &TaskPrincipal,
tool_name: &str,
) -> Result<PublishedAgentTool, TaskApplicationError> {
let catalog = self
.catalog
.get_catalog(principal.workspace_slug(), principal.agent_slug())
.await
.map_err(|_| TaskApplicationError::NotFound)?;
let tool = catalog
.tools
.into_iter()
.find(|tool| tool.tool_name == tool_name)
.ok_or(TaskApplicationError::NotFound)?;
if let TaskPrincipal::Admin { workspace_slug, .. } = principal
&& tool.workspace_slug != *workspace_slug
{
return Err(TaskApplicationError::Denied);
}
if let TaskPrincipal::Agent { credential, .. } = principal
&& !credential_allows_security_level(credential, tool.operation.security_level)
{
return Err(TaskApplicationError::Denied);
}
Ok(tool)
}
}
/// Bounded retry snapshot pinned from the exact published operation at acceptance. A later
/// publication, a restart or a new binary never changes the numbers an accepted Task was admitted
/// with; a missing or oversized policy is clamped to the closed story bound.
fn task_retry_policy(tool: &PublishedAgentTool) -> crank_core::TaskRetryPolicy {
let max_attempts = tool
.operation
.execution_config
.retry_policy
.as_ref()
.map_or(1, |policy| {
policy.max_attempts.clamp(1, crank_core::TASK_MAX_ATTEMPTS)
});
crank_core::TaskRetryPolicy::new(max_attempts, 1)
}
@@ -415,7 +415,7 @@ pub(super) fn agent_id() -> AgentId {
AgentId::new(format!("agent_{AGENT_SLUG}"))
}
pub(crate) async fn publish_agent(
pub(super) async fn publish_agent(
registry: &PostgresRegistry,
operation: &Operation<Schema, MappingSet>,
) {
@@ -470,7 +470,7 @@ pub(crate) async fn publish_agent(
.expect("the agent must be published");
}
pub(crate) async fn create_api_key(registry: &PostgresRegistry) -> String {
pub(super) async fn create_api_key(registry: &PostgresRegistry) -> String {
let secret = format!("crk_dispatch_observer_{}", uuid::Uuid::now_v7().simple());
let api_key = PlatformApiKey {
id: PlatformApiKeyId::new("pk_dispatch_observer"),
@@ -500,7 +500,7 @@ pub(super) fn hash_access_secret(secret: &str) -> String {
URL_SAFE_NO_PAD.encode(Sha256::digest(secret.as_bytes()))
}
pub(crate) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
Operation {
id: OperationId::new(format!("op_{TOOL_NAME}")),
name: TOOL_NAME.to_owned(),
@@ -550,7 +550,6 @@ pub(crate) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
+1 -3
View File
@@ -16,8 +16,6 @@ mod tool_search;
mod transport;
pub use app::{
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler,
VerifiedIdempotentWrite, build_app, build_app_with_artifacts,
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
build_app, build_app_with_artifacts, build_app_with_background_workers,
build_app_with_background_workers_and_limits,
};
@@ -33,7 +33,6 @@ use crate::{
access::credential_allows_security_level,
app::{
AppState, ApprovalPolicyContext, ApprovalPolicyDecision, InvocationRecord,
lifecycle::{Phase, ToolCallLifecycle},
maybe_handle_approval_policy, persist_invocation_for_key, resolve_generated_tool,
resolve_operation_auth, runtime_operation, take_confirmation_token,
},
@@ -141,15 +140,10 @@ pub(crate) enum ToolCallOutcome {
/// Handle one shared Tools command. No wire envelope, `ResponseMode`, `axum::Response`, session or
/// protocol revision crosses this boundary.
///
/// `lifecycle` is the request-scoped progress/cancellation handle of a `tools/call` (Story 3.4);
/// a command that cannot have one (`tools/list`) passes `None`, and the service then simply never
/// produces a progress event.
pub(crate) async fn handle_tools_command(
state: &Arc<AppState>,
context: ServiceContext<'_>,
command: ToolsCommand,
lifecycle: Option<ToolCallLifecycle>,
) -> ToolsOutcome {
// One marker per entered owning Tools service/handler (Story 3.1 section 7.2). It is a
// `#[cfg(test)]`-only, request-scoped counter and has no production effect.
@@ -172,9 +166,9 @@ pub(crate) async fn handle_tools_command(
Err(message) => ToolsOutcome::InvalidParams(message.to_owned()),
}
}
ToolsCommand::Call(call) => ToolsOutcome::Call(
execute_tool_command(state, &context, call, &catalog, lifecycle.as_ref()).await,
),
ToolsCommand::Call(call) => {
ToolsOutcome::Call(execute_tool_command(state, &context, call, &catalog).await)
}
}
}
@@ -184,7 +178,6 @@ async fn execute_tool_command(
context: &ServiceContext<'_>,
command: ToolCallCommand,
catalog: &PublishedAgentCatalog,
lifecycle: Option<&ToolCallLifecycle>,
) -> ToolCallOutcome {
match catalog.tool_selection_policy.mode {
ToolAccessMode::Direct => {
@@ -195,7 +188,6 @@ async fn execute_tool_command(
&command.tool_name,
command.arguments,
command.confirmation_token,
lifecycle,
)
.await
}
@@ -230,7 +222,6 @@ async fn execute_tool_command(
&proxy.name,
arguments,
confirmation_token,
lifecycle,
)
.await
}
@@ -249,7 +240,6 @@ async fn execute_resolved_tool(
tool_name: &str,
arguments: Value,
confirmation_token: Option<String>,
lifecycle: Option<&ToolCallLifecycle>,
) -> ToolCallOutcome {
let correlation = context.correlation;
let transport_request_id = correlation.request_id().as_str();
@@ -260,10 +250,6 @@ async fn execute_resolved_tool(
Some(resolved) => {
StageOutcome::Success.record(&resolve_span);
drop(resolve_span);
// First real execution step: the addressed tool of the published catalog is resolved.
if let Some(lifecycle) = lifecycle {
lifecycle.advance(Phase::Resolved);
}
resolved.tool
}
None => {
@@ -363,14 +349,8 @@ async fn execute_resolved_tool(
let resolved_auth =
resolve_operation_auth(state, &tool.workspace_id, &operation.execution_config).await;
// A cancellation can only stop the call before it reaches its terminal outcome: nothing is
// persisted for a stopped call, so Invocation History can never claim a false `completed`.
let result = match resolved_auth {
Ok(resolved_auth) => {
// Second real step: the operation's own authorization is resolved, so the runtime may run.
if let Some(lifecycle) = lifecycle {
lifecycle.advance(Phase::Authorized);
}
match RuntimeExecutionRequest::try_new(
&tool.workspace_id,
ExecutionOrigin::AgentSnapshot,
@@ -385,15 +365,7 @@ async fn execute_resolved_tool(
operation.execution_config.timeout_ms.max(1),
),
) {
Ok(request) => {
// Third real step: the request is about to enter the runtime, so from here on an
// outbound side effect may happen and a cancellation can no longer promise that
// nothing was sent.
if let Some(lifecycle) = lifecycle {
lifecycle.advance(Phase::Dispatching);
}
state.runtime.execute_outcome(request).await
}
Ok(request) => state.runtime.execute_outcome(request).await,
Err(_) => Err(ExecutionFailure::new(
ExecutionErrorCode::RuntimeInternal,
correlation.clone(),
@@ -192,46 +192,6 @@ pub(crate) fn set_active_level(level: Option<LogLevel>) {
});
}
/// The request's logging context, carried into a streamed response body (Story 3.4).
///
/// A streamed `tools/call` response is assembled after the request handler returned, so its terminal
/// event would be projected outside the task-local scope of the request. This snapshot carries the
/// only two facts the projection needs — the level the client opted in to and whether the request
/// offered the SSE carrier — and never a request value.
#[derive(Clone, Copy, Default)]
pub(crate) struct CarriedScope {
active: Option<LogLevel>,
sse_acceptable: bool,
}
/// Snapshot the scope of the request that is being served now. Outside a scope the snapshot is
/// "not opted in", which delivers nothing.
pub(crate) fn carry_scope() -> CarriedScope {
REQUEST_LOG
.try_with(|log| {
let log = log.borrow();
CarriedScope {
active: log.active,
sse_acceptable: log.sse_acceptable,
}
})
.unwrap_or_default()
}
/// Run one step of the streamed response inside a scope carrying the request's logging context.
pub(crate) async fn in_carried_scope<F: Future>(scope: CarriedScope, future: F) -> F::Output {
REQUEST_LOG
.scope(
RefCell::new(RequestLog {
active: scope.active,
sse_acceptable: scope.sse_acceptable,
pending: Vec::new(),
}),
future,
)
.await
}
/// Remember that this request offered `text/event-stream`, the only carrier of a notification.
pub(crate) fn set_sse_acceptable() {
let _ = REQUEST_LOG.try_with(|log| {
@@ -30,7 +30,6 @@ mod logging;
mod prompts;
mod registry;
mod resources;
mod signals;
pub(crate) use command::{
ContinuationSupport, ServiceContext, ToolCallCommand, ToolCallOutcome, ToolsCommand,
@@ -38,10 +37,9 @@ pub(crate) use command::{
};
pub(crate) use completion::{CompletionCommand, CompletionOutcome, handle_completion_command};
pub(crate) use logging::{
CarriedScope, EVENT_FAILED, EVENT_LEVEL_APPLIED, EVENT_REJECTED, EVENT_SERVED, INVALID_LEVEL,
LogLevel, LogLevelStore, LoggingOutcome, SetLevelCommand, carry_scope, handle_logging_command,
in_carried_scope, modern_request_level, note, scoped as logging_scope, set_active_level,
set_sse_acceptable, take_notifications,
EVENT_FAILED, EVENT_LEVEL_APPLIED, EVENT_REJECTED, EVENT_SERVED, INVALID_LEVEL, LogLevel,
LogLevelStore, LoggingOutcome, SetLevelCommand, handle_logging_command, modern_request_level,
note, scoped as logging_scope, set_active_level, set_sse_acceptable, take_notifications,
};
pub(crate) use prompts::{
PromptGetCommand, PromptsCommand, PromptsListCommand, PromptsOutcome, handle_prompts_command,
@@ -50,10 +48,6 @@ pub(crate) use resources::{
ResourceReadCommand, ResourceTemplatesListError, ResourcesCommand, ResourcesListCommand,
ResourcesOutcome, handle_resources_command, list_resource_templates,
};
pub(crate) use signals::{
FilterOutcome, INVALID_LISTEN_FILTER, ListenParams, Signals, StreamRegistry, Watch, Wire,
acknowledged_frame, frames_stream, spawn as spawn_signal_watch, subscription_id,
};
/// Resolved era of an incoming POST request.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -22,11 +22,6 @@ pub(crate) enum CapabilityId {
Prompts,
/// Bounded argument completion for bound Published Prompts and Resource templates (Story 3.17).
Completions,
/// `subscriptions/listen` of the modern revision: a *client-sent* request that opens the
/// notification stream (Story 3.19). It is a dispatch-only registration without an advertisement
/// fragment because the pinned modern schema has no server `subscriptions` capability object;
/// there is no such method in the legacy revision at all.
Subscriptions,
/// `logging` server capability: safe structured `notifications/message` events of the negotiated
/// client context (Story 3.18). Advertised in both revisions; the pinned modern revision carries
/// the level per request in `_meta` and marks the capability deprecated (SEP-2577).
@@ -95,9 +90,6 @@ pub(crate) enum MethodHandler {
/// `completion/complete`: bounded suggestions from the typed schema of one exact bound Prompt
/// argument or one exact bound Resource template parameter (Story 3.17).
Completion,
/// `subscriptions/listen`: open the long-lived modern notification stream and deliver the
/// Resource/Prompt list-change signals the client opted in to (Story 3.19).
SubscriptionsListen,
/// `logging/setLevel`: validate the pinned level and record it for this legacy session, so the
/// adapter can deliver safe structured `notifications/message` events of that context
/// (Story 3.18). The handler never widens the accepted vocabulary and never echoes the input.
@@ -105,10 +97,6 @@ pub(crate) enum MethodHandler {
Ping,
Discover,
Lifecycle,
/// `notifications/cancelled`: the legacy client cancels a `tools/call` it already issued
/// (Story 3.4). The handler only routes the id to the owner's in-flight call; it never widens
/// what may be cancelled and never authorizes by the progress token.
Cancelled,
}
#[derive(Clone, Copy, Debug)]
@@ -373,9 +361,7 @@ pub(crate) const CAPABILITY_DESCRIPTORS: &[CapabilityIdDescriptor] = &[
extension: None,
deprecated: None,
sub_features: FeatureSet(&["listChanged"]),
// Story 3.5: the adapter emits `notifications/tools/list_changed` from the committed
// published Tool bindings of the Agent snapshot, so the declaration is now true.
advertised: Some(("tools", &[("listChanged", true)])),
advertised: Some(("tools", &[("listChanged", false)])),
revisions: MODERN_AND_LEGACY,
},
CapabilityIdDescriptor {
@@ -385,9 +371,7 @@ pub(crate) const CAPABILITY_DESCRIPTORS: &[CapabilityIdDescriptor] = &[
extension: None,
deprecated: None,
sub_features: FeatureSet(&["subscribe", "listChanged"]),
// Story 3.19: the server really emits `notifications/resources/list_changed`; URI-content
// subscriptions (`subscribe`) remain unimplemented and are still advertised as false.
advertised: Some(("resources", &[("subscribe", false), ("listChanged", true)])),
advertised: Some(("resources", &[("subscribe", false), ("listChanged", false)])),
revisions: MODERN_AND_LEGACY,
},
CapabilityIdDescriptor {
@@ -397,8 +381,7 @@ pub(crate) const CAPABILITY_DESCRIPTORS: &[CapabilityIdDescriptor] = &[
extension: None,
deprecated: None,
sub_features: FeatureSet(&["listChanged"]),
// Story 3.19: the server really emits `notifications/prompts/list_changed`.
advertised: Some(("prompts", &[("listChanged", true)])),
advertised: Some(("prompts", &[("listChanged", false)])),
revisions: MODERN_AND_LEGACY,
},
CapabilityIdDescriptor {
@@ -439,19 +422,6 @@ pub(crate) const CAPABILITY_DESCRIPTORS: &[CapabilityIdDescriptor] = &[
advertised: None,
revisions: LEGACY_ONLY,
},
CapabilityIdDescriptor {
// Story 3.19: dispatch-only registration of the modern `subscriptions/listen` request. The
// client sends it; the pinned modern schema declares no server `subscriptions` capability
// object, so it carries no advertisement fragment. The legacy revision has no such method.
id: CapabilityId::Subscriptions,
roster: Roster::Client,
kind: Kind::Method,
extension: None,
deprecated: None,
sub_features: FeatureSet(&[]),
advertised: None,
revisions: MODERN_ONLY,
},
CapabilityIdDescriptor {
id: CapabilityId::Ping,
roster: Roster::Server,
@@ -566,17 +536,6 @@ pub(crate) const METHOD_DESCRIPTORS: &[MethodDescriptor] = &[
required_headers: HEADERS_METHOD_ONLY,
handler: MethodHandler::LoggingSetLevel,
},
MethodDescriptor {
// Story 3.19: the modern long-lived notification stream. It declares the notification filter
// in its params, not a server-side name, so only the method header applies. The legacy
// revision opens its notification stream with `GET` instead, so this name is a wrong-era
// request there and must never be served.
name: "subscriptions/listen",
capability: CapabilityId::Subscriptions,
kind: Kind::Method,
required_headers: HEADERS_METHOD_ONLY,
handler: MethodHandler::SubscriptionsListen,
},
MethodDescriptor {
name: "ping",
capability: CapabilityId::Ping,
@@ -605,17 +564,6 @@ pub(crate) const METHOD_DESCRIPTORS: &[MethodDescriptor] = &[
required_headers: HEADERS_NONE,
handler: MethodHandler::Lifecycle,
},
MethodDescriptor {
// Story 3.4: the legacy revision's client-side cancellation of a request it issued. The
// pinned modern revision has no client-to-server notification over Streamable HTTP (closing
// the response stream is its cancellation signal), so this name exists in the legacy era
// only. It is a dispatch-only notification registration and advertises nothing.
name: "notifications/cancelled",
capability: CapabilityId::LegacyLifecycle,
kind: Kind::Notification,
required_headers: HEADERS_NONE,
handler: MethodHandler::Cancelled,
},
];
#[cfg(test)]
@@ -37,9 +37,9 @@ fn static_registration_is_unique_and_advertises_only_the_registered_capability()
json!({
"completions": {},
"logging": {},
"prompts": {"listChanged": true},
"resources": {"listChanged": true, "subscribe": false},
"tools": {"listChanged": true}
"prompts": {"listChanged": false},
"resources": {"listChanged": false, "subscribe": false},
"tools": {"listChanged": false}
}),
"the registered advertisement must be the same reviewed object in {revision}"
);
@@ -48,9 +48,9 @@ fn static_registration_is_unique_and_advertises_only_the_registered_capability()
json!({
"completions": {},
"logging": {},
"prompts": {"listChanged": true},
"resources": {"listChanged": true, "subscribe": false},
"tools": {"listChanged": true}
"prompts": {"listChanged": false},
"resources": {"listChanged": false, "subscribe": false},
"tools": {"listChanged": false}
}),
"the production advertisement entry point must serve {revision}"
);
@@ -262,9 +262,9 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
json!({
"completions": {},
"logging": {},
"prompts": {"listChanged": true},
"resources": {"listChanged": true, "subscribe": false},
"tools": {"listChanged": true}
"prompts": {"listChanged": false},
"resources": {"listChanged": false, "subscribe": false},
"tools": {"listChanged": false}
})
);
assert_eq!(
@@ -273,9 +273,9 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
"inert_test_capability": {"enabled": true},
"completions": {},
"logging": {},
"prompts": {"listChanged": true},
"resources": {"listChanged": true, "subscribe": false},
"tools": {"listChanged": true}
"prompts": {"listChanged": false},
"resources": {"listChanged": false, "subscribe": false},
"tools": {"listChanged": false}
})
);
assert_eq!(
@@ -284,9 +284,9 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
"inert_test_capability": {"enabled": true},
"completions": {},
"logging": {},
"prompts": {"listChanged": true},
"resources": {"listChanged": true, "subscribe": false},
"tools": {"listChanged": true}
"prompts": {"listChanged": false},
"resources": {"listChanged": false, "subscribe": false},
"tools": {"listChanged": false}
})
);
@@ -468,9 +468,9 @@ fn a_single_era_descriptor_never_leaks_into_the_other_era_advertisement() {
json!({
"completions": {},
"logging": {},
"prompts": {"listChanged": true},
"resources": {"listChanged": true, "subscribe": false},
"tools": {"listChanged": true}
"prompts": {"listChanged": false},
"resources": {"listChanged": false, "subscribe": false},
"tools": {"listChanged": false}
}),
"the other era advertisement stays the reviewed object"
);
@@ -759,9 +759,9 @@ fn adding_and_removing_a_method_descriptor_changes_the_dispatch_decision() {
"inert_test_capability": {"enabled": true},
"completions": {},
"logging": {},
"prompts": {"listChanged": true},
"resources": {"listChanged": true, "subscribe": false},
"tools": {"listChanged": true}
"prompts": {"listChanged": false},
"resources": {"listChanged": false, "subscribe": false},
"tools": {"listChanged": false}
})
);
}
@@ -1,501 +0,0 @@
//! Story 3.19: Resource and Prompt list-change signals (FR-19); Story 3.5 adds the Tools signal of
//! the same seam (FR-14).
//!
//! The positive result of the story is that a client which asked for the list-change signal of its
//! revision learns that the *committed* Resource, Prompt or Tool catalog changed, and that the
//! ordinary `resources/list`, `resources/read`, `resources/templates/list`, `prompts/list`,
//! `prompts/get` and `tools/list` stay authoritative for a client that misses the notification.
//!
//! Shape of the chosen seam (ARC decision, kanban t_c12863fb): a **stream-local watcher**. One open
//! notification stream owns one bounded periodic read of the published Agent catalog the stream is
//! scoped to; a change of the visible Resource/Prompt/Tool projection of that snapshot (the Resource
//! and Prompt bindings plus the exact Tool definitions `tools/list` serves) produces the revision's
//! list-change notification. There is no post-commit hook, no cross-process bus and no durable
//! queue: the committed catalog is the source of truth, so a signal can never be produced before its
//! commit, and a failed read can never roll the commit back — it only skips that tick and stays
//! observable in tracing.
//!
//! Bounds (issue requirement: bounded queues/frequency/bytes):
//! * one in-flight read per stream, one fixed poll interval, a read timeout and skipped ticks;
//! * at most one frame per change type per tick, so a burst coalesces by the latest observed
//! revision instead of queueing every intermediate commit;
//! * the queue is a fixed-capacity channel; a slow reader drops the newest frame instead of blocking
//! the producer or growing memory, and recovers through the authoritative list methods;
//! * every frame has a fixed shape — a closed method name plus the request id of the subscription —
//! so the byte bound follows from the id bound ([`MAX_SUBSCRIPTION_ID_BYTES`]).
//!
//! Scope boundary: this module serves list-change signals only. URI-content subscriptions
//! (`resources/subscribe` / `notifications/resources/updated`) stay unimplemented and are never
//! acknowledged or emitted here.
use std::{
collections::{HashMap, VecDeque},
future::Future,
pin::Pin,
sync::{
Arc, Mutex, MutexGuard,
atomic::{AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use axum::response::sse::Event;
use crank_registry::PublishedAgentCatalog;
use futures_util::stream;
use serde::Deserialize;
use serde_json::{Map, Value, json};
use tokio::sync::mpsc;
use tracing::warn;
use crate::app::AppState;
/// The pinned key that correlates every message of one subscription stream with its
/// `subscriptions/listen` request id.
pub(crate) const SUBSCRIPTION_ID_KEY: &str = "io.modelcontextprotocol/subscriptionId";
/// `notifications/resources/list_changed` (both approved revisions).
pub(crate) const RESOURCE_LIST_CHANGED: &str = "notifications/resources/list_changed";
/// `notifications/prompts/list_changed` (both approved revisions).
pub(crate) const PROMPT_LIST_CHANGED: &str = "notifications/prompts/list_changed";
/// `notifications/tools/list_changed` (both approved revisions, Story 3.5).
pub(crate) const TOOL_LIST_CHANGED: &str = "notifications/tools/list_changed";
/// `notifications/subscriptions/acknowledged` (modern revision only).
pub(crate) const SUBSCRIPTIONS_ACKNOWLEDGED: &str = "notifications/subscriptions/acknowledged";
/// Engineering default of the watcher tick. It is not a measured latency SLO: the story only needs
/// "after commit", and the client always keeps the authoritative list methods for recovery.
pub(crate) const POLL_INTERVAL: Duration = Duration::from_secs(1);
/// Upper bound of one watcher read. A read that exceeds it is a skipped tick, exactly like a failed
/// read: no signal, no rollback, one safe tracing event.
pub(crate) const READ_TIMEOUT: Duration = Duration::from_secs(5);
/// Frames a slow reader may fall behind before the newest frame is dropped (`coalescing`).
pub(crate) const SIGNAL_QUEUE_CAPACITY: usize = 4;
/// Bound of the encoded `subscriptions/listen` id carried by every frame of one stream.
pub(crate) const MAX_SUBSCRIPTION_ID_BYTES: usize = 64;
/// Bound of the legacy session stream table (memory bound; see [`StreamRegistry`]).
pub(crate) const MAX_TRACKED_STREAMS: usize = 1_024;
/// Bound of the URIs a modern filter may name. The server does not implement URI subscriptions, so
/// this many names is already far more than a well-behaved client asks for.
pub(crate) const MAX_FILTER_URIS: usize = 32;
/// Bound of one URI in the modern filter: longer values are rejected without being echoed back.
pub(crate) const MAX_FILTER_URI_BYTES: usize = 2_048;
/// The one safe detail of a malformed or oversized `subscriptions/listen` filter.
pub(crate) const INVALID_LISTEN_FILTER: &str = "invalid subscriptions/listen notification filter";
/// The three list-change signal types this server implements.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct Signals {
pub(crate) resources: bool,
pub(crate) prompts: bool,
pub(crate) tools: bool,
}
impl Signals {
/// All types: the legacy revision defines no client opt-in, so its stream carries every type.
pub(crate) const ALL: Self = Self {
resources: true,
prompts: true,
tools: true,
};
#[cfg_attr(not(test), allow(dead_code))] // read by the unit oracle of the projection
pub(crate) fn any(self) -> bool {
self.resources || self.prompts || self.tools
}
fn intersect(self, other: Self) -> Self {
Self {
resources: self.resources && other.resources,
prompts: self.prompts && other.prompts,
tools: self.tools && other.tools,
}
}
}
/// The pinned modern `SubscriptionFilter`, as far as this server reads it.
///
/// Unknown members are ignored and a malformed shape is a deserialization error, so a client cannot
/// reach the watcher with anything but the reviewed field set.
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ListenFilter {
#[serde(default)]
tools_list_changed: bool,
#[serde(default)]
prompts_list_changed: bool,
#[serde(default)]
resources_list_changed: bool,
#[serde(default)]
resource_subscriptions: Vec<String>,
}
/// The pinned `SubscriptionsListenRequestParams`: the notification filter is a required member.
///
/// Only the reviewed member is read; `_meta` and any other member of the request params is not part
/// of this contract.
#[derive(Debug, Deserialize)]
pub(crate) struct ListenParams {
pub(crate) notifications: ListenFilter,
}
/// The bounded input contract of a `subscriptions/listen` filter.
pub(crate) enum FilterOutcome {
/// The list-change subset this server honours for the stream.
Accepted(Signals),
/// One safe detail: no URI, name or value the client sent is echoed back.
Invalid(&'static str),
}
impl ListenFilter {
/// Validate the bounded inputs and keep only the honoured types.
///
/// `resourceSubscriptions` is not implemented in the current scope, which is what the
/// acknowledgment reports: it is simply not honoured, and the pinned schema forbids the server
/// from sending a type it did not agree to.
pub(crate) fn accepted(&self) -> FilterOutcome {
if self.resource_subscriptions.len() > MAX_FILTER_URIS
|| self
.resource_subscriptions
.iter()
.any(|uri| uri.len() > MAX_FILTER_URI_BYTES)
{
return FilterOutcome::Invalid(INVALID_LISTEN_FILTER);
}
FilterOutcome::Accepted(Signals {
resources: self.resources_list_changed,
prompts: self.prompts_list_changed,
tools: self.tools_list_changed,
})
}
}
/// The `notifications` object of the acknowledgment: exactly the subset that will be delivered.
fn acknowledged_filter_wire(accepted: Signals) -> Value {
let mut filter = Map::new();
if accepted.prompts {
filter.insert("promptsListChanged".to_owned(), Value::Bool(true));
}
if accepted.resources {
filter.insert("resourcesListChanged".to_owned(), Value::Bool(true));
}
if accepted.tools {
filter.insert("toolsListChanged".to_owned(), Value::Bool(true));
}
Value::Object(filter)
}
/// The exact bound Resource/Prompt/Tool projection of one published Agent catalog.
///
/// The projection is derived from the same committed catalog read that already feeds
/// `resources/list`, `resources/templates/list`, `prompts/list`, `prompts/get` and `tools/list`: the
/// exact Resource and Prompt bindings of the published Agent snapshot, in deterministic binding
/// order, and the exact Tool definitions that `tools/list` would serve. The Tool part is taken from
/// [`crate::manifest::catalog_tool_definitions`] rather than from the bindings, because that is the
/// projection the client sees: it also depends on the Tool selection policy (mode, groups, search
/// settings), and in `Search` mode it does not depend on the bindings at all. `catalog_revision`
/// alone is *not* the comparison key — a commit that only changes Tools must not produce a Resource
/// or Prompt signal, and vice versa — so the three projections are compared on their own.
///
/// shortcut: a Resource revocation invalidates a bound publication without a new Agent snapshot, so
/// it does not change this projection and produces no list-change signal; the revocation gate at
/// read time (Story 3.8) still fails closed, and a faithful revocation signal needs a bounded
/// registry read that the current scope does not have.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct CatalogFingerprint {
resources: Vec<(String, u32)>,
prompts: Vec<(String, u32)>,
/// Canonical JSON of every definition `tools/list` serves, sorted so a re-ordered read is not a
/// false change.
tools: Vec<String>,
}
impl CatalogFingerprint {
pub(crate) fn of(catalog: &PublishedAgentCatalog) -> Self {
fn sorted<T: Ord>(values: impl Iterator<Item = T>) -> Vec<T> {
let mut sorted = values.collect::<Vec<_>>();
sorted.sort_unstable();
sorted
}
Self {
resources: sorted(catalog.resources.iter().map(|binding| {
(
binding.resource_id.as_str().to_owned(),
binding.resource_version,
)
})),
prompts: sorted(catalog.prompts.iter().map(|binding| {
(
binding.prompt_id.as_str().to_owned(),
binding.prompt_version,
)
})),
tools: sorted(
crate::manifest::catalog_tool_definitions(catalog)
.into_iter()
.map(|definition| definition.to_string()),
),
}
}
/// The list-change types whose visible projection differs between `previous` and `self`.
pub(crate) fn changes_since(&self, previous: &Self) -> Signals {
Signals {
resources: self.resources != previous.resources,
prompts: self.prompts != previous.prompts,
tools: self.tools != previous.tools,
}
}
}
/// The wire contract of one stream.
#[derive(Clone, Debug)]
pub(crate) enum Wire {
/// `2025-11-25`: a bare server notification without parameters.
Legacy,
/// `2026-07-28`: every message carries the subscription id of its `subscriptions/listen` request.
Modern(Value),
}
impl Wire {
fn signal_frames(&self, signals: Signals) -> Vec<String> {
let mut frames = Vec::with_capacity(3);
for (changed, method) in [
(signals.resources, RESOURCE_LIST_CHANGED),
(signals.prompts, PROMPT_LIST_CHANGED),
(signals.tools, TOOL_LIST_CHANGED),
] {
if !changed {
continue;
}
let frame = match self {
Wire::Legacy => json!({ "jsonrpc": "2.0", "method": method }),
Wire::Modern(subscription_id) => json!({
"jsonrpc": "2.0",
"method": method,
"params": { "_meta": { SUBSCRIPTION_ID_KEY: subscription_id } }
}),
};
frames.push(frame.to_string());
}
frames
}
}
/// The first modern frame: the accepted subset, before any list-change notification of the stream.
pub(crate) fn acknowledged_frame(subscription_id: &Value, accepted: Signals) -> String {
json!({
"jsonrpc": "2.0",
"method": SUBSCRIPTIONS_ACKNOWLEDGED,
"params": {
"_meta": { SUBSCRIPTION_ID_KEY: subscription_id },
"notifications": acknowledged_filter_wire(accepted),
}
})
.to_string()
}
/// The subscription id of a `subscriptions/listen` request, or `None` when its encoding is not a
/// bounded identity carrier.
pub(crate) fn subscription_id(request_id: &Value) -> Option<Value> {
let encoded = request_id.to_string();
(encoded.len() <= MAX_SUBSCRIPTION_ID_BYTES).then(|| request_id.clone())
}
/// Bounded table of the legacy session that currently owns the notification stream.
///
/// The pinned legacy transport forbids broadcasting one message across several SSE streams of the
/// same session, and the community scope keeps exactly one producer per session: the newest admitted
/// `GET` takes the signals over and the older stream ends. A retired stream (session `DELETE`) or a
/// superseded generation never writes into a reconnect.
#[derive(Default)]
pub(crate) struct StreamRegistry {
next_generation: AtomicU64,
owners: Mutex<HashMap<String, u64>>,
}
impl StreamRegistry {
pub(crate) fn new() -> Self {
Self::default()
}
/// Admit the notification stream of one session and return its generation.
pub(crate) fn admit(&self, session_id: &str) -> Option<u64> {
let generation = self.next_generation.fetch_add(1, Ordering::Relaxed) + 1;
let mut owners = self.lock()?;
if owners.len() >= MAX_TRACKED_STREAMS && !owners.contains_key(session_id) {
// Bounded table: forget the whole owner set instead of growing without a limit. Each
// running producer still holds its own generation, so the worst case is a stream that
// can no longer be told apart from a newer one — never unbounded memory.
owners.clear();
}
owners.insert(session_id.to_owned(), generation);
Some(generation)
}
/// Whether `generation` still owns the stream of `session_id`.
pub(crate) fn is_current(&self, session_id: &str, generation: u64) -> bool {
self.lock()
.and_then(|owners| owners.get(session_id).copied())
.is_some_and(|current| current == generation)
}
/// Stop the producer of one session (`DELETE`, or a session that no longer exists).
pub(crate) fn retire(&self, session_id: &str) {
if let Some(mut owners) = self.lock() {
owners.remove(session_id);
}
}
fn lock(&self) -> Option<MutexGuard<'_, HashMap<String, u64>>> {
self.owners.try_lock().ok()
}
}
/// Everything one watcher needs.
pub(crate) struct Watch {
pub(crate) workspace_slug: String,
pub(crate) agent_slug: String,
/// The signal types this stream may deliver.
pub(crate) signals: Signals,
pub(crate) wire: Wire,
/// The legacy session lease: the producer stops when the stream is superseded or retired.
pub(crate) legacy_stream: Option<(Arc<StreamRegistry>, String, u64)>,
/// Lifetime bound inherited from the legacy transport session TTL.
pub(crate) deadline: Option<Instant>,
}
/// One committed-catalog read of a watcher tick.
///
/// The reader is a closure so the failure path is driveable in a unit oracle without a live
/// registry: production passes [`read`] over the shared [`AppState`], and the oracle passes a
/// scripted sequence.
pub(crate) type ReadFuture =
Pin<Box<dyn Future<Output = Result<CatalogFingerprint, ReadError>> + Send>>;
/// A source of committed-catalog reads. One call per tick.
pub(crate) type Reader = Box<dyn FnMut() -> ReadFuture + Send>;
/// Start the stream-local watcher and return the bounded frame queue of the response stream.
///
/// The first read is the baseline taken **before** the acknowledgment is written; it is never reset
/// afterwards, so a publication between the baseline and the acknowledgment is still delivered.
pub(crate) fn spawn(state: Arc<AppState>, watch: Watch) -> mpsc::Receiver<String> {
let scope = (watch.workspace_slug.clone(), watch.agent_slug.clone());
let reader: Reader = Box::new(move || {
let state = Arc::clone(&state);
let scope = scope.clone();
Box::pin(async move { read(&state, &scope).await })
});
let (sender, receiver) = mpsc::channel(SIGNAL_QUEUE_CAPACITY);
tokio::spawn(run(watch, sender, reader));
receiver
}
async fn run(watch: Watch, sender: mpsc::Sender<String>, mut read: Reader) {
let mut baseline = match read().await {
Ok(fingerprint) => Some(fingerprint),
Err(ReadError::Gone) => return,
Err(ReadError::Retry) => {
observe_failure();
None
}
};
loop {
if !still_owned(&watch) || sender.is_closed() {
return;
}
if watch
.deadline
.is_some_and(|deadline| Instant::now() >= deadline)
{
return;
}
tokio::time::sleep(POLL_INTERVAL).await;
if !still_owned(&watch) {
return;
}
match read().await {
Ok(fingerprint) => {
if let Some(previous) = baseline.as_ref() {
let changes = fingerprint.changes_since(previous).intersect(watch.signals);
for frame in watch.wire.signal_frames(changes) {
// A slow reader coalesces: the newest frame is dropped instead of blocking
// the producer or growing a queue, and the client recovers through the
// authoritative list methods.
let _ = sender.try_send(frame);
}
}
baseline = Some(fingerprint);
}
// The Agent is no longer published: fail closed by ending the stream instead of serving a
// catalog nobody may read any more.
Err(ReadError::Gone) => return,
// The committed catalog is untouched: the failure is observable and this tick is skipped.
Err(_) => observe_failure(),
}
}
}
pub(crate) enum ReadError {
/// The published Agent behind this stream is gone: the producer ends.
Gone,
/// The catalog could not be read this tick: skip it, keep the committed state.
Retry,
}
fn still_owned(watch: &Watch) -> bool {
match &watch.legacy_stream {
Some((registry, session_id, generation)) => registry.is_current(session_id, *generation),
None => true,
}
}
async fn read(state: &AppState, scope: &(String, String)) -> Result<CatalogFingerprint, ReadError> {
let catalog = tokio::time::timeout(READ_TIMEOUT, state.catalog.get_catalog(&scope.0, &scope.1))
.await
.map_err(|_| ReadError::Retry)?
.map_err(|error| match error {
crank_registry::RegistryError::PublishedAgentNotFound { .. } => ReadError::Gone,
_ => ReadError::Retry,
})?;
Ok(CatalogFingerprint::of(&catalog))
}
/// One safe, closed-vocabulary observation of a skipped tick: no URI, Agent name, credential or raw
/// error text is read here.
fn observe_failure() {
warn!(
name: "mcp.signals.read_failed",
error_category = "catalog",
"catalog change signal read failed"
);
}
/// The bounded SSE body of one signal stream: the leading frames (acknowledgment), then the queue,
/// ending when the producer ends. The caller's stream permit lives in `guard` and is released when
/// the response stream is dropped.
pub(crate) fn frames_stream<G: Send + 'static>(
first: Vec<String>,
receiver: mpsc::Receiver<String>,
guard: G,
) -> impl stream::Stream<Item = Result<Event, std::convert::Infallible>> {
let pending: VecDeque<String> = first.into();
stream::unfold(
(pending, receiver, guard),
|(mut pending, mut receiver, guard)| async move {
loop {
if let Some(frame) = pending.pop_front() {
return Some((Ok(Event::default().data(frame)), (pending, receiver, guard)));
}
let frame = receiver.recv().await?;
pending.push_back(frame);
}
},
)
}
#[cfg(test)]
#[path = "signals_tests.rs"]
mod tests;
@@ -1,741 +0,0 @@
//! Unit oracles of the Story 3.19 signal seam: the catalog projection, the bounded filter, the exact
//! wire frames of both approved revisions and the bounded legacy stream ownership.
use std::{
collections::BTreeMap,
io,
sync::{Arc, Mutex},
time::{Duration, Instant},
};
use crank_core::ids::{PromptId, ResourceId};
use crank_core::{
ExecutionConfig, HttpMethod, Operation, OperationId, OperationSecurityLevel, OperationStatus,
Protocol, RestTarget, Target, ToolAccessMode, ToolDescription, ToolGroup, ToolSearchSettings,
ToolSelectionPolicy, WorkspaceId,
};
use crank_mapping::MappingSet;
use crank_registry::PublishedAgentCatalog;
use crank_registry::model::{AgentPromptBinding, AgentResourceBinding, PublishedAgentTool};
use crank_schema::{Schema, SchemaKind};
use serde_json::{Value, json};
use time::OffsetDateTime;
use tokio::sync::mpsc;
use super::{
CatalogFingerprint, FilterOutcome, INVALID_LISTEN_FILTER, ListenFilter, MAX_FILTER_URIS,
MAX_SUBSCRIPTION_ID_BYTES, PROMPT_LIST_CHANGED, RESOURCE_LIST_CHANGED, ReadError, Reader,
SIGNAL_QUEUE_CAPACITY, SUBSCRIPTION_ID_KEY, SUBSCRIPTIONS_ACKNOWLEDGED, Signals,
StreamRegistry, TOOL_LIST_CHANGED, Watch, Wire, acknowledged_frame, run, subscription_id,
};
fn catalog(
resources: &[(&str, u32)],
prompts: &[(&str, u32)],
catalog_revision: &str,
) -> PublishedAgentCatalog {
catalog_with(
ToolSelectionPolicy::default(),
Vec::new(),
resources,
prompts,
catalog_revision,
)
}
fn catalog_with(
tool_selection_policy: ToolSelectionPolicy,
tools: Vec<PublishedAgentTool>,
resources: &[(&str, u32)],
prompts: &[(&str, u32)],
catalog_revision: &str,
) -> PublishedAgentCatalog {
PublishedAgentCatalog {
agent_version: 1,
catalog_revision: catalog_revision.to_owned(),
tool_selection_policy,
tools,
resources: resources
.iter()
.map(|(id, version)| AgentResourceBinding {
resource_id: ResourceId::new(*id),
resource_version: *version,
})
.collect(),
prompts: prompts
.iter()
.map(|(id, version)| AgentPromptBinding {
prompt_id: PromptId::new(*id),
prompt_version: *version,
})
.collect(),
}
}
fn search_policy(tool_names: &[&str]) -> ToolSelectionPolicy {
ToolSelectionPolicy {
mode: ToolAccessMode::Search,
groups: vec![ToolGroup {
id: "crm".to_owned(),
name: "CRM".to_owned(),
description: "CRM operations".to_owned(),
tool_names: tool_names.iter().map(|name| (*name).to_owned()).collect(),
}],
search: ToolSearchSettings::default(),
}
}
fn empty_object_schema() -> Schema {
Schema {
kind: SchemaKind::Object,
format: None,
description: None,
required: false,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
}
}
/// One published Tool binding whose served definition is exactly `title`/`description`.
fn published_tool(name: &str, title: &str, description: &str, version: u32) -> PublishedAgentTool {
PublishedAgentTool {
workspace_id: WorkspaceId::new("ws_test"),
workspace_slug: "ws-test".to_owned(),
agent_id: "agent_test".into(),
agent_slug: "agent".to_owned(),
operation: Operation {
id: OperationId::new(format!("op_{name}")),
name: name.to_owned(),
display_name: title.to_owned(),
category: "test".to_owned(),
protocol: Protocol::Rest,
security_level: OperationSecurityLevel::Standard,
status: OperationStatus::Published,
version,
target: Target::Rest(RestTarget {
base_url: "https://example.test".to_owned(),
method: HttpMethod::Get,
path_template: "/leads".to_owned(),
static_headers: BTreeMap::new(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: BTreeMap::new(),
}),
input_schema: empty_object_schema(),
output_schema: empty_object_schema(),
input_mapping: MappingSet { rules: Vec::new() },
output_mapping: MappingSet { rules: Vec::new() },
execution_config: ExecutionConfig {
timeout_ms: 10_000,
retry_policy: None,
response_cache: None,
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
tool_description: ToolDescription {
title: title.to_owned(),
description: description.to_owned(),
tags: Vec::new(),
examples: Vec::new(),
},
samples: None,
generated_draft: None,
config_export: None,
wizard_state: None,
created_at: OffsetDateTime::UNIX_EPOCH,
updated_at: OffsetDateTime::UNIX_EPOCH,
published_at: None,
},
tool_name: name.to_owned(),
tool_title: title.to_owned(),
tool_description: description.to_owned(),
}
}
#[test]
fn a_committed_resource_binding_is_a_resource_change_only() {
let before = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-1"));
let after = CatalogFingerprint::of(&catalog(
&[("res_a", 1), ("res_b", 1)],
&[("prm_a", 1)],
"rev-2",
));
assert_eq!(
after.changes_since(&before),
Signals {
resources: true,
prompts: false,
tools: false
}
);
assert!(!before.changes_since(&after).prompts);
assert_eq!(
before.changes_since(&after),
Signals {
resources: true,
prompts: false,
tools: false
}
);
}
#[test]
fn a_committed_prompt_binding_is_a_prompt_change_only() {
let before = CatalogFingerprint::of(&catalog(&[], &[("prm_a", 1)], "rev-1"));
let after = CatalogFingerprint::of(&catalog(&[], &[("prm_a", 1), ("prm_b", 2)], "rev-2"));
assert_eq!(
after.changes_since(&before),
Signals {
resources: false,
prompts: true,
tools: false
}
);
}
#[test]
fn a_pinned_version_bump_is_a_change_even_when_the_binding_names_are_stable() {
let before = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-1"));
let after = CatalogFingerprint::of(&catalog(&[("res_a", 2)], &[("prm_a", 1)], "rev-1"));
assert_eq!(
after.changes_since(&before),
Signals {
resources: true,
prompts: false,
tools: false
}
);
}
#[test]
fn a_committed_tool_definition_change_is_a_tool_change_only() {
let lead = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
let before = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![lead()],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-1",
));
let added = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![
lead(),
published_tool(
"crm_create_second",
"Create Second",
"Creates a second lead",
1,
),
],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-2",
));
let retitled = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![published_tool(
"crm_create_lead",
"Create a lead",
"Creates a lead",
1,
)],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-2",
));
let tools_only = Signals {
resources: false,
prompts: false,
tools: true,
};
assert_eq!(added.changes_since(&before), tools_only);
assert_eq!(
retitled.changes_since(&before),
tools_only,
"a retitled binding changes the visible Tool projection without a new Operation version"
);
// The reverse direction is a Tool change as well: a removed binding is a real change.
assert!(before.changes_since(&added).tools);
}
#[test]
fn a_republished_operation_version_with_an_identical_definition_is_silent() {
// `tools/list` serves the definition, not the Operation version: a version bump that leaves the
// served definition unchanged is not a client-visible Tool catalog change.
let before = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![published_tool(
"crm_create_lead",
"Create Lead",
"Creates a lead",
1,
)],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-1",
));
let after = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![published_tool(
"crm_create_lead",
"Create Lead",
"Creates a lead",
2,
)],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-2",
));
assert!(!after.changes_since(&before).any());
}
#[test]
fn a_policy_only_commit_that_changes_tools_list_is_a_tool_change_only() {
// The Tool selection policy is part of the served `tools/list` projection: switching the same
// bindings from direct to search mode replaces the served tools with `search_tools`/`call_tool`.
let tool = published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
let direct = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![tool.clone()],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-1",
));
let search = CatalogFingerprint::of(&catalog_with(
search_policy(&["crm_create_lead"]),
vec![tool],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-2",
));
let tools_only = Signals {
resources: false,
prompts: false,
tools: true,
};
assert_eq!(search.changes_since(&direct), tools_only);
assert_eq!(direct.changes_since(&search), tools_only);
}
#[test]
fn a_policy_field_invisible_to_tools_list_is_silent() {
// `search.max_results` is only served inside the search-mode definitions: in direct mode it
// changes nothing the client receives.
let tool = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
let before = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![tool()],
&[],
&[],
"rev-1",
));
let after = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy {
search: ToolSearchSettings { max_results: 5 },
..ToolSelectionPolicy::default()
},
vec![tool()],
&[],
&[],
"rev-2",
));
assert!(!after.changes_since(&before).any());
}
#[test]
fn a_search_mode_binding_change_is_silent_when_tools_list_is_unchanged() {
// In search mode the served list is `search_tools`/`call_tool`; the bindings behind it are not
// part of `tools/list`, so a binding-only commit must not claim a visible Tool change.
let policy = search_policy(&["crm_create_lead"]);
let before = CatalogFingerprint::of(&catalog_with(
policy.clone(),
vec![published_tool(
"crm_create_lead",
"Create Lead",
"Creates a lead",
1,
)],
&[],
&[],
"rev-1",
));
let after = CatalogFingerprint::of(&catalog_with(
policy,
vec![
published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1),
published_tool(
"crm_create_second",
"Create Second",
"Creates a second lead",
1,
),
],
&[],
&[],
"rev-2",
));
assert!(!after.changes_since(&before).any());
}
#[test]
fn a_resource_or_prompt_change_is_not_a_tool_change() {
let tool = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
let before = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![tool()],
&[("res_a", 1)],
&[("prm_a", 1)],
"rev-1",
));
let changed = CatalogFingerprint::of(&catalog_with(
ToolSelectionPolicy::default(),
vec![tool()],
&[("res_a", 2)],
&[("prm_a", 1), ("prm_b", 1)],
"rev-1",
));
assert_eq!(
changed.changes_since(&before),
Signals {
resources: true,
prompts: true,
tools: false
}
);
}
#[test]
fn a_catalog_revision_bump_without_a_visible_change_is_silent() {
// `catalog_revision` moved, but no visible Resource/Prompt/Tool projection did: no signal.
let before = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-1"));
let after = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-2"));
assert!(!after.changes_since(&before).any());
}
#[test]
fn the_projection_is_order_independent_and_has_no_duplicate_growth() {
let forward = CatalogFingerprint::of(&catalog(
&[("res_a", 1), ("res_b", 2)],
&[("prm_a", 1), ("prm_b", 2)],
"rev-1",
));
let reversed = CatalogFingerprint::of(&catalog(
&[("res_b", 2), ("res_a", 1)],
&[("prm_b", 2), ("prm_a", 1)],
"rev-1",
));
assert!(!reversed.changes_since(&forward).any());
}
#[test]
fn the_legacy_frames_carry_a_closed_method_name_and_no_parameters() {
let frames = Wire::Legacy.signal_frames(Signals::ALL);
assert_eq!(
frames,
vec![
json!({"jsonrpc": "2.0", "method": RESOURCE_LIST_CHANGED}).to_string(),
json!({"jsonrpc": "2.0", "method": PROMPT_LIST_CHANGED}).to_string(),
json!({"jsonrpc": "2.0", "method": TOOL_LIST_CHANGED}).to_string(),
]
);
// No URI, name or payload of a changed object is part of a list-change notification.
assert!(frames.iter().all(|frame| !frame.contains("uri")));
}
#[test]
fn the_modern_frames_carry_the_subscription_id_of_their_listen_request() {
let id = json!(41);
let frames = Wire::Modern(id.clone()).signal_frames(Signals::ALL);
assert_eq!(
frames,
vec![
json!({
"jsonrpc": "2.0",
"method": RESOURCE_LIST_CHANGED,
"params": { "_meta": { SUBSCRIPTION_ID_KEY: 41 } }
})
.to_string(),
json!({
"jsonrpc": "2.0",
"method": PROMPT_LIST_CHANGED,
"params": { "_meta": { SUBSCRIPTION_ID_KEY: 41 } }
})
.to_string(),
json!({
"jsonrpc": "2.0",
"method": TOOL_LIST_CHANGED,
"params": { "_meta": { SUBSCRIPTION_ID_KEY: 41 } }
})
.to_string(),
]
);
assert_eq!(
serde_json::from_str::<Value>(&frames[0]).unwrap()["params"]["_meta"][SUBSCRIPTION_ID_KEY],
id
);
}
#[test]
fn only_the_requested_and_honoured_types_are_encoded() {
let frames = Wire::Modern(json!(1)).signal_frames(Signals {
resources: false,
prompts: true,
tools: false,
});
assert_eq!(frames.len(), 1);
assert!(frames[0].contains(PROMPT_LIST_CHANGED));
}
#[test]
fn the_acknowledgment_reports_exactly_the_honoured_subset() {
let frame: Value = serde_json::from_str(&acknowledged_frame(
&json!(7),
Signals {
resources: true,
prompts: false,
tools: true,
},
))
.unwrap();
assert_eq!(frame["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
assert_eq!(frame["params"]["_meta"][SUBSCRIPTION_ID_KEY], json!(7));
assert_eq!(
frame["params"]["notifications"],
json!({ "resourcesListChanged": true, "toolsListChanged": true })
);
}
#[test]
fn an_empty_honoured_subset_is_an_empty_filter_object() {
let frame: Value =
serde_json::from_str(&acknowledged_frame(&json!("sub-1"), Signals::default())).unwrap();
assert_eq!(frame["params"]["notifications"], json!({}));
}
#[test]
fn the_filter_keeps_only_the_implemented_list_change_types() {
let filter: ListenFilter = serde_json::from_value(json!({
"toolsListChanged": true,
"promptsListChanged": true,
"resourcesListChanged": false,
"resourceSubscriptions": ["file:///project/config.json"]
}))
.unwrap();
match filter.accepted() {
FilterOutcome::Accepted(signals) => assert_eq!(
signals,
Signals {
resources: false,
prompts: true,
tools: true
},
"the honoured set is the requested list-change subset: {signals:?}"
),
FilterOutcome::Invalid(detail) => panic!("a bounded filter must be accepted: {detail}"),
}
}
#[test]
fn an_absent_filter_field_means_not_subscribed() {
let filter: ListenFilter = serde_json::from_value(json!({})).unwrap();
match filter.accepted() {
FilterOutcome::Accepted(signals) => assert!(!signals.any()),
FilterOutcome::Invalid(detail) => panic!("an empty filter must be accepted: {detail}"),
}
}
#[test]
fn an_oversized_uri_filter_is_refused_without_echoing_it() {
let filter: ListenFilter = serde_json::from_value(json!({
"resourceSubscriptions": (0..=MAX_FILTER_URIS)
.map(|index| format!("file:///resource-{index}"))
.collect::<Vec<_>>()
}))
.unwrap();
match filter.accepted() {
FilterOutcome::Invalid(detail) => assert_eq!(detail, INVALID_LISTEN_FILTER),
FilterOutcome::Accepted(_) => panic!("an oversized filter must be refused"),
}
let oversized: ListenFilter = serde_json::from_value(json!({
"resourceSubscriptions": ["u".repeat(super::MAX_FILTER_URI_BYTES + 1)]
}))
.unwrap();
assert!(matches!(
oversized.accepted(),
FilterOutcome::Invalid(INVALID_LISTEN_FILTER)
));
}
#[test]
fn a_non_object_filter_is_a_deserialization_error() {
assert!(serde_json::from_value::<ListenFilter>(json!(["toolsListChanged"])).is_err());
assert!(serde_json::from_value::<ListenFilter>(json!({"promptsListChanged": "yes"})).is_err());
}
#[test]
fn the_subscription_id_is_bound_by_its_encoded_size() {
assert_eq!(subscription_id(&json!(1)), Some(json!(1)));
assert_eq!(subscription_id(&json!("sub-1")), Some(json!("sub-1")));
assert_eq!(
subscription_id(&json!("x".repeat(MAX_SUBSCRIPTION_ID_BYTES))),
None,
"an id longer than the frame bound must not be used as a subscription id"
);
}
#[test]
fn a_newer_legacy_stream_supersedes_the_previous_one() {
let streams = StreamRegistry::new();
let first = streams.admit("session-1").unwrap();
let second = streams.admit("session-1").unwrap();
assert!(!streams.is_current("session-1", first));
assert!(streams.is_current("session-1", second));
// Another session is unaffected.
let other = streams.admit("session-2").unwrap();
assert!(streams.is_current("session-2", other));
streams.retire("session-1");
assert!(!streams.is_current("session-1", second));
assert!(streams.is_current("session-2", other));
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
/// AC-2: a failed committed-catalog read is observable and loses nothing — the tick is skipped (no
/// fabricated signal, no write path that could roll the catalog back) while the baseline survives, so
/// the next successful read still reports the change of the committed catalog.
#[tokio::test]
async fn a_failed_catalog_read_is_observable_and_loses_no_committed_change() {
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
crank_observability::ObservabilityConfig::new(
crank_observability::ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
"info",
crank_observability::RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
let baseline = CatalogFingerprint::of(&catalog(&[], &[("prm_a", 1)], "rev-1"));
let committed = CatalogFingerprint::of(&catalog(&[], &[("prm_a", 1), ("prm_b", 1)], "rev-2"));
let script = Arc::new(Mutex::new(
vec![Ok(baseline), Err(ReadError::Retry), Ok(committed)].into_iter(),
));
let reader: Reader = Box::new(move || {
let next = script
.lock()
.unwrap()
.next()
.unwrap_or(Err(ReadError::Retry));
Box::pin(async move { next })
});
let (sender, mut receiver) = mpsc::channel(SIGNAL_QUEUE_CAPACITY);
tokio::spawn(run(
Watch {
workspace_slug: "workspace-canary".to_owned(),
agent_slug: "agent-canary".to_owned(),
signals: Signals::ALL,
wire: Wire::Legacy,
legacy_stream: None,
deadline: Some(Instant::now() + Duration::from_millis(2_100)),
},
sender,
reader,
));
let mut frames = Vec::new();
while let Some(frame) = receiver.recv().await {
frames.push(frame);
}
// The failed tick produced no frame; the committed change of the next successful read did.
assert_eq!(
frames,
vec![json!({"jsonrpc": "2.0", "method": PROMPT_LIST_CHANGED}).to_string()]
);
let output = writer.output();
let failures: Vec<Value> = output
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.filter(|event| event["event"] == "mcp.signals.read_failed")
.collect();
assert!(
!failures.is_empty(),
"a skipped tick must be observable: {output}"
);
for event in &failures {
assert_eq!(event["fields"]["error_category"], "catalog");
}
// Closed vocabulary: no workspace, Agent, URI or raw database text is read into the event.
assert!(!output.contains("workspace-canary"));
assert!(!output.contains("agent-canary"));
}
@@ -1,409 +0,0 @@
use super::*;
#[tokio::test]
async fn rmcp_http_catalog_follows_new_publication_without_reinitializing() {
let h = Harness::new().await;
let session = h.initialize().await;
let list = json!({"jsonrpc":"2.0","id":60,"method":"tools/list"});
let listed = frames(
h.request("2025-11-25", Some(&session), &list)
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
listed.last().unwrap()["result"]["tools"][0]["name"],
"create_lead"
);
let workspace = crank_core::WorkspaceId::new("ws_default");
let agent = crank_core::AgentId::new("agent_dispatch-observer");
let now = time::OffsetDateTime::now_utc();
let version = crank_core::AgentVersion {
agent_id: agent.clone(),
version: 2,
status: crank_core::AgentStatus::Draft,
instructions: json!({}),
tool_selection_policy: crank_core::ToolSelectionPolicy::default(),
created_at: now,
};
let bindings = [crank_core::AgentOperationBinding {
operation_id: crank_core::OperationId::new("op_create_lead"),
operation_version: 1,
tool_name: "new_publication_tool".into(),
tool_title: "New Tool".into(),
tool_description_override: None,
enabled: true,
}];
h.state
.registry
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
workspace_id: &workspace,
agent_id: &agent,
version: &version,
bindings: &bindings,
updated_at: &now,
})
.await
.unwrap();
h.state
.registry
.publish_agent(crank_registry::PublishAgentRequest {
workspace_id: &workspace,
agent_id: &agent,
version: 2,
published_at: &now,
published_by: Some("alice"),
expected_state: None,
})
.await
.unwrap();
let listed = frames(
h.request("2025-11-25", Some(&session), &list)
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
listed.last().unwrap()["result"]["tools"][0]["name"],
"new_publication_tool"
);
let mut message = call(61, "new-publication@example.test", false, false);
message["params"]["name"] = json!("new_publication_tool");
let result = frames(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap(),
)
.await;
assert_eq!(
result.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_new-publication@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn rmcp_http_identical_request_ids_in_two_sessions_cancel_independently() {
let h = Harness::new().await;
let a = h.initialize().await;
let b = h.initialize().await;
let message = call(62, "slow-same-id@example.test", false, true);
let first = h
.request("2025-11-25", Some(&a), &message)
.send()
.await
.unwrap();
let second = h
.request("2025-11-25", Some(&b), &message)
.send()
.await
.unwrap();
wait_count(&h.calls, 2).await;
let cancel =
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":62}});
assert_eq!(
h.request("2025-11-25", Some(&a), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
wait_count(&h.active, 1).await;
assert_eq!(
h.request("2025-11-25", Some(&b), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
wait_count(&h.active, 0).await;
for response in [first, second] {
assert_eq!(
frames(response).await.last().unwrap()["error"]["message"],
"request cancelled"
);
}
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
// Independent QA: dropped legacy response bodies must not bypass the stream ceiling.
#[tokio::test]
async fn rmcp_http_qa_legacy_disconnect_does_not_bypass_admission() {
let h = Harness::new().await;
let session = h.initialize().await;
let mut admitted = Vec::new();
for id in 100..117 {
let response = h
.request(
"2025-11-25",
Some(&session),
&call(id, "slow-qa-limit@example.test", false, true),
)
.send()
.await
.unwrap();
if response.status() == 429 {
break;
}
assert_eq!(response.status(), 200);
admitted.push(id);
wait_count(&h.calls, admitted.len()).await;
drop(response);
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
let active_before_cleanup = h.active.load(Ordering::SeqCst);
let free_slots = h.state.session_slots.available_permits();
let calls_before_cleanup = h.calls.load(Ordering::SeqCst);
for id in &admitted {
let cancel =
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":id}});
assert_eq!(
h.request("2025-11-25", Some(&session), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
}
wait_count(&h.active, 0).await;
assert!(
admitted.len() <= 16 && calls_before_cleanup <= 16 && active_before_cleanup <= 16,
"legacy disconnect bypassed admission: admitted={}, calls={calls_before_cleanup}, active={active_before_cleanup}, free_slots={free_slots}",
admitted.len()
);
tokio::time::timeout(std::time::Duration::from_secs(2), async {
while h.state.session_slots.available_permits() != 16 {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("owned cancellation cleanup must release all admission permits");
assert_eq!(h.state.session_slots.available_permits(), 16);
// After cleanup, normal admission and REST execution must work again.
let calls_before = h.calls.load(Ordering::SeqCst);
let message = call(199, "recovery-after-cleanup@example.test", false, false);
let response = h
.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let result = frames(response).await;
assert_eq!(
result.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_recovery-after-cleanup@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), calls_before + 1);
}
#[tokio::test]
async fn rmcp_http_correlation_headers_preserve_each_concurrent_request() {
let h = Harness::new().await;
let first = call(50, "correlation-first@example.test", true, false);
let second = call(51, "correlation-second@example.test", true, false);
let (a, b) = tokio::join!(
h.request("2026-07-28", None, &first)
.header("x-request-id", "rmcp-oracle-first")
.send(),
h.request("2026-07-28", None, &second)
.header("x-request-id", "rmcp-oracle-second")
.send()
);
let a = a.unwrap();
let b = b.unwrap();
assert_eq!(a.headers()["x-request-id"], "rmcp-oracle-first");
assert_eq!(b.headers()["x-request-id"], "rmcp-oracle-second");
assert_eq!(
frames(a).await[0]["result"]["structuredContent"]["id"],
"lead_for_correlation-first@example.test"
);
assert_eq!(
frames(b).await[0]["result"]["structuredContent"]["id"],
"lead_for_correlation-second@example.test"
);
}
#[tokio::test]
async fn rmcp_http_stream_limit_rejects_without_dispatch_then_recovers() {
let h = Harness::new().await;
let permits = h
.state
.session_slots
.clone()
.acquire_many_owned(16)
.await
.unwrap();
let message = call(40, "limit@example.test", true, false);
assert_eq!(
h.request("2026-07-28", None, &message)
.send()
.await
.unwrap()
.status(),
429
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
drop(permits);
let response = h
.request("2026-07-28", None, &message)
.send()
.await
.unwrap();
assert_eq!(
frames(response).await[0]["result"]["structuredContent"]["id"],
"lead_for_limit@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn rmcp_http_rate_limit_is_existing_admission_not_sdk_default() {
let h = Harness::with_rate(1).await;
let list = json!({"jsonrpc":"2.0","id":41,"method":"tools/list","params":{"_meta":meta()}});
assert_eq!(
h.request("2026-07-28", None, &list)
.send()
.await
.unwrap()
.status(),
200
);
assert_eq!(
h.request(
"2026-07-28",
None,
&call(42, "rate@example.test", true, false)
)
.send()
.await
.unwrap()
.status(),
429
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_current_durable_store_survives_recreation_but_is_not_sdk_store() {
use crate::session::{PostgresTransportSessionStore, TransportSessionStore};
let h = Harness::new().await;
let store = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
.await
.unwrap();
let now = time::OffsetDateTime::now_utc();
let id = store
.create(
"2025-11-25",
"default",
"dispatch-observer",
false,
now,
Some(now + time::Duration::hours(24)),
)
.await
.unwrap();
assert!(store.mark_initialized(&id, now).await.unwrap());
drop(store);
let restarted = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
.await
.unwrap();
let persisted = restarted.get(&id).await.unwrap().unwrap();
assert!(persisted.initialized);
assert_eq!(persisted.workspace_slug, "default");
assert_eq!(persisted.protocol_version, "2025-11-25");
assert_eq!(
h.request(
"2025-11-25",
Some(&id),
&call(43, "durable@example.test", false, false)
)
.send()
.await
.unwrap()
.status(),
404
);
assert!(
restarted.get(&id).await.unwrap().is_some(),
"SDK experiment never mutates existing storage"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_json_modern_and_legacy_sse_only_gap_are_observed() {
let h = Harness::new().await;
let session = h.initialize().await;
let response = h
.request(
"2026-07-28",
None,
&call(44, "json@example.test", true, false),
)
.send()
.await
.unwrap();
assert!(
response.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("application/json")
);
assert_eq!(
frames(response).await[0]["result"]["structuredContent"]["id"],
"lead_for_json@example.test"
);
// Build a JSON-only request (not two contradictory Accept carriers).
let response = reqwest::Client::new()
.post(format!("{}/v1/default/dispatch-observer", h.server.1))
.bearer_auth(&h.key)
.header("Accept", "application/json")
.header("MCP-Protocol-Version", "2025-11-25")
.header("MCP-Session-Id", &session)
.json(&call(45, "legacy-json@example.test", false, false))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
406,
"bundled SDK legacy mode requires SSE Accept; not transport parity"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn rmcp_http_concurrent_keys_keep_outputs_and_cancel_ownership_separate() {
let h = Harness::new().await;
let session = h.initialize().await;
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
let first = call(46, "first@example.test", false, false);
let second = call(47, "second@example.test", false, false);
let (a, b) = tokio::join!(
h.request("2025-11-25", Some(&session), &first).send(),
h.request_as(
"2025-11-25",
Some(&session),
&second,
"test-second-valid-key"
)
.send()
);
assert_eq!(
frames(a.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_first@example.test"
);
assert_eq!(
frames(b.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_second@example.test"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
@@ -1,563 +0,0 @@
//! Wire oracle for the isolated SDK adapter, with real registry and REST execution.
#[path = "rmcp_http_limits.rs"]
mod limits;
use super::{AgentRoutePath, dispatch_observer::support, rmcp_http, rmcp_tools_tests::test_state};
use axum::{Json, Router, extract::State, routing::post};
use serde_json::{Value, json};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
struct Server(tokio::task::JoinHandle<()>, String);
impl Drop for Server {
fn drop(&mut self) {
self.0.abort();
}
}
async fn serve(router: Router) -> Server {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
Server(
tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
}),
url,
)
}
async fn upstream(State(count): State<Arc<AtomicUsize>>, Json(body): Json<Value>) -> Json<Value> {
count.fetch_add(1, Ordering::SeqCst);
if body["email"].as_str().unwrap().starts_with("slow") {
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
}
Json(json!({"id": format!("lead_for_{}", body["email"].as_str().unwrap())}))
}
struct Harness {
server: Server,
_upstream: Server,
key: String,
calls: Arc<AtomicUsize>,
state: Arc<super::AppState>,
active: Arc<AtomicUsize>,
}
impl Harness {
async fn new() -> Self {
Self::with_rate(10_000).await
}
async fn with_rate(burst: u32) -> Self {
let mut state = test_state().await;
Arc::get_mut(&mut state).unwrap().api_rate_limiter = crank_runtime::RequestRateLimiter::new(
crank_runtime::RequestRateLimitConfig::new(burst, burst).unwrap(),
);
let calls = Arc::new(AtomicUsize::new(0));
let upstream = serve(
Router::new()
.route("/crm/leads", post(upstream))
.with_state(calls.clone()),
)
.await;
let mut operation = support::test_operation(&upstream.1);
operation.execution_config.timeout_ms = 10_000;
state
.registry
.create_operation(
&crank_core::WorkspaceId::new("ws_default"),
&operation,
Some("alice"),
)
.await
.unwrap();
state
.registry
.publish_operation(crank_registry::PublishRequest {
workspace_id: &crank_core::WorkspaceId::new("ws_default"),
operation_id: &operation.id,
version: 1,
published_at: &operation.created_at,
published_by: Some("alice"),
})
.await
.unwrap();
support::publish_agent(&state.registry, &operation).await;
let key = support::create_api_key(&state.registry).await;
let active = Arc::new(AtomicUsize::new(0));
let server = serve(rmcp_http::router(
state.clone(),
AgentRoutePath {
workspace_slug: "default".into(),
agent_slug: "dispatch-observer".into(),
},
active.clone(),
))
.await;
Self {
server,
_upstream: upstream,
key,
calls,
state,
active,
}
}
fn request(
&self,
revision: &str,
session: Option<&str>,
message: &Value,
) -> reqwest::RequestBuilder {
self.request_as(revision, session, message, &self.key)
}
fn request_as(
&self,
revision: &str,
session: Option<&str>,
message: &Value,
key: &str,
) -> reqwest::RequestBuilder {
let mut request = reqwest::Client::new()
.post(format!("{}/v1/default/dispatch-observer", self.server.1))
.bearer_auth(key)
.header("Accept", "application/json, text/event-stream")
.header("MCP-Protocol-Version", revision);
if let Some(session) = session {
request = request.header("MCP-Session-Id", session);
}
if revision == "2026-07-28" {
request = request.header("Mcp-Method", message["method"].as_str().unwrap());
if let Some(name) = message.pointer("/params/name").and_then(Value::as_str) {
request = request.header("Mcp-Name", name);
}
}
request
.json(message)
.timeout(std::time::Duration::from_secs(10))
}
async fn initialize(&self) -> String {
let message = json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"oracle","version":"1"}}});
let response = self
.request("2025-11-25", None, &message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let session = response.headers()["MCP-Session-Id"]
.to_str()
.unwrap()
.to_owned();
let body = frames(response).await;
assert_eq!(
body.last().unwrap()["result"]["protocolVersion"],
"2025-11-25"
);
let response = self
.request(
"2025-11-25",
Some(&session),
&json!({"jsonrpc":"2.0","method":"notifications/initialized"}),
)
.send()
.await
.unwrap();
assert_eq!(response.status(), 202);
session
}
}
fn meta() -> Value {
json!({"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{},"io.modelcontextprotocol/clientInfo":{"name":"oracle","version":"1"}})
}
async fn frames(response: reqwest::Response) -> Vec<Value> {
let status = response.status();
let text = response.text().await.unwrap();
if let Ok(json) = serde_json::from_str(&text) {
return vec![json];
}
let values: Vec<Value> = text
.lines()
.filter_map(|line| line.strip_prefix("data:"))
.filter_map(|data| serde_json::from_str(data.trim()).ok())
.collect();
assert!(!values.is_empty(), "{status}: {text}");
values
}
#[tokio::test]
async fn rmcp_http_positive_tools_both_revisions() {
let h = Harness::new().await;
let session = h.initialize().await;
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
if session.is_none() {
let discover = json!({"jsonrpc":"2.0","id":2,"method":"server/discover","params":{"_meta":meta()}});
let response = h.request(revision, None, &discover).send().await.unwrap();
assert_eq!(response.status(), 200);
assert!(!frames(response).await[0]["result"].is_null());
}
let list = json!({"jsonrpc":"2.0","id":3,"method":"tools/list","params":{"_meta": if session.is_none() { meta() } else { json!({}) }}});
let response = h.request(revision, session, &list).send().await.unwrap();
assert_eq!(response.status(), 200);
assert!(response.headers().contains_key("x-request-id"));
let listed = frames(response).await;
assert_eq!(
listed.last().unwrap()["result"]["tools"][0]["name"],
"create_lead"
);
let call = json!({"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":"rmcp@example.test"},"_meta":if session.is_none() { meta() } else { json!({}) }}});
let response = h.request(revision, session, &call).send().await.unwrap();
assert_eq!(response.status(), 200);
if session.is_none() {
assert!(!response.headers().contains_key("MCP-Session-Id"));
}
let result = frames(response).await;
assert_eq!(
result.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_rmcp@example.test",
"{result:?}"
);
}
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
fn call(id: i64, email: &str, modern: bool, progress: bool) -> Value {
let mut metadata = if modern { meta() } else { json!({}) };
if progress {
metadata["progressToken"] = json!("wire-progress");
}
json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":email},"_meta":metadata}})
}
#[tokio::test]
async fn rmcp_http_credentials_revalidated_after_initialize_and_no_outbound() {
let h = Harness::new().await;
let session = h.initialize().await;
let message = call(10, "denied@example.test", false, false);
assert_eq!(
h.request_as("2025-11-25", Some(&session), &message, "invalid-test-key")
.send()
.await
.unwrap()
.status(),
401
);
// A scope change and expiry are authoritative registry changes, not session credentials.
sqlx::query("update platform_api_keys set scopes_json = '[\"read\"]'::jsonb where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
assert_eq!(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap()
.status(),
403
);
sqlx::query("update platform_api_keys set scopes_json = '[\"read\",\"write\"]'::jsonb, expires_at = now() - interval '1 second' where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
assert_eq!(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap()
.status(),
401
);
sqlx::query("update platform_api_keys set expires_at = null where id = 'pk_dispatch_observer'")
.execute(h.state.registry.pool())
.await
.unwrap();
h.state
.registry
.revoke_platform_api_key(
&crank_core::WorkspaceId::new("ws_default"),
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
&time::OffsetDateTime::now_utc(),
)
.await
.unwrap();
for method in [reqwest::Method::GET, reqwest::Method::DELETE] {
assert_eq!(
reqwest::Client::new()
.request(
method,
format!("{}/v1/default/dispatch-observer", h.server.1)
)
.bearer_auth(&h.key)
.header("MCP-Session-Id", &session)
.header("MCP-Protocol-Version", "2025-11-25")
.header("Accept", "text/event-stream")
.send()
.await
.unwrap()
.status(),
401
);
}
assert_eq!(
h.request("2025-11-25", Some(&session), &message)
.send()
.await
.unwrap()
.status(),
401
);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_admission_rejects_foreign_route_revision_headers_and_body_overflow() {
let h = Harness::new().await;
let message = call(11, "denied@example.test", true, false);
let response = h
.request("2026-07-28", Some("foreign"), &message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 400);
let response = h
.request("2026-07-28", None, &message)
.header("Mcp-Name", "foreign")
.send()
.await
.unwrap();
assert_eq!(response.status(), 400);
let response = h
.request("2026-07-28", None, &message)
.header("Origin", "https://evil.example")
.send()
.await
.unwrap();
assert_eq!(response.status(), 403);
assert_eq!(
h.request("2030-01-01", None, &message)
.send()
.await
.unwrap()
.status(),
400
);
let response = reqwest::Client::new()
.post(format!("{}/v1/default/foreign-agent", h.server.1))
.bearer_auth(&h.key)
.json(&message)
.send()
.await
.unwrap();
assert_eq!(response.status(), 404);
let response = h
.request("2026-07-28", None, &message)
.body(vec![b' '; 2 * 1024 * 1024 + 1])
.send()
.await
.unwrap();
assert_eq!(response.status(), 413);
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn rmcp_http_progress_is_real_and_tool_error_is_not_protocol_error() {
let h = Harness::new().await;
let session = h.initialize().await;
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
let response = h
.request(
revision,
session,
&call(20, "progress@example.test", session.is_none(), true),
)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
assert!(
response.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("text/event-stream")
);
let messages = frames(response).await;
let progress: Vec<_> = messages
.iter()
.filter(|m| m["method"] == "notifications/progress")
.collect();
assert_eq!(progress.len(), 3, "{messages:?}");
for (index, phase) in [
"tool_resolved",
"authorization_resolved",
"execution_dispatched",
]
.iter()
.enumerate()
{
assert_eq!(progress[index]["params"]["message"], *phase);
assert_eq!(
progress[index]["params"]["progress"].as_f64(),
Some((index + 1) as f64)
);
assert_eq!(progress[index]["params"]["progressToken"], "wire-progress");
}
let mut invalid = call(21, "unused", session.is_none(), false);
invalid["params"]["arguments"] = json!({"email":123});
let response = h.request(revision, session, &invalid).send().await.unwrap();
let messages = frames(response).await;
let result = messages.last().unwrap();
assert_eq!(result["result"]["isError"], true, "{messages:?}");
assert!(result.get("error").is_none());
}
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn rmcp_http_reconnect_live_manager_and_restart_loses_sdk_session() {
let h = Harness::new().await;
let session = h.initialize().await;
for _ in 0..2 {
let response = reqwest::Client::new()
.get(format!("{}/v1/default/dispatch-observer", h.server.1))
.bearer_auth(&h.key)
.header("MCP-Session-Id", &session)
.header("MCP-Protocol-Version", "2025-11-25")
.header("Accept", "text/event-stream")
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
drop(response);
}
let response = h
.request(
"2025-11-25",
Some(&session),
&call(22, "reconnect@example.test", false, false),
)
.send()
.await
.unwrap();
assert_eq!(
frames(response).await.last().unwrap()["result"]["structuredContent"]["id"],
"lead_for_reconnect@example.test"
);
let fresh = serve(rmcp_http::router(
h.state.clone(),
AgentRoutePath {
workspace_slug: "default".into(),
agent_slug: "dispatch-observer".into(),
},
h.active.clone(),
))
.await;
let response = reqwest::Client::new()
.post(format!("{}/v1/default/dispatch-observer", fresh.1))
.bearer_auth(&h.key)
.header("MCP-Session-Id", &session)
.header("MCP-Protocol-Version", "2025-11-25")
.header("Accept", "application/json, text/event-stream")
.json(&call(23, "restart@example.test", false, false))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
404,
"new SDK manager cannot restore from Crank session storage"
);
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
}
async fn wait_count(counter: &AtomicUsize, expected: usize) {
tokio::time::timeout(std::time::Duration::from_secs(3), async {
while counter.load(Ordering::SeqCst) != expected {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("bounded wait for the real execution/cleanup");
}
#[tokio::test]
async fn rmcp_http_legacy_cancel_is_owned_by_session_and_current_key() {
let h = Harness::new().await;
let session = h.initialize().await;
let foreign_session = h.initialize().await;
// A second valid credential can use this Agent/session but does not own the first call.
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
let response = h
.request(
"2025-11-25",
Some(&session),
&call(30, "slow-cancel@example.test", false, true),
)
.send()
.await
.unwrap();
wait_count(&h.calls, 1).await;
let cancel =
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":30}});
assert_eq!(
h.request_as(
"2025-11-25",
Some(&session),
&cancel,
"test-second-valid-key"
)
.send()
.await
.unwrap()
.status(),
202
);
assert_eq!(
h.request("2025-11-25", Some(&foreign_session), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
tokio::time::sleep(std::time::Duration::from_millis(75)).await;
assert_eq!(
h.active.load(Ordering::SeqCst),
1,
"foreign cancel must not stop call"
);
assert_eq!(
h.request("2025-11-25", Some(&session), &cancel)
.send()
.await
.unwrap()
.status(),
202
);
wait_count(&h.active, 0).await;
// Gap: SDK emits an error frame where the existing transport suppresses all terminal frames.
let messages = frames(response).await;
assert_eq!(
messages.last().unwrap()["error"]["message"],
"request cancelled"
);
assert!(!messages.iter().any(|m| m.get("result").is_some()));
}
#[tokio::test]
async fn rmcp_http_modern_disconnect_drops_execution_and_releases_stream_slot() {
let h = Harness::new().await;
let response = h
.request(
"2026-07-28",
None,
&call(31, "slow-disconnect@example.test", true, true),
)
.send()
.await
.unwrap();
wait_count(&h.calls, 1).await;
assert_eq!(h.active.load(Ordering::SeqCst), 1);
drop(response);
wait_count(&h.active, 0).await;
tokio::time::timeout(std::time::Duration::from_secs(2), async {
while h.state.session_slots.available_permits() != 16 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert_eq!(
h.calls.load(Ordering::SeqCst),
1,
"disconnect never retries a possible external effect"
);
}
@@ -1,178 +0,0 @@
//! Behavioural ATDD for the rmcp 3.7.0 Tools adapter over the existing typed Tools service.
use std::sync::Arc;
use std::time::Duration;
use crank_core::{
AdmissionSchedulerConfig, CoordinationStateStore, CorrelationContext, ExecutionLocale,
MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
};
use crank_registry::{AdmissionLimits, MigrationAuthority, PostgresRegistry, PostgresTaskStore};
use crank_runtime::{
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
TaskService,
};
use rmcp::model::{CallToolRequestParams, PaginatedRequestParams, RequestId};
use rmcp::service::{RequestContext, serve_directly};
use rmcp::{RoleServer, ServerHandler};
use super::{
AppState,
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
};
use crate::app::TaskApplication;
use crate::auth::{CommunityMachineCredentialVerifier, VerifiedMachineCredential};
use crate::catalog::PublishedToolCatalog;
use crate::session::{ActiveSessionMetrics, InMemorySessionStore};
use crate::transport::AllowedOrigins;
pub(super) async fn test_state() -> Arc<AppState> {
let database_url =
crank_test_support::postgres_schema_url_for_task("rmcp_tools_tests", "t_fdcc7df8").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let coordination: Arc<dyn CoordinationStateStore> =
Arc::new(InMemoryCoordinationStateStore::default());
let sessions = Arc::new(InMemorySessionStore::default());
let session_metrics = ActiveSessionMetrics::start(sessions.clone());
let catalog = PublishedToolCatalog::new(registry.clone(), Duration::ZERO, coordination.clone());
let tasks_store = PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits::default(),
AdmissionSchedulerConfig::default(),
)
.expect("default admission limits and scheduler weights are valid");
let tasks = Arc::new(TaskApplication::new(
registry.clone(),
Duration::from_secs(60),
coordination.clone(),
TaskService::new(Arc::new(tasks_store)),
));
Arc::new(AppState {
registry,
catalog,
runtime: crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
api_rate_limiter: RequestRateLimiter::new(RequestRateLimitConfig {
requests_per_second: 10_000,
burst: 10_000,
}),
artifact_store: None,
secret_crypto: SecretCrypto::new("01234567890123456789012345678901")
.expect("test master key"),
sessions,
session_metrics,
session_slots: Arc::new(tokio::sync::Semaphore::new(16)),
credential_verifier: Arc::new(CommunityMachineCredentialVerifier),
allowed_origins: AllowedOrigins::new(None),
logging_levels: Arc::new(crate::protocol::LogLevelStore::new()),
signal_streams: Arc::new(crate::protocol::StreamRegistry::new()),
in_flight: Arc::new(super::lifecycle::InFlightCalls::new(
super::lifecycle::MAX_IN_FLIGHT_CALLS,
)),
tasks,
})
}
fn verified_credential() -> VerifiedMachineCredential {
VerifiedMachineCredential {
machine_access_mode: MachineAccessMode::StaticAgentKey,
max_security_level: OperationSecurityLevel::Standard,
scopes: vec![PlatformApiKeyScope::Write],
platform_api_key_id: None,
}
}
fn rmcp_tools(state: Arc<AppState>) -> RmcpTools {
RmcpTools {
state,
active: Arc::default(),
}
}
async fn request_context(adapter: &RmcpTools) -> RequestContext<RoleServer> {
let (client_io, server_io) = tokio::io::duplex(1024);
let (server_reader, server_writer) = tokio::io::split(server_io);
let running = serve_directly::<RoleServer, _, _, _, _>(
adapter.clone(),
(server_reader, server_writer),
None,
);
// keep client side alive for transport lifetime
let _ = client_io;
let mut context = RequestContext::new(RequestId::Number(1), running.peer().clone());
let (mut parts, _) = axum::http::Request::new(()).into_parts();
parts.extensions.insert(RmcpRequestIdentity {
workspace_slug: "missing-workspace".into(),
agent_slug: "missing-agent".into(),
credential: verified_credential(),
correlation: CorrelationContext::generate(),
continuation: crate::protocol::ContinuationSupport::from_supported(false),
locale: ExecutionLocale::En,
session_id: None,
progress_meta: None,
sse_acceptable: false,
});
context.extensions.insert(parts);
context
}
#[tokio::test]
async fn rmcp_adapter_rejects_missing_request_identity() {
let adapter = rmcp_tools(test_state().await);
let mut context = request_context(&adapter).await;
context.extensions.remove::<axum::http::request::Parts>();
let result = <RmcpTools as ServerHandler>::list_tools(&adapter, None, context).await;
assert!(
result
.unwrap_err()
.message
.contains("missing authenticated request context")
);
}
#[tokio::test]
async fn rmcp_adapter_list_tools_reports_catalog_unavailable_for_unknown_agent() {
let state = test_state().await;
let adapter = rmcp_tools(state);
let ctx = request_context(&adapter).await;
match <RmcpTools as ServerHandler>::list_tools(
&adapter,
Some(PaginatedRequestParams::default()),
ctx,
)
.await
{
Err(err) => assert!(
err.message.contains("crate catalog unavailable"),
"unexpected error: {}",
err.message
),
Ok(_) => panic!("expected error outcome for unknown agent"),
}
}
#[tokio::test]
async fn rmcp_adapter_call_tool_reports_catalog_unavailable_for_unknown_agent() {
let state = test_state().await;
let adapter = rmcp_tools(state);
let ctx = request_context(&adapter).await;
let request = CallToolRequestParams::new("missing_tool");
match <RmcpTools as ServerHandler>::call_tool(&adapter, request, ctx).await {
Err(err) => assert!(
err.message.contains("crate catalog unavailable"),
"unexpected error: {}",
err.message
),
Ok(_) => panic!("expected error outcome for unknown agent"),
}
}
+3 -6
View File
@@ -313,15 +313,12 @@ pub(super) fn transport_response(
/// Record the closed safe event of one response: served, rejected as invalid, failed, or a tool-level
/// error. Every label is a fixed string of the closed vocabulary; nothing from the payload or the
/// request is copied into it.
pub(super) fn note_logging_outcome(payload: &Value) {
fn note_logging_outcome(payload: &Value) {
use crate::protocol::{EVENT_FAILED, EVENT_REJECTED, EVENT_SERVED, LogLevel, note};
if payload.get("error").is_some() {
match payload.pointer("/error/code").and_then(Value::as_i64) {
// The pinned client-error codes: a malformed request, a method that this era does not
// implement, and invalid params. The client can correct all three, so they are a
// rejection rather than a server failure; the closed label stays the one client-error
// label of the vocabulary instead of growing a new one.
Some(-32602..=-32600) => note(LogLevel::Warning, EVENT_REJECTED, "invalid_params"),
// The pinned client-error codes of a malformed request.
Some(-32600 | -32602) => note(LogLevel::Warning, EVENT_REJECTED, "invalid_params"),
_ => note(LogLevel::Error, EVENT_FAILED, "internal"),
}
return;
@@ -284,7 +284,6 @@ fn operation() -> RegistryOperation {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
-531
View File
@@ -1,531 +0,0 @@
//! Typed execution-admission seam (Story 4.1, FR-21/FR-24).
//!
//! Admission is the single gate in front of every real Tool execution: it decides whether the
//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide
//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives
//! in PostgreSQL (`crank-registry`), so several processes share one budget.
//!
//! Deliberately absent from this contract: credentials, connection or protocol revision (they must
//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2).
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use crate::{NoTasksFallback, ToolExecutionMode};
pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128;
pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64;
pub const MAX_RETRY_AFTER_MS: u64 = 60_000;
/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for
/// the store and every runtime so the client contract stays stable.
pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250;
/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms).
pub fn current_unix_ms() -> i64 {
i64::try_from(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_millis())
.unwrap_or_default(),
)
.unwrap_or(i64::MAX)
}
/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin
/// bucket that is still bounded by the global limit.
pub const ADMIN_ORIGIN_KEY: &str = "admin";
/// Exact execution identity pinned at admission time.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AdmissionScope {
pub workspace_id: String,
/// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution.
pub agent_key: String,
pub operation_id: String,
/// Quota keys exclude the version; the version is pinned for execution semantics only.
pub operation_version: u32,
}
impl AdmissionScope {
/// Quota key of the Agent bucket: `(workspace, agent)`.
pub fn agent_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.agent_key)
}
/// Quota key of the Tool bucket: `(workspace, operation)` without version.
pub fn tool_bucket(&self) -> String {
format!("{}:{}", self.workspace_id, self.operation_id)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AdmissionMode {
/// The caller waits for a scoped permit and dispatches inside it.
Synchronous,
/// The store commits a durable enqueue decision and reserves queue capacity.
Background,
}
impl AdmissionMode {
pub const fn as_str(self) -> &'static str {
match self {
Self::Synchronous => "synchronous",
Self::Background => "background",
}
}
}
/// Non-consuming approval evidence gathered by the runtime before admission.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ApprovalEvidence {
/// The operation does not require confirmation.
NotRequired,
/// A verified approval exists for this exact scope and input.
Granted,
/// Confirmation is required and no verified approval exists yet.
NotGranted,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionRequest {
pub scope: AdmissionScope,
pub mode: AdmissionMode,
/// Published policy mode that produced this request.
pub policy_mode: ToolExecutionMode,
pub no_tasks_fallback: NoTasksFallback,
pub approval: ApprovalEvidence,
/// Bounded request identity used only to correlate one admission entry.
pub fingerprint: String,
/// Absolute deadline (unix milliseconds), already including queue waiting time.
pub expires_at_unix_ms: i64,
}
impl AdmissionRequest {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if self.fingerprint.is_empty()
|| self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES
|| self
.fingerprint
.bytes()
.any(|byte| byte.is_ascii_control() || byte == b' ')
{
return Err(AdmissionStoreError::invalid(
"admission fingerprint is outside its allowed bounds",
));
}
if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission workspace is invalid",
));
}
if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 {
return Err(AdmissionStoreError::invalid("admission agent is invalid"));
}
if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 {
return Err(AdmissionStoreError::invalid(
"admission operation is invalid",
));
}
if self.scope.operation_version == 0 {
return Err(AdmissionStoreError::invalid(
"admission operation version must be positive",
));
}
if self.expires_at_unix_ms <= 0 {
return Err(AdmissionStoreError::invalid(
"admission deadline is invalid",
));
}
Ok(())
}
}
/// Scoped permit. Held for the whole local execution future (including retries) and released
/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionPermit {
pub ticket_id: u64,
pub owner_token: String,
pub scope: AdmissionScope,
pub mode: AdmissionMode,
pub policy_mode: ToolExecutionMode,
}
/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics.
/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionDecision {
pub ticket_id: u64,
pub scope: AdmissionScope,
pub expires_at_unix_ms: i64,
pub policy_mode: ToolExecutionMode,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AdmissionOutcome {
Permitted(AdmissionPermit),
Enqueued(AdmissionDecision),
/// Capacity or queue bounds were exhausted before any unbounded allocation.
Overloaded {
retry_after_ms: u64,
},
/// The queue wait reached the absolute deadline; nothing was dispatched.
DeadlineElapsed,
/// Confirmation is mandatory for this operation and no verified approval exists.
ApprovalRequired,
/// The published policy does not advertise durable Tasks and refuses the fallback.
Unsupported,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionQueueSnapshot {
pub waiting: u64,
pub executing: u64,
/// Age of the oldest waiting request in the authoritative queue, in milliseconds.
pub oldest_waiting_ms: Option<u64>,
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum AdmissionStoreError {
/// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed.
#[error("execution admission store is unavailable: {message}")]
Unavailable { message: String },
/// The store rejected a malformed admission request.
#[error("execution admission request is invalid: {message}")]
Invalid { message: String },
}
impl AdmissionStoreError {
pub fn unavailable(message: impl Into<String>) -> Self {
Self::Unavailable {
message: message.into(),
}
}
pub fn invalid(message: impl Into<String>) -> Self {
Self::Invalid {
message: message.into(),
}
}
}
#[async_trait]
pub trait ExecutionAdmission: Send + Sync {
/// Proves the mandatory admission state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), AdmissionStoreError>;
/// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue
/// decision for a background request, or returns a typed refusal.
async fn admit(
&self,
request: AdmissionRequest,
) -> Result<AdmissionOutcome, AdmissionStoreError>;
/// Releases the caller's own permit. Idempotent: a repeated release returns `false`.
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError>;
/// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket.
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError>;
/// Bounded authoritative queue readings for low-cardinality metrics.
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError>;
}
/// One waiting entry as seen by the fairness scheduler.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionQueueEntry {
pub ticket_id: u64,
/// Authoritative insertion order inside `(Agent, Tool)`; never rewritten.
pub sequence: u64,
pub agent_key: String,
pub operation_id: String,
}
/// Weighted round-robin state persisted with the authoritative queue.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct AdmissionCursor {
pub agent: Option<String>,
pub agent_credit: u32,
pub tool: Option<String>,
pub tool_credit: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionSchedulerConfig {
pub agent_weight: u32,
pub tool_weight: u32,
}
impl Default for AdmissionSchedulerConfig {
fn default() -> Self {
Self {
agent_weight: 1,
tool_weight: 1,
}
}
}
impl AdmissionSchedulerConfig {
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) {
return Err(AdmissionStoreError::invalid(
"admission weights must be between 1 and 64",
));
}
Ok(())
}
}
/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the
/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its
/// turn instead of being skipped by newer arrivals.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionSelection {
pub index: usize,
pub cursor: AdmissionCursor,
}
/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside
/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit.
pub fn select_next_grant(
queue: &[AdmissionQueueEntry],
cursor: &AdmissionCursor,
config: AdmissionSchedulerConfig,
) -> Option<AdmissionSelection> {
if queue.is_empty() {
return None;
}
let agent_weight = config.agent_weight.max(1);
let tool_weight = config.tool_weight.max(1);
let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect();
agents.sort_unstable();
agents.dedup();
let mut next = cursor.clone();
let agent = match cursor.agent.as_deref() {
Some(current) if cursor.agent_credit > 0 && agents.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.agent.as_deref() {
None => agents[0].to_owned(),
Some(current) => agents
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(agents[0])
.to_owned(),
};
if cursor.agent.as_deref() != Some(chosen.as_str()) {
next.tool = None;
next.tool_credit = 0;
}
next.agent = Some(chosen.clone());
next.agent_credit = agent_weight;
chosen
}
};
let mut tools: Vec<&str> = queue
.iter()
.filter(|entry| entry.agent_key == agent)
.map(|entry| entry.operation_id.as_str())
.collect();
tools.sort_unstable();
tools.dedup();
let tool = match cursor.tool.as_deref() {
Some(current) if cursor.tool_credit > 0 && tools.contains(&current) => current.to_owned(),
_ => {
let chosen = match cursor.tool.as_deref() {
None => tools[0].to_owned(),
Some(current) => tools
.iter()
.copied()
.find(|candidate| *candidate > current)
.unwrap_or(tools[0])
.to_owned(),
};
next.tool = Some(chosen.clone());
next.tool_credit = tool_weight;
chosen
}
};
let index = queue
.iter()
.position(|entry| entry.agent_key == agent && entry.operation_id == tool)?;
next.agent_credit = next.agent_credit.saturating_sub(1);
next.tool_credit = next.tool_credit.saturating_sub(1);
Some(AdmissionSelection {
index,
cursor: next,
})
}
#[cfg(test)]
mod tests {
use super::{
ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest,
AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence,
select_next_grant,
};
use crate::{NoTasksFallback, ToolExecutionMode};
fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry {
super::AdmissionQueueEntry {
ticket_id,
sequence: ticket_id,
agent_key: agent.to_owned(),
operation_id: tool.to_owned(),
}
}
fn request() -> AdmissionRequest {
AdmissionRequest {
scope: AdmissionScope {
workspace_id: "ws_default".to_owned(),
agent_key: ADMIN_ORIGIN_KEY.to_owned(),
operation_id: "op_leads".to_owned(),
operation_version: 3,
},
mode: AdmissionMode::Synchronous,
policy_mode: ToolExecutionMode::Synchronous,
no_tasks_fallback: NoTasksFallback::Synchronous,
approval: ApprovalEvidence::NotRequired,
fingerprint: "fp_0123456789".to_owned(),
expires_at_unix_ms: 1_790_000_000_000,
}
}
#[test]
fn quota_keys_exclude_version_and_keep_agent_bucket() {
let scope = request().scope;
assert_eq!(scope.agent_bucket(), "ws_default:admin");
assert_eq!(scope.tool_bucket(), "ws_default:op_leads");
}
#[test]
fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() {
let mut invalid = request();
invalid.expires_at_unix_ms = 0;
assert!(matches!(
invalid.validate(),
Err(AdmissionStoreError::Invalid { .. })
));
let mut invalid = request();
invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1);
assert!(invalid.validate().is_err());
let mut invalid = request();
invalid.scope.operation_version = 0;
assert!(invalid.validate().is_err());
assert!(request().validate().is_ok());
}
#[test]
fn weights_are_bounded_positive_configuration() {
assert!(
AdmissionSchedulerConfig {
agent_weight: 0,
tool_weight: 1
}
.validate()
.is_err()
);
assert!(
AdmissionSchedulerConfig {
agent_weight: 1,
tool_weight: 65
}
.validate()
.is_err()
);
assert!(AdmissionSchedulerConfig::default().validate().is_ok());
}
#[test]
fn weighted_round_robin_serves_each_agent_its_quantum() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_1"),
entry(4, "agent_b", "op_1"),
entry(5, "agent_b", "op_1"),
];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
order.push(queue.remove(selection.index).agent_key.clone());
cursor = selection.cursor;
}
assert_eq!(
order,
vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"]
);
}
#[test]
fn fifo_inside_one_tool_and_round_robin_between_tools() {
let mut queue = vec![
entry(1, "agent_a", "op_1"),
entry(2, "agent_a", "op_1"),
entry(3, "agent_a", "op_2"),
];
let config = AdmissionSchedulerConfig::default();
let mut cursor = AdmissionCursor::default();
let mut order = Vec::new();
while !queue.is_empty() {
let selection = select_next_grant(&queue, &cursor, config).unwrap();
// A granted ticket leaves the waiting queue: granted entries are never re-granted.
order.push(queue.remove(selection.index).ticket_id);
cursor = selection.cursor;
}
// One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue.
assert_eq!(order, vec![1, 3, 2]);
}
#[test]
fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() {
let queue = vec![entry(1, "agent_a", "op_1")];
let config = AdmissionSchedulerConfig {
agent_weight: 2,
tool_weight: 1,
};
let cursor = AdmissionCursor::default();
let first = select_next_grant(&queue, &cursor, config).unwrap();
assert_eq!(first.cursor.agent.as_deref(), Some("agent_a"));
assert_eq!(first.cursor.agent_credit, 1);
// A later Agent joins; the cursor keeps its remaining quantum for `agent_a`.
let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")];
let second = select_next_grant(&queue, &first.cursor, config).unwrap();
assert_eq!(queue[second.index].agent_key, "agent_a");
assert_eq!(second.cursor.agent_credit, 0);
let third = select_next_grant(&queue, &second.cursor, config).unwrap();
assert_eq!(queue[third.index].agent_key, "agent_b");
assert_eq!(third.cursor.agent_credit, 1);
}
#[test]
fn empty_queue_keeps_cursor_untouched() {
let cursor = AdmissionCursor {
agent: Some("agent_b".to_owned()),
agent_credit: 1,
tool: Some("op_1".to_owned()),
tool_credit: 0,
};
assert_eq!(
select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()),
None
);
assert_eq!(cursor.agent.as_deref(), Some("agent_b"));
}
}
+9 -46
View File
@@ -1,5 +1,4 @@
pub mod access;
pub mod admission;
pub mod agent;
pub mod approval;
pub mod auth;
@@ -18,7 +17,6 @@ pub mod prompt_composition;
pub mod protocol;
pub mod resource;
pub mod secret;
pub mod task;
pub mod tool_catalog;
pub mod tool_quality;
pub mod tool_search;
@@ -62,11 +60,10 @@ pub mod domain {
pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use crate::operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, WizardState,
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState,
};
pub use crate::product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
@@ -74,18 +71,6 @@ pub mod domain {
};
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use crate::task::{
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TASK_APPROVAL_REFERENCE_MAX_BYTES,
TASK_CANCEL_REASON_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
TASK_IDENTITY_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind,
TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskSupportIds,
};
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
pub use crate::tool_quality::{
ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
@@ -123,20 +108,12 @@ pub mod ports {
PreparedRequest, ProtocolAdapter, ProtocolAdapterError, ResponseCacheScope,
RuntimeRequestContext, SharedProtocolAdapter,
};
pub use crate::task::TaskStore;
}
pub use access::{
InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
};
pub use admission::{
ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS,
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission,
MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant,
};
pub use agent::{
Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
@@ -191,14 +168,13 @@ pub use observability::{
pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
pub use operation::{
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation,
OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy,
OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract,
RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget,
RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
ToolExecutionPolicy, ToolExecutionPolicyError, WizardState,
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError,
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState,
ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor,
RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample,
WizardState,
};
pub use product_event::{
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
@@ -206,19 +182,6 @@ pub use product_event::{
};
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use task::{
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
TASK_INPUT_PROMPT_CODE_MAX_BYTES, TASK_INPUT_SCHEMA_FORMAT_VERSION,
TASK_INPUT_WAIT_ID_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest,
TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskRetryPolicy, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
redact_task_envelope, sensitive_task_key, validate_task_envelope,
};
pub use tool_catalog::{
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
ToolCatalogBudget, analyze_tool_catalog,
+2 -144
View File
@@ -270,72 +270,6 @@ pub struct OperationApprovalPolicy {
pub elicitation_message: Option<String>,
}
/// Published execution mode of one Tool (Story 4.1, FR-21).
///
/// This is a scheduling policy of the Published Tool version. It is deliberately separate from
/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolExecutionMode {
#[default]
Synchronous,
Background,
Adaptive,
}
/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NoTasksFallback {
/// Execute through the same synchronous admission path.
#[default]
Synchronous,
/// Reject with a typed unsupported-execution-mode failure; never a Task ID.
Unsupported,
}
pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000;
/// Execution policy pinned by a Published Tool version.
///
/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by
/// `ExecutionConfig::timeout_ms`, with no background path.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ToolExecutionPolicy {
#[serde(default)]
pub mode: ToolExecutionMode,
/// Absolute budget of one execution, counted from the start of admission.
pub deadline_ms: u64,
#[serde(default)]
pub no_tasks_fallback: NoTasksFallback,
}
impl Default for ToolExecutionPolicy {
fn default() -> Self {
Self {
mode: ToolExecutionMode::Synchronous,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
}
}
impl ToolExecutionPolicy {
pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> {
if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) {
return Err(ToolExecutionPolicyError::DeadlineOutOfRange);
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ToolExecutionPolicyError {
#[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")]
DeadlineOutOfRange,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExecutionConfig {
@@ -351,8 +285,6 @@ pub struct ExecutionConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub approval_policy: Option<OperationApprovalPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub execution_policy: Option<ToolExecutionPolicy>,
#[serde(skip_serializing_if = "Option::is_none")]
pub auth_profile_ref: Option<AuthProfileId>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub headers: BTreeMap<String, String>,
@@ -512,10 +444,9 @@ mod tests {
edition::OperationSecurityLevel,
ids::{AuthProfileId, OperationId},
operation::{
ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability,
OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus,
ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle,
OperationLifecycleAction, OperationLifecycleError, OperationStatus,
OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample,
ToolExecutionMode, ToolExecutionPolicy,
},
protocol::{AuthKind, ExportMode, HttpMethod, Protocol},
};
@@ -733,7 +664,6 @@ updated_at: 2026-03-25T08:10:00Z
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: Some(AuthProfileId::new("auth_01")),
headers: BTreeMap::new(),
},
@@ -754,76 +684,4 @@ updated_at: 2026-03-25T08:10:00Z
published_at: None,
}
}
#[test]
fn execution_policy_round_trips_all_modes_and_fallbacks() {
for (mode, fallback) in [
(ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Synchronous),
(ToolExecutionMode::Background, NoTasksFallback::Unsupported),
(ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported),
] {
let config = ExecutionConfig {
execution_policy: Some(ToolExecutionPolicy {
mode,
deadline_ms: 45_000,
no_tasks_fallback: fallback,
}),
..ExecutionConfig::default()
};
let encoded = serde_json::to_value(&config).unwrap();
assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000));
let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap();
assert_eq!(decoded, config);
assert_eq!(
decoded.execution_policy.unwrap().mode,
mode,
"the published mode must survive the JSON round-trip"
);
}
}
#[test]
fn legacy_execution_config_without_policy_stays_synchronous() {
let legacy: ExecutionConfig =
serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap();
assert_eq!(legacy.execution_policy, None);
assert_eq!(legacy.timeout_ms, 12_000);
}
#[test]
fn execution_policy_rejects_explicit_invalid_deadline() {
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 0,
no_tasks_fallback: NoTasksFallback::Synchronous,
}
.validate()
.is_err()
);
assert!(
ToolExecutionPolicy {
mode: ToolExecutionMode::Adaptive,
deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1,
no_tasks_fallback: NoTasksFallback::Unsupported,
}
.validate()
.is_err()
);
assert!(ToolExecutionPolicy::default().validate().is_ok());
}
#[test]
fn execution_policy_rejects_unknown_fields() {
assert!(
serde_json::from_value::<ToolExecutionPolicy>(json!({
"mode": "background",
"deadline_ms": 1000,
"no_tasks_fallback": "synchronous",
"task_queue": "default"
}))
.is_err()
);
}
}
-285
View File
@@ -1,285 +0,0 @@
//! Durable attempt, retry-snapshot and cancellation vocabulary of a Task (Story 4.4).
use serde::{Deserialize, Serialize};
use super::{TASK_IDENTITY_MAX_BYTES, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskRecord};
/// Upper bound of the retry budget a Task may pin at acceptance (Story 4.4).
pub const TASK_MAX_ATTEMPTS: u32 = 5;
/// Upper bound of the pinned backoff step between two durable attempts (Story 4.4).
pub const TASK_MAX_BACKOFF_STEP_MS: i64 = 1_000;
/// Bound of a stored cancellation reason.
pub const TASK_CANCEL_REASON_MAX_BYTES: usize = 160;
/// Immutable retry snapshot of one accepted Task.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TaskRetryPolicy {
/// Total durable outbound attempts (1 = a single attempt, never retried).
pub max_attempts: u32,
/// Pinned base backoff between two attempts; the delay grows by this step per attempt.
pub backoff_step_ms: i64,
}
impl TaskRetryPolicy {
/// A Task that may be dispatched exactly once (legacy Tasks and unverified writes).
pub const fn single_attempt() -> Self {
Self {
max_attempts: 1,
backoff_step_ms: 0,
}
}
pub fn new(max_attempts: u32, backoff_step_ms: i64) -> Self {
Self {
max_attempts,
backoff_step_ms,
}
}
pub fn validate(&self) -> Result<(), TaskError> {
if !(1..=TASK_MAX_ATTEMPTS).contains(&self.max_attempts) {
return Err(TaskError::invalid("task retry budget is invalid"));
}
if !(0..=TASK_MAX_BACKOFF_STEP_MS).contains(&self.backoff_step_ms) {
return Err(TaskError::invalid("task retry backoff is invalid"));
}
Ok(())
}
/// Delay before attempt number `next_ordinal` (2..=max_attempts), capped so a pinned snapshot can
/// never stretch the accepted execution deadline on its own.
pub fn backoff_before(&self, next_ordinal: u32) -> i64 {
let steps = i64::from(next_ordinal.saturating_sub(1)).max(1);
self.backoff_step_ms
.saturating_mul(steps)
.min(self.backoff_step_ms.saturating_mul(4))
}
}
/// Certainty of one durable attempt outcome.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptCertainty {
/// The outcome of the outbound operation is proven (success or a definite failure).
Known,
/// The operation may have been applied upstream; the effect cannot be proven locally.
Unknown,
}
impl AttemptCertainty {
pub const fn as_str(self) -> &'static str {
match self {
Self::Known => "known",
Self::Unknown => "unknown",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"known" => Ok(Self::Known),
"unknown" => Ok(Self::Unknown),
_ => Err(TaskError::invalid("unknown attempt certainty")),
}
}
}
/// Phase of one durable attempt at the moment it was resolved.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptPhase {
/// The durable checkpoint exists but no outbound send was observed.
BeforeDispatch,
/// The request was handed to the transport; its effect may exist upstream.
Dispatched,
}
impl AttemptPhase {
pub const fn as_str(self) -> &'static str {
match self {
Self::BeforeDispatch => "before_dispatch",
Self::Dispatched => "dispatched",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"before_dispatch" => Ok(Self::BeforeDispatch),
"dispatched" => Ok(Self::Dispatched),
_ => Err(TaskError::invalid("unknown attempt phase")),
}
}
}
/// Closed outcome vocabulary of one resolved attempt.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptOutcomeClass {
Completed,
Failed,
/// A bounded further attempt was scheduled instead of a terminal outcome.
RetryScheduled,
OutcomeUnknown,
}
impl AttemptOutcomeClass {
pub const fn as_str(self) -> &'static str {
match self {
Self::Completed => "completed",
Self::Failed => "failed",
Self::RetryScheduled => "retry_scheduled",
Self::OutcomeUnknown => "outcome_unknown",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"retry_scheduled" => Ok(Self::RetryScheduled),
"outcome_unknown" => Ok(Self::OutcomeUnknown),
_ => Err(TaskError::invalid("unknown attempt outcome")),
}
}
}
/// One durable outbound attempt. The row is created by the dispatch checkpoint before any network
/// use and becomes immutable once resolved, so a stale worker can never append to the history.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAttempt {
pub attempt_id: String,
pub task_id: TaskId,
pub ordinal: u32,
pub owner: String,
pub fence: u64,
/// Canonical fingerprint of the sealed executable input this attempt was admitted for.
pub input_fingerprint: String,
/// Digest of the stable upstream idempotency key (never the raw key or headers).
pub upstream_key_digest: Option<String>,
/// Whether a retry of this attempt is safe (read or server-verified idempotent write).
pub retry_allowed: bool,
pub checkpoint_at_unix_ms: i64,
pub resolved_at_unix_ms: Option<i64>,
pub outcome: Option<AttemptOutcomeClass>,
pub certainty: Option<AttemptCertainty>,
pub phase: Option<AttemptPhase>,
}
/// Committed dispatch intent of one durable attempt (Story 4.4). The store owns the transaction: the
/// attempt row, the incremented budget and the new Task version commit together before the network is
/// touched, so an uncommitted checkpoint can never produce an outbound call.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BeginAttempt {
pub task_id: TaskId,
pub owner: String,
pub fence: u64,
/// Version the worker read from the lease; a concurrent change rejects the checkpoint.
pub expected_version: u32,
pub attempt_id: String,
pub input_fingerprint: String,
pub upstream_key_digest: Option<String>,
/// The runtime's decision that a retry of this attempt is safe; the store still enforces the
/// pinned budget and the absolute deadline.
pub retry_allowed: bool,
}
impl BeginAttempt {
pub fn validate(&self) -> Result<(), TaskError> {
if self.owner.is_empty() || self.owner.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task attempt owner is invalid"));
}
if self.attempt_id.is_empty() || self.attempt_id.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task attempt identity is invalid"));
}
if self.input_fingerprint.len() != 64 {
return Err(TaskError::invalid("task attempt fingerprint is invalid"));
}
if let Some(digest) = self.upstream_key_digest.as_deref()
&& (digest.is_empty() || digest.len() > TASK_IDENTITY_MAX_BYTES)
{
return Err(TaskError::invalid(
"task attempt upstream key digest is invalid",
));
}
Ok(())
}
}
/// Leased, committed view of one in-flight attempt and the Task version it was admitted under.
#[derive(Clone, Debug, PartialEq)]
pub struct AttemptLease {
pub attempt: TaskAttempt,
pub task: TaskRecord,
}
/// Typed outcome of one attempt, handed to the store for a single atomic resolution.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AttemptOutcome {
/// Proven success; the Task becomes `completed` with this result.
Completed(TaskEnvelope),
/// Proven failure. `retryable` only schedules another attempt when the store's pinned budget,
/// the attempt's safe-retry decision and the absolute deadline all still allow it; otherwise the
/// Task fails with this error.
Failed {
error: TaskEnvelope,
retryable: bool,
phase: AttemptPhase,
},
/// The outbound effect cannot be proven. `repeatable` only schedules another bounded attempt when
/// repeating it is safe and the pinned budget/deadline still allow it; otherwise the Task becomes
/// the non-terminal `outcome_unknown`, never a plain failure.
Uncertain {
error: TaskEnvelope,
repeatable: bool,
phase: AttemptPhase,
},
}
/// Single-winner result of resolving one attempt.
#[derive(Clone, Debug, PartialEq)]
pub enum AttemptResolution {
/// A terminal transition committed (completed or failed).
Finished(TaskRecord),
/// The attempt failed in a known way and another bounded attempt was armed.
RetryAt {
task: TaskRecord,
next_attempt_at_unix_ms: i64,
},
/// The attempt resolved to the non-terminal uncertain state.
Unknown(TaskRecord),
}
/// Owner-side cancellation request. `expected_version` is optional so a repeated cancel of the same
/// accepted Task stays idempotent while a stale version is reported as a conflict.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskCancelRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub expected_version: Option<u32>,
pub reason: String,
}
impl TaskCancelRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if self.reason.is_empty()
|| self.reason.len() > TASK_CANCEL_REASON_MAX_BYTES
|| self.reason.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task cancellation reason is invalid"));
}
Ok(())
}
}
/// Typed result of an owner-scoped cancellation.
#[derive(Clone, Debug, PartialEq)]
pub enum TaskCancelOutcome {
/// Cancelled before the dispatch checkpoint: no outbound call can have happened.
Cancelled(TaskRecord),
/// The Task already had a checkpointed attempt: non-terminal uncertain, never a plain cancel.
OutcomeUnknown(TaskRecord),
/// The Task was already in a terminal state; its accepted result is unchanged.
AlreadyTerminal(TaskRecord),
/// The caller's expected version is stale; `current` is the authorized current state.
Conflict(TaskRecord),
}
-295
View File
@@ -1,295 +0,0 @@
//! Continuation of an accepted Task through a validated additional input (Story 4.5, FR-25).
//!
//! A Task that cannot run because the pinned operation still lacks a required input item is
//! suspended as `input_required` before any outbound dispatch. The durable request keeps the exact
//! immutable schema snapshot, its JSON projection, safe prompt metadata, the absolute TTL and the
//! update fingerprint contract. A valid update moves the Task back to `working` exactly once; an
//! identical replay returns the committed outcome and a conflicting or stale reply is typed.
//!
//! Deliberately absent: raw input in the request/receipt/audit read models, a second cipher store and
//! any public transport. The regular executable-input ciphertext carries the continuation binding.
use serde_json::Value;
use crate::task::{TaskAccess, TaskError, TaskId};
/// Version of the persisted input-continuation contract. A stored snapshot always declares the
/// version it was written with, so a later binary never reinterprets it silently.
pub const TASK_INPUT_SCHEMA_FORMAT_VERSION: u32 = 1;
/// Bound of a wait identity before it is persisted.
pub const TASK_INPUT_WAIT_ID_MAX_BYTES: usize = 64;
/// Bound of the safe prompt code (never the prompt text or an input value).
pub const TASK_INPUT_PROMPT_CODE_MAX_BYTES: usize = 64;
/// Bound of the missing-field path metadata.
pub const TASK_INPUT_MAX_MISSING_FIELDS: usize = 32;
/// Bound of one missing-field path.
pub const TASK_INPUT_FIELD_PATH_MAX_BYTES: usize = 160;
/// Durable request of additional input for one suspended Task.
///
/// `schema_snapshot` is the native serialized schema subset the server validates against; the JSON
/// projection is presentation metadata only and never the validation authority. `expected_version`
/// is the Task version a caller must present with the answer.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskInputRequest {
pub wait_id: String,
pub expected_version: u32,
pub schema_revision: u32,
pub schema_format_version: u32,
pub schema_snapshot: Value,
pub projected_schema: Value,
pub prompt_code: String,
pub missing_fields: Vec<String>,
pub expires_at_unix_ms: i64,
pub fingerprint_format_version: u32,
}
impl TaskInputRequest {
pub fn validate(&self) -> Result<(), TaskError> {
if self.wait_id.is_empty()
|| self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES
|| self.wait_id.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task input wait identity is invalid"));
}
if self.expected_version == 0 || self.schema_revision == 0 {
return Err(TaskError::invalid(
"task input request generation is invalid",
));
}
if self.schema_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
|| self.fingerprint_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
{
return Err(TaskError::invalid(
"task input request format version is invalid",
));
}
if self.schema_snapshot.as_object().is_none() || self.projected_schema.as_object().is_none()
{
return Err(TaskError::invalid("task input schema snapshot is invalid"));
}
if self.prompt_code.is_empty() || self.prompt_code.len() > TASK_INPUT_PROMPT_CODE_MAX_BYTES
{
return Err(TaskError::invalid("task input prompt code is invalid"));
}
if self.missing_fields.len() > TASK_INPUT_MAX_MISSING_FIELDS
|| self
.missing_fields
.iter()
.any(|field| field.is_empty() || field.len() > TASK_INPUT_FIELD_PATH_MAX_BYTES)
{
return Err(TaskError::invalid(
"task input missing-field metadata is invalid",
));
}
if self.expires_at_unix_ms <= 0 {
return Err(TaskError::invalid("task input TTL is invalid"));
}
Ok(())
}
}
/// Authoritative write of one validated answer. The transaction owns schema-generation, TTL,
/// fingerprint and CAS checks; this request only carries the committed values.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskInputUpdateRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub wait_id: String,
pub expected_version: u32,
/// Digest of the scoped update key; the caller key is never persisted.
pub update_key_digest: String,
/// Canonical digest of the validated full executable input.
pub canonical_fingerprint: String,
/// Effective execution fingerprint of the resumed payload (never the acceptance fingerprint).
pub effective_input_fingerprint: String,
pub executable_input_ciphertext: String,
pub executable_input_key_version: String,
pub executable_input_master_key_epoch: i64,
}
impl TaskInputUpdateRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if self.wait_id.is_empty() || self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES {
return Err(TaskError::invalid("task input wait identity is invalid"));
}
if self.expected_version == 0 {
return Err(TaskError::invalid("task input version is invalid"));
}
if !valid_digest(&self.update_key_digest)
|| !valid_digest(&self.canonical_fingerprint)
|| !valid_digest(&self.effective_input_fingerprint)
{
return Err(TaskError::invalid("task input digest is invalid"));
}
if self.executable_input_ciphertext.is_empty()
|| self.executable_input_ciphertext.len()
> crate::task::TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
{
return Err(TaskError::invalid(
"task continuation ciphertext is invalid",
));
}
if self.executable_input_key_version.is_empty()
|| self.executable_input_key_version.len() > 64
|| self.executable_input_master_key_epoch < 1
{
return Err(TaskError::invalid(
"task continuation ciphertext is invalid",
));
}
Ok(())
}
}
/// Committed outcome of one input update. An identical replay returns the same stored outcome with
/// `replayed = true` instead of repeating the CAS, audit or dispatch.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskInputUpdateOutcome {
pub task_id: TaskId,
pub wait_id: String,
pub state: crate::task::TaskState,
pub version: u32,
pub replayed: bool,
}
fn valid_digest(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
}
/// Durable input-continuation operations of the Task store.
///
/// Split out of the general [`crate::task::TaskStore`] port so the lifecycle store stays readable and
/// the continuation contract (suspend, answer, receipt, expiry) is implemented and audited as one
/// cohesive surface. [`crate::task::TaskStore`] extends this trait, so an owned `Arc<dyn TaskStore>`
/// exposes both halves through one object.
#[async_trait::async_trait]
pub trait TaskInputStore: Send + Sync {
/// Suspends a leased Task that still lacks a required input of its pinned operation. The
/// transaction persists the immutable request and its audit row, moves `working` to
/// `input_required` exactly once, releases the worker lease and keeps the existing admission
/// permit. No outbound dispatch may have happened in this generation.
async fn suspend_for_input(
&self,
lease: &crate::task::TaskLease,
request: TaskInputRequest,
) -> Result<crate::task::TaskRecord, TaskError>;
/// Owner-scoped read of the durable input request of one Task.
async fn input_request(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<TaskInputRequest, TaskError>;
/// Owner-scoped read of the executable-input binding that a committed continuation may extend.
/// It is not lease-authorized: it only serves the update transaction and never dispatch.
async fn read_executable_input_for_update(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<(String, String, i64), TaskError>;
/// Applies exactly one validated answer under a row lock: authority recheck, receipt lookup,
/// generation/TTL check, atomic ciphertext/effective-fingerprint/receipt/`working` transition and
/// audit. An identical replay returns the stored outcome without a second transition.
async fn apply_input(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError>;
/// Bounded sweep of `input_required` Tasks whose TTL elapsed: `failed` with an audit reason and
/// the release of the admission permit, without any dispatch.
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError>;
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn request() -> TaskInputRequest {
TaskInputRequest {
wait_id: "wait-1".to_owned(),
expected_version: 2,
schema_revision: 1,
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
schema_snapshot: json!({"type": "object", "fields": {}}),
projected_schema: json!({"type": "object", "properties": {}}),
prompt_code: "required_input_missing".to_owned(),
missing_fields: vec!["$.email".to_owned()],
expires_at_unix_ms: 1_900_000_000_000,
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
}
}
fn update() -> TaskInputUpdateRequest {
TaskInputUpdateRequest {
access: crate::task::TaskAccess {
workspace_id: "ws_default".to_owned(),
owner_kind: crate::task::TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
},
task_id: crate::task::TaskId::generate(),
wait_id: "wait-1".to_owned(),
expected_version: 2,
update_key_digest: "a".repeat(64),
canonical_fingerprint: "b".repeat(64),
effective_input_fingerprint: "c".repeat(64),
executable_input_ciphertext: "sealed".to_owned(),
executable_input_key_version: "key-v1".to_owned(),
executable_input_master_key_epoch: 1,
}
}
#[test]
fn input_request_holds_generation_schema_ttl_and_bounds() {
assert!(request().validate().is_ok());
let mut no_wait = request();
no_wait.wait_id.clear();
assert!(no_wait.validate().is_err());
let mut no_generation = request();
no_generation.expected_version = 0;
assert!(no_generation.validate().is_err());
let mut bad_schema = request();
bad_schema.schema_snapshot = json!("not-an-object");
assert!(bad_schema.validate().is_err());
let mut no_ttl = request();
no_ttl.expires_at_unix_ms = 0;
assert!(no_ttl.validate().is_err());
let mut too_many_fields = request();
too_many_fields.missing_fields = vec!["$.x".to_owned(); TASK_INPUT_MAX_MISSING_FIELDS + 1];
assert!(too_many_fields.validate().is_err());
}
#[test]
fn update_request_rejects_invalid_digests_and_ciphertext_bounds() {
assert!(update().validate().is_ok());
let mut short_digest = update();
short_digest.canonical_fingerprint = "abc".to_owned();
assert!(short_digest.validate().is_err());
let mut upper_digest = update();
upper_digest.update_key_digest = "A".repeat(64);
assert!(upper_digest.validate().is_err());
let mut no_ciphertext = update();
no_ciphertext.executable_input_ciphertext.clear();
assert!(no_ciphertext.validate().is_err());
let mut no_epoch = update();
no_epoch.executable_input_master_key_epoch = 0;
assert!(no_epoch.validate().is_err());
}
}
-964
View File
@@ -1,964 +0,0 @@
//! Durable background Task seam (Story 4.2, FR-22/FR-23).
//!
//! A Task is the internal, protocol-independent record of one accepted background request. This
//! module owns the domain vocabulary only: the lifecycle states, the accepted identity/pins, the
//! bounded persisted envelopes and the `TaskStore` port. The protocol adapter (Story 4.7) and the
//! worker (Story 4.3) do not define these semantics; the authoritative store lives in PostgreSQL
//! (`crank-registry`), so an accepted Task survives a crash or restart.
//!
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
//! Task was executed. A terminal state is written by a conditional transition only.
mod attempt;
mod input;
use async_trait::async_trait;
pub use attempt::*;
pub use input::*;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
use crate::correlation::{RequestId, TraceContext, TraceId};
use crate::ids::PlatformApiKeyId;
use crate::operation::ToolExecutionMode;
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
/// Maximum byte budget of the decrypted executable Task input.
pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES;
/// Maximum byte budget of the stored encrypted executable Task input ciphertext.
pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304;
/// Maximum nesting depth of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
/// Maximum number of JSON members/nodes of a persisted Task envelope.
pub const TASK_ENVELOPE_MAX_NODES: usize = 256;
/// Bound of every scope/owner/operation identity column.
pub const TASK_IDENTITY_MAX_BYTES: usize = 160;
/// Bound of the caller-supplied idempotency key before it becomes a scoped digest.
pub const TASK_SCOPED_KEY_MAX_BYTES: usize = 128;
/// Bound of a saved approval reference (an identifier, never a bearer token).
pub const TASK_APPROVAL_REFERENCE_MAX_BYTES: usize = 160;
/// Default retention of a terminal Task after its execution deadline (internal bounded default).
pub const TASK_DEFAULT_RETENTION_MS: i64 = 86_400_000;
/// Upper bound on the retention an accepted Task may pin.
pub const TASK_MAX_RETENTION_MS: i64 = 30 * 86_400_000;
/// Upper bound of one retention/housekeeping batch.
pub const TASK_PURGE_BATCH_LIMIT: u32 = 128;
/// Stable identity of one accepted Task. Generated as UUIDv7 and returned only after the accepting
/// transaction committed, so the ID never exists without its durable record.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct TaskId(Uuid);
impl TaskId {
pub fn generate() -> Self {
Self(Uuid::now_v7())
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
Uuid::parse_str(value)
.map(Self)
.map_err(|_| TaskError::invalid("task identity is not a UUID"))
}
pub fn as_uuid(self) -> Uuid {
self.0
}
}
impl std::fmt::Display for TaskId {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(formatter)
}
}
/// Closed Task lifecycle vocabulary. `outcome_unknown` is a non-terminal state: a value commit that
/// failed to confirm its dispatch is never collapsed into a plain failure.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskState {
Queued,
Working,
InputRequired,
OutcomeUnknown,
Completed,
Failed,
Cancelled,
}
impl TaskState {
pub const fn as_str(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Working => "working",
Self::InputRequired => "input_required",
Self::OutcomeUnknown => "outcome_unknown",
Self::Completed => "completed",
Self::Failed => "failed",
Self::Cancelled => "cancelled",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"queued" => Ok(Self::Queued),
"working" => Ok(Self::Working),
"input_required" => Ok(Self::InputRequired),
"outcome_unknown" => Ok(Self::OutcomeUnknown),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"cancelled" => Ok(Self::Cancelled),
_ => Err(TaskError::invalid("unknown task state")),
}
}
pub const fn is_terminal(self) -> bool {
matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
}
/// One conditional transition may only move along this table. A terminal state has no successor,
/// so a repeated terminal write can never rewrite an accepted result.
pub const fn allows_transition_to(self, next: Self) -> bool {
match self {
Self::Queued => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
Self::Working => matches!(
next,
Self::InputRequired
| Self::OutcomeUnknown
| Self::Completed
| Self::Failed
| Self::Cancelled
),
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
// Story 4.4: an uncertain outbound outcome is never resolved by a generic transition.
// Reconciliation is Story 4.6, so no ordinary finish/cancel/retry may leave this state.
Self::OutcomeUnknown => false,
Self::Completed | Self::Failed | Self::Cancelled => false,
}
}
}
/// Who owns the accepted Task. An administrator is never a wildcard owner of every Task: the owner
/// identity and the authorization scope are separate, both-pinned columns.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskOwnerKind {
Agent,
Admin,
}
impl TaskOwnerKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Agent => "agent",
Self::Admin => "admin",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"agent" => Ok(Self::Agent),
"admin" => Ok(Self::Admin),
_ => Err(TaskError::invalid("unknown task owner kind")),
}
}
}
/// Support identities captured at acceptance. They are stored and returned independently of
/// sampling, and a replay never rewrites them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskSupportIds {
pub request_id: RequestId,
pub trace_id: TraceId,
pub traceparent: String,
pub trace_sampled: bool,
}
impl TaskSupportIds {
pub fn new(request_id: RequestId, trace_context: &TraceContext) -> Self {
Self {
request_id,
trace_id: trace_context.trace_id().clone(),
traceparent: trace_context.traceparent().to_owned(),
trace_sampled: trace_context.is_sampled(),
}
}
/// Re-validates both the trace identity and the parent context: a Task must never persist an
/// inconsistent or malformed support pair, and an unsampled context is still a valid parent.
pub fn validate(&self) -> Result<(), TaskError> {
if self.trace_id.as_str().is_empty() {
return Err(TaskError::invalid("task trace identity is missing"));
}
let context = TraceContext::parse(&self.traceparent)
.map_err(|_| TaskError::invalid("task trace parent is invalid"))?;
if context.trace_id() != &self.trace_id || context.is_sampled() != self.trace_sampled {
return Err(TaskError::invalid("task trace parent is inconsistent"));
}
Ok(())
}
}
/// Execution semantics pinned at acceptance: a later publication never rewrites them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskPins {
pub operation_version: u32,
pub policy_mode: ToolExecutionMode,
pub execution_deadline_unix_ms: i64,
/// Retention deadline of the record; never the same thing as the execution deadline.
pub retain_until_unix_ms: i64,
/// Verified approval evidence reference (identifier only, never a bearer token).
pub approval_reference: Option<String>,
/// Bounded retry budget pinned at acceptance (Story 4.4). A later publication, a restart or a
/// new binary never changes the numbers an accepted Task was admitted with.
pub retry: TaskRetryPolicy,
}
impl TaskPins {
pub fn validate(&self) -> Result<(), TaskError> {
if self.operation_version == 0 {
return Err(TaskError::invalid(
"task operation version must be positive",
));
}
if self.execution_deadline_unix_ms <= 0 {
return Err(TaskError::invalid("task execution deadline is invalid"));
}
if self.retain_until_unix_ms < self.execution_deadline_unix_ms {
return Err(TaskError::invalid(
"task retention deadline precedes execution",
));
}
if self.retain_until_unix_ms - self.execution_deadline_unix_ms > TASK_MAX_RETENTION_MS {
return Err(TaskError::invalid("task retention exceeds its bound"));
}
if let Some(reference) = self.approval_reference.as_deref()
&& (reference.is_empty()
|| reference.len() > TASK_APPROVAL_REFERENCE_MAX_BYTES
|| reference.bytes().any(|byte| byte.is_ascii_control()))
{
return Err(TaskError::invalid("task approval reference is invalid"));
}
self.retry.validate()?;
Ok(())
}
}
/// Bounded, classified payload of a Task. Construction validates the byte/depth/node budget and
/// redacts secret-bearing fields before anything is hashed or persisted.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskEnvelope {
value: Value,
}
impl TaskEnvelope {
/// Validates the caller-provided value, then stores a redacted copy of it. The caller hashes the
/// original value separately, so redaction never collapses two different requests into one.
pub fn try_redacted(input: &Value) -> Result<Self, TaskError> {
validate_task_envelope(input)?;
Ok(Self {
value: redact_task_envelope(input),
})
}
/// Accepts a value that was already validated and redacted at a trust boundary.
pub fn from_persisted(value: Value) -> Result<Self, TaskError> {
validate_task_envelope(&value)?;
Ok(Self { value })
}
pub fn value(&self) -> &Value {
&self.value
}
pub fn into_value(self) -> Value {
self.value
}
}
/// Validates the size, nesting depth and node count of one Task envelope. Every byte, collection and
/// node is counted before the value is cloned, hashed or persisted.
pub fn validate_task_envelope(value: &Value) -> Result<(), TaskError> {
let encoded = serde_json::to_vec(value)
.map_err(|_| TaskError::invalid("task envelope is not serializable"))?;
if encoded.len() > TASK_ENVELOPE_MAX_BYTES {
return Err(TaskError::invalid("task envelope exceeds its byte budget"));
}
let mut nodes = 0usize;
check_budget(value, 1, &mut nodes)?;
Ok(())
}
fn check_budget(value: &Value, depth: usize, nodes: &mut usize) -> Result<(), TaskError> {
if depth > TASK_ENVELOPE_MAX_DEPTH {
return Err(TaskError::invalid("task envelope exceeds its depth budget"));
}
*nodes += 1;
if *nodes > TASK_ENVELOPE_MAX_NODES {
return Err(TaskError::invalid("task envelope exceeds its node budget"));
}
match value {
Value::Object(object) => {
for (key, child) in object {
check_budget_name(key)?;
check_budget(child, depth + 1, nodes)?;
}
}
Value::Array(values) => {
for child in values {
check_budget(child, depth + 1, nodes)?;
}
}
_ => {}
}
Ok(())
}
fn check_budget_name(key: &str) -> Result<(), TaskError> {
if key.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task envelope key exceeds its budget"));
}
Ok(())
}
/// Field-name classification of secret-bearing values. A matching field keeps its position but never
/// its plaintext in the persisted envelope.
pub fn sensitive_task_key(name: &str) -> bool {
let lowered = name.to_ascii_lowercase();
[
"password",
"passwd",
"secret",
"token",
"api_key",
"apikey",
"authorization",
"credential",
"private_key",
"cookie",
"signature",
]
.iter()
.any(|marker| lowered.contains(marker))
}
/// Redacted projection of an already validated envelope.
pub fn redact_task_envelope(value: &Value) -> Value {
match value {
Value::Object(object) => Value::Object(
object
.iter()
.map(|(key, child)| {
let value = if sensitive_task_key(key) {
Value::String("[redacted]".to_owned())
} else {
redact_task_envelope(child)
};
(key.clone(), value)
})
.collect(),
),
Value::Array(values) => Value::Array(values.iter().map(redact_task_envelope).collect()),
other => other.clone(),
}
}
/// Recursive check used to fail closed on sensitive fields before encryption.
pub fn contains_sensitive_task_keys(value: &Value) -> bool {
match value {
Value::Object(object) => object
.iter()
.any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)),
Value::Array(values) => values.iter().any(contains_sensitive_task_keys),
_ => false,
}
}
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
/// queue capacity commit together, and no acknowledgement exists before that commit.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAcceptRequest {
pub task_id: TaskId,
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
/// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the
/// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively.
/// `None` means the accepting principal has no machine key (administrator) and fail-closes.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
pub pins: TaskPins,
/// Digest of the caller key over the full owner/authorization/Tool scope.
pub scoped_key: String,
/// Digest of the canonical validated executable input.
pub input_fingerprint: String,
pub input: TaskEnvelope,
pub support: TaskSupportIds,
/// Bounded encrypted executable input, persisted alongside the redacted input projection.
pub executable_input_ciphertext: Option<String>,
pub executable_input_key_version: Option<String>,
pub executable_input_master_key_epoch: Option<i64>,
}
impl TaskAcceptRequest {
pub fn validate(&self) -> Result<(), TaskError> {
for (field, value) in [
("workspace", self.workspace_id.as_str()),
("owner", self.owner_id.as_str()),
("authorization scope", self.authorization_scope.as_str()),
("operation", self.operation_id.as_str()),
] {
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
if let Some(key_id) = self.platform_api_key_id.as_ref()
&& (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES)
{
return Err(TaskError::invalid(
"task platform api key reference is invalid",
));
}
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
return Err(TaskError::invalid("task scoped key digest is invalid"));
}
if self.input_fingerprint.len() != 64
|| !self
.input_fingerprint
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
{
return Err(TaskError::invalid("task input fingerprint is invalid"));
}
self.pins.validate()?;
self.support.validate()?;
let has_ciphertext = self.executable_input_ciphertext.is_some();
let has_key = self.executable_input_key_version.is_some();
let has_epoch = self.executable_input_master_key_epoch.is_some();
match (has_ciphertext, has_key, has_epoch) {
(false, false, false) => {}
(true, true, true) => {
let ciphertext = self.executable_input_ciphertext.as_deref().unwrap();
if ciphertext.is_empty()
|| ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
{
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
let key_version = self.executable_input_key_version.as_deref().unwrap();
if key_version.is_empty() || key_version.len() > 64 {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
if self.executable_input_master_key_epoch.unwrap_or_default() < 1 {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
}
_ => {
return Err(TaskError::invalid(
"task executable input ciphertext is invalid",
));
}
}
Ok(())
}
}
/// Successful acceptance, returned only after the accepting transaction committed. A replay returns
/// the stored identity and support IDs of the original acceptance.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAcceptance {
pub task_id: TaskId,
pub state: TaskState,
/// Exact Published Version pinned by this acceptance; a replay returns the stored pin.
pub operation_version: u32,
pub support: TaskSupportIds,
pub created_at_unix_ms: i64,
pub retain_until_unix_ms: i64,
pub replayed: bool,
}
/// Owner-side access proof of a read or a terminal write. It is built by the application layer from
/// the verified principal, never from a caller-supplied marker, and every store query carries the
/// whole predicate so a foreign row is never fetched.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAccess {
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
}
impl TaskAccess {
pub fn validate(&self) -> Result<(), TaskError> {
for (field, value) in [
("workspace", self.workspace_id.as_str()),
("owner", self.owner_id.as_str()),
("authorization scope", self.authorization_scope.as_str()),
] {
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid(format!("task {field} is invalid")));
}
}
Ok(())
}
}
/// Read model of one Task. Lifecycle fields, pins and support IDs stay readable together; the
/// protocol adapter only maps them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskRecord {
pub task_id: TaskId,
pub state: TaskState,
pub workspace_id: String,
pub owner_kind: TaskOwnerKind,
pub owner_id: String,
pub authorization_scope: String,
/// Typed immutable identity of the Platform API key that authorized the acceptance; the worker
/// revalidates this exact key against the registry before any outbound dispatch.
pub platform_api_key_id: Option<PlatformApiKeyId>,
pub operation_id: String,
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
pub input_fingerprint: String,
/// Effective fingerprint of the payload the next attempt will execute. It equals the acceptance
/// fingerprint until a committed input continuation replaces the effective payload; the
/// acceptance fingerprint above is never overwritten, so an original accept replay stays replay.
pub effective_input_fingerprint: String,
pub pins: TaskPins,
pub support: TaskSupportIds,
pub input: TaskEnvelope,
pub progress: Option<TaskEnvelope>,
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
pub attempts: u32,
/// Monotonic lease/fencing token. It never resets and protects Task state mutations.
pub fence: u64,
/// Random process-instance lease owner, if a worker currently owns the Task.
pub lease_owner: Option<String>,
/// Lease expiry epoch milliseconds, if the Task is claimed.
pub lease_until_unix_ms: Option<i64>,
/// Current dispatch attempt identity, committed before any outbound adapter call.
pub current_attempt_id: Option<String>,
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
pub dispatch_started_at_unix_ms: Option<i64>,
/// Earliest moment the next durable attempt may be dispatched (pinned backoff).
pub next_attempt_at_unix_ms: Option<i64>,
/// Durable cancellation request that arrived while an attempt was already checkpointed.
pub cancel_requested_at_unix_ms: Option<i64>,
/// Bounded reason of the cancellation request.
pub cancel_reason: Option<String>,
/// Monotonic lifecycle version used by the terminal CAS.
pub version: u32,
pub created_at_unix_ms: i64,
pub updated_at_unix_ms: i64,
}
/// One conditional terminal transition. Exactly one concurrent writer wins; the losers observe the
/// accepted state and never rewrite the result.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskFinishRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub expected_state: TaskState,
pub expected_version: u32,
pub target_state: TaskState,
pub result: Option<TaskEnvelope>,
pub error: Option<TaskEnvelope>,
}
impl TaskFinishRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if !self.target_state.is_terminal() {
return Err(TaskError::invalid("task finish target is not terminal"));
}
if self.expected_state.is_terminal() {
return Err(TaskError::invalid("task finish source is already terminal"));
}
if !self.expected_state.allows_transition_to(self.target_state) {
return Err(TaskError::invalid("task state transition is not allowed"));
}
if self.target_state == TaskState::Completed && self.result.is_none() {
return Err(TaskError::invalid("a completed task requires a result"));
}
Ok(())
}
}
/// Bounded retention outcome.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct TaskPurgeReport {
/// Queued Tasks that passed their execution deadline and were failed with their intent removed.
pub expired_queued: u64,
/// Terminal Tasks that passed their retention deadline.
pub expired_terminal: u64,
}
/// Typed Task failure. A denial and a missing Task share one outward shape, so a foreign caller can
/// never distinguish "not yours" from "does not exist".
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum TaskError {
#[error("task request is invalid: {message}")]
Invalid { message: String },
#[error("task store is unavailable: {message}")]
Unavailable { message: String },
#[error("task was not found")]
NotFound,
#[error("task request conflicts with the accepted request")]
FingerprintConflict,
#[error("task lifecycle transition was rejected")]
StateConflict,
#[error("task input request has expired")]
InputExpired,
#[error("background task capacity is exhausted")]
Overloaded { retry_after_ms: u64 },
}
impl TaskError {
pub fn invalid(message: impl Into<String>) -> Self {
Self::Invalid {
message: message.into(),
}
}
pub fn unavailable(message: impl Into<String>) -> Self {
Self::Unavailable {
message: message.into(),
}
}
/// Whether the caller may tell this failure apart from an absent Task.
pub fn is_not_found(&self) -> bool {
matches!(self, Self::NotFound)
}
}
/// Leased view of a claimed Task: its durable record plus lease fencing identity.
#[derive(Clone, Debug, PartialEq)]
pub struct TaskLease {
pub record: TaskRecord,
pub owner: String,
pub fence: u64,
pub lease_until: i64,
}
/// Authoritative durable Task port.
#[async_trait]
pub trait TaskStore: TaskInputStore + Send + Sync {
/// Proves the durable Task state is usable (schema and operations reachable).
async fn health(&self) -> Result<(), TaskError>;
/// Atomically accepts a Task: Task record, dispatch intent and one queue slot commit together.
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError>;
/// Reads one Task inside the caller's owner scope.
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError>;
/// Scoped lookup of an already accepted Task by its full-scope idempotency digest. Used to keep a
/// retry of an explicitly pinned version from becoming new work; `None` means "not accepted yet".
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError>;
/// Applies exactly one conditional terminal transition and releases its intent atomically.
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
/// Reads encrypted executable input only under a valid, unexpired current lease.
async fn read_executable_input(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
) -> Result<(String, String, i64), TaskError>;
/// Bounded retention...
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
/// Claims a bounded batch of queued or safely recoverable Tasks under leases.
async fn claim_batch(
&self,
owner: &str,
batch_limit: u32,
lease_ms: i64,
) -> Result<Vec<TaskLease>, TaskError>;
/// Renews a lease owned by `owner` when fence and expiry still match.
async fn renew_lease(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
lease_ms: i64,
) -> Result<TaskLease, TaskError>;
/// Persists redacted progress under a valid lease.
async fn write_progress(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
progress: TaskEnvelope,
) -> Result<TaskRecord, TaskError>;
/// Commits one durable outbound attempt before network use: attempt row, incremented budget and
/// the new Task version commit in one transaction. A returned lease is the only authority to
/// dispatch that attempt; an unknown commit means zero outbound calls.
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError>;
/// Resolves exactly one committed attempt under its lease/fence/version. Only the current attempt
/// of the current owner may be resolved, and the attempt becomes immutable afterwards.
async fn resolve_attempt(
&self,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError>;
/// Owner-scoped atomic cancellation. It serializes with claim and checkpoint on the same Task.
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError>;
/// Owner-scoped bounded history of the durable attempts of one Task.
async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<TaskAttempt>, TaskError>;
/// Applies a terminal transition only under a valid lease and releases intent atomically.
async fn finish_leased(
&self,
request: TaskFinishRequest,
owner: &str,
fence: u64,
) -> Result<TaskRecord, TaskError>;
/// Resolves a leased Task whose next attempt was refused before its dispatch checkpoint (elapsed
/// deadline, rejected authority). A Task that still carries a proven ambiguity is never downgraded
/// to a plain failure: it ends non-terminal `outcome_unknown` with `code`; otherwise `failed`.
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError>;
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn request() -> TaskAcceptRequest {
TaskAcceptRequest {
task_id: TaskId::generate(),
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
operation_id: "op_report".to_owned(),
pins: TaskPins {
operation_version: 3,
policy_mode: ToolExecutionMode::Background,
execution_deadline_unix_ms: 1_790_000_000_000,
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
approval_reference: Some("appr_1".to_owned()),
retry: TaskRetryPolicy::new(3, 1),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
support: TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
),
executable_input_ciphertext: None,
executable_input_key_version: None,
executable_input_master_key_epoch: None,
}
}
#[test]
fn uuid_v7_task_ids_are_unique_and_round_trip() {
let first = TaskId::generate();
let second = TaskId::generate();
assert_ne!(first, second);
assert_eq!(TaskId::parse(&first.to_string()).unwrap(), first);
}
#[test]
fn terminal_states_have_no_successor() {
for terminal in [
TaskState::Completed,
TaskState::Failed,
TaskState::Cancelled,
] {
assert!(terminal.is_terminal());
for next in [
TaskState::Queued,
TaskState::Working,
TaskState::Completed,
TaskState::Failed,
] {
assert!(!terminal.allows_transition_to(next));
}
}
assert!(TaskState::Queued.allows_transition_to(TaskState::Working));
assert!(TaskState::Working.allows_transition_to(TaskState::OutcomeUnknown));
assert!(!TaskState::Queued.allows_transition_to(TaskState::Completed));
assert!(!TaskState::InputRequired.allows_transition_to(TaskState::Completed));
}
#[test]
fn envelope_bounds_reject_bytes_depth_and_nodes() {
assert!(TaskEnvelope::try_redacted(&json!({"a": 1})).is_ok());
assert!(
TaskEnvelope::try_redacted(&json!({"a": "x".repeat(TASK_ENVELOPE_MAX_BYTES)})).is_err()
);
let mut deep = json!(1);
for _ in 0..=TASK_ENVELOPE_MAX_DEPTH {
deep = json!({"n": deep});
}
assert!(TaskEnvelope::try_redacted(&deep).is_err());
let wide = Value::Array((0..=TASK_ENVELOPE_MAX_NODES).map(|_| json!(1)).collect());
assert!(TaskEnvelope::try_redacted(&wide).is_err());
}
#[test]
fn secret_bearing_fields_are_redacted_and_keep_their_shape() {
let envelope = TaskEnvelope::try_redacted(&json!({
"query": "leads",
"auth": {"api_key": "sk_live_1", "nested": [{"password": "p"}]}
}))
.unwrap();
let value = envelope.value();
assert_eq!(value["query"], json!("leads"));
assert_eq!(value["auth"]["api_key"], json!("[redacted]"));
assert_eq!(value["auth"]["nested"][0]["password"], json!("[redacted]"));
}
#[test]
fn unsampled_support_ids_are_valid_and_consistent() {
let support = TaskSupportIds::new(
RequestId::generate(),
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
.unwrap(),
);
assert!(!support.trace_sampled);
assert!(support.traceparent.ends_with("-00"));
support.validate().unwrap();
let mut inconsistent = support.clone();
inconsistent.traceparent = inconsistent.traceparent.replace("-00", "-01");
assert!(inconsistent.validate().is_err());
}
#[test]
fn accept_request_validation_holds_the_identity_and_pin_bounds() {
assert!(request().validate().is_ok());
let mut oversized = request();
oversized.workspace_id = "w".repeat(TASK_IDENTITY_MAX_BYTES + 1);
assert!(oversized.validate().is_err());
let mut bad_fingerprint = request();
bad_fingerprint.input_fingerprint = "XYZ".to_owned();
assert!(bad_fingerprint.validate().is_err());
let mut bad_retention = request();
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
assert!(bad_retention.validate().is_err());
// The persisted authority reference is bounded and never empty.
let mut empty_key_reference = request();
empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new(""));
assert!(empty_key_reference.validate().is_err());
}
#[test]
fn outcome_unknown_has_no_generic_successor() {
// Story 4.4: reconciliation of an uncertain outcome is Story 4.6. Until then no ordinary
// finish/cancel/retry may leave the non-terminal uncertain state.
for next in [
TaskState::Queued,
TaskState::Working,
TaskState::InputRequired,
TaskState::OutcomeUnknown,
TaskState::Completed,
TaskState::Failed,
TaskState::Cancelled,
] {
assert!(
!TaskState::OutcomeUnknown.allows_transition_to(next),
"outcome_unknown must not transition to {next:?}"
);
}
assert!(!TaskState::OutcomeUnknown.is_terminal());
}
#[test]
fn retry_policy_bounds_are_pinned_and_bounded() {
assert!(TaskRetryPolicy::single_attempt().validate().is_ok());
assert!(
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS)
.validate()
.is_ok()
);
assert!(TaskRetryPolicy::new(0, 0).validate().is_err());
assert!(
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS + 1, 0)
.validate()
.is_err()
);
assert!(
TaskRetryPolicy::new(1, TASK_MAX_BACKOFF_STEP_MS + 1)
.validate()
.is_err()
);
// The pinned step grows the delay but never beyond four steps.
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(2), 2);
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(5), 8);
assert_eq!(TaskRetryPolicy::single_attempt().backoff_before(2), 0);
// A pin whose retry snapshot is invalid is rejected at acceptance.
let mut invalid = request();
invalid.pins.retry = TaskRetryPolicy::new(0, 0);
assert!(invalid.validate().is_err());
}
#[test]
fn finish_requires_a_terminal_reachable_transition() {
let access = TaskAccess {
workspace_id: "ws_default".to_owned(),
owner_kind: TaskOwnerKind::Agent,
owner_id: "agent_a".to_owned(),
authorization_scope: "credential:pk_1".to_owned(),
};
let base = TaskFinishRequest {
access,
task_id: TaskId::generate(),
expected_state: TaskState::Working,
expected_version: 1,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
};
assert!(base.validate().is_ok());
let mut non_terminal = base.clone();
non_terminal.target_state = TaskState::Working;
assert!(non_terminal.validate().is_err());
let mut terminal_source = base.clone();
terminal_source.expected_state = TaskState::Completed;
assert!(terminal_source.validate().is_err());
let mut completed_without_result = base;
completed_without_result.result = None;
assert!(completed_without_result.validate().is_err());
}
}
@@ -231,7 +231,6 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
-1
View File
@@ -564,7 +564,6 @@ fn default_execution_config() -> ExecutionConfig {
idempotency: None,
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
}
-28
View File
@@ -217,34 +217,6 @@ string_enum! {
Concurrency => "concurrency",
RateLimit => "rate_limit",
McpStream => "mcp_stream",
Admission => "admission",
}
}
string_enum! {
pub enum RejectionReason {
Limit => "limit",
Capacity => "capacity",
QueueFull => "queue_full",
Deadline => "deadline",
Unsupported => "unsupported",
DependencyUnavailable => "dependency_unavailable",
}
}
string_enum! {
pub enum ScopeKind {
Process => "process",
Global => "global",
Agent => "agent",
Tool => "tool",
}
}
string_enum! {
pub enum AdmissionModeLabel {
Synchronous => "synchronous",
Background => "background",
}
}
+5 -6
View File
@@ -16,15 +16,14 @@ mod schema;
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
pub use labels::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
UpstreamOutcome,
};
pub use record::{
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service,
bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting,
record_admission_queue_wait, record_admission_rejection, record_cache_outcome,
bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
+5 -41
View File
@@ -5,10 +5,10 @@ use metrics::Gauge;
use crate::schema::{MetricDefinition, metric_schema};
use crate::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter,
HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason,
ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
static BINDING: OnceLock<MetricService> = OnceLock::new();
@@ -352,47 +352,11 @@ pub fn record_limit_rejection(stage: LimitStage) {
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => stage.as_str(),
"reason" => RejectionReason::Limit.as_str(),
"scope_kind" => ScopeKind::Process.as_str()
"stage" => stage.as_str()
)
.increment(1);
}
/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool,
/// request id, fingerprint or arbitrary error text ever becomes a label.
pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) {
if bound_family("crank_runtime_limit_rejections_total").is_none() {
return;
}
metrics::counter!(
"crank_runtime_limit_rejections_total",
"stage" => LimitStage::Admission.as_str(),
"reason" => reason.as_str(),
"scope_kind" => scope_kind.as_str()
)
.increment(1);
}
pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) {
if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() {
return;
}
metrics::gauge!(
"crank_runtime_admission_queue_wait_seconds",
"mode" => mode.as_str()
)
.set(waited_ms as f64 / 1000.0);
}
pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option<u64>) {
if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() {
return;
}
let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0;
metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds);
}
pub fn record_cache_outcome(outcome: CacheOutcome) {
if bound_family("crank_runtime_cache_total").is_none() {
return;
+8 -38
View File
@@ -3,17 +3,16 @@
use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP};
use crate::labels::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome,
UpstreamOperationKind, UpstreamOutcome,
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
};
pub const METRIC_SCHEMA_VERSION: u32 = 2;
pub const MAX_METRIC_FAMILIES: usize = 128;
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611;
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285;
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515;
pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348;
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
@@ -180,9 +179,6 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
const REJECTION_REASONS: &[&str] = RejectionReason::VALUES;
const SCOPE_KINDS: &[&str] = ScopeKind::VALUES;
const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES;
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
@@ -407,39 +403,13 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[
"crank_runtime_limit_rejections_total",
MetricKind::Counter,
MetricUnit::Count,
&["stage", "reason", "scope_kind"],
&[
domain("stage", LIMIT_STAGES),
domain("reason", REJECTION_REASONS),
domain("scope_kind", SCOPE_KINDS),
],
&["stage"],
&[domain("stage", LIMIT_STAGES)],
&[],
BOTH,
96,
3,
"Runtime executions rejected by a bounded limit.",
),
metric(
"crank_runtime_admission_queue_wait_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&["mode"],
&[domain("mode", ADMISSION_MODES)],
&[],
BOTH,
2,
"Wait of the most recently admitted execution before it received a scoped permit.",
),
metric(
"crank_runtime_admission_oldest_waiting_seconds",
MetricKind::Gauge,
MetricUnit::Seconds,
&[],
&[],
&[],
BOTH,
1,
"Age of the oldest waiting admission request.",
),
metric(
"crank_runtime_cache_total",
MetricKind::Counter,
+6 -13
View File
@@ -1,13 +1,12 @@
use std::{collections::BTreeSet, time::Duration};
use crank_metrics::{
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage,
McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType,
ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema,
record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection,
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics,
ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics,
bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome,
record_confirmation_outcome, record_export_failure, record_http_request,
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
@@ -72,9 +71,6 @@ fn schema_names_are_unique_and_labels_are_closed() {
"state",
"signal_type",
"exporter",
"reason",
"scope_kind",
"mode",
];
for metric in schema {
assert!(
@@ -269,9 +265,6 @@ fn every_recording_api_matches_the_declared_schema() {
Duration::from_millis(1),
);
record_limit_rejection(LimitStage::Concurrency);
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5);
record_admission_oldest_waiting(Some(5));
record_cache_outcome(CacheOutcome::Hit);
record_idempotency_outcome(IdempotencyOutcome::Replay);
record_confirmation_outcome(ConfirmationOutcome::Required);
+8 -8
View File
@@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
#[test]
fn frozen_budget_edges_are_exact() {
assert!(validate_budget(6_380, 19_610).is_ok());
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_000).is_err());
assert!(validate_budget(6_000, 19_612).is_err());
assert!(validate_budget(6_284, 19_514).is_ok());
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 16_000).is_err());
assert!(validate_budget(6_000, 19_516).is_err());
assert_eq!(
crank_metrics::max_exemplar_slots(MetricService::AdminApi),
12_348
@@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() {
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_381,
rendered_series: 19_611
logical_series: 6_285,
rendered_series: 19_515
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_179,
rendered_series: 3_654
logical_series: 1_083,
rendered_series: 3_558
})
);
}
+11 -11
View File
@@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() {
"root обязан объявлять ровно один ключ budgets"
);
assert!(rendered.ends_with(
",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\
\"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n"
",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\
\"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
\"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n"
));
assert_eq!(
schema_budget(MetricService::AdminApi),
Ok(crank_metrics::SchemaBudget {
logical_series: 6_381,
rendered_series: 19_611
logical_series: 6_285,
rendered_series: 19_515
})
);
assert_eq!(
schema_budget(MetricService::McpServer),
Ok(crank_metrics::SchemaBudget {
logical_series: 1_179,
rendered_series: 3_654
logical_series: 1_083,
rendered_series: 3_558
})
);
}
@@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() {
/// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны.
#[test]
fn frozen_caps_and_edges_are_exact() {
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611);
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285);
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515);
assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348);
assert!(validate_budget(6_381, 19_611).is_ok());
assert!(validate_budget(6_382, 19_611).is_err());
assert!(validate_budget(6_285, 19_515).is_ok());
assert!(validate_budget(6_286, 19_515).is_err());
assert!(validate_exemplar_budget(12_347).is_ok());
assert!(validate_exemplar_budget(12_348).is_ok());
assert!(validate_exemplar_budget(12_349).is_err());
+1 -1
View File
@@ -13,7 +13,7 @@ mod schema;
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
pub use crank_metrics::{
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit,
bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting,
bind_metric_service, bound_metric_service, metric_schema,
};
pub use error_reporting::{
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
-1
View File
@@ -7,7 +7,6 @@ publish.workspace = true
version.workspace = true
[dependencies]
async-trait = "0.1"
crank-artifacts = { path = "../crank-artifacts" }
crank-core = { path = "../crank-core" }
crank-mapping = { path = "../crank-mapping" }
+2 -3
View File
@@ -116,7 +116,6 @@ pub use model::{
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
};
pub use postgres::{
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, PostgresTaskStore,
admission_health, admission_snapshot, tasks_health,
AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError,
PostgresRegistry,
};
-14
View File
@@ -6,13 +6,6 @@ mod artifact_cleanup_indexes_v13;
mod artifact_metadata_v12;
mod authority;
mod baseline_v1;
mod durable_task_input_v30;
mod durable_tasks_v25;
mod durable_tasks_v26;
mod durable_tasks_v27;
mod durable_tasks_v28;
mod durable_tasks_v29;
mod execution_admission_v24;
mod execution_outcome_v5;
mod import_plan_v14;
mod master_key_identity_v7;
@@ -42,13 +35,6 @@ mod schema_guard_v19;
mod schema_guard_v20;
mod schema_guard_v22;
mod schema_guard_v23;
mod schema_guard_v24;
mod schema_guard_v25;
mod schema_guard_v26;
mod schema_guard_v27;
mod schema_guard_v28;
mod schema_guard_v29;
mod schema_guard_v30;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -4,13 +4,6 @@ use super::agent_resource_bindings_v18;
use super::approval_side_effects_v10;
use super::artifact_cleanup_indexes_v13;
use super::artifact_metadata_v12;
use super::durable_task_input_v30;
use super::durable_tasks_v25;
use super::durable_tasks_v26;
use super::durable_tasks_v27;
use super::durable_tasks_v28;
use super::durable_tasks_v29;
use super::execution_admission_v24;
use super::execution_outcome_v5;
use super::import_plan_v14;
use super::master_key_identity_v7;
@@ -46,10 +39,9 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 30;
const CURRENT_VERSION: i64 = 23;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -338,48 +330,6 @@ impl MigrationAuthority {
prompt_dependencies_v23::SOURCE_SHA256,
22,
),
expand_descriptor(
24,
execution_admission_v24::NAME,
execution_admission_v24::SOURCE_SHA256,
23,
),
expand_descriptor(
25,
durable_tasks_v25::NAME,
durable_tasks_v25::SOURCE_SHA256,
24,
),
expand_descriptor(
26,
durable_tasks_v26::NAME,
durable_tasks_v26::SOURCE_SHA256,
25,
),
expand_descriptor(
27,
durable_tasks_v27::NAME,
durable_tasks_v27::SOURCE_SHA256,
26,
),
expand_descriptor(
28,
durable_tasks_v28::NAME,
durable_tasks_v28::SOURCE_SHA256,
27,
),
expand_descriptor(
29,
durable_tasks_v29::NAME,
durable_tasks_v29::SOURCE_SHA256,
28,
),
expand_descriptor(
30,
durable_task_input_v30::NAME,
durable_task_input_v30::SOURCE_SHA256,
29,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -539,27 +489,6 @@ impl MigrationAuthority {
if from < 23 {
prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?;
}
if from < 24 {
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
}
if from < 25 {
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
}
if from < 26 {
durable_tasks_v26::apply(&mut transaction, &Self::sequence()[25]).await?;
}
if from < 27 {
durable_tasks_v27::apply(&mut transaction, &Self::sequence()[26]).await?;
}
if from < 28 {
durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?;
}
if from < 29 {
durable_tasks_v29::apply(&mut transaction, &Self::sequence()[28]).await?;
}
if from < 30 {
durable_task_input_v30::apply(&mut transaction, &Self::sequence()[29]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -140,8 +140,6 @@ pub(super) fn validate_descriptors(
!= agent_resource_bindings_v18::SOURCE_SHA256
|| sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes())
!= resource_security_revocations_v19::SOURCE_SHA256
|| sha256_hex(execution_admission_v24::SOURCE.as_bytes())
!= execution_admission_v24::SOURCE_SHA256
{
return Err(MigrationError::new(
"invalid_contract",
@@ -148,7 +148,6 @@ pub(super) async fn inspect(
owned_relations::PROMPTS,
owned_relations::PROMPT_PUBLICATIONS,
owned_relations::PROMPT_DEPENDENCIES,
owned_relations::EXECUTION_ADMISSION,
] {
validate_required_relations(connection, relations, CURRENT_VERSION).await?;
}
@@ -1,45 +0,0 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-task-input-v30";
pub(super) const SOURCE: &str = include_str!("durable_task_input_v30.sql");
pub(super) const SOURCE_SHA256: &str =
"84a51f712fd0bc439acba2ab3849232777c29e3b181586bbb4b06fd129843a38";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_task_input_v30",
Some(30),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_task_input_v30",
Some(30),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -1,59 +0,0 @@
-- V30: continuation through validated input (Story 4.5).
alter table tasks add column if not exists effective_input_fingerprint text;
update tasks set effective_input_fingerprint = input_fingerprint where effective_input_fingerprint is null;
alter table tasks alter column effective_input_fingerprint set not null;
alter table tasks add constraint tasks_effective_input_fingerprint_check check (
length(effective_input_fingerprint) = 64 and effective_input_fingerprint ~ '^[0-9a-f]{64}$'
);
create table if not exists task_input_requests (
task_id uuid primary key references tasks (task_id) on delete cascade,
wait_id text not null,
expected_version integer not null,
schema_revision integer not null,
schema_format_version integer not null,
fingerprint_format_version integer not null,
schema_snapshot jsonb not null,
projected_schema jsonb not null,
prompt_code text not null,
missing_fields jsonb not null default '[]'::jsonb,
expires_at timestamptz not null,
created_at timestamptz not null default now(),
accepted_at timestamptz,
update_key_digest text,
canonical_fingerprint text,
effective_input_fingerprint text,
committed_state text,
committed_version integer,
constraint task_input_requests_wait_unique unique (wait_id),
constraint task_input_requests_prompt_check check (octet_length(prompt_code) between 1 and 64),
constraint task_input_requests_receipt_check check (
(accepted_at is null and update_key_digest is null and canonical_fingerprint is null
and effective_input_fingerprint is null and committed_state is null and committed_version is null)
or (accepted_at is not null and update_key_digest is not null and canonical_fingerprint is not null
and effective_input_fingerprint is not null and committed_state is not null and committed_version is not null)
)
);
-- Not unique on purpose: several waiting Tasks may share one absolute TTL value (the same pinned
-- policy deadline), and an expired/cancelled wait keeps its row. Per-Task uniqueness comes from the
-- primary key and task_input_requests_wait_unique.
create index if not exists task_input_requests_active_ttl_idx
on task_input_requests (expires_at) where accepted_at is null;
create table if not exists task_input_audit (
id bigint generated always as identity primary key,
task_id uuid not null references tasks (task_id) on delete cascade,
wait_id text not null,
event text not null,
from_state text,
to_state text,
from_version integer,
to_version integer,
actor text,
reason_code text,
created_at timestamptz not null default now(),
constraint task_input_audit_event_check check (event in ('input_required','input_accepted','input_expired')),
constraint task_input_audit_actor_check check (actor is null or octet_length(actor) between 1 and 160),
constraint task_input_audit_reason_check check (reason_code is null or octet_length(reason_code) between 1 and 64)
);
create index if not exists task_input_audit_task_idx on task_input_audit (task_id, id);
@@ -1,45 +0,0 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v25";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v25.sql");
pub(super) const SOURCE_SHA256: &str =
"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v25",
Some(25),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.canonical_ledger",
Some(25),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -1,90 +0,0 @@
-- V25: durable background Tasks (Story 4.2, FR-22, FR-23).
--
-- Одна строка `tasks` — принятая durable работа. Идентификатор Task (UUIDv7), owner scope,
-- authorization scope, запрошенный Tool и точная published-версия, canonical fingerprint входа,
-- support IDs (Request/Trace/parent TraceContext независимо от sampling) и bounded envelopes
-- закрепляются в той же PostgreSQL transaction, что и dispatch intent
-- (`execution_admission_entries.task_id`) и его queue slot. До commit наружу не выдаётся ни ID, ни ack.
--
-- `scoped_key` — digest caller-ключа по полному (workspace, owner kind/id, authorization scope,
-- operation) scope; UNIQUE по этому набору и есть защита идемпотентности. Повтор того же ключа с тем
-- же canonical fingerprint возвращает сохранённую Task, другой fingerprint — typed conflict.
--
-- `retain_until` — срок хранения записи, а не execution deadline: пока запись в retention, replay
-- находит тот же Task. Удаление terminal-записей возможно только после retain_until; queued/working/
-- input_required/outcome_unknown не удаляются никогда ради лимита.
--
-- Строка не хранит credentials, bearer/prepared headers и confirmation tokens: секретные поля входа
-- редактируются до записи, а approval хранится безопасной ссылкой.
create table tasks (
task_id uuid not null,
workspace_id text not null,
owner_kind text not null,
owner_id text not null,
authorization_scope text not null,
operation_id text not null,
operation_version integer not null,
scoped_key text not null,
input_fingerprint text not null,
request_id text not null,
trace_id text not null,
traceparent text not null,
trace_sampled boolean not null,
policy_mode text not null,
approval_reference text,
state text not null,
version integer not null default 1,
attempts integer not null default 0,
input_envelope jsonb not null,
progress_envelope jsonb,
result_envelope jsonb,
error_envelope jsonb,
execution_deadline timestamptz not null,
retain_until timestamptz not null,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
constraint tasks_pkey primary key (task_id),
constraint tasks_workspace_check check (octet_length(workspace_id) between 1 and 160),
constraint tasks_owner_kind_check check (owner_kind in ('agent', 'admin')),
constraint tasks_owner_check check (octet_length(owner_id) between 1 and 160),
constraint tasks_authorization_scope_check check (
octet_length(authorization_scope) between 1 and 160
),
constraint tasks_operation_check check (octet_length(operation_id) between 1 and 160),
constraint tasks_operation_version_check check (operation_version >= 1),
constraint tasks_scoped_key_check check (octet_length(scoped_key) between 16 and 128),
constraint tasks_fingerprint_check check (octet_length(input_fingerprint) = 64),
constraint tasks_request_check check (octet_length(request_id) between 1 and 128),
constraint tasks_trace_check check (octet_length(trace_id) = 32),
constraint tasks_traceparent_check check (octet_length(traceparent) = 55),
constraint tasks_policy_mode_check check (
policy_mode in ('synchronous', 'background', 'adaptive')
),
constraint tasks_state_check check (
state in ('queued', 'working', 'input_required', 'outcome_unknown',
'completed', 'failed', 'cancelled')
),
constraint tasks_lifecycle_check check (version >= 1 and attempts >= 0),
constraint tasks_retention_check check (retain_until >= execution_deadline),
constraint tasks_approval_check check (
approval_reference is null or octet_length(approval_reference) between 1 and 160
)
);
create unique index tasks_scoped_key_uidx
on tasks (workspace_id, owner_kind, owner_id, authorization_scope, operation_id, scoped_key);
create index tasks_retention_idx on tasks (state, retain_until, task_id);
create index tasks_deadline_idx on tasks (state, execution_deadline, task_id);
-- Dispatch intent: прежняя reserved-строка admission становится точкой связи Task и capacity.
-- FK без cascade: удаление принятой Task не маскируется housekeeping-удалением intent.
alter table execution_admission_entries
add column task_id uuid,
add constraint execution_admission_entries_task_fkey
foreign key (task_id) references tasks (task_id);
create unique index execution_admission_entries_task_uidx
on execution_admission_entries (task_id) where task_id is not null;
Loaded 100 of 190 files, more files were not shown because too many files have changed in this diff. Show more