Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
35a21f8ab5 | ||
|
|
2d0deb8963 | ||
|
|
f0a79362eb | ||
|
|
27f86d4916 | ||
|
|
f728d31dbf | ||
|
|
29ee01b6ef | ||
|
|
bcf8b43516 | ||
|
|
1c256e6c75 | ||
|
|
e4953abcd2 | ||
|
|
90e5357a08 | ||
|
|
d80eda7329 | ||
|
|
a3057de51e | ||
|
|
8c02380762 | ||
|
|
3b5fcc51cc | ||
|
|
433987413e | ||
|
|
5488eb73ec | ||
|
|
f9f8c625d1 | ||
|
|
6df6ffac4e | ||
|
|
2219a7217d | ||
|
|
cc612516a2 | ||
|
|
599dc1867b | ||
|
|
4e7440f25e | ||
|
|
e80d30ddb0 | ||
|
|
44774883a0 | ||
|
|
a0365bb80b |
No files matched your search
+28
-10
@@ -11,6 +11,12 @@ on:
|
||||
options: [story, feature]
|
||||
default: story
|
||||
required: true
|
||||
deploy_target:
|
||||
description: Deployment target for this run
|
||||
type: choice
|
||||
options: [none, production]
|
||||
default: none
|
||||
required: true
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
@@ -42,9 +48,13 @@ jobs:
|
||||
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
|
||||
exit 1
|
||||
fi
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
rustup default "$toolchain"
|
||||
host="$(rustc -vV | sed -n 's/^host: //p')"
|
||||
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
fi
|
||||
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
|
||||
toolchain_bin="$toolchain_dir/bin"
|
||||
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
|
||||
@@ -181,9 +191,13 @@ jobs:
|
||||
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
|
||||
exit 1
|
||||
fi
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
rustup default "$toolchain"
|
||||
host="$(rustc -vV | sed -n 's/^host: //p')"
|
||||
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
fi
|
||||
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
|
||||
toolchain_bin="$toolchain_dir/bin"
|
||||
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
|
||||
@@ -260,9 +274,13 @@ jobs:
|
||||
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
|
||||
exit 1
|
||||
fi
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
rustup default "$toolchain"
|
||||
host="$(rustc -vV | sed -n 's/^host: //p')"
|
||||
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
fi
|
||||
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
|
||||
toolchain_bin="$toolchain_dir/bin"
|
||||
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
|
||||
@@ -499,7 +517,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- required-checks
|
||||
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
|
||||
if: ${{ gitea.event_name == 'workflow_dispatch' && gitea.ref == 'refs/heads/main' && inputs.validation_level == 'feature' && inputs.deploy_target == 'production' }}
|
||||
env:
|
||||
REGISTRY: git.itexp.me
|
||||
IMAGE_TAG: ${{ gitea.sha }}
|
||||
|
||||
@@ -12,10 +12,20 @@ on:
|
||||
options: [story, feature]
|
||||
default: story
|
||||
required: true
|
||||
resume_tag:
|
||||
description: Existing draft release tag to resume with image-only steps
|
||||
type: string
|
||||
default: ""
|
||||
required: false
|
||||
resume_source_sha:
|
||||
description: Exact accepted source commit SHA for the resume tag
|
||||
type: string
|
||||
default: ""
|
||||
required: false
|
||||
|
||||
env:
|
||||
REGISTRY: git.itexp.me
|
||||
IMAGE_TAG: ${{ gitea.ref_name }}
|
||||
IMAGE_TAG: ${{ inputs.resume_tag || gitea.ref_name }}
|
||||
ADMIN_API_IMAGE: git.itexp.me/bsodfather/crank-community-admin-api
|
||||
MCP_SERVER_IMAGE: git.itexp.me/bsodfather/crank-community-mcp-server
|
||||
UI_IMAGE: git.itexp.me/bsodfather/crank-community-ui
|
||||
@@ -31,12 +41,47 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Verify existing release before resume
|
||||
id: resume
|
||||
if: inputs.resume_tag != ''
|
||||
env:
|
||||
GITEA_SERVER_URL: https://git.itexp.me
|
||||
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
RELEASE_TAG: ${{ inputs.resume_tag }}
|
||||
RESUME_SOURCE_SHA: ${{ inputs.resume_source_sha }}
|
||||
run: python3 scripts/ci-release-resume.py
|
||||
- name: Preserve updated image scanner for resume
|
||||
id: pin_resume_scanner
|
||||
if: inputs.resume_tag != ''
|
||||
run: |
|
||||
set -eu
|
||||
scanner_dir="$(mktemp -d "${RUNNER_TEMP:?RUNNER_TEMP is unavailable}/crank-release-scanner.XXXXXX")"
|
||||
cp scripts/scan-images.sh "$scanner_dir/scan-images.sh"
|
||||
cp scripts/check-image-scan.py "$scanner_dir/check-image-scan.py"
|
||||
chmod +x "$scanner_dir/scan-images.sh"
|
||||
printf 'path=%s\n' "$scanner_dir/scan-images.sh" >> "$GITHUB_OUTPUT"
|
||||
- name: Checkout validated release source
|
||||
if: inputs.resume_tag != ''
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
ref: ${{ steps.resume.outputs.release_tag }}
|
||||
fetch-depth: 0
|
||||
- name: Assert exact release source checkout
|
||||
if: inputs.resume_tag != ''
|
||||
env:
|
||||
EXPECTED_SOURCE_SHA: ${{ steps.resume.outputs.source_sha }}
|
||||
run: test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"
|
||||
- name: Clear optional release evidence
|
||||
if: inputs.resume_tag == ''
|
||||
run: rm -f .tmp/openapi-playwright.json .tmp/openapi-ui-evidence.json
|
||||
|
||||
|
||||
|
||||
- name: Use preinstalled Rust toolchain
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
set -eu
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/1.96.1-x86_64-unknown-linux-gnu"
|
||||
@@ -54,6 +99,7 @@ jobs:
|
||||
"$toolchain_bin/cargo-clippy" --version
|
||||
|
||||
- name: Verify runner toolchain
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
python3 --version
|
||||
rustc --version
|
||||
@@ -65,19 +111,23 @@ jobs:
|
||||
bao version
|
||||
|
||||
- name: Install dependency policy tool
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo install cargo-deny --version 0.20.2 --locked
|
||||
|
||||
- name: Check formatting
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo fmt --all --check
|
||||
|
||||
- name: Audit Rust dependencies
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo deny --locked check advisories bans licenses sources
|
||||
|
||||
- name: Run tooling unit tests
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
run: python3 -m unittest discover -s tests/unit
|
||||
|
||||
- name: Check typed runtime configuration contract
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
cargo run -p crank-config --bin crank-config-contract -- --check
|
||||
python3 scripts/check-runtime-config.py --root .
|
||||
@@ -85,14 +135,17 @@ jobs:
|
||||
scripts/check-rust-boundaries.sh
|
||||
|
||||
- name: Check canonical migration contract
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo run -p admin-api --bin crank-migrate -- plan --check
|
||||
|
||||
- name: Check typed metrics contract
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
|
||||
python3 scripts/check-metrics-boundaries.py --root .
|
||||
|
||||
- name: Check Capability Inventory
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
required_args=""
|
||||
for number in $(seq 1 54); do
|
||||
@@ -105,6 +158,7 @@ jobs:
|
||||
$required_args
|
||||
|
||||
- name: Check Capability Baseline
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
python3 scripts/validate-capability-baseline.py \
|
||||
--root . \
|
||||
@@ -112,40 +166,46 @@ jobs:
|
||||
--schema docs/schemas/capability-baseline.schema.json
|
||||
|
||||
- name: Check Community scope
|
||||
if: inputs.resume_tag == ''
|
||||
run: scripts/check-community-scope.sh
|
||||
|
||||
- name: Run Crank story ATDD
|
||||
if: inputs.resume_tag == ''
|
||||
run: scripts/ci-story-checks.sh
|
||||
|
||||
- name: Run release quality gates
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
run: |
|
||||
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
cargo test --workspace --all-targets -- --test-threads=1
|
||||
|
||||
- name: Build release binaries
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo build --release -p admin-api -p mcp-server
|
||||
|
||||
|
||||
- name: Install UI dependencies
|
||||
if: inputs.resume_tag == ''
|
||||
working-directory: apps/ui
|
||||
run: npm ci
|
||||
|
||||
- name: Audit UI dependencies
|
||||
if: inputs.resume_tag == ''
|
||||
working-directory: apps/ui
|
||||
run: npm audit --audit-level=high
|
||||
|
||||
- name: Build UI dist
|
||||
if: inputs.resume_tag == ''
|
||||
working-directory: apps/ui
|
||||
run: npm run build
|
||||
|
||||
- name: Install Playwright browser
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
working-directory: apps/ui
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run release end-to-end tests
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
working-directory: apps/ui
|
||||
run: |
|
||||
mkdir -p ../../.tmp
|
||||
@@ -153,7 +213,7 @@ jobs:
|
||||
PLAYWRIGHT_JSON_OUTPUT=../../.tmp/openapi-playwright.json npm run e2e
|
||||
|
||||
- name: Collect sanitized OpenAPI UI release evidence
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
working-directory: apps/ui
|
||||
run: |
|
||||
trap 'rm -f ../../.tmp/openapi-playwright.json' EXIT
|
||||
@@ -175,12 +235,14 @@ jobs:
|
||||
--require-accepted
|
||||
|
||||
- name: Validate deployment manifests
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
docker compose -f docker-compose.yml --env-file .env.example config -q
|
||||
docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
|
||||
docker compose -f deploy/community/docker-compose.images.yml --env-file deploy/community/.env.images.example --profile local-db config -q
|
||||
|
||||
- name: Package release artifacts
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p dist/release
|
||||
@@ -204,13 +266,14 @@ jobs:
|
||||
sha256sum "${files[@]}" > "dist/crank-community-${IMAGE_TAG}-checksums.txt"
|
||||
|
||||
- name: Generate SBOM
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
docker run --rm -v "$PWD:/workspace" anchore/syft:latest \
|
||||
dir:/workspace -o spdx-json \
|
||||
> dist/crank-community-${IMAGE_TAG}-sbom.spdx.json
|
||||
|
||||
- name: Publish release bundles, checksums, evidence, and SBOM to Gitea
|
||||
if: startsWith(gitea.ref, 'refs/tags/v')
|
||||
if: inputs.resume_tag == '' && startsWith(gitea.ref, 'refs/tags/v')
|
||||
env:
|
||||
# gitea.server_url inside the job container is the internal http://gitea:3000 origin;
|
||||
# the publisher only forwards its token to the public HTTPS origin.
|
||||
@@ -249,6 +312,8 @@ jobs:
|
||||
docker login '${{ env.REGISTRY }}' -u "$DEPLOY_REGISTRY_USER" --password-stdin
|
||||
|
||||
- name: Build and push release images
|
||||
env:
|
||||
RELEASE_IMAGE_SCANNER: ${{ steps.pin_resume_scanner.outputs.path || 'scripts/scan-images.sh' }}
|
||||
run: |
|
||||
docker build -f apps/admin-api/Dockerfile \
|
||||
-t '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
|
||||
@@ -285,7 +350,7 @@ jobs:
|
||||
--env-file .tmp/release-migration-smoke.env --profile local-db up -d --wait
|
||||
docker compose -f deploy/community/docker-compose.images.yml \
|
||||
--env-file .tmp/release-migration-smoke.env --profile local-db logs migrate | grep '"status":"applied"'
|
||||
scripts/scan-images.sh \
|
||||
"$RELEASE_IMAGE_SCANNER" \
|
||||
'${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
|
||||
'${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
|
||||
'${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
|
||||
@@ -297,5 +362,6 @@ jobs:
|
||||
docker push '${{ env.UI_IMAGE }}:latest'
|
||||
|
||||
- name: Show release bundle summary
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
ls -lh dist
|
||||
Generated
+205
-62
@@ -25,7 +25,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"axum-extra",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-artifacts",
|
||||
"crank-community-auth",
|
||||
"crank-community-mcp",
|
||||
@@ -186,7 +186,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -197,7 +197,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -349,6 +349,12 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
@@ -398,7 +404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bitflags",
|
||||
"bollard-buildkit-proto",
|
||||
"bollard-stubs",
|
||||
@@ -455,7 +461,7 @@ version = "1.52.1-rc.29.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bollard-buildkit-proto",
|
||||
"bytes",
|
||||
"prost",
|
||||
@@ -653,7 +659,7 @@ version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-core",
|
||||
"crank-metrics",
|
||||
"crank-trace",
|
||||
@@ -688,7 +694,7 @@ dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"axum-extra",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-core",
|
||||
"crank-registry",
|
||||
"rand 0.10.2",
|
||||
@@ -705,7 +711,7 @@ version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-adapter-rest",
|
||||
"crank-artifacts",
|
||||
"crank-core",
|
||||
@@ -722,6 +728,7 @@ dependencies = [
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"reqwest 0.12.28",
|
||||
"rmcp",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
@@ -826,6 +833,7 @@ dependencies = [
|
||||
name = "crank-registry"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"crank-artifacts",
|
||||
"crank-core",
|
||||
"crank-mapping",
|
||||
@@ -849,7 +857,7 @@ dependencies = [
|
||||
"aes-gcm",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-adapter-rest",
|
||||
"crank-core",
|
||||
"crank-import",
|
||||
@@ -991,8 +999,18 @@ version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
"darling_core 0.23.0",
|
||||
"darling_macro 0.23.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"darling_macro 0.24.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1005,7 +1023,20 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff"
|
||||
dependencies = [
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1014,9 +1045,20 @@ version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_core 0.23.0",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1068,7 +1110,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1077,7 +1119,7 @@ version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
@@ -1146,7 +1188,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1308,7 +1350,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1660,7 +1702,7 @@ version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
@@ -1671,7 +1713,7 @@ dependencies = [
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
@@ -1919,7 +1961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2055,7 +2097,7 @@ version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-artifacts",
|
||||
"crank-community-mcp",
|
||||
"crank-config",
|
||||
@@ -2120,7 +2162,7 @@ version = "0.18.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"evmap",
|
||||
"indexmap 2.14.0",
|
||||
"metrics",
|
||||
@@ -2456,7 +2498,7 @@ dependencies = [
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"structmeta",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2470,6 +2512,12 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pastey"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
@@ -2493,7 +2541,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2579,7 +2627,7 @@ dependencies = [
|
||||
"itertools",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2635,7 +2683,7 @@ dependencies = [
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -2674,9 +2722,9 @@ dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2851,7 +2899,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2889,7 +2937,7 @@ version = "0.12.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
@@ -2933,7 +2981,7 @@ version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -2980,6 +3028,52 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmcp"
|
||||
version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ece64e19631fc38758181e637c15509a2c2709ee53bfde1e1d58f9dc991f919e"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"indexmap 2.14.0",
|
||||
"libc",
|
||||
"pastey",
|
||||
"pin-project-lite",
|
||||
"rand 0.10.2",
|
||||
"rmcp-macros",
|
||||
"schemars 1.2.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sse-stream",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmcp-macros"
|
||||
version = "3.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3140f24a766f80f9ec1607bde85680959a69b83d46fa9ce87dadab7e67ab029"
|
||||
dependencies = [
|
||||
"darling 0.24.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_json",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.28"
|
||||
@@ -3015,7 +3109,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3074,7 +3168,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3143,12 +3237,26 @@ version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scoped-tls"
|
||||
version = "1.0.1"
|
||||
@@ -3292,7 +3400,18 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3327,7 +3446,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3348,7 +3467,7 @@ version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bs58",
|
||||
"chrono",
|
||||
"hex",
|
||||
@@ -3368,10 +3487,10 @@ version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3409,7 +3528,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3543,7 +3662,7 @@ dependencies = [
|
||||
name = "sqlx-core"
|
||||
version = "0.9.0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"crc",
|
||||
@@ -3585,7 +3704,7 @@ dependencies = [
|
||||
"quote",
|
||||
"sqlx-core",
|
||||
"sqlx-macros-core",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3608,7 +3727,7 @@ dependencies = [
|
||||
"sqlx-mysql",
|
||||
"sqlx-postgres",
|
||||
"sqlx-sqlite",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"url",
|
||||
@@ -3649,7 +3768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e"
|
||||
dependencies = [
|
||||
"atoi",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"crc",
|
||||
@@ -3705,6 +3824,19 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sse-stream"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
@@ -3737,7 +3869,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"structmeta-derive",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3748,7 +3880,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3768,6 +3900,17 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sync_wrapper"
|
||||
version = "1.0.2"
|
||||
@@ -3785,7 +3928,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3854,7 +3997,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3865,7 +4008,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3956,7 +4099,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4001,7 +4144,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"h2",
|
||||
"http",
|
||||
@@ -4102,7 +4245,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4231,7 +4374,7 @@ version = "3.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
@@ -4247,7 +4390,7 @@ version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
@@ -4384,7 +4527,7 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -4476,7 +4619,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4506,7 +4649,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4517,7 +4660,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4733,7 +4876,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4754,7 +4897,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4774,7 +4917,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4814,7 +4957,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
mod admin_auth_command;
|
||||
#[path = "crank_migrate/db_connect.rs"]
|
||||
mod db_connect;
|
||||
#[path = "crank_migrate/reporting.rs"]
|
||||
mod reporting;
|
||||
|
||||
use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json};
|
||||
|
||||
use crank_config::{ConfigSource, parse_migrator};
|
||||
use crank_registry::{
|
||||
@@ -11,7 +15,7 @@ use crank_registry::{
|
||||
};
|
||||
use crank_runtime::SecretCrypto;
|
||||
use serde_json::json;
|
||||
use std::{path::Path, process::ExitCode};
|
||||
use std::process::ExitCode;
|
||||
use time::OffsetDateTime;
|
||||
const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000;
|
||||
#[tokio::main]
|
||||
@@ -765,6 +769,59 @@ async fn process_rotation_batch(
|
||||
processed += 1;
|
||||
}
|
||||
}
|
||||
let mut after_task_id: Option<String> = None;
|
||||
loop {
|
||||
let tasks = registry
|
||||
.list_tasks_for_master_key_epoch_page(
|
||||
rotation.source_epoch,
|
||||
after_task_id.as_deref(),
|
||||
MASTER_KEY_ROTATION_PAGE_SIZE,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
if tasks.is_empty() {
|
||||
break;
|
||||
}
|
||||
for task in tasks {
|
||||
after_task_id = Some(task.task_id.clone());
|
||||
if task.target_epoch == Some(rotation.target_epoch) {
|
||||
continue;
|
||||
}
|
||||
if max_versions.is_some_and(|limit| processed as usize >= limit) {
|
||||
return Ok(processed);
|
||||
}
|
||||
let plaintext = current_crypto
|
||||
.decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext)
|
||||
.map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.stage_master_key_rotation_task_ciphertext(
|
||||
&rotation.id,
|
||||
&task.task_id,
|
||||
rotation.source_epoch,
|
||||
&target_ciphertext,
|
||||
target_crypto.key_version(),
|
||||
rotation.target_epoch,
|
||||
&now,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
processed += 1;
|
||||
}
|
||||
}
|
||||
Ok(processed)
|
||||
}
|
||||
|
||||
@@ -818,6 +875,47 @@ async fn verify_staged_targets(
|
||||
verified += 1;
|
||||
}
|
||||
}
|
||||
let mut after_task_id: Option<String> = None;
|
||||
loop {
|
||||
let tasks = registry
|
||||
.list_target_tasks_for_master_key_rotation_page(
|
||||
rotation.target_epoch,
|
||||
after_task_id.as_deref(),
|
||||
MASTER_KEY_ROTATION_PAGE_SIZE,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
if tasks.is_empty() {
|
||||
break;
|
||||
}
|
||||
for task in tasks {
|
||||
after_task_id = Some(task.task_id.clone());
|
||||
let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let key_version = task.target_key_version.as_deref().ok_or_else(|| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
target_crypto
|
||||
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
|
||||
.map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
verified += 1;
|
||||
}
|
||||
}
|
||||
if verified != rotation.total_secret_versions {
|
||||
return Err(CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
@@ -877,89 +975,3 @@ fn decrypt_current(
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_rotation(
|
||||
registry: &PostgresRegistry,
|
||||
allowed_states: &[&str],
|
||||
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
|
||||
let status = registry
|
||||
.master_key_rotation_status()
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
Ok(status
|
||||
.rotations
|
||||
.into_iter()
|
||||
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
|
||||
}
|
||||
|
||||
fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
|
||||
json!({
|
||||
"id": rotation.id,
|
||||
"source_epoch": rotation.source_epoch,
|
||||
"target_epoch": rotation.target_epoch,
|
||||
"state": rotation.state,
|
||||
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
|
||||
"checkpoint_secret_id": rotation.checkpoint_secret_id,
|
||||
"total_secret_versions": rotation.total_secret_versions,
|
||||
"processed_secret_versions": rotation.processed_secret_versions,
|
||||
"verified_secret_versions": rotation.verified_secret_versions,
|
||||
"failure_code": rotation.failure_code,
|
||||
})
|
||||
}
|
||||
|
||||
fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
|
||||
let path = path.ok_or_else(|| {
|
||||
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
|
||||
})?;
|
||||
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if !metadata.is_file() || metadata.len() > 16_384 {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if value.trim().is_empty() {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
|
||||
if options.current_key_file.is_some()
|
||||
|| options.target_key_file.is_some()
|
||||
|| options.backup_ref.is_some()
|
||||
|| options.rotation_id.is_some()
|
||||
|| options.max_versions.is_some()
|
||||
{
|
||||
return Err(CliError::new(
|
||||
"invalid_command",
|
||||
"cli.arguments",
|
||||
"run_master_key_status",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Leaf/reporting helpers of the migrator CLI.
|
||||
//!
|
||||
//! Extracted from `crank-migrate.rs` without behaviour changes so the binary entry stays within
|
||||
//! the code-health line limit.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crank_registry::{MasterKeyRotationRecord, PostgresRegistry};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{CliError, MasterKeyOptions};
|
||||
|
||||
pub(super) async fn active_rotation(
|
||||
registry: &PostgresRegistry,
|
||||
allowed_states: &[&str],
|
||||
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
|
||||
let status = registry
|
||||
.master_key_rotation_status()
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
Ok(status
|
||||
.rotations
|
||||
.into_iter()
|
||||
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
|
||||
}
|
||||
|
||||
pub(super) fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
|
||||
json!({
|
||||
"id": rotation.id,
|
||||
"source_epoch": rotation.source_epoch,
|
||||
"target_epoch": rotation.target_epoch,
|
||||
"state": rotation.state,
|
||||
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
|
||||
"checkpoint_secret_id": rotation.checkpoint_secret_id,
|
||||
"total_secret_versions": rotation.total_secret_versions,
|
||||
"processed_secret_versions": rotation.processed_secret_versions,
|
||||
"verified_secret_versions": rotation.verified_secret_versions,
|
||||
"failure_code": rotation.failure_code,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
|
||||
let path = path.ok_or_else(|| {
|
||||
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
|
||||
})?;
|
||||
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if !metadata.is_file() || metadata.len() > 16_384 {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if value.trim().is_empty() {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub(super) fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
|
||||
if options.current_key_file.is_some()
|
||||
|| options.target_key_file.is_some()
|
||||
|| options.backup_ref.is_some()
|
||||
|| options.rotation_id.is_some()
|
||||
|| options.max_versions.is_some()
|
||||
{
|
||||
return Err(CliError::new(
|
||||
"invalid_command",
|
||||
"cli.arguments",
|
||||
"run_master_key_status",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -181,6 +181,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
|
||||
use admin_api::{
|
||||
app::build_app,
|
||||
auth::{AuthSettings, BootstrapAdminConfig},
|
||||
pool_metrics::spawn_postgres_pool_metrics,
|
||||
pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics},
|
||||
reconciliation::{open_reconciliation_store, spawn_artifact_reconciliation},
|
||||
service::AdminServiceBuilder,
|
||||
state::AppState,
|
||||
@@ -169,6 +169,7 @@ async fn run(
|
||||
};
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
spawn_admission_metrics(registry.pool().clone());
|
||||
}
|
||||
let base_url = config
|
||||
.runtime
|
||||
@@ -214,6 +215,16 @@ async fn run(
|
||||
.with_limits(runtime_limits)
|
||||
.with_response_cache(cache_stores.response.clone())
|
||||
.with_coordination_store(cache_stores.coordination.clone())
|
||||
.with_admission(std::sync::Arc::new(
|
||||
crank_registry::PostgresExecutionAdmission::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?,
|
||||
))
|
||||
.build();
|
||||
let identity_provider =
|
||||
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
|
||||
@@ -321,6 +332,12 @@ fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::Con
|
||||
config.runtime.max_concurrent_sessions,
|
||||
)
|
||||
.map_err(|_| invalid("runtime.limits"))?;
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: config.runtime.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
}
|
||||
.validate()
|
||||
.map_err(|_| invalid("runtime.admission"))?;
|
||||
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
|
||||
RequestRateLimitConfig::new(
|
||||
config.rate_limit.requests_per_second,
|
||||
|
||||
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
|
||||
/// to zero) without waiting for a new request.
|
||||
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
|
||||
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -37,13 +37,22 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
let execution_admission = if checks.execution_admission {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
};
|
||||
if checks.is_ready() {
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
"checks": {
|
||||
"postgres": postgres,
|
||||
"artifact_storage": artifact_storage,
|
||||
"execution_admission": execution_admission
|
||||
}
|
||||
})),
|
||||
)
|
||||
} else {
|
||||
@@ -52,7 +61,11 @@ pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": postgres, "artifact_storage": artifact_storage }
|
||||
"checks": {
|
||||
"postgres": postgres,
|
||||
"artifact_storage": artifact_storage,
|
||||
"execution_admission": execution_admission
|
||||
}
|
||||
})),
|
||||
)
|
||||
}
|
||||
@@ -67,6 +80,7 @@ mod tests {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: false,
|
||||
artifact_storage: true,
|
||||
execution_admission: true,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
@@ -92,6 +106,7 @@ mod tests {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: true,
|
||||
artifact_storage: false,
|
||||
execution_admission: true,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
assert_eq!(
|
||||
@@ -111,4 +126,14 @@ mod tests {
|
||||
"not_ready"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readiness_checks_identify_an_admission_state_failure() {
|
||||
let checks = ReadinessChecks {
|
||||
postgres: true,
|
||||
artifact_storage: true,
|
||||
execution_admission: false,
|
||||
};
|
||||
assert!(!checks.is_ready());
|
||||
}
|
||||
}
|
||||
@@ -55,8 +55,8 @@ use crate::{
|
||||
storage::LocalArtifactStorage,
|
||||
};
|
||||
use operation_validation::{
|
||||
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
|
||||
validate_protocol_target, validate_response_cache_policy,
|
||||
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
|
||||
validate_idempotency_policy, validate_protocol_target, validate_response_cache_policy,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -104,11 +104,12 @@ pub use crate::dto::*;
|
||||
pub struct ReadinessChecks {
|
||||
pub postgres: bool,
|
||||
pub artifact_storage: bool,
|
||||
pub execution_admission: bool,
|
||||
}
|
||||
|
||||
impl ReadinessChecks {
|
||||
pub const fn is_ready(self) -> bool {
|
||||
self.postgres && self.artifact_storage
|
||||
self.postgres && self.artifact_storage && self.execution_admission
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +151,8 @@ pub(super) struct CredentialAuditRecord<'a> {
|
||||
impl AdminService {
|
||||
pub async fn readiness(&self) -> ReadinessChecks {
|
||||
let postgres = self.registry.ping().await.is_ok();
|
||||
// Story 4.1: the mandatory admission state is a readiness dependency of every execution.
|
||||
let execution_admission = self.registry.execution_admission_health().await.is_ok();
|
||||
let store = Arc::clone(&self.artifact_store);
|
||||
let artifact_storage = tokio::task::spawn_blocking(move || store.check_health())
|
||||
.await
|
||||
@@ -157,6 +160,7 @@ impl AdminService {
|
||||
ReadinessChecks {
|
||||
postgres,
|
||||
artifact_storage,
|
||||
execution_admission,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -725,6 +729,7 @@ impl AdminService {
|
||||
validate_response_cache_policy(&payload.target, &payload.execution_config)?;
|
||||
validate_idempotency_policy(&payload.target, &payload.execution_config)?;
|
||||
validate_approval_policy(&payload.execution_config)?;
|
||||
validate_execution_policy(&payload.execution_config)?;
|
||||
payload.input_mapping.validate_paths()?;
|
||||
payload.output_mapping.validate_paths()?;
|
||||
Ok(())
|
||||
@@ -738,6 +743,7 @@ impl AdminService {
|
||||
validate_response_cache_policy(&operation.target, &operation.execution_config)?;
|
||||
validate_idempotency_policy(&operation.target, &operation.execution_config)?;
|
||||
validate_approval_policy(&operation.execution_config)?;
|
||||
validate_execution_policy(&operation.execution_config)?;
|
||||
operation.input_mapping.validate_paths()?;
|
||||
operation.output_mapping.validate_paths()?;
|
||||
Ok(())
|
||||
|
||||
@@ -441,6 +441,7 @@ fn demo_rest_operation_payload() -> OperationPayload {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -635,6 +635,7 @@ impl AdminService {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -610,6 +610,7 @@ impl AdminService {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -160,6 +160,23 @@ pub(super) fn validate_approval_policy(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_execution_policy(
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<(), ApiError> {
|
||||
let Some(policy) = execution_config.execution_policy.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Err(error) = policy.validate() {
|
||||
return Err(ApiError::validation_with_context(
|
||||
error.to_string(),
|
||||
json!({
|
||||
"field": "execution_config.execution_policy.deadline_ms",
|
||||
}),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
@@ -171,8 +188,8 @@ mod tests {
|
||||
};
|
||||
|
||||
use super::{
|
||||
validate_approval_policy, validate_execution_timeout, validate_idempotency_policy,
|
||||
validate_response_cache_policy,
|
||||
validate_approval_policy, validate_execution_policy, validate_execution_timeout,
|
||||
validate_idempotency_policy, validate_response_cache_policy,
|
||||
};
|
||||
|
||||
fn cacheable_execution_config() -> ExecutionConfig {
|
||||
@@ -183,6 +200,7 @@ mod tests {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
@@ -201,6 +219,7 @@ mod tests {
|
||||
}),
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
@@ -365,4 +384,36 @@ mod tests {
|
||||
"approval ttl must be between 1 and 300 seconds"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_published_execution_policy_and_absent_legacy_policy() {
|
||||
let mut config = cacheable_execution_config();
|
||||
validate_execution_policy(&config).unwrap();
|
||||
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Adaptive,
|
||||
deadline_ms: 120_000,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
});
|
||||
validate_execution_policy(&config).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_and_overflowing_execution_policy_deadline() {
|
||||
let mut config = cacheable_execution_config();
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 0,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
|
||||
});
|
||||
let error = validate_execution_policy(&config).unwrap_err();
|
||||
assert!(matches!(error, crate::error::ApiError::Validation { .. }));
|
||||
|
||||
config.execution_policy = Some(crank_core::ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Background,
|
||||
deadline_ms: crank_core::MAX_TOOL_DEADLINE_MS + 1,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Synchronous,
|
||||
});
|
||||
assert!(validate_execution_policy(&config).is_err());
|
||||
}
|
||||
}
|
||||
@@ -495,6 +495,7 @@ pub(super) fn test_operation_payload(base_url: &str, name: &str) -> OperationPay
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -29,7 +29,7 @@ sha2.workspace = true
|
||||
sqlx.workspace = true
|
||||
thiserror.workspace = true
|
||||
time.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio = { workspace = true, features = ["sync", "signal"] }
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
@@ -45,5 +45,6 @@ opentelemetry_sdk.workspace = true
|
||||
prost.workspace = true
|
||||
reqwest.workspace = true
|
||||
tower.workspace = true
|
||||
tokio = { workspace = true, features = ["net", "time"] }
|
||||
tracing-opentelemetry.workspace = true
|
||||
metrics-util = "0.20.4"
|
||||
+193
-11
@@ -1,7 +1,7 @@
|
||||
use std::{io, process::ExitCode, time::Duration};
|
||||
|
||||
use crank_community_mcp::{
|
||||
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts,
|
||||
auth::CommunityMachineCredentialVerifier, build_app_with_artifacts_and_tasks,
|
||||
session::PostgresTransportSessionStore,
|
||||
};
|
||||
use crank_config::{
|
||||
@@ -14,13 +14,14 @@ use crank_observability::{
|
||||
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
|
||||
};
|
||||
use crank_registry::{
|
||||
MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate, PostgresPoolConfig, PostgresRegistry,
|
||||
AdmissionLimits, MASTER_KEY_CIPHER_CONTRACT, MasterKeyIdentityCandidate,
|
||||
PostgresExecutionAdmission, PostgresPoolConfig, PostgresRegistry, PostgresTaskStore,
|
||||
};
|
||||
use crank_runtime::{
|
||||
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
|
||||
RuntimeLimits, SecretCrypto,
|
||||
};
|
||||
use mcp_server::pool_metrics::spawn_postgres_pool_metrics;
|
||||
use mcp_server::pool_metrics::{spawn_admission_metrics, spawn_postgres_pool_metrics};
|
||||
use sqlx::postgres::PgConnectOptions;
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::info;
|
||||
@@ -175,6 +176,7 @@ async fn run(
|
||||
};
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
spawn_admission_metrics(registry.pool().clone());
|
||||
}
|
||||
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
|
||||
let secret_crypto =
|
||||
@@ -190,12 +192,55 @@ async fn run(
|
||||
.with_limits(runtime_limits)
|
||||
.with_response_cache(cache_stores.response.clone())
|
||||
.with_coordination_store(cache_stores.coordination.clone())
|
||||
.with_admission(execution_admission(
|
||||
registry.pool().clone(),
|
||||
runtime_limits,
|
||||
)?)
|
||||
.build();
|
||||
// Published static Resources are read from the same immutable content-addressed root the
|
||||
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
|
||||
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
|
||||
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
|
||||
let app = build_app_with_artifacts(
|
||||
// Story 4.2/4.3: the durable Task application entry shares the registry pool, the published
|
||||
// catalog refresh interval and the exact admission limits of the runtime; the public wire stays
|
||||
// synchronous/unsupported while a fenced in-process worker executes accepted Tasks.
|
||||
let task_store = std::sync::Arc::new(PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
admission_limits(runtime_limits),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?);
|
||||
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
cache_stores.coordination.clone(),
|
||||
crank_runtime::TaskService::with_secret_crypto(
|
||||
task_store.clone(),
|
||||
std::sync::Arc::new(secret_crypto.clone()),
|
||||
),
|
||||
));
|
||||
// Story 4.3: the durable Task worker runs in this process over the same PostgreSQL store,
|
||||
// published catalog, executor and encrypted-input authority as the accepting application.
|
||||
let task_handler =
|
||||
std::sync::Arc::new(crank_community_mcp::TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime.clone(),
|
||||
secret_crypto.clone(),
|
||||
));
|
||||
let task_worker = std::sync::Arc::new(crank_runtime::TaskWorker::new(
|
||||
task_store,
|
||||
task_handler,
|
||||
crank_runtime::TaskWorkerConfig {
|
||||
owner: format!("mcp-server-{}", std::process::id()),
|
||||
..crank_runtime::TaskWorkerConfig::default()
|
||||
},
|
||||
)?);
|
||||
let (task_shutdown_tx, task_shutdown_rx) = tokio::sync::watch::channel(false);
|
||||
let task_worker_handle = tokio::spawn({
|
||||
let worker = task_worker.clone();
|
||||
async move { worker.run_until_shutdown(task_shutdown_rx).await }
|
||||
});
|
||||
let app = build_app_with_artifacts_and_tasks(
|
||||
registry,
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
config.runtime.base_url.clone(),
|
||||
@@ -211,6 +256,7 @@ async fn run(
|
||||
std::sync::Arc::new(CommunityMachineCredentialVerifier),
|
||||
runtime_limits.max_concurrent_sessions,
|
||||
artifact_store,
|
||||
tasks,
|
||||
);
|
||||
let listener = TcpListener::bind(config.bind_addr).await?;
|
||||
|
||||
@@ -231,15 +277,77 @@ async fn run(
|
||||
info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening");
|
||||
*startup_completed = true;
|
||||
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = axum::serve(listener, app) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
let serve_result: Result<(), Box<dyn std::error::Error>> = async {
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
}
|
||||
} else {
|
||||
serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone())
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
// Stop new claims; active attempts checkpoint/drain or are cancelled inside the bounded worker
|
||||
// shutdown timeout (<=30s), and queued PostgreSQL work is left for recovery.
|
||||
task_shutdown_tx.send(true).ok();
|
||||
if tokio::time::timeout(Duration::from_secs(30), task_worker_handle)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
name: "mcp.task_worker.shutdown_timeout",
|
||||
"durable Task worker did not stop within the story shutdown budget"
|
||||
);
|
||||
}
|
||||
serve_result
|
||||
}
|
||||
|
||||
/// Gracefully stop the durable Task worker as soon as shutdown is requested, while still
|
||||
/// draining in-flight HTTP connections before returning.
|
||||
async fn serve_with_worker_shutdown<S>(
|
||||
app: axum::Router,
|
||||
listener: tokio::net::TcpListener,
|
||||
signal: S,
|
||||
worker_stop: tokio::sync::watch::Sender<bool>,
|
||||
) -> std::io::Result<()>
|
||||
where
|
||||
S: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let worker_stop_graceful = worker_stop.clone();
|
||||
let graceful = async move {
|
||||
signal.await;
|
||||
let _ = worker_stop_graceful.send(true);
|
||||
};
|
||||
let result = axum::serve(listener, app)
|
||||
.with_graceful_shutdown(graceful)
|
||||
.await;
|
||||
let _ = worker_stop.send(true);
|
||||
result
|
||||
}
|
||||
|
||||
/// Process shutdown trigger for the HTTP surface and the durable Task worker.
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
if let Ok(mut signal) =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
{
|
||||
signal.recv().await;
|
||||
}
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {}
|
||||
_ = terminate => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_observability(
|
||||
@@ -287,6 +395,25 @@ fn init_observability(
|
||||
)?)
|
||||
}
|
||||
|
||||
fn admission_limits(runtime_limits: RuntimeLimits) -> AdmissionLimits {
|
||||
AdmissionLimits {
|
||||
global_limit: runtime_limits.max_concurrent_unary,
|
||||
..AdmissionLimits::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn execution_admission(
|
||||
pool: sqlx::PgPool,
|
||||
runtime_limits: RuntimeLimits,
|
||||
) -> Result<std::sync::Arc<dyn crank_core::ExecutionAdmission>, Box<dyn std::error::Error>> {
|
||||
let store = PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
admission_limits(runtime_limits),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?;
|
||||
Ok(std::sync::Arc::new(store))
|
||||
}
|
||||
|
||||
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
|
||||
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
|
||||
database_options(&config.database).map_err(|_| invalid("database.source"))?;
|
||||
@@ -307,6 +434,12 @@ fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::Confi
|
||||
config.runtime.max_concurrent_sessions,
|
||||
)
|
||||
.map_err(|_| invalid("runtime.limits"))?;
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: config.runtime.max_concurrent_unary,
|
||||
..crank_registry::AdmissionLimits::default()
|
||||
}
|
||||
.validate()
|
||||
.map_err(|_| invalid("runtime.admission"))?;
|
||||
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
|
||||
RequestRateLimitConfig::new(
|
||||
config.rate_limit.requests_per_second,
|
||||
@@ -460,3 +593,52 @@ fn runtime_cache_config(
|
||||
.map(|value| value.expose_secret().to_owned()),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{Router, routing::get};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{oneshot, watch};
|
||||
|
||||
async fn slow_handler() -> &'static str {
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
"ok"
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_stops_before_http_drain_completes() {
|
||||
let app = Router::new().route("/slow", get(slow_handler));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (stop_tx, mut stop_rx) = watch::channel(false);
|
||||
let (signal_tx, signal_rx) = oneshot::channel::<()>();
|
||||
|
||||
let handle = tokio::spawn(serve_with_worker_shutdown(
|
||||
app,
|
||||
listener,
|
||||
async {
|
||||
let _ = signal_rx.await;
|
||||
},
|
||||
stop_tx,
|
||||
));
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let request = client.get(format!("http://{}/slow", addr));
|
||||
let in_flight = tokio::spawn(async move { request.send().await });
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
let _ = signal_tx.send(());
|
||||
|
||||
tokio::time::timeout(Duration::from_millis(250), stop_rx.changed())
|
||||
.await
|
||||
.expect("worker stop signal should arrive before HTTP drain")
|
||||
.expect("watch changed");
|
||||
assert!(*stop_rx.borrow());
|
||||
|
||||
let response = in_flight.await.unwrap().unwrap();
|
||||
assert!(response.status().is_success());
|
||||
let serve_result = handle.await.unwrap();
|
||||
assert!(serve_result.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -11,3 +11,17 @@ pub fn spawn_postgres_pool_metrics(pool: PgPool) -> tokio::task::JoinHandle<()>
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Story 4.1: samples the authoritative admission queue so the oldest-waiting age grows (and returns
|
||||
/// to zero) without waiting for a new request.
|
||||
pub fn spawn_admission_metrics(pool: PgPool) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Ok(snapshot) = crank_registry::admission_snapshot(&pool).await {
|
||||
crank_observability::record_admission_oldest_waiting(snapshot.oldest_waiting_ms);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,874 @@
|
||||
//! ATDD Story 4.1 (FR-21/FR-24): the runtime asks one authoritative store for admission.
|
||||
//!
|
||||
//! Real PostgreSQL state, two independent `RuntimeExecutor` instances (the multi-process shape) and a
|
||||
//! counted adapter: one shared global budget, typed overload with retry guidance, typed unsupported
|
||||
//! fallback, deadline that includes queue waiting and zero outbound calls on every refusal.
|
||||
//!
|
||||
//! Lives in the `mcp-server` app crate because the runtime must not depend on the registry crate; this
|
||||
//! app already wires both and this is the real deployment shape.
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdapterResponse, AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence,
|
||||
ConfirmationPolicy, ExecutionAdmission, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
|
||||
ExecutionOrigin, ExecutionStage, HttpMethod, InvocationSource, NoTasksFallback, Operation,
|
||||
OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
|
||||
OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter, RestTarget,
|
||||
RuntimeRequestContext, Target, ToolDescription, ToolExecutionMode, ToolExecutionPolicy,
|
||||
WorkspaceId,
|
||||
};
|
||||
use crank_mapping::MappingSet;
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission};
|
||||
use crank_runtime::{RuntimeExecutor, RuntimeExecutorBuilder, RuntimeLimits};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
struct CountingAdapter {
|
||||
calls: Arc<AtomicUsize>,
|
||||
current: Arc<AtomicUsize>,
|
||||
peak: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ProtocolAdapter for CountingAdapter {
|
||||
fn protocol(&self) -> Protocol {
|
||||
Protocol::Rest
|
||||
}
|
||||
|
||||
fn supports_mode(&self, mode: ExecutionMode) -> bool {
|
||||
mode == ExecutionMode::Unary
|
||||
}
|
||||
|
||||
async fn invoke_unary(
|
||||
&self,
|
||||
_target: &Target,
|
||||
_prepared: &crank_core::PreparedRequest,
|
||||
context: &RuntimeRequestContext,
|
||||
) -> Result<AdapterResponse, crank_core::ProtocolAdapterError> {
|
||||
context.mark_dispatch_started();
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let current = self.current.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
self.peak.fetch_max(current, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
self.current.fetch_sub(1, Ordering::SeqCst);
|
||||
Ok(AdapterResponse {
|
||||
status_code: 200,
|
||||
headers: BTreeMap::new(),
|
||||
body: json!({"ok": true}),
|
||||
data: json!({"ok": true}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn schema(kind: SchemaKind) -> Schema {
|
||||
Schema {
|
||||
kind,
|
||||
format: None,
|
||||
description: None,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: None,
|
||||
fields: BTreeMap::new(),
|
||||
items: None,
|
||||
enum_values: Vec::new(),
|
||||
variants: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn operation(policy: Option<ToolExecutionPolicy>) -> Operation<Schema, MappingSet> {
|
||||
Operation {
|
||||
id: OperationId::new("op_admission_budget"),
|
||||
name: "admission_budget".to_owned(),
|
||||
display_name: "Admission budget".to_owned(),
|
||||
category: "general".to_owned(),
|
||||
protocol: Protocol::Rest,
|
||||
security_level: OperationSecurityLevel::Standard,
|
||||
status: OperationStatus::Published,
|
||||
version: 7,
|
||||
target: Target::Rest(RestTarget {
|
||||
base_url: "https://api.example.com".to_owned(),
|
||||
method: HttpMethod::Get,
|
||||
path_template: "/v1/items".to_owned(),
|
||||
static_headers: BTreeMap::new(),
|
||||
request_contract: None,
|
||||
response_contract: Vec::new(),
|
||||
multipart_field_mime_types: BTreeMap::new(),
|
||||
}),
|
||||
input_schema: schema(SchemaKind::Object),
|
||||
output_schema: schema(SchemaKind::Object),
|
||||
input_mapping: MappingSet::default(),
|
||||
output_mapping: MappingSet::default(),
|
||||
execution_config: ExecutionConfig {
|
||||
timeout_ms: 5_000,
|
||||
execution_policy: policy,
|
||||
..ExecutionConfig::default()
|
||||
},
|
||||
tool_description: ToolDescription {
|
||||
title: "Admission".to_owned(),
|
||||
description: "Admission budget probe".to_owned(),
|
||||
tags: Vec::new(),
|
||||
examples: Vec::new(),
|
||||
},
|
||||
samples: None,
|
||||
generated_draft: None,
|
||||
config_export: None,
|
||||
wizard_state: None,
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
updated_at: OffsetDateTime::now_utc(),
|
||||
published_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
executor: &RuntimeExecutor,
|
||||
operation: &crank_runtime::RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
|
||||
let workspace = WorkspaceId::new("ws_test");
|
||||
let context = crank_runtime::RuntimeRequestContext::from_correlation(
|
||||
&crank_core::CorrelationContext::generate(),
|
||||
)
|
||||
.with_metering_context(workspace.clone(), None, InvocationSource::AdminTestRun);
|
||||
execute_in(executor, operation, input, &context).await
|
||||
}
|
||||
|
||||
async fn execute_in(
|
||||
executor: &RuntimeExecutor,
|
||||
operation: &crank_runtime::RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
context: &crank_runtime::RuntimeRequestContext,
|
||||
) -> Result<serde_json::Value, crank_core::ExecutionFailure> {
|
||||
let workspace = context
|
||||
.metering_context()
|
||||
.map(|metering| metering.workspace_id.clone())
|
||||
.unwrap_or_else(|| WorkspaceId::new("ws_test"));
|
||||
let request = crank_runtime::RuntimeExecutionRequest::try_new(
|
||||
&workspace,
|
||||
ExecutionOrigin::AdminDraft,
|
||||
None,
|
||||
operation,
|
||||
input,
|
||||
crank_runtime::ExecutionAuthorization::Authorized,
|
||||
None,
|
||||
context,
|
||||
Instant::now() + Duration::from_secs(30),
|
||||
)
|
||||
.map_err(|_| {
|
||||
crank_core::ExecutionFailure::new(
|
||||
ExecutionErrorCode::RuntimeInternal,
|
||||
crank_core::CorrelationContext::new(
|
||||
context.request_id.clone(),
|
||||
context.trace_context.clone(),
|
||||
),
|
||||
)
|
||||
})?;
|
||||
executor
|
||||
.execute_outcome(request)
|
||||
.await
|
||||
.map(|success| success.output)
|
||||
}
|
||||
|
||||
async fn schema_pool(schema_url: &str) -> PgPool {
|
||||
let pool = PgPool::connect(schema_url)
|
||||
.await
|
||||
.expect("admission schema pool");
|
||||
crank_registry::MigrationAuthority::apply(&pool)
|
||||
.await
|
||||
.expect("migration authority applies the admission schema");
|
||||
pool
|
||||
}
|
||||
|
||||
fn admission_store(pool: PgPool, global: usize, queue: usize) -> Arc<PostgresExecutionAdmission> {
|
||||
Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: global,
|
||||
agent_limit: 8,
|
||||
tool_limit: 8,
|
||||
queue_limit: queue,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)
|
||||
}
|
||||
|
||||
fn executor(
|
||||
store: Arc<PostgresExecutionAdmission>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
current: Arc<AtomicUsize>,
|
||||
peak: Arc<AtomicUsize>,
|
||||
) -> RuntimeExecutor {
|
||||
RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(CountingAdapter {
|
||||
calls,
|
||||
current,
|
||||
peak,
|
||||
}))
|
||||
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
|
||||
.with_admission(store)
|
||||
.build()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_executors_share_one_global_budget() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_budget").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool.clone(), 2, 8);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let first = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
let second = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(None).into();
|
||||
let mut tasks = Vec::new();
|
||||
for index in 0..4 {
|
||||
let executor = if index % 2 == 0 {
|
||||
first.clone()
|
||||
} else {
|
||||
second.clone()
|
||||
};
|
||||
let operation = operation.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
execute(&executor, &operation, &json!({})).await
|
||||
}));
|
||||
}
|
||||
let mut completed = 0;
|
||||
for task in tasks {
|
||||
match task.await.expect("execution task") {
|
||||
Ok(_) => completed += 1,
|
||||
Err(failure) => panic!("admission must grant every request: {failure:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(completed, 4);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 4);
|
||||
assert!(
|
||||
peak.load(Ordering::SeqCst) <= 2,
|
||||
"peak concurrency {} exceeded the shared global limit",
|
||||
peak.load(Ordering::SeqCst)
|
||||
);
|
||||
assert_eq!(store.snapshot().await.unwrap().executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saturation_returns_typed_overload_without_an_outbound_call() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_overload").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 1, 1);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
// Another process already holds the only global permit.
|
||||
let held = match store
|
||||
.admit(AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: "ws_test".to_owned(),
|
||||
agent_key: "admin".to_owned(),
|
||||
operation_id: "op_admission_budget".to_owned(),
|
||||
operation_version: 7,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_holder".to_owned(),
|
||||
expires_at_unix_ms: crank_core::current_unix_ms() + 10_000,
|
||||
})
|
||||
.await
|
||||
.expect("admission store is reachable")
|
||||
{
|
||||
crank_core::AdmissionOutcome::Permitted(permit) => permit,
|
||||
other => panic!("expected a permit, got {other:?}"),
|
||||
};
|
||||
|
||||
let waiting_operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 250,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}))
|
||||
.into();
|
||||
let waiting_runtime = runtime.clone();
|
||||
let first_waiting_operation = waiting_operation.clone();
|
||||
let waiting = tokio::spawn(async move {
|
||||
execute(&waiting_runtime, &first_waiting_operation, &json!({})).await
|
||||
});
|
||||
// Let the first request occupy the bounded queue before the next one arrives.
|
||||
while store.snapshot().await.unwrap().waiting == 0 {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
let overloaded = execute(&runtime, &waiting_operation, &json!({})).await;
|
||||
let failure = overloaded.expect_err("a full queue must refuse the request");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::ExecutionOverloaded
|
||||
);
|
||||
assert_eq!(failure.stage(), ExecutionStage::Admission);
|
||||
assert_eq!(failure.retry_after_ms(), Some(250));
|
||||
|
||||
// The queued request hits its absolute deadline instead of dispatching. That refusal happened
|
||||
// inside admission (nothing was ever dispatched), so the client must see an admission-stage
|
||||
// overload with retry guidance — not an `upstream_timeout`.
|
||||
let expired = waiting
|
||||
.await
|
||||
.expect("waiting task")
|
||||
.expect_err("deadline must refuse");
|
||||
assert_eq!(
|
||||
expired.error_code(),
|
||||
ExecutionErrorCode::ExecutionOverloaded,
|
||||
"queue waiting counts against the published deadline"
|
||||
);
|
||||
assert_eq!(expired.stage(), ExecutionStage::Admission);
|
||||
assert_eq!(expired.retry_after_ms(), Some(250));
|
||||
assert_eq!(expired.outcome_certainty(), OutcomeCertainty::Certain);
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"no outbound call on refusal"
|
||||
);
|
||||
assert!(store.release(&held).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_policy_without_a_task_transport_is_typed_unsupported() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_unsupported").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 2_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}))
|
||||
.into();
|
||||
let failure = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect_err("no Task transport exists in Story 4.1");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::ExecutionModeUnsupported
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
let snapshot = store.snapshot().await.unwrap();
|
||||
assert_eq!(snapshot.waiting, 0);
|
||||
assert_eq!(snapshot.executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_policy_with_a_synchronous_fallback_keeps_the_sync_response_shape() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_fallback").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool, 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
|
||||
let operation: crank_runtime::RuntimeOperation = operation(Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Adaptive,
|
||||
deadline_ms: 2_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}))
|
||||
.into();
|
||||
let output = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect("adaptive resolves through the published synchronous fallback");
|
||||
// The response shape is unchanged by admission: an operation without output mapping still
|
||||
// returns the same mapped output object it returned before Story 4.1.
|
||||
assert_eq!(output, json!({}));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(store.snapshot().await.unwrap().executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admission_store_outage_fails_closed_before_dispatch() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_outage").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let store = admission_store(pool.clone(), 2, 4);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let current = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = executor(
|
||||
Arc::clone(&store),
|
||||
Arc::clone(&calls),
|
||||
Arc::clone(¤t),
|
||||
Arc::clone(&peak),
|
||||
);
|
||||
// The mandatory state disappears after the store was built: every execution must fail closed.
|
||||
sqlx::query("drop table execution_admission_entries")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("drop the mandatory admission state");
|
||||
let operation: crank_runtime::RuntimeOperation = operation(None).into();
|
||||
let failure = execute(&runtime, &operation, &json!({}))
|
||||
.await
|
||||
.expect_err("a missing mandatory store must fail closed");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert!(store.health().await.is_err());
|
||||
}
|
||||
|
||||
/// Admission store that records the snapshot it was asked to decide on (Story 4.1, AC2/F2).
|
||||
struct RecordingAdmission {
|
||||
inner: Arc<PostgresExecutionAdmission>,
|
||||
requests: Mutex<Vec<AdmissionRequest>>,
|
||||
permits: AtomicUsize,
|
||||
releases: AtomicUsize,
|
||||
}
|
||||
|
||||
impl RecordingAdmission {
|
||||
fn new(inner: Arc<PostgresExecutionAdmission>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
permits: AtomicUsize::new(0),
|
||||
releases: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn requests(&self) -> Vec<AdmissionRequest> {
|
||||
self.requests.lock().expect("admission log").clone()
|
||||
}
|
||||
|
||||
fn permits(&self) -> usize {
|
||||
self.permits.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn releases(&self) -> usize {
|
||||
self.releases.load(Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionAdmission for RecordingAdmission {
|
||||
async fn health(&self) -> Result<(), crank_core::AdmissionStoreError> {
|
||||
self.inner.health().await
|
||||
}
|
||||
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<crank_core::AdmissionOutcome, crank_core::AdmissionStoreError> {
|
||||
self.requests
|
||||
.lock()
|
||||
.expect("admission log")
|
||||
.push(request.clone());
|
||||
let outcome = self.inner.admit(request).await?;
|
||||
if matches!(outcome, crank_core::AdmissionOutcome::Permitted(_)) {
|
||||
self.permits.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn release(
|
||||
&self,
|
||||
permit: &crank_core::AdmissionPermit,
|
||||
) -> Result<bool, crank_core::AdmissionStoreError> {
|
||||
let released = self.inner.release(permit).await?;
|
||||
if released {
|
||||
self.releases.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(released)
|
||||
}
|
||||
|
||||
async fn cancel(
|
||||
&self,
|
||||
ticket_id: u64,
|
||||
owner_token: &str,
|
||||
) -> Result<bool, crank_core::AdmissionStoreError> {
|
||||
self.inner.cancel(ticket_id, owner_token).await
|
||||
}
|
||||
|
||||
async fn snapshot(
|
||||
&self,
|
||||
) -> Result<crank_core::AdmissionQueueSnapshot, crank_core::AdmissionStoreError> {
|
||||
self.inner.snapshot().await
|
||||
}
|
||||
}
|
||||
|
||||
fn confirmation_operation(version: u32) -> crank_runtime::RuntimeOperation {
|
||||
let mut source = operation(None);
|
||||
source.id = OperationId::new("op_admission_confirmation");
|
||||
source.name = "admission_confirmation".to_owned();
|
||||
source.version = version;
|
||||
let Target::Rest(target) = &mut source.target;
|
||||
target.method = HttpMethod::Delete;
|
||||
source.execution_config.safety = Some(OperationSafetyPolicy {
|
||||
class: OperationSafetyClass::Destructive,
|
||||
confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
|
||||
});
|
||||
source.into()
|
||||
}
|
||||
|
||||
fn confirmation_runtime(
|
||||
admission: Arc<RecordingAdmission>,
|
||||
coordination: Option<Arc<dyn crank_core::CoordinationStateStore>>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
) -> RuntimeExecutor {
|
||||
let store: Arc<dyn ExecutionAdmission> = admission;
|
||||
let builder = RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(CountingAdapter {
|
||||
calls,
|
||||
current: Arc::new(AtomicUsize::new(0)),
|
||||
peak: Arc::new(AtomicUsize::new(0)),
|
||||
}))
|
||||
.with_limits(RuntimeLimits::try_new(8, 2).expect("local limits"))
|
||||
.with_admission(store);
|
||||
match coordination {
|
||||
Some(coordination) => builder.with_coordination_store(coordination).build(),
|
||||
None => builder.build(),
|
||||
}
|
||||
}
|
||||
|
||||
fn confirmation_context() -> crank_runtime::RuntimeRequestContext {
|
||||
crank_runtime::RuntimeRequestContext::from_correlation(
|
||||
&crank_core::CorrelationContext::generate(),
|
||||
)
|
||||
.with_response_cache_scope("ws_test", "agent_a")
|
||||
.with_metering_context(
|
||||
WorkspaceId::new("ws_test"),
|
||||
None,
|
||||
InvocationSource::AdminTestRun,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn confirmation_requiring_call_takes_a_permit_only_with_verified_approval() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let recording = Arc::new(RecordingAdmission::new(Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)));
|
||||
let coordination: Arc<dyn crank_core::CoordinationStateStore> =
|
||||
Arc::new(crank_runtime::InMemoryCoordinationStateStore::default());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let runtime = confirmation_runtime(
|
||||
Arc::clone(&recording),
|
||||
Some(Arc::clone(&coordination)),
|
||||
Arc::clone(&calls),
|
||||
);
|
||||
let operation = confirmation_operation(3);
|
||||
let context = confirmation_context();
|
||||
let input = json!({ "reason": "remove the stale item" });
|
||||
|
||||
// 1. No approval yet: the existing challenge is returned and no execution capacity is taken.
|
||||
let challenge = execute_in(&runtime, &operation, &input, &context)
|
||||
.await
|
||||
.expect_err("an unapproved destructive call must only stage confirmation");
|
||||
assert_eq!(
|
||||
challenge.error_code(),
|
||||
ExecutionErrorCode::ConfirmationRequired
|
||||
);
|
||||
let token = challenge
|
||||
.confirmation()
|
||||
.expect("expected the confirmation challenge")
|
||||
.token()
|
||||
.to_owned();
|
||||
assert_eq!(
|
||||
recording.permits(),
|
||||
0,
|
||||
"an unapproved request must not take an execution permit"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(
|
||||
requests.len(),
|
||||
1,
|
||||
"the one admission snapshot is consulted once"
|
||||
);
|
||||
assert_eq!(
|
||||
requests[0].approval,
|
||||
ApprovalEvidence::NotGranted,
|
||||
"admission must see the real approval state, not a fixed NotRequired"
|
||||
);
|
||||
assert_eq!(requests[0].scope.operation_version, 3);
|
||||
let snapshot = recording.snapshot().await.expect("admission snapshot");
|
||||
assert_eq!((snapshot.executing, snapshot.waiting), (0, 0));
|
||||
|
||||
// 2. An approval issued for another published version never opens this execution.
|
||||
let other_version = confirmation_operation(4);
|
||||
let foreign = execute_in(
|
||||
&runtime,
|
||||
&other_version,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect_err("a foreign approval must not authorise this version");
|
||||
assert_eq!(
|
||||
foreign.error_code(),
|
||||
ExecutionErrorCode::ConfirmationInvalid
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
1,
|
||||
"an unusable approval is refused before any capacity request"
|
||||
);
|
||||
|
||||
// 3. An approval is bound to its exact input as well.
|
||||
let mismatched_input = json!({ "reason": "remove another item" });
|
||||
let mismatched = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&mismatched_input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect_err("a confirmation token must not authorise different input");
|
||||
assert_eq!(
|
||||
mismatched.error_code(),
|
||||
ExecutionErrorCode::ConfirmationInvalid
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(recording.requests().len(), 1);
|
||||
|
||||
// 4. The matching approval passes admission with truthful evidence and executes exactly once.
|
||||
let executed = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("a verified approval executes");
|
||||
assert!(executed.is_object());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(requests[1].approval, ApprovalEvidence::Granted);
|
||||
assert_eq!(requests[1].scope.operation_version, 3);
|
||||
assert_eq!(recording.permits(), 1);
|
||||
assert_eq!(
|
||||
recording.releases(),
|
||||
1,
|
||||
"the permit is released after execution"
|
||||
);
|
||||
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
|
||||
|
||||
// 5. Replay of the consumed confirmation is refused before capacity and without a side effect.
|
||||
let replay = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&input,
|
||||
&context.clone().with_confirmation_token(token),
|
||||
)
|
||||
.await
|
||||
.expect_err("a confirmation token is single-use");
|
||||
assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(recording.permits(), 1);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
2,
|
||||
"a replayed approval never reaches admission"
|
||||
);
|
||||
|
||||
// 6. The trusted approved-recovery path still passes the same gate with real evidence.
|
||||
let approved = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&json!({ "reason": "approved by an operator" }),
|
||||
&context.with_approval_granted(),
|
||||
)
|
||||
.await;
|
||||
assert!(approved.is_ok());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
let requests = recording.requests();
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert_eq!(requests[2].approval, ApprovalEvidence::Granted);
|
||||
assert_eq!(recording.permits(), 2);
|
||||
assert_eq!(recording.snapshot().await.expect("snapshot").executing, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mandatory_safety_store_outage_fails_closed_before_admission() {
|
||||
let schema_url = crank_test_support::postgres_schema_url("admission_confirmation_outage").await;
|
||||
let pool = schema_pool(&schema_url).await;
|
||||
let recording = Arc::new(RecordingAdmission::new(Arc::new(
|
||||
PostgresExecutionAdmission::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("valid admission limits"),
|
||||
)));
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
// Confirmation is mandatory for this Tool, but no safety store is wired: the request must fail
|
||||
// closed instead of reserving capacity and hoping for an in-memory fallback.
|
||||
let runtime = confirmation_runtime(Arc::clone(&recording), None, Arc::clone(&calls));
|
||||
let operation = confirmation_operation(3);
|
||||
|
||||
let failure = execute_in(
|
||||
&runtime,
|
||||
&operation,
|
||||
&json!({ "reason": "no store" }),
|
||||
&confirmation_context(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a mandatory safety store outage must fail closed");
|
||||
assert_eq!(
|
||||
failure.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
let after_missing_store = recording.requests().len();
|
||||
|
||||
// The same Tool with a claimed approval and a *failing* safety store never reaches the capacity
|
||||
// store at all: the approval claim stays unverified, so no permit and no outbound call exist.
|
||||
let failing = confirmation_runtime(
|
||||
Arc::clone(&recording),
|
||||
Some(Arc::new(UnavailableSafetyStore)),
|
||||
Arc::clone(&calls),
|
||||
);
|
||||
let outage = execute_in(
|
||||
&failing,
|
||||
&operation,
|
||||
&json!({ "reason": "failing store" }),
|
||||
&confirmation_context().with_confirmation_token("ct_unavailable"),
|
||||
)
|
||||
.await
|
||||
.expect_err("an unreachable safety store must refuse the claimed approval");
|
||||
assert_eq!(
|
||||
outage.error_code(),
|
||||
ExecutionErrorCode::SafetyStoreUnavailable
|
||||
);
|
||||
assert_eq!(
|
||||
recording.requests().len(),
|
||||
after_missing_store,
|
||||
"an unverified approval never becomes a capacity request"
|
||||
);
|
||||
assert_eq!(recording.permits(), 0);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Coordination store whose mandatory backend is down: every operation fails closed.
|
||||
struct UnavailableSafetyStore;
|
||||
|
||||
impl UnavailableSafetyStore {
|
||||
fn error() -> crank_core::CacheStoreError {
|
||||
crank_core::CacheStoreError::Unavailable {
|
||||
message: "test backend unavailable".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crank_core::CoordinationStateStore for UnavailableSafetyStore {
|
||||
async fn get_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn put_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<(), crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn delete_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<(), crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn take_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
) -> Result<Option<crank_core::CoordinationStateValue>, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn reserve_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<crank_core::CoordinationStateReservation, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
|
||||
async fn compare_and_set_value(
|
||||
&self,
|
||||
_scope: crank_core::CacheScope,
|
||||
_key: &str,
|
||||
_expected: &crank_core::CoordinationStateValue,
|
||||
_value: crank_core::CoordinationStateValue,
|
||||
_ttl: Duration,
|
||||
) -> Result<bool, crank_core::CacheStoreError> {
|
||||
Err(Self::error())
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
mod integration {
|
||||
mod catalog_access;
|
||||
mod common;
|
||||
mod durable_tasks;
|
||||
mod era_isolation;
|
||||
mod execution_certainty;
|
||||
mod execution_stages;
|
||||
|
||||
@@ -621,6 +621,7 @@ pub(super) fn test_operation(base_url: &str, name: &str) -> Operation<Schema, Ma
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,555 @@
|
||||
//! Story 4.4 ATDD: no outbound send exists without a committed durable attempt, and an elapsed
|
||||
//! accepted deadline fails before dispatch.
|
||||
//!
|
||||
//! Both oracles drive the production composition (application entry → PG store → runtime executor →
|
||||
//! real local HTTP upstream); only the upstream and the checkpoint failure are test-controlled.
|
||||
|
||||
use super::*;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{
|
||||
AgentOperationBinding, PlatformApiKeyScope, TaskState, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
async fn composition(
|
||||
registry: &PostgresRegistry,
|
||||
upstream: &str,
|
||||
policy: ToolExecutionPolicy,
|
||||
caller_name: &str,
|
||||
scoped_key: &str,
|
||||
) -> (
|
||||
Arc<PostgresTaskStore>,
|
||||
crank_community_mcp::TaskApplication,
|
||||
SecretCrypto,
|
||||
crank_community_mcp::TaskPrincipal,
|
||||
) {
|
||||
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(
|
||||
registry,
|
||||
caller_name,
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let _ = scoped_key;
|
||||
(store, tasks, crypto, caller)
|
||||
}
|
||||
|
||||
fn worker_for(
|
||||
registry: &PostgresRegistry,
|
||||
store: &Arc<PostgresTaskStore>,
|
||||
tasks: &TaskApplication,
|
||||
crypto: &SecretCrypto,
|
||||
owner: &str,
|
||||
) -> TaskWorker {
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto.clone(),
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: owner.to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
TaskWorker::new(store.clone(), handler, config).expect("valid worker")
|
||||
}
|
||||
|
||||
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// AC1: a dispatch checkpoint that cannot commit produces zero outbound calls and no attempt row.
|
||||
#[tokio::test]
|
||||
async fn checkpoint_commit_failure_never_reaches_upstream() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let (store, tasks, crypto, caller) = composition(
|
||||
®istry,
|
||||
&upstream,
|
||||
background_policy(),
|
||||
"attempt-guards-checkpoint",
|
||||
"guards-checkpoint-key",
|
||||
)
|
||||
.await;
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"guards-checkpoint-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
// The checkpoint is the only statement that commits an attempt; it is refused here.
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"create function refuse_task_checkpoint() returns trigger language plpgsql as $$
|
||||
begin
|
||||
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
||||
raise exception 'durable checkpoint refused for test';
|
||||
end if;
|
||||
return new;
|
||||
end $$;
|
||||
create trigger refuse_task_checkpoint before update on tasks
|
||||
for each row execute function refuse_task_checkpoint();",
|
||||
))
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.expect("checkpoint refusal trigger");
|
||||
|
||||
let worker = worker_for(
|
||||
®istry,
|
||||
&store,
|
||||
&tasks,
|
||||
&crypto,
|
||||
"worker-guards-checkpoint",
|
||||
);
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"an uncommitted durable checkpoint must never produce an outbound call"
|
||||
);
|
||||
let attempts: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
attempts, 0,
|
||||
"no attempt history without a committed checkpoint"
|
||||
);
|
||||
assert_ne!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("completed"),
|
||||
"the refused attempt cannot complete the Task"
|
||||
);
|
||||
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"drop trigger refuse_task_checkpoint on tasks; drop function refuse_task_checkpoint();",
|
||||
))
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.expect("checkpoint refusal trigger removed");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
/// AC6: an accepted absolute deadline that already elapsed fails the Task before any dispatch.
|
||||
#[tokio::test]
|
||||
async fn elapsed_deadline_fails_before_dispatch_without_upstream_call() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let short_deadline = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 300,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
};
|
||||
let (store, tasks, crypto, caller) = composition(
|
||||
®istry,
|
||||
&upstream,
|
||||
short_deadline,
|
||||
"attempt-guards-deadline",
|
||||
"guards-deadline-key",
|
||||
)
|
||||
.await;
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"guards-deadline-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
// The accepted deadline elapses before any worker claims the Task.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
let worker = worker_for(®istry, &store, &tasks, &crypto, "worker-guards-deadline");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
state = state_of(®istry, acceptance.task_id).await;
|
||||
if state.as_deref() == Some("failed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state.as_deref(), Some("failed"));
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"an elapsed deadline must fail before dispatch"
|
||||
);
|
||||
let attempts: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts, 0, "no attempt is checkpointed after the deadline");
|
||||
let code: Option<String> =
|
||||
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(code.as_deref(), Some("task_deadline_elapsed"));
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, String>("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
TaskState::Failed.as_str()
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
/// AC2: a write whose upstream idempotency is backed by server-owned evidence is retried with the
|
||||
/// same stable key, in a bounded number of distinct durable attempts, and applies exactly one effect.
|
||||
#[tokio::test]
|
||||
async fn verified_idempotent_write_retries_with_stable_key_and_one_effect() {
|
||||
use crank_community_mcp::VerifiedIdempotentWrite;
|
||||
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let effects = Arc::new(AtomicUsize::new(0));
|
||||
let keys = Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
let seen = Arc::new(Mutex::new(HashSet::<String>::new()));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let requests = Arc::clone(&requests);
|
||||
let effects = Arc::clone(&effects);
|
||||
let keys = Arc::clone(&keys);
|
||||
let seen = Arc::clone(&seen);
|
||||
move |headers: axum::http::HeaderMap| {
|
||||
let requests = Arc::clone(&requests);
|
||||
let effects = Arc::clone(&effects);
|
||||
let keys = Arc::clone(&keys);
|
||||
let seen = Arc::clone(&seen);
|
||||
async move {
|
||||
let key = headers
|
||||
.get("idempotency-key")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
keys.lock().expect("keys").push(key.clone());
|
||||
let first_application = seen.lock().expect("seen").insert(key);
|
||||
if first_application {
|
||||
effects.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
||||
if attempt == 0 {
|
||||
// The first application is stored but its acknowledgement is lost.
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(json!({"error": "acknowledgement lost"})),
|
||||
);
|
||||
}
|
||||
(
|
||||
StatusCode::OK,
|
||||
axum::Json(json!({"id": "lead-1", "deduped": !first_application})),
|
||||
)
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
||||
mode: IdempotencyMode::Required,
|
||||
ttl_ms: 60_000,
|
||||
input_field: Some("idempotency_key".to_owned()),
|
||||
header_name: Some("Idempotency-Key".to_owned()),
|
||||
});
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"attempt-guards-verified",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"guards-verified-key",
|
||||
&json!({"email": "user@example.com", "idempotency_key": "k-1"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
||||
.build();
|
||||
let handler = Arc::new(
|
||||
TaskWorkerHandler::from_application(&tasks, registry.clone(), runtime, crypto.clone())
|
||||
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
||||
operation_id: operation.id.as_str().to_owned(),
|
||||
operation_version: 1,
|
||||
key_header: "Idempotency-Key".to_owned(),
|
||||
validity_ms: 60_000,
|
||||
}]),
|
||||
);
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "worker-guards-verified".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
state = state_of(®istry, acceptance.task_id).await;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
2,
|
||||
"exactly one retry of the write"
|
||||
);
|
||||
assert_eq!(
|
||||
effects.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the upstream applied the write exactly once"
|
||||
);
|
||||
assert_eq!(
|
||||
keys.lock().expect("keys").as_slice(),
|
||||
["k-1".to_owned(), "k-1".to_owned()],
|
||||
"the same stable key is delivered on both attempts"
|
||||
);
|
||||
|
||||
let attempts = store
|
||||
.attempts(
|
||||
&crank_core::TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: crank_core::TaskOwnerKind::Agent,
|
||||
owner_id: "agent_attempt_guards".to_owned(),
|
||||
authorization_scope: String::new(),
|
||||
},
|
||||
acceptance.task_id,
|
||||
)
|
||||
.await;
|
||||
// Owner scope of this principal is the agent of the published binding; read the history through
|
||||
// the application entry instead of guessing the authorization scope string.
|
||||
assert!(attempts.is_err(), "a foreign scope never reads the history");
|
||||
let attempts = tasks
|
||||
.attempts(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.expect("owner-scoped attempt history");
|
||||
assert_eq!(attempts.len(), 2, "two distinct durable attempts");
|
||||
assert_eq!(attempts[0].ordinal, 1);
|
||||
assert_eq!(attempts[1].ordinal, 2);
|
||||
assert!(attempts.iter().all(|attempt| attempt.retry_allowed));
|
||||
assert_eq!(
|
||||
attempts[0].outcome,
|
||||
Some(crank_core::AttemptOutcomeClass::RetryScheduled)
|
||||
);
|
||||
assert_eq!(
|
||||
attempts[1].outcome,
|
||||
Some(crank_core::AttemptOutcomeClass::Completed)
|
||||
);
|
||||
assert_eq!(
|
||||
attempts[0].upstream_key_digest, attempts[1].upstream_key_digest,
|
||||
"the same stable key digest is recorded on both attempts"
|
||||
);
|
||||
assert!(attempts[0].upstream_key_digest.is_some());
|
||||
let record = tasks
|
||||
.status(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.expect("status");
|
||||
assert_eq!(record.pins.retry.max_attempts, 2);
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1).
|
||||
//!
|
||||
//! End-to-end: a real durable Task worker dispatches a retrying GET through a real local
|
||||
//! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is
|
||||
//! persisted in `tasks.current_attempt_id` on both attempts.
|
||||
|
||||
use super::super::common;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::get;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{
|
||||
AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_runtime::{
|
||||
OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy,
|
||||
};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{Id, Subscriber, field::Visit};
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
const TOOL_NAME: &str = "attempt_trace";
|
||||
const KEY: &str = "attempt-trace-key-1";
|
||||
|
||||
async fn publish(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
operation_id: &crank_core::OperationId,
|
||||
version: u32,
|
||||
) {
|
||||
registry
|
||||
.publish_operation(crank_registry::PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id,
|
||||
version,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn application_with_store(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
) -> (
|
||||
crank_community_mcp::TaskApplication,
|
||||
std::sync::Arc<crank_registry::PostgresTaskStore>,
|
||||
SecretCrypto,
|
||||
) {
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = std::sync::Arc::new(
|
||||
crank_registry::PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = crank_community_mcp::TaskApplication::new(
|
||||
registry.clone(),
|
||||
std::time::Duration::ZERO,
|
||||
std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()),
|
||||
crank_runtime::TaskService::with_secret_crypto(
|
||||
store.clone(),
|
||||
std::sync::Arc::new(crypto.clone()),
|
||||
),
|
||||
);
|
||||
(tasks, store, crypto)
|
||||
}
|
||||
|
||||
async fn principal(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
name: &str,
|
||||
scopes: &[crank_core::PlatformApiKeyScope],
|
||||
) -> crank_community_mcp::TaskPrincipal {
|
||||
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("an active platform key resolves");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crank_community_mcp::TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: crank_core::VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn adapter_attempt_span_carries_durable_attempt_id() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
get({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
let n = calls.fetch_add(1, Ordering::SeqCst);
|
||||
if n == 0 {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(serde_json::json!({"error":"boom"})),
|
||||
);
|
||||
}
|
||||
(
|
||||
StatusCode::OK,
|
||||
axum::Json(serde_json::json!({"id":"lead-1"})),
|
||||
)
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
let Target::Rest(target) = &mut operation.target;
|
||||
target.method = crank_core::HttpMethod::Get;
|
||||
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
});
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"attempt-trace",
|
||||
&[
|
||||
crank_core::PlatformApiKeyScope::Read,
|
||||
crank_core::PlatformApiKeyScope::Write,
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let (tasks, store, crypto) = application_with_store(®istry).await;
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&serde_json::json!({"email": "user@example.com"}),
|
||||
&CorrelationContext::generate(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
let runtime =
|
||||
community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-attempt-trace".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: std::time::Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: std::time::Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
|
||||
let capture = TraceCapture::default();
|
||||
let subscriber = tracing_subscriber::registry().with(capture.clone());
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
server.abort();
|
||||
|
||||
let durable_attempt_id: Option<String> = sqlx::query_scalar::<_, Option<String>>(
|
||||
"select current_attempt_id from tasks where task_id = $1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
|
||||
// Story 4.4: every outbound send is its own durable attempt, so the two retries of one lease carry
|
||||
// two distinct attempt identities, both recorded in the immutable attempt history.
|
||||
let attempt_rows: Vec<(String, i32)> = sqlx::query_as(
|
||||
"select attempt_id, ordinal from task_attempts where task_id = $1 order by ordinal",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_all(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempt_rows.len(), 2, "two durable attempts were committed");
|
||||
assert_eq!(attempt_rows[0].1, 1);
|
||||
assert_eq!(attempt_rows[1].1, 2);
|
||||
assert_eq!(
|
||||
Some(attempt_rows[1].0.as_str()),
|
||||
durable_attempt_id.as_deref(),
|
||||
"the Task points at its latest durable attempt"
|
||||
);
|
||||
|
||||
let spans = capture.snapshot();
|
||||
let attempt_spans: Vec<_> = spans
|
||||
.iter()
|
||||
.filter(|s| s.name == "adapter_attempt")
|
||||
.collect();
|
||||
assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans");
|
||||
for span in &attempt_spans {
|
||||
let aid = span.fields.get("attempt_id").cloned().unwrap_or_default();
|
||||
assert!(!aid.is_empty(), "attempt_id must not be empty");
|
||||
assert!(
|
||||
!aid.contains("traceparent"),
|
||||
"attempt_id must not contain traceparent"
|
||||
);
|
||||
assert!(
|
||||
attempt_rows.iter().any(|(id, _)| id == &aid),
|
||||
"every attempt span names an attempt that was durably committed"
|
||||
);
|
||||
}
|
||||
let ids: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert_ne!(ids[0], ids[1], "each retry has its own durable attempt id");
|
||||
assert_eq!(
|
||||
spans
|
||||
.iter()
|
||||
.filter(|span| span.name == "durable_task_attempt")
|
||||
.count(),
|
||||
1,
|
||||
"one durable attempt span per dispatch"
|
||||
);
|
||||
let runtime_span = spans
|
||||
.iter()
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime.execute span");
|
||||
assert_eq!(
|
||||
runtime_span.parent_name,
|
||||
Some("durable_task_attempt"),
|
||||
"the execution continues the durable attempt trace"
|
||||
);
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(
|
||||
span.parent_name,
|
||||
Some("runtime.execute"),
|
||||
"each retry is a child attempt span under the durable execution"
|
||||
);
|
||||
}
|
||||
|
||||
let attempts: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|s| s.fields.get("attempt").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert!(attempts.contains(&"1".to_string()));
|
||||
assert!(attempts.contains(&"2".to_string()));
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(
|
||||
span.fields.get("max_attempts").cloned().unwrap_or_default(),
|
||||
"2"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// TraceCapture minimal port from crates/crank-runtime/tests/stages.rs
|
||||
#[derive(Clone, Default)]
|
||||
struct TraceCapture {
|
||||
spans: Arc<Mutex<Vec<CapturedSpan>>>,
|
||||
}
|
||||
|
||||
impl TraceCapture {
|
||||
fn snapshot(&self) -> Vec<CapturedSpan> {
|
||||
self.spans.lock().expect("span lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturedSpan {
|
||||
name: &'static str,
|
||||
parent_name: Option<&'static str>,
|
||||
fields: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for TraceCapture
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
{
|
||||
fn on_new_span(
|
||||
&self,
|
||||
attributes: &tracing::span::Attributes<'_>,
|
||||
id: &Id,
|
||||
context: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let parent = attributes
|
||||
.parent()
|
||||
.and_then(|parent| context.span(parent))
|
||||
.or_else(|| {
|
||||
attributes
|
||||
.is_contextual()
|
||||
.then(|| context.lookup_current())
|
||||
.flatten()
|
||||
});
|
||||
let mut visitor = FieldVisitor::default();
|
||||
attributes.record(&mut visitor);
|
||||
let mut spans = self.spans.lock().expect("span lock");
|
||||
let index = spans.len();
|
||||
spans.push(CapturedSpan {
|
||||
name: attributes.metadata().name(),
|
||||
parent_name: parent.map(|span| span.metadata().name()),
|
||||
fields: visitor.fields,
|
||||
});
|
||||
context
|
||||
.span(id)
|
||||
.expect("span exists")
|
||||
.extensions_mut()
|
||||
.insert(index);
|
||||
}
|
||||
|
||||
fn on_record(
|
||||
&self,
|
||||
id: &Id,
|
||||
values: &tracing::span::Record<'_>,
|
||||
context: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let mut visitor = FieldVisitor::default();
|
||||
values.record(&mut visitor);
|
||||
let span = context.span(id).expect("span exists");
|
||||
let index = *span.extensions().get::<usize>().expect("capture index");
|
||||
self.spans.lock().expect("span lock")[index]
|
||||
.fields
|
||||
.extend(visitor.fields);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FieldVisitor {
|
||||
fields: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl Visit for FieldVisitor {
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), value.to_owned());
|
||||
}
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), format!("{value:?}"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,846 @@
|
||||
//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream.
|
||||
|
||||
use super::common;
|
||||
use super::unsampled_correlation;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
|
||||
use crank_core::{
|
||||
AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState,
|
||||
TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
const TOOL_NAME: &str = "report_gen";
|
||||
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
||||
|
||||
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) {
|
||||
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal {
|
||||
let token = common::create_platform_api_key(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
name,
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("active key");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crank_community_mcp::TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store, Arc::new(crypto)),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let signal = Arc::new(std::sync::Mutex::new(Some(tx)));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
let signal = Arc::clone(&signal);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
let signal = Arc::clone(&signal);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(tx) = signal.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &upstream, policy).await;
|
||||
let caller = principal(®istry, "cancel-s1").await;
|
||||
let foreign = principal(®istry, "cancel-s1-foreign").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s1-key",
|
||||
&json!({"email":"u"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&app,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto.clone(),
|
||||
));
|
||||
let worker = TaskWorker::new(
|
||||
store,
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-s1".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
renewal_interval: Duration::from_millis(100),
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), rx).await;
|
||||
|
||||
let cancel = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
match cancel {
|
||||
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
|
||||
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
||||
}
|
||||
other => panic!("expected OutcomeUnknown got {other:?}"),
|
||||
}
|
||||
|
||||
let foreign_cancel = app
|
||||
.cancel(&foreign, TOOL_NAME, acceptance.task_id, None)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
foreign_cancel,
|
||||
Err(crank_community_mcp::TaskApplicationError::NotFound)
|
||||
));
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(1600)).await;
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("outcome_unknown")
|
||||
);
|
||||
let certainty: Option<String> = sqlx::query_scalar(
|
||||
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(certainty.as_deref(), Some("unknown"));
|
||||
let code: Option<String> =
|
||||
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
|
||||
let attempts_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts_count, 1);
|
||||
|
||||
let cancel2 = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel2");
|
||||
assert!(matches!(
|
||||
cancel2,
|
||||
crank_core::TaskCancelOutcome::OutcomeUnknown(_)
|
||||
));
|
||||
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &upstream, policy).await;
|
||||
let caller = principal(®istry, "cancel-s3").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s3-key",
|
||||
&json!({"email":"u"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let cancel = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
match cancel {
|
||||
crank_core::TaskCancelOutcome::Cancelled(record) => {
|
||||
assert_eq!(record.state, TaskState::Cancelled)
|
||||
}
|
||||
other => panic!("expected Cancelled got {other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
let attempts = app
|
||||
.attempts(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.expect("attempts");
|
||||
assert!(attempts.is_empty());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() {
|
||||
use crank_community_mcp::VerifiedIdempotentWrite;
|
||||
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let requests = Arc::clone(&requests);
|
||||
move |headers: axum::http::HeaderMap| {
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
let _ = headers.get("idempotency-key");
|
||||
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
||||
if attempt == 0 {
|
||||
// Applied upstream, acknowledgement lost after the caller had time to observe that
|
||||
// the request really left the worker.
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(json!({"error":"acknowledgement lost"})),
|
||||
);
|
||||
}
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
});
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
||||
mode: IdempotencyMode::Required,
|
||||
ttl_ms: 60_000,
|
||||
input_field: Some("idempotency_key".to_owned()),
|
||||
header_name: Some("Idempotency-Key".to_owned()),
|
||||
});
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(®istry, "cancel-s2").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s2-key",
|
||||
&json!({"email":"u","idempotency_key":"k-2"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
||||
.build();
|
||||
let handler = Arc::new(
|
||||
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
|
||||
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
||||
operation_id: operation.id.as_str().to_owned(),
|
||||
operation_version: 1,
|
||||
key_header: "Idempotency-Key".to_owned(),
|
||||
validity_ms: 60_000,
|
||||
}]),
|
||||
);
|
||||
// Reproduce deterministically the durable state the executor commits after an ambiguous attempt
|
||||
// whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and
|
||||
// `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a
|
||||
// running executor against the cancel would not be a repeatable oracle.
|
||||
let lease = store
|
||||
.claim_batch("worker-s2-prep", 1, 10_000)
|
||||
.await
|
||||
.expect("claim")
|
||||
.pop()
|
||||
.expect("one claimed lease");
|
||||
let attempt = store
|
||||
.checkpoint_dispatch(crank_core::BeginAttempt {
|
||||
task_id: acceptance.task_id,
|
||||
owner: lease.owner.clone(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: true,
|
||||
})
|
||||
.await
|
||||
.expect("durable checkpoint");
|
||||
let resolution = store
|
||||
.resolve_attempt(
|
||||
&attempt,
|
||||
crank_core::AttemptOutcome::Uncertain {
|
||||
error: crank_core::TaskEnvelope::try_redacted(
|
||||
&json!({"code": "upstream_ack_lost", "phase": "dispatched"}),
|
||||
)
|
||||
.expect("bounded attempt error"),
|
||||
repeatable: true,
|
||||
phase: crank_core::AttemptPhase::Dispatched,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("attempt resolution");
|
||||
assert!(
|
||||
matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }),
|
||||
"an ambiguous but safely repeatable attempt arms the pinned retry"
|
||||
);
|
||||
|
||||
let cancel = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
match cancel {
|
||||
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
|
||||
assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown)
|
||||
}
|
||||
other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"),
|
||||
}
|
||||
|
||||
// A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not
|
||||
// claimable and no second terminal write is possible.
|
||||
let worker = TaskWorker::new(
|
||||
store.clone(),
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-s2".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
0,
|
||||
"no outbound send after the cancel"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("outcome_unknown")
|
||||
);
|
||||
let attempts_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts_count, 1, "no second durable attempt");
|
||||
let certainty: Option<String> = sqlx::query_scalar(
|
||||
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(certainty.as_deref(), Some("unknown"));
|
||||
let code: Option<String> =
|
||||
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() {
|
||||
use crank_community_mcp::VerifiedIdempotentWrite;
|
||||
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let requests = Arc::clone(&requests);
|
||||
move |headers: axum::http::HeaderMap| {
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
let _ = headers.get("idempotency-key");
|
||||
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
||||
if attempt == 0 {
|
||||
// Applied upstream, acknowledgement lost after the caller had time to observe that
|
||||
// the request really left the worker.
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(json!({"error":"acknowledgement lost"})),
|
||||
);
|
||||
}
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
});
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
||||
mode: IdempotencyMode::Required,
|
||||
ttl_ms: 60_000,
|
||||
input_field: Some("idempotency_key".to_owned()),
|
||||
header_name: Some("Idempotency-Key".to_owned()),
|
||||
});
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(®istry, "cancel-s4").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s4-key",
|
||||
&json!({"email":"u","idempotency_key":"k-4"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
||||
.build();
|
||||
let handler = Arc::new(
|
||||
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
|
||||
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
||||
operation_id: operation.id.as_str().to_owned(),
|
||||
operation_version: 1,
|
||||
key_header: "Idempotency-Key".to_owned(),
|
||||
validity_ms: 60_000,
|
||||
}]),
|
||||
);
|
||||
let worker = TaskWorker::new(
|
||||
store.clone(),
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-s4".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
// Revoke the key the Task was accepted with while the first outbound write is provably in flight:
|
||||
// that attempt still resolves as an uncertain write, and the next attempt is refused by the
|
||||
// authority before its checkpoint.
|
||||
for _ in 0..200 {
|
||||
if requests.load(Ordering::SeqCst) >= 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the first attempt is dispatched before the refusal"
|
||||
);
|
||||
registry
|
||||
.revoke_platform_api_key_for_agent(
|
||||
&common::test_workspace_id(),
|
||||
&common::test_agent_id(AGENT_SLUG),
|
||||
&PlatformApiKeyId::new("pk_cancel-s4".to_owned()),
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.expect("revoke");
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
state = state_of(®istry, acceptance.task_id).await;
|
||||
if state.as_deref() == Some("outcome_unknown") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(
|
||||
state.as_deref(),
|
||||
Some("outcome_unknown"),
|
||||
"a refused attempt after a lost acknowledgement must stay uncertain"
|
||||
);
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the refused attempt is never dispatched"
|
||||
);
|
||||
let attempts_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts_count, 1, "no second durable attempt");
|
||||
let certainty: Option<String> = sqlx::query_scalar(
|
||||
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(certainty.as_deref(), Some("unknown"));
|
||||
let phase: Option<String> = sqlx::query_scalar(
|
||||
"select phase from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(phase.as_deref(), Some("dispatched"));
|
||||
let error: Option<(Option<String>, Option<String>)> = sqlx::query_as(
|
||||
"select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let (code, error_phase) = error.expect("the refused Task carries its bounded error");
|
||||
assert_eq!(code.as_deref(), Some("task_authority_rejected"));
|
||||
assert_eq!(
|
||||
error_phase.as_deref(),
|
||||
Some("dispatched"),
|
||||
"the error keeps the phase of the attempt that may have reached the upstream"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,965 @@
|
||||
//! ATDD Story 4.5 (Beam): durable input continuation through TaskApplication -> PG -> worker -> upstream.
|
||||
|
||||
use super::common;
|
||||
use super::unsampled_correlation;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
|
||||
use crank_core::{
|
||||
NoTasksFallback, PlatformApiKeyScope, TaskId, TaskState, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
|
||||
};
|
||||
use serde_json::json;
|
||||
use sqlx::Row;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
pub(crate) const TOOL_NAME: &str = "report_gen";
|
||||
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
||||
|
||||
pub(crate) async fn state_of(registry: &PostgresRegistry, task_id: TaskId) -> Option<String> {
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_state(
|
||||
registry: &PostgresRegistry,
|
||||
task_id: TaskId,
|
||||
desired: &str,
|
||||
limit: Duration,
|
||||
) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + limit;
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
if state_of(registry, task_id).await.as_deref() == Some(desired) {
|
||||
return true;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
state_of(registry, task_id).await.as_deref() == Some(desired)
|
||||
}
|
||||
|
||||
pub(crate) async fn seed_tool(
|
||||
registry: &PostgresRegistry,
|
||||
upstream: &str,
|
||||
policy: ToolExecutionPolicy,
|
||||
) {
|
||||
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
vec![crank_core::AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn principal(
|
||||
registry: &PostgresRegistry,
|
||||
name: &str,
|
||||
) -> crank_community_mcp::TaskPrincipal {
|
||||
let token = common::create_platform_api_key(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
name,
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("active key");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crank_community_mcp::TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: crank_core::VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store, Arc::new(crypto)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_worker(
|
||||
registry: &PostgresRegistry,
|
||||
app: &TaskApplication,
|
||||
) -> (
|
||||
Arc<std::sync::atomic::AtomicBool>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
) {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
let pool = registry.pool().clone();
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
app,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let worker = TaskWorker::new(
|
||||
store,
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-input".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
renewal_interval: Duration::from_millis(100),
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let jh = tokio::spawn(async move {
|
||||
let _ = worker.run().await;
|
||||
});
|
||||
(stop_flag, jh)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn input_continuation_happy_path_accept_update_resume_replay() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &upstream, policy).await;
|
||||
let caller = principal(®istry, "input-happy").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"input-key-1",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("acceptance");
|
||||
assert_eq!(acceptance.state, TaskState::Queued);
|
||||
let task_id = acceptance.task_id;
|
||||
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(
|
||||
®istry,
|
||||
task_id,
|
||||
"input_required",
|
||||
Duration::from_secs(10)
|
||||
)
|
||||
.await,
|
||||
"worker must suspend for missing email"
|
||||
);
|
||||
// positive control: before update no outbound request
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
// persisted request fields
|
||||
let row = sqlx::query(
|
||||
"select wait_id, expected_version, schema_snapshot, projected_schema, prompt_code, expires_at from task_input_requests where task_id = $1",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let wait_id: String = row.get("wait_id");
|
||||
let expected_version: i32 = row.get("expected_version");
|
||||
let schema_snapshot: serde_json::Value = row.get("schema_snapshot");
|
||||
let projected_schema: serde_json::Value = row.get("projected_schema");
|
||||
let prompt_code: String = row.get("prompt_code");
|
||||
let _expires_at: OffsetDateTime = row.get("expires_at");
|
||||
assert!(!wait_id.is_empty());
|
||||
assert!(expected_version > 0);
|
||||
assert!(schema_snapshot.is_object());
|
||||
assert!(projected_schema.is_object());
|
||||
assert_eq!(prompt_code, "required_input_missing");
|
||||
let audit_count_before: i64 = sqlx::query_scalar(
|
||||
"select count(*)::bigint from task_input_audit where task_id = $1 and event = 'input_required'",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(audit_count_before >= 1);
|
||||
|
||||
// stop worker before update to avoid races
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
let outcome = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version as u32,
|
||||
"update-key-1",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("valid update");
|
||||
assert_eq!(outcome.state, TaskState::Working);
|
||||
assert!(!outcome.replayed);
|
||||
assert_eq!(outcome.task_id, task_id);
|
||||
assert_eq!(outcome.wait_id, wait_id);
|
||||
|
||||
let (stop2, worker_task2) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
||||
"worker must complete after update"
|
||||
);
|
||||
stop2.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let attempts = app
|
||||
.attempts(&caller, TOOL_NAME, task_id)
|
||||
.await
|
||||
.expect("attempts");
|
||||
assert_eq!(attempts.len(), 1);
|
||||
let effective_fp: String =
|
||||
sqlx::query_scalar("select effective_input_fingerprint from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts[0].input_fingerprint, effective_fp);
|
||||
let input_fp: String =
|
||||
sqlx::query_scalar("select input_fingerprint from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_ne!(effective_fp, input_fp);
|
||||
|
||||
// An identical answer replay still returns the committed outcome after the Task completed:
|
||||
// no second transition, audit row or outbound call.
|
||||
let completed_replay = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version as u32,
|
||||
"update-key-1",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("identical replay after completion");
|
||||
assert!(completed_replay.replayed);
|
||||
assert_eq!(completed_replay.version, outcome.version);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
// replay with original scoped key/partial input: same Task, no fingerprint conflict
|
||||
let replay = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"input-key-1",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("replay accepted");
|
||||
assert_eq!(replay.task_id, task_id);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
pub(crate) async fn start_and_suspend(
|
||||
registry: &PostgresRegistry,
|
||||
app: &TaskApplication,
|
||||
caller: &crank_community_mcp::TaskPrincipal,
|
||||
scoped_key: &str,
|
||||
) -> (TaskId, String, u32) {
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
scoped_key,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("acceptance");
|
||||
let (stop, worker_task) = spawn_worker(registry, app);
|
||||
assert!(
|
||||
wait_for_state(
|
||||
registry,
|
||||
acceptance.task_id,
|
||||
"input_required",
|
||||
Duration::from_secs(10)
|
||||
)
|
||||
.await,
|
||||
"worker must suspend for missing email"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
let (wait_id, expected_version) = wait_row(registry, acceptance.task_id).await;
|
||||
(acceptance.task_id, wait_id, expected_version)
|
||||
}
|
||||
|
||||
async fn wait_row(registry: &PostgresRegistry, task_id: TaskId) -> (String, u32) {
|
||||
let row =
|
||||
sqlx::query("select wait_id, expected_version from task_input_requests where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
(
|
||||
row.get::<String, _>("wait_id"),
|
||||
u32::try_from(row.get::<i32, _>("expected_version")).unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn audit_events(
|
||||
registry: &PostgresRegistry,
|
||||
task_id: TaskId,
|
||||
) -> Vec<(String, Option<String>)> {
|
||||
sqlx::query("select event, reason_code from task_input_audit where task_id = $1 order by id")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_all(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|row| (row.get("event"), row.get("reason_code")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn admission_entries(
|
||||
registry: &PostgresRegistry,
|
||||
task_id: TaskId,
|
||||
) -> Vec<(String, i64)> {
|
||||
sqlx::query(
|
||||
"select state, count(*)::bigint as count from execution_admission_entries
|
||||
where task_id = $1 group by state order by state",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_all(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|row| (row.get("state"), row.get("count")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn key_id(principal: &crank_community_mcp::TaskPrincipal) -> crank_core::PlatformApiKeyId {
|
||||
match principal {
|
||||
crank_community_mcp::TaskPrincipal::Agent { credential, .. } => credential
|
||||
.platform_api_key_id
|
||||
.clone()
|
||||
.expect("agent principal carries a key id"),
|
||||
crank_community_mcp::TaskPrincipal::Admin { .. } => panic!("agent principal expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_schema_foreign_scope_and_revoked_key_never_transition_or_dispatch() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "input-scope").await;
|
||||
let app = application(®istry).await;
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-scope").await;
|
||||
|
||||
// Invalid answer: server-side schema validation refuses it, nothing transitions and nothing leaks.
|
||||
let invalid = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-invalid",
|
||||
&json!({}),
|
||||
)
|
||||
.await
|
||||
.expect_err("the pinned schema requires email");
|
||||
assert!(
|
||||
matches!(
|
||||
invalid,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::Invalid { .. })
|
||||
),
|
||||
"unexpected error: {invalid:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert!(
|
||||
audit_events(®istry, task_id)
|
||||
.await
|
||||
.iter()
|
||||
.all(|(event, _)| event == "input_required"),
|
||||
"a rejected answer must not be audited as accepted"
|
||||
);
|
||||
let stored_snapshot: serde_json::Value =
|
||||
sqlx::query_scalar("select schema_snapshot from task_input_requests where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!stored_snapshot.to_string().contains("user@example.com"));
|
||||
|
||||
// A different key of the same agent carries a different authorization scope: opaque not-found.
|
||||
let foreign = principal(®istry, "input-foreign").await;
|
||||
let foreign_result = app
|
||||
.update_input(
|
||||
&foreign,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-foreign",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a foreign scope cannot continue the Task");
|
||||
assert_eq!(
|
||||
foreign_result,
|
||||
crank_community_mcp::TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
|
||||
// Positive control: the accepting principal (same key/scope) completes the transition.
|
||||
let accepted = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-valid",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("valid answer for the accepting scope");
|
||||
assert_eq!(accepted.state, TaskState::Working);
|
||||
assert!(!accepted.replayed);
|
||||
// Finish the positive-control Task through a real worker so the revocation case starts from a
|
||||
// known dispatch count instead of leaving resumable work behind.
|
||||
let (finish_stop, finish_worker) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
||||
"the positive-control Task completes"
|
||||
);
|
||||
finish_stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), finish_worker).await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
// Revocation race: the accepting key is revoked after suspension. The same waiting scope must
|
||||
// fail closed inside the update transaction and never revive or dispatch the Task.
|
||||
let revocable = principal(®istry, "input-revoked").await;
|
||||
let (revoked_task, revoked_wait, revoked_version) =
|
||||
start_and_suspend(®istry, &app, &revocable, "input-key-revoked").await;
|
||||
registry
|
||||
.revoke_platform_api_key(
|
||||
&common::test_workspace_id(),
|
||||
&key_id(&revocable),
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let revoked_result = app
|
||||
.update_input(
|
||||
&revocable,
|
||||
TOOL_NAME,
|
||||
revoked_task,
|
||||
&revoked_wait,
|
||||
revoked_version,
|
||||
"update-revoked",
|
||||
&json!({"email": "revoked@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a revoked key cannot continue the Task");
|
||||
assert_eq!(
|
||||
revoked_result,
|
||||
crank_community_mcp::TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, revoked_task).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the revoked answer must not add an outbound call"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replay_conflict_and_stale_answers_have_one_typed_outcome() {
|
||||
let registry = common::test_registry().await;
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, "http://127.0.0.1:9", policy).await;
|
||||
let caller = principal(®istry, "input-replay").await;
|
||||
let app = application(®istry).await;
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-replay").await;
|
||||
|
||||
let first = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("first valid answer");
|
||||
assert!(!first.replayed);
|
||||
assert_eq!(first.state, TaskState::Working);
|
||||
|
||||
// Identical replay returns the stored outcome without a second CAS/audit.
|
||||
let replay = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("identical replay");
|
||||
assert!(replay.replayed);
|
||||
assert_eq!(replay.version, first.version);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("working")
|
||||
);
|
||||
let accepted_audit = audit_events(®istry, task_id)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|(event, _)| event == "input_accepted")
|
||||
.count();
|
||||
assert_eq!(accepted_audit, 1);
|
||||
|
||||
// Same key with another payload is a fingerprint conflict.
|
||||
let conflict = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-key",
|
||||
&json!({"email": "other@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("same key with another payload conflicts");
|
||||
assert_eq!(
|
||||
conflict,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::FingerprintConflict)
|
||||
);
|
||||
|
||||
// Another key after a committed answer, and a foreign wait id, are state conflicts.
|
||||
let other_key = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
first.version,
|
||||
"update-key-2",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("another key after commit conflicts");
|
||||
assert_eq!(
|
||||
other_key,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
||||
);
|
||||
let foreign_wait = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
"wait-does-not-exist",
|
||||
first.version,
|
||||
"update-key-3",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a foreign wait id conflicts");
|
||||
assert_eq!(
|
||||
foreign_wait,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
||||
);
|
||||
|
||||
// A stale generation on a fresh wait is refused; the valid generation still works.
|
||||
let (stale_task, stale_wait, stale_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-stale").await;
|
||||
let stale = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
stale_task,
|
||||
&stale_wait,
|
||||
stale_version.saturating_sub(1).max(1),
|
||||
"update-stale",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a stale generation cannot be continued");
|
||||
assert_eq!(
|
||||
stale,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
||||
);
|
||||
let repaired = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
stale_task,
|
||||
&stale_wait,
|
||||
stale_version,
|
||||
"update-stale",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("the current generation still accepts the answer");
|
||||
assert!(!repaired.replayed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ttl_expiry_fails_with_audit_and_releases_capacity_without_dispatch() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
// The wait TTL is the remaining pinned absolute execution deadline.
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 1_200,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "input-ttl").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"input-key-ttl",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("acceptance");
|
||||
let task_id = acceptance.task_id;
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "input_required", Duration::from_secs(5)).await,
|
||||
"worker must suspend"
|
||||
);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "failed", Duration::from_secs(6)).await,
|
||||
"the per-poll sweeper must fail the expired wait"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
let (wait_id, expected_version) = wait_row(®istry, task_id).await;
|
||||
let events = audit_events(®istry, task_id).await;
|
||||
assert!(
|
||||
events.iter().any(|(event, reason)| event == "input_expired"
|
||||
&& reason.as_deref() == Some("input_ttl_expired")),
|
||||
"expiry reason must be audited: {events:?}"
|
||||
);
|
||||
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
// A late answer cannot revive the failed Task.
|
||||
let late = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-late",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("an expired wait never revives");
|
||||
assert!(
|
||||
matches!(
|
||||
late,
|
||||
crank_community_mcp::TaskApplicationError::Task(
|
||||
crank_core::TaskError::InputExpired | crank_core::TaskError::StateConflict
|
||||
)
|
||||
),
|
||||
"unexpected late answer error: {late:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("failed")
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn competing_identical_updates_commit_once_and_keep_one_permit() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "input-race").await;
|
||||
let app = application(®istry).await;
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-race").await;
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)],
|
||||
"the waiting Task keeps exactly one executing permit"
|
||||
);
|
||||
|
||||
let payload = json!({"email": "user@example.com"});
|
||||
let first = app.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-race",
|
||||
&payload,
|
||||
);
|
||||
let second = app.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-race",
|
||||
&payload,
|
||||
);
|
||||
let (first, second) = tokio::join!(first, second);
|
||||
let mut outcomes = [
|
||||
first.expect("competing answer"),
|
||||
second.expect("competing answer"),
|
||||
];
|
||||
outcomes.sort_by_key(|outcome| outcome.replayed);
|
||||
assert!(!outcomes[0].replayed);
|
||||
assert!(outcomes[1].replayed);
|
||||
assert_eq!(outcomes[0].version, outcomes[1].version);
|
||||
assert_eq!(
|
||||
audit_events(®istry, task_id)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|(event, _)| event == "input_accepted")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)],
|
||||
"the update must not grant a second permit"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
||||
"the resumed Task completes"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
||||
server.abort();
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
//! ATDD Story 4.5 QA1 guards (F1/F3): one pinned deadline for several waiting Tasks, cancellation
|
||||
//! while the Task waits for input and refusal of a stale lease/fence around the continuation seam.
|
||||
|
||||
use super::common;
|
||||
use super::input_continuation::{
|
||||
TOOL_NAME, admission_entries, application, audit_events, principal, seed_tool, spawn_worker,
|
||||
start_and_suspend, state_of, wait_for_state,
|
||||
};
|
||||
use super::unsampled_correlation;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_core::{
|
||||
BeginAttempt, NoTasksFallback, TASK_INPUT_SCHEMA_FORMAT_VERSION, TaskInputRequest,
|
||||
TaskInputStore, TaskState, TaskStore, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresTaskStore};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_waits_with_one_pinned_deadline_and_cancelled_leftover_do_not_block_suspension() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "deadline-cancel").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acc_a = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"deadline-key-a",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept a");
|
||||
let acc_b = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"deadline-key-b",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept b");
|
||||
let task_a = acc_a.task_id;
|
||||
let task_b = acc_b.task_id;
|
||||
|
||||
let pinned_ms = (OffsetDateTime::now_utc().unix_timestamp() as f64 * 1000.0) + 60_000.0;
|
||||
for tid in [task_a, task_b] {
|
||||
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
|
||||
.bind(pinned_ms)
|
||||
.bind(tid.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_a, "input_required", Duration::from_secs(10)).await,
|
||||
"a must suspend"
|
||||
);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_b, "input_required", Duration::from_secs(10)).await,
|
||||
"b must suspend"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
// Both active waits carry the same absolute TTL value: the per-Task uniqueness above is the only
|
||||
// uniqueness, so this must not be a UNIQUE index on expires_at.
|
||||
let same_deadline_wait: i64 = sqlx::query_scalar(
|
||||
"select count(*)::bigint from task_input_requests a
|
||||
join task_input_requests b on b.task_id = $2
|
||||
where a.task_id = $1 and a.accepted_at is null and b.accepted_at is null
|
||||
and a.expires_at = b.expires_at",
|
||||
)
|
||||
.bind(task_a.as_uuid())
|
||||
.bind(task_b.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
same_deadline_wait, 1,
|
||||
"both waits share the pinned deadline"
|
||||
);
|
||||
|
||||
let outcome = app
|
||||
.cancel(&caller, TOOL_NAME, task_a, None)
|
||||
.await
|
||||
.expect("cancel a");
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
crank_core::TaskCancelOutcome::Cancelled(_)
|
||||
));
|
||||
assert_eq!(
|
||||
state_of(®istry, task_a).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
assert_eq!(admission_entries(®istry, task_a).await, Vec::new());
|
||||
|
||||
// accept C with same pinned deadline
|
||||
let acc_c = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"deadline-key-c",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept c");
|
||||
let task_c = acc_c.task_id;
|
||||
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
|
||||
.bind(pinned_ms)
|
||||
.bind(task_c.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (stop2, worker_task2) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_c, "input_required", Duration::from_secs(10)).await,
|
||||
"c must suspend despite leftover a"
|
||||
);
|
||||
stop2.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
|
||||
|
||||
assert_eq!(
|
||||
state_of(®istry, task_b).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
let c_deadline_ms: i64 = sqlx::query_scalar(
|
||||
"select (extract(epoch from expires_at) * 1000)::bigint
|
||||
from task_input_requests where task_id = $1",
|
||||
)
|
||||
.bind(task_c.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
c_deadline_ms, pinned_ms as i64,
|
||||
"the third wait reuses the same pinned deadline value"
|
||||
);
|
||||
let a_accepted: Option<OffsetDateTime> =
|
||||
sqlx::query_scalar("select accepted_at from task_input_requests where task_id = $1")
|
||||
.bind(task_a.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
a_accepted.is_none(),
|
||||
"cancellation leaves the wait row without an accepted receipt"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_while_input_required_never_revives_or_dispatches() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "cancel-wait").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-cancel").await;
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)]
|
||||
);
|
||||
|
||||
let outcome = app
|
||||
.cancel(&caller, TOOL_NAME, task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
crank_core::TaskCancelOutcome::Cancelled(_)
|
||||
));
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
||||
|
||||
let late = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-late",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("late answer must not revive");
|
||||
assert!(
|
||||
matches!(
|
||||
late,
|
||||
crank_community_mcp::TaskApplicationError::Task(
|
||||
crank_core::TaskError::StateConflict | crank_core::TaskError::InputExpired
|
||||
)
|
||||
),
|
||||
"unexpected: {late:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let events = audit_events(®istry, task_id).await;
|
||||
assert!(events.iter().any(|(e, _)| e == "input_required"));
|
||||
assert!(!events.iter().any(|(e, _)| e == "input_accepted"));
|
||||
let attempts = app
|
||||
.attempts(&caller, TOOL_NAME, task_id)
|
||||
.await
|
||||
.expect("attempts");
|
||||
assert!(attempts.is_empty());
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_lease_fence_cannot_suspend_or_dispatch() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "stale-fence").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"stale-key",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
let task_id = acceptance.task_id;
|
||||
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let mut old_claims = store
|
||||
.claim_batch("worker-old", 1, 50)
|
||||
.await
|
||||
.expect("old claim");
|
||||
assert!(!old_claims.is_empty());
|
||||
let old = old_claims.remove(0);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(120)).await;
|
||||
let mut new_claims = store
|
||||
.claim_batch("worker-new", 1, 10_000)
|
||||
.await
|
||||
.expect("new claim");
|
||||
assert!(!new_claims.is_empty());
|
||||
let new = new_claims.remove(0);
|
||||
assert!(new.fence > old.fence);
|
||||
assert_eq!(new.record.task_id, task_id);
|
||||
|
||||
let request = TaskInputRequest {
|
||||
wait_id: "stale-wait".to_string(),
|
||||
expected_version: old.record.version + 1,
|
||||
schema_revision: 1,
|
||||
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot: json!({}),
|
||||
projected_schema: json!({}),
|
||||
prompt_code: "required_input_missing".to_string(),
|
||||
missing_fields: vec!["$.email".to_string()],
|
||||
expires_at_unix_ms: OffsetDateTime::now_utc().unix_timestamp() * 1000 + 30_000,
|
||||
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
};
|
||||
|
||||
let suspended =
|
||||
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &old, request).await;
|
||||
assert!(
|
||||
suspended.is_err(),
|
||||
"stale fence must not suspend: {suspended:?}"
|
||||
);
|
||||
|
||||
let attempt_id = "attempt-stale".to_string();
|
||||
let checkpoint = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id,
|
||||
owner: old.owner.clone(),
|
||||
fence: old.fence,
|
||||
expected_version: old.record.version,
|
||||
attempt_id,
|
||||
input_fingerprint: old.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
checkpoint.is_err(),
|
||||
"stale fence must not checkpoint: {checkpoint:?}"
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
let attempt_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempt_count, 0);
|
||||
let req_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_input_requests where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(req_count, 0);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
// Positive control: the same request under the current lease suspends the Task exactly once, so
|
||||
// the refusal above is the stale fence and not an invalid request shape.
|
||||
let request = TaskInputRequest {
|
||||
wait_id: "current-wait".to_string(),
|
||||
expected_version: new.record.version + 1,
|
||||
schema_revision: 1,
|
||||
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot: json!({}),
|
||||
projected_schema: json!({}),
|
||||
prompt_code: "required_input_missing".to_string(),
|
||||
missing_fields: vec!["$.email".to_string()],
|
||||
expires_at_unix_ms: new.record.pins.execution_deadline_unix_ms,
|
||||
};
|
||||
let suspended =
|
||||
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &new, request)
|
||||
.await
|
||||
.expect("the current lease suspends");
|
||||
assert_eq!(suspended.state, TaskState::InputRequired);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)]
|
||||
);
|
||||
// The stale fence still cannot checkpoint a dispatch of this generation.
|
||||
let checkpoint = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id,
|
||||
owner: old.owner.clone(),
|
||||
fence: old.fence,
|
||||
expected_version: old.record.version,
|
||||
attempt_id: "attempt-stale-after-suspend".to_string(),
|
||||
input_fingerprint: old.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
checkpoint.is_err(),
|
||||
"stale fence must not checkpoint: {checkpoint:?}"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
@@ -0,0 +1,659 @@
|
||||
//! ATDD Story 4.2 (FR-22/FR-23): the durable Task application entry over a real published catalog
|
||||
//! and a real PostgreSQL schema.
|
||||
//!
|
||||
//! These oracles drive the production application entry (`TaskApplication`) that the mcp-server
|
||||
//! composition root constructs: a verified machine credential is checked against the published Tool,
|
||||
//! the Task is accepted durably, a freshly constructed application instance reads the same identity
|
||||
//! and pins, replay/version conflicts never rebind, foreign callers receive no metadata, and a missing
|
||||
//! Task store fails closed and reports readiness as not ready. The public `tools/call` wire stays
|
||||
//! synchronous/unsupported: this suite never calls it.
|
||||
|
||||
use super::common;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
use crank_core::{
|
||||
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId,
|
||||
TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{
|
||||
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
|
||||
};
|
||||
use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService};
|
||||
use serde_json::json;
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
const TOOL_NAME: &str = "report_gen";
|
||||
const KEY: &str = "retry-key-1";
|
||||
|
||||
fn operation(policy: ToolExecutionPolicy) -> crank_registry::RegistryOperation {
|
||||
let mut operation = common::test_operation("https://api.example.com", TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
operation
|
||||
}
|
||||
|
||||
fn background_policy() -> ToolExecutionPolicy {
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_tool(registry: &PostgresRegistry, policy: ToolExecutionPolicy) {
|
||||
let operation = operation(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &operation.id, 1).await;
|
||||
// A second, explicitly synchronous Tool of the same Agent: the durable entry must refuse it.
|
||||
let synchronous = common::test_operation("https://api.example.com", "report_sync");
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &synchronous, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &synchronous.id, 1).await;
|
||||
let binding = AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
};
|
||||
let synchronous_binding = AgentOperationBinding {
|
||||
operation_id: synchronous.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: synchronous.name.clone(),
|
||||
tool_title: synchronous.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
};
|
||||
common::publish_agent_with_bindings(registry, AGENT_SLUG, vec![binding, synchronous_binding])
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn publish(
|
||||
registry: &PostgresRegistry,
|
||||
operation_id: &crank_core::OperationId,
|
||||
version: u32,
|
||||
) {
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id,
|
||||
version,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
|
||||
let mut next = operation(background_policy());
|
||||
next.version = version;
|
||||
next.status = crank_core::OperationStatus::Draft;
|
||||
next.published_at = None;
|
||||
registry
|
||||
.create_version(CreateVersionRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
snapshot: &next,
|
||||
change_note: Some("next publication"),
|
||||
created_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &next.id, version).await;
|
||||
// The agent catalog must bind the published Tool version for the new publication to be addressed.
|
||||
let agent_id = common::test_agent_id(AGENT_SLUG);
|
||||
let draft = crank_core::AgentVersion {
|
||||
agent_id: agent_id.clone(),
|
||||
version: 2,
|
||||
status: crank_core::AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: Default::default(),
|
||||
created_at: OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
|
||||
};
|
||||
registry
|
||||
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: &draft,
|
||||
bindings: &[AgentOperationBinding {
|
||||
operation_id: next.id.clone(),
|
||||
operation_version: version,
|
||||
tool_name: next.name.clone(),
|
||||
tool_title: next.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
updated_at: &OffsetDateTime::parse("2026-03-26T10:01:00Z", &Rfc3339).unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_agent(crank_registry::PublishAgentRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: 2,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:02:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
||||
|
||||
fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService {
|
||||
let store = PostgresTaskStore::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration");
|
||||
TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto))
|
||||
}
|
||||
|
||||
/// The application entry under the same encrypted-input authority the production process uses.
|
||||
async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key");
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(0),
|
||||
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
task_service(registry.pool().clone(), crypto),
|
||||
)
|
||||
}
|
||||
|
||||
async fn principal(
|
||||
registry: &PostgresRegistry,
|
||||
name: &str,
|
||||
scopes: &[PlatformApiKeyScope],
|
||||
) -> TaskPrincipal {
|
||||
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
|
||||
// The production machine-access path resolves the presented secret against the active platform
|
||||
// key of the addressed Agent and derives the scope identity from that verified record.
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("an active platform key resolves");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: crank_core::VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn unsampled_correlation() -> CorrelationContext {
|
||||
CorrelationContext::new(
|
||||
crank_core::RequestId::generate(),
|
||||
crank_core::TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn counts(pool: &PgPool) -> (i64, i64) {
|
||||
let row = sqlx::query(
|
||||
"select
|
||||
(select count(*)::bigint from tasks) as tasks,
|
||||
(select count(*)::bigint from execution_admission_entries where task_id is not null)
|
||||
as intents",
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(row.get("tasks"), row.get("intents"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-accept",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("acceptance over the real publication");
|
||||
assert_eq!(acceptance.state, TaskState::Queued);
|
||||
assert!(!acceptance.replayed);
|
||||
assert_eq!(acceptance.operation_version, 1);
|
||||
assert_eq!(acceptance.support.request_id, *correlation.request_id());
|
||||
assert_eq!(acceptance.support.trace_id, *correlation.trace_id());
|
||||
assert!(!acceptance.support.trace_sampled);
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(&acceptance.task_id.to_string())
|
||||
.unwrap()
|
||||
.get_version_num(),
|
||||
7,
|
||||
"the durable Task id is UUIDv7"
|
||||
);
|
||||
// Task, dispatch intent and queue slot committed together.
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// A freshly constructed application + service reads the same durable identity and pins.
|
||||
let reader = application(®istry).await;
|
||||
let record = reader
|
||||
.status(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(record.state, TaskState::Queued);
|
||||
assert_eq!(record.pins.operation_version, 1);
|
||||
assert_eq!(record.pins.policy_mode, ToolExecutionMode::Background);
|
||||
assert_eq!(record.support.request_id, acceptance.support.request_id);
|
||||
assert_eq!(record.support.traceparent, acceptance.support.traceparent);
|
||||
assert!(!record.support.trace_sampled);
|
||||
let outcome = reader
|
||||
.result(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(outcome.result.is_none() && outcome.error.is_none());
|
||||
|
||||
// Replay of the same scoped key and input returns the accepted Task instead of a new one.
|
||||
let replay = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replay.task_id, acceptance.task_id);
|
||||
assert!(replay.replayed);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_new_publication_never_rebinds_an_accepted_task() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-rebind",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry).await;
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let before = app
|
||||
.status(&caller, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
publish_tool_version(®istry, 2).await;
|
||||
|
||||
// The same retry keeps the accepted Task and its v1 pin; it never becomes a v2 Task.
|
||||
let replay = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replay.task_id, accepted.task_id);
|
||||
assert_eq!(replay.operation_version, 1);
|
||||
let after = app
|
||||
.status(&caller, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(after.pins.operation_version, 1);
|
||||
assert_eq!(after.support.request_id, before.support.request_id);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// An explicit other exact version is a typed conflict, not a rebind.
|
||||
let explicit = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
Some(2),
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a different explicit pin cannot rebind the accepted Task");
|
||||
assert_eq!(explicit, TaskApplicationError::VersionUnavailable);
|
||||
// The original explicit pin still resolves to the accepted Task.
|
||||
let pinned = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
Some(1),
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(pinned.task_id, accepted.task_id);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let owner = principal(
|
||||
®istry,
|
||||
"task-owner",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry).await;
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&owner,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Positive control: the owner reads its Task.
|
||||
assert!(
|
||||
app.status(&owner, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
// A second verified key of the same Agent is another authorization scope: no payload, no metadata.
|
||||
let foreign_scope =
|
||||
principal(®istry, "task-other-scope", &[PlatformApiKeyScope::Read]).await;
|
||||
match app
|
||||
.status(&foreign_scope, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
{
|
||||
Err(TaskApplicationError::NotFound) => {}
|
||||
other => panic!("foreign scope must be not-found without metadata, got {other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
app.result(&foreign_scope, TOOL_NAME, accepted.task_id)
|
||||
.await,
|
||||
Err(TaskApplicationError::NotFound)
|
||||
));
|
||||
|
||||
// A synchronous publication is not accepted as background work at all.
|
||||
let synchronous_app = application(®istry).await;
|
||||
let refused = synchronous_app
|
||||
.accept_background(
|
||||
&owner,
|
||||
"report_sync",
|
||||
None,
|
||||
"sync-key",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a synchronous policy has no durable acceptance");
|
||||
assert_eq!(refused, TaskApplicationError::NotBackground);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let app = application(®istry).await;
|
||||
let admin = TaskPrincipal::Admin {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
user_id: "user_admin_1".to_owned(),
|
||||
};
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&admin,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("an authenticated administrator accepts durable work");
|
||||
assert_eq!(accepted.state, TaskState::Queued);
|
||||
// Positive control: the same administrator reads its own Task.
|
||||
assert!(
|
||||
app.status(&admin, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
// An administrator is never a wildcard owner: another admin user and the owning Agent of the
|
||||
// same workspace both see nothing.
|
||||
let other_admin = TaskPrincipal::Admin {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
user_id: "user_admin_2".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
app.status(&other_admin, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
app.result(&other_admin, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
let agent = principal(®istry, "task-admin-cross", &[PlatformApiKeyScope::Read]).await;
|
||||
assert_eq!(
|
||||
app.status(&agent, TOOL_NAME, accepted.task_id)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
|
||||
// An admin identity addressed to another workspace cannot resolve the publication at all.
|
||||
let foreign_workspace = TaskPrincipal::Admin {
|
||||
workspace_slug: "other-workspace".to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
user_id: "user_admin_1".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
app.accept_background(
|
||||
&foreign_workspace,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"foreign-key",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
|
||||
let database_url = crank_test_support::postgres_schema_url("durable_tasks_outage").await;
|
||||
let pool = PgPool::connect(&database_url).await.unwrap();
|
||||
// No migration: the durable Task state does not exist.
|
||||
let outage_registry = PostgresRegistry::connect(&database_url).await;
|
||||
assert!(
|
||||
outage_registry.is_err(),
|
||||
"an unmigrated registry cannot be used"
|
||||
);
|
||||
|
||||
crank_registry::MigrationAuthority::apply(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-outage",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry).await;
|
||||
assert!(app.health().await.is_ok());
|
||||
|
||||
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
|
||||
sqlx::query("drop table tasks cascade")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(app.health().await.is_err());
|
||||
let error = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect_err("no durable Task store, no acceptance");
|
||||
assert!(matches!(
|
||||
error,
|
||||
TaskApplicationError::Task(crank_core::TaskError::Unavailable { .. })
|
||||
));
|
||||
|
||||
// Readiness of the app that wires the same store reports the durable Task dependency as not ready.
|
||||
let router = common::build_test_app(registry.clone(), Duration::from_millis(0), None);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
});
|
||||
let response = reqwest::get(format!("http://{address}/ready"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE);
|
||||
let body: serde_json::Value = response.json().await.unwrap();
|
||||
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
/// Commits and resolves exactly one durable attempt of the current lease through the real store.
|
||||
///
|
||||
/// Story 4.4 moved the dispatch checkpoint and the attempt resolution into the runtime seam, so a
|
||||
/// worker-level test double persists exactly one attempt instead of inventing a second retry loop.
|
||||
pub(crate) async fn commit_attempt(
|
||||
context: &crank_runtime::TaskAttemptContext,
|
||||
outcome: crank_core::AttemptOutcome,
|
||||
) -> crank_runtime::TaskAttemptOutcome {
|
||||
use crank_core::{AttemptResolution, BeginAttempt, TaskState};
|
||||
|
||||
let lease = context.lease();
|
||||
let begin = BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: lease.owner.clone(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: lease.record.pins.retry.max_attempts > 1,
|
||||
};
|
||||
match context.store().checkpoint_dispatch(begin).await {
|
||||
Ok(attempt) => match context.store().resolve_attempt(&attempt, outcome).await {
|
||||
Ok(AttemptResolution::Finished(task)) if task.state == TaskState::Completed => {
|
||||
crank_runtime::TaskAttemptOutcome::Completed
|
||||
}
|
||||
Ok(AttemptResolution::Finished(_)) => crank_runtime::TaskAttemptOutcome::Failed {
|
||||
code: "test_failure".to_owned(),
|
||||
},
|
||||
Ok(AttemptResolution::Unknown(_)) => crank_runtime::TaskAttemptOutcome::OutcomeUnknown,
|
||||
Ok(AttemptResolution::RetryAt { .. }) | Err(_) => {
|
||||
crank_runtime::TaskAttemptOutcome::Stopped
|
||||
}
|
||||
},
|
||||
Err(_) => crank_runtime::TaskAttemptOutcome::Stopped,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded success outcome used by the worker-level test doubles.
|
||||
pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::AttemptOutcome {
|
||||
crank_core::AttemptOutcome::Completed(
|
||||
crank_core::TaskEnvelope::try_redacted(&value).expect("test result is bounded"),
|
||||
)
|
||||
}
|
||||
|
||||
mod attempt_guards;
|
||||
mod attempt_trace;
|
||||
mod cancel_chain;
|
||||
mod input_continuation;
|
||||
mod input_continuation_guards;
|
||||
mod shutdown_window;
|
||||
mod task_worker;
|
||||
@@ -0,0 +1,220 @@
|
||||
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
|
||||
//!
|
||||
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
|
||||
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
|
||||
/// does not consume work that is still queued.
|
||||
///
|
||||
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
|
||||
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
|
||||
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
|
||||
#[tokio::test]
|
||||
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
|
||||
// statement keeps the window deterministic without competing for a second connection.
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
|
||||
begin
|
||||
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
||||
perform pg_sleep(1.5);
|
||||
end if;
|
||||
return new;
|
||||
end $$;
|
||||
create trigger delay_task_dispatch_checkpoint before update on tasks
|
||||
for each row execute function delay_task_dispatch_checkpoint();",
|
||||
))
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.expect("checkpoint delay trigger");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let mut accepted = Vec::new();
|
||||
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_stop_window".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: scoped_key.repeat(64),
|
||||
input_fingerprint: scoped_key.repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
accepted.push(store.accept(request).await.expect("PG accepts"));
|
||||
}
|
||||
|
||||
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
|
||||
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let executed = Arc::new(AtomicUsize::new(0));
|
||||
struct GatedHandler {
|
||||
entered: Arc<tokio::sync::Semaphore>,
|
||||
release: Arc<tokio::sync::Semaphore>,
|
||||
executed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for GatedHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
self.entered.add_permits(1);
|
||||
let _ = self.release.acquire().await;
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
// The trigger delays the durable checkpoint; a stop that lands while it is in flight must
|
||||
// finalize failed(shutdown_before_dispatch) without an outbound send.
|
||||
let begin = crank_core::BeginAttempt {
|
||||
task_id: context.lease().record.task_id,
|
||||
owner: context.lease().owner.clone(),
|
||||
fence: context.lease().fence,
|
||||
expected_version: context.lease().record.version,
|
||||
attempt_id: "attempt-stop-window".to_owned(),
|
||||
input_fingerprint: context.lease().record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
};
|
||||
let Ok(attempt) = context.store().checkpoint_dispatch(begin).await else {
|
||||
return TaskAttemptOutcome::Stopped;
|
||||
};
|
||||
if context.is_stopping() {
|
||||
let _ = context
|
||||
.store()
|
||||
.resolve_attempt(
|
||||
&attempt,
|
||||
crank_core::AttemptOutcome::Failed {
|
||||
error: crank_core::TaskEnvelope::try_redacted(&json!({
|
||||
"code": "shutdown_before_dispatch",
|
||||
"phase": "before_dispatch",
|
||||
}))
|
||||
.expect("bounded error envelope"),
|
||||
retryable: false,
|
||||
phase: crank_core::AttemptPhase::BeforeDispatch,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return TaskAttemptOutcome::Failed {
|
||||
code: "shutdown_before_dispatch".to_owned(),
|
||||
};
|
||||
}
|
||||
self.executed.fetch_add(1, Ordering::SeqCst);
|
||||
commit_attempt(&context, completed_outcome(json!({ "ok": true }))).await
|
||||
}
|
||||
}
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
|
||||
entered: revalidate_entered.clone(),
|
||||
release: revalidate_release.clone(),
|
||||
executed: executed.clone(),
|
||||
});
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-stop-window".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 30_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
renewal_interval: Duration::from_secs(20),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(20),
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
|
||||
.await
|
||||
.expect("worker must claim the first Task and enter revalidation")
|
||||
.expect("revalidation gate is open")
|
||||
.forget();
|
||||
|
||||
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
|
||||
revalidate_release.add_permits(1);
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
|
||||
// finalize the Task, but no permit exists yet, so no outbound call can start.
|
||||
let stop_requested = std::time::Instant::now();
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the 30s budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
assert!(
|
||||
stop_requested.elapsed() >= Duration::from_millis(500),
|
||||
"the checkpoint must have still been in flight when the stop arrived"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
executed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"a stop between checkpoint and permit must never reach the upstream adapter"
|
||||
);
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_stop_window".to_owned(),
|
||||
};
|
||||
let record = store.read(&access, accepted[0].task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Failed);
|
||||
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
|
||||
assert!(record.current_attempt_id.is_some());
|
||||
assert_eq!(
|
||||
record
|
||||
.error
|
||||
.as_ref()
|
||||
.and_then(|error| error.value().get("code"))
|
||||
.and_then(|code| code.as_str()),
|
||||
Some("shutdown_before_dispatch")
|
||||
);
|
||||
|
||||
// Work that was still queued is not consumed by the stopped worker.
|
||||
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
|
||||
assert_eq!(queued.state, TaskState::Queued);
|
||||
let recovered = store
|
||||
.claim_batch("worker_after_stop", 4, 10_000)
|
||||
.await
|
||||
.expect("queued work stays claimable");
|
||||
assert_eq!(recovered.len(), 1);
|
||||
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
|
||||
}
|
||||
@@ -0,0 +1,821 @@
|
||||
//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch.
|
||||
//!
|
||||
//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a
|
||||
/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan,
|
||||
/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented.
|
||||
#[tokio::test]
|
||||
async fn task_worker_store_lease_recovery_contract() {
|
||||
let registry = common::test_registry().await;
|
||||
let store = PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration");
|
||||
|
||||
// Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and
|
||||
// leases durable work, preserving the same admission-backed intent.
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "a".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
// This store-level control exercises lease/fence behavior, not the encrypted acceptance path.
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
let leases = store
|
||||
.claim_batch("worker_1", 4, 10_000)
|
||||
.await
|
||||
.expect("claim");
|
||||
assert_eq!(leases.len(), 1);
|
||||
assert_eq!(leases[0].record.task_id, accepted.task_id);
|
||||
let lease = leases[0].clone();
|
||||
|
||||
// A stale or competing owner cannot win CAS with the issued fencing token.
|
||||
assert!(
|
||||
store
|
||||
.renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.write_progress(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence + 1,
|
||||
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
|
||||
assert!(
|
||||
store
|
||||
.checkpoint_dispatch(crank_core::BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker_1".to_owned(),
|
||||
fence: lease.fence + 1,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-stale".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
// The valid owner still commits the dispatch checkpoint before any outbound work.
|
||||
let committed = store
|
||||
.checkpoint_dispatch(crank_core::BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker_1".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-1".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(committed.attempt.attempt_id, "attempt-1");
|
||||
assert_eq!(
|
||||
committed.task.current_attempt_id.as_deref(),
|
||||
Some("attempt-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_worker_runtime_dispatch_and_recovery_contract() {
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let pool = registry.pool().clone();
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
pool.clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// Positive control: PostgreSQL alone accepts the durable Task.
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_runtime".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "r".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
let task_id = accepted.task_id;
|
||||
|
||||
struct RuntimeHandler {
|
||||
observed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl TaskHandler for RuntimeHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.observed.fetch_add(1, Ordering::SeqCst);
|
||||
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
|
||||
observed: observed.clone(),
|
||||
});
|
||||
|
||||
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-1".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(100),
|
||||
renewal_interval: Duration::from_millis(1_000),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_runtime".to_owned(),
|
||||
};
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
if let Ok(record) = store.read(&access, task_id).await {
|
||||
state = Some(record.state);
|
||||
if record.state == TaskState::Completed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state, Some(TaskState::Completed));
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
1,
|
||||
"exactly one outbound attempt"
|
||||
);
|
||||
// Terminal state is not duplicated.
|
||||
let record = store.read(&access, task_id).await.expect("terminal read");
|
||||
assert_eq!(record.state, TaskState::Completed);
|
||||
|
||||
// Bounded shutdown with no fresh work returns promptly.
|
||||
let idle_config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-idle".to_owned(),
|
||||
poll_interval: Duration::from_millis(50),
|
||||
shutdown_timeout: Duration::from_secs(2),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let idle_worker = TaskWorker::new(
|
||||
store.clone(),
|
||||
handler_for_no_work(store.clone()),
|
||||
idle_config,
|
||||
)
|
||||
.expect("valid idle worker");
|
||||
idle_worker.stop();
|
||||
let started = std::time::Instant::now();
|
||||
idle_worker.run().await.expect("idle worker stops");
|
||||
assert!(started.elapsed() <= Duration::from_secs(30));
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call.
|
||||
#[tokio::test]
|
||||
async fn task_worker_revalidation_denial_never_dispatches() {
|
||||
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_denied".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "d".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
|
||||
struct DeniedHandler {
|
||||
observed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for DeniedHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Err(TaskWorkerError::Store(TaskError::invalid(
|
||||
"publication or credential revoked",
|
||||
)))
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.observed.fetch_add(1, Ordering::SeqCst);
|
||||
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
|
||||
observed: observed.clone(),
|
||||
});
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-denied".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_denied".to_owned(),
|
||||
};
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
if let Ok(record) = store.read(&access, accepted.task_id).await {
|
||||
state = Some(record.state);
|
||||
if record.state == TaskState::Failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state, Some(TaskState::Failed));
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"invalidated work never reaches the upstream"
|
||||
);
|
||||
}
|
||||
|
||||
fn handler_for_no_work(
|
||||
_store: std::sync::Arc<PostgresTaskStore>,
|
||||
) -> std::sync::Arc<dyn crank_runtime::TaskHandler> {
|
||||
struct NoWorkHandler;
|
||||
#[async_trait::async_trait]
|
||||
impl crank_runtime::TaskHandler for NoWorkHandler {
|
||||
async fn revalidate(
|
||||
&self,
|
||||
_lease: &crank_core::TaskLease,
|
||||
) -> Result<(), crank_runtime::TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
context: crank_runtime::TaskAttemptContext,
|
||||
) -> crank_runtime::TaskAttemptOutcome {
|
||||
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
std::sync::Arc::new(NoWorkHandler)
|
||||
}
|
||||
|
||||
/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a
|
||||
/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the
|
||||
/// accepted parent trace is continued on the wire.
|
||||
#[tokio::test]
|
||||
async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
|
||||
use axum::Router;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let seen_traceparent = Arc::new(Mutex::new(None::<String>));
|
||||
|
||||
let app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
let seen = Arc::clone(&seen_traceparent);
|
||||
move |headers: HeaderMap| {
|
||||
let calls = Arc::clone(&calls);
|
||||
let seen = Arc::clone(&seen);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(value) = headers
|
||||
.get("traceparent")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
{
|
||||
*seen.lock().unwrap() = Some(value.to_owned());
|
||||
}
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-worker-e2e",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"worker-e2e-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance with executable input");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-e2e".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call");
|
||||
assert_eq!(
|
||||
seen_traceparent.lock().unwrap().as_deref(),
|
||||
Some(correlation.trace_context().traceparent()),
|
||||
"the durable attempt continues the accepted parent trace"
|
||||
);
|
||||
}
|
||||
|
||||
/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as
|
||||
/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically.
|
||||
#[tokio::test]
|
||||
async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-worker-uncertain",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"worker-uncertain-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance with executable input");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-uncertain".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("outcome_unknown") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("outcome_unknown"));
|
||||
// No automatic retry: another authoritative scan never redispaches the uncertain attempt.
|
||||
assert!(
|
||||
store
|
||||
.claim_batch("worker_after", 4, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced
|
||||
/// terminal commit within the 30s budget.
|
||||
#[tokio::test]
|
||||
async fn task_worker_graceful_stop_drains_active_attempt() {
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_drain".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "s".repeat(64),
|
||||
input_fingerprint: "d".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
|
||||
struct SlowHandler {
|
||||
started: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for SlowHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.started.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
let started = Arc::new(AtomicUsize::new(0));
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
|
||||
started: started.clone(),
|
||||
});
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-drain".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(20),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
for _ in 0..100 {
|
||||
if started.load(Ordering::SeqCst) > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started");
|
||||
let stop_requested = std::time::Instant::now();
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the 30s budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
assert!(stop_requested.elapsed() <= Duration::from_secs(30));
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_drain".to_owned(),
|
||||
};
|
||||
let record = store.read(&access, accepted.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Completed);
|
||||
}
|
||||
@@ -20,7 +20,8 @@ use std::{
|
||||
use axum::http::header;
|
||||
use crank_artifacts::ArtifactStore;
|
||||
use crank_core::{
|
||||
Agent, AgentId, AgentStatus, AgentVersion, PlatformApiKeyScope, ToolSelectionPolicy,
|
||||
Agent, AgentId, AgentStatus, AgentVersion, PlatformApiKeyScope, ToolAccessMode, ToolGroup,
|
||||
ToolSearchSettings, ToolSelectionPolicy,
|
||||
ids::{PromptId, ResourceId},
|
||||
prompt::{PromptDefinition, PromptSensitivity},
|
||||
};
|
||||
@@ -47,10 +48,13 @@ const META_CLIENT_INFO: &str = "io.modelcontextprotocol/clientInfo";
|
||||
|
||||
const RESOURCE_LIST_CHANGED: &str = "notifications/resources/list_changed";
|
||||
const PROMPT_LIST_CHANGED: &str = "notifications/prompts/list_changed";
|
||||
const TOOL_LIST_CHANGED: &str = "notifications/tools/list_changed";
|
||||
const SUBSCRIPTIONS_ACKNOWLEDGED: &str = "notifications/subscriptions/acknowledged";
|
||||
const SUBSCRIPTION_ID_KEY: &str = "io.modelcontextprotocol/subscriptionId";
|
||||
|
||||
const AGENT_SLUG: &str = "sales-signals";
|
||||
/// A second published Agent of the same workspace: its catalog must never reach another Agent.
|
||||
const OTHER_AGENT_SLUG: &str = "sales-other";
|
||||
|
||||
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
@@ -254,6 +258,26 @@ impl SignalsEndpoint {
|
||||
tool_bindings: &[crank_core::AgentOperationBinding],
|
||||
resource_bindings: &[AgentResourceBinding],
|
||||
prompt_bindings: &[AgentPromptBinding],
|
||||
) {
|
||||
self.configure_draft_with_policy(
|
||||
version,
|
||||
tool_bindings,
|
||||
&ToolSelectionPolicy::default(),
|
||||
resource_bindings,
|
||||
prompt_bindings,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// The same transition with an explicit Tool selection policy (Story 3.5: the policy is part of
|
||||
/// the served `tools/list` projection).
|
||||
async fn configure_draft_with_policy(
|
||||
&self,
|
||||
version: u32,
|
||||
tool_bindings: &[crank_core::AgentOperationBinding],
|
||||
tool_selection_policy: &ToolSelectionPolicy,
|
||||
resource_bindings: &[AgentResourceBinding],
|
||||
prompt_bindings: &[AgentPromptBinding],
|
||||
) {
|
||||
self.registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
@@ -262,7 +286,7 @@ impl SignalsEndpoint {
|
||||
agent_id: &self.agent_id,
|
||||
agent_version: version,
|
||||
bindings: tool_bindings,
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
tool_selection_policy,
|
||||
resource_bindings,
|
||||
expected_state: None,
|
||||
},
|
||||
@@ -410,6 +434,169 @@ impl SignalsEndpoint {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// The reverse Tool-only commit: the second Tool binding leaves the published snapshot while the
|
||||
/// visible Resource/Prompt projection stays identical.
|
||||
async fn commit_tool_only_revert(&self) {
|
||||
let version = self.next_draft_version().await;
|
||||
self.open_draft(version).await;
|
||||
self.configure_draft(
|
||||
version,
|
||||
&self.tool_bindings(),
|
||||
&[self.notice_binding()],
|
||||
&self.extended_prompts(),
|
||||
)
|
||||
.await;
|
||||
self.publish_version(version).await;
|
||||
}
|
||||
|
||||
/// The search-mode Tool selection policy of this Agent: one group over the bound Tool.
|
||||
fn search_policy(&self) -> ToolSelectionPolicy {
|
||||
ToolSelectionPolicy {
|
||||
mode: ToolAccessMode::Search,
|
||||
groups: vec![ToolGroup {
|
||||
id: "crm".to_owned(),
|
||||
name: "CRM".to_owned(),
|
||||
description: "CRM operations".to_owned(),
|
||||
tool_names: vec![self.operation.name.clone()],
|
||||
}],
|
||||
search: ToolSearchSettings::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A commit that changes only the Tool selection policy: same Tool, Resource and Prompt
|
||||
/// bindings, direct mode -> search mode. The served `tools/list` changes from the bound Tool to
|
||||
/// `search_tools`/`call_tool`, so this is a visible Tool catalog change.
|
||||
async fn commit_policy_only_change(&self) {
|
||||
let version = self.next_draft_version().await;
|
||||
self.open_draft(version).await;
|
||||
self.configure_draft_with_policy(
|
||||
version,
|
||||
&self.tool_bindings(),
|
||||
&self.search_policy(),
|
||||
&[],
|
||||
&self.base_prompts(),
|
||||
)
|
||||
.await;
|
||||
self.publish_version(version).await;
|
||||
}
|
||||
|
||||
/// A search-mode commit that only adds a Tool binding: `tools/list` still serves exactly
|
||||
/// `search_tools`/`call_tool`, so it must not announce a Tools change.
|
||||
async fn commit_search_mode_binding_change(&self) {
|
||||
let version = self.next_draft_version().await;
|
||||
self.open_draft(version).await;
|
||||
self.configure_draft_with_policy(
|
||||
version,
|
||||
&self.both_tool_bindings(),
|
||||
&self.search_policy(),
|
||||
&[],
|
||||
&self.base_prompts(),
|
||||
)
|
||||
.await;
|
||||
self.publish_version(version).await;
|
||||
}
|
||||
|
||||
/// Publish an independent Agent of the same workspace, bound to the first fixture operation, and
|
||||
/// return its read-only MCP client key.
|
||||
async fn publish_other_agent(&self, slug: &str) -> String {
|
||||
let agent_id = test_agent_id(slug);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
self.registry
|
||||
.create_agent(CreateAgentRequest {
|
||||
agent: &Agent {
|
||||
id: agent_id.clone(),
|
||||
workspace_id: test_workspace_id(),
|
||||
slug: slug.to_owned(),
|
||||
display_name: slug.to_owned(),
|
||||
description: "Story 3.5 isolation fixture".to_owned(),
|
||||
status: AgentStatus::Draft,
|
||||
current_draft_version: 1,
|
||||
latest_published_version: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
published_at: None,
|
||||
},
|
||||
version: &AgentVersion {
|
||||
agent_id: agent_id.clone(),
|
||||
version: 1,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: now,
|
||||
},
|
||||
bindings: &[binding_for_operation(&self.operation)],
|
||||
})
|
||||
.await
|
||||
.expect("the second Agent is created");
|
||||
self.registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::now_utc(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.expect("the second Agent publishes");
|
||||
create_platform_api_key(
|
||||
&self.registry,
|
||||
slug,
|
||||
"mcp-signals-other",
|
||||
&[PlatformApiKeyScope::Read],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// A committed Tool-only change of the independent Agent: it binds the second fixture Tool.
|
||||
async fn commit_other_agent_tool_change(&self, slug: &str) {
|
||||
let agent_id = test_agent_id(slug);
|
||||
let version = AgentVersion {
|
||||
agent_id: agent_id.clone(),
|
||||
version: 2,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
};
|
||||
self.registry
|
||||
.create_agent_draft_version(CreateAgentDraftVersionRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: &version,
|
||||
bindings: &[],
|
||||
updated_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("a second draft version of the other Agent opens");
|
||||
self.registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
agent_version: 2,
|
||||
bindings: &self.both_tool_bindings(),
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[],
|
||||
expected_state: None,
|
||||
},
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.expect("the other Agent's second catalog config saves");
|
||||
self.registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
agent_id: &agent_id,
|
||||
version: 2,
|
||||
published_at: &OffsetDateTime::now_utc(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.expect("the other Agent's second version publishes");
|
||||
}
|
||||
}
|
||||
|
||||
/// A live SSE response, read event by event.
|
||||
@@ -714,6 +901,17 @@ fn resource_uris(list: &Value) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn tool_names(list: &Value) -> Vec<String> {
|
||||
let mut names: Vec<String> = list["result"]["tools"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| panic!("tools/list must return a page: {list}"))
|
||||
.iter()
|
||||
.map(|tool| tool["name"].as_str().unwrap().to_owned())
|
||||
.collect();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
/// Write a stand file that only the current user may read (credential files live here).
|
||||
fn write_private(path: &PathBuf, contents: &str) {
|
||||
use std::io::Write as _;
|
||||
|
||||
@@ -127,12 +127,20 @@ async fn legacy_stream_announces_a_committed_catalog_change_after_the_commit() {
|
||||
"resources/list must serve the newly bound Resource: {resources}"
|
||||
);
|
||||
|
||||
// A Tool-only commit keeps the Resource/Prompt projection: no further signal.
|
||||
// A Tool-only commit keeps the Resource/Prompt projection: only a Tools frame follows (Story 3.5).
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let frame = stream
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
assert!(
|
||||
frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {frame}"
|
||||
);
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"signal for a Tool-only commit",
|
||||
"a Resource or Prompt signal for a Tool-only commit",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -162,7 +170,11 @@ async fn modern_listen_stream_acknowledges_then_announces_coalesced_changes() {
|
||||
);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "promptsListChanged": true, "resourcesListChanged": true }),
|
||||
json!({
|
||||
"promptsListChanged": true,
|
||||
"resourcesListChanged": true,
|
||||
"toolsListChanged": true
|
||||
}),
|
||||
"only the implemented list-change types are acknowledged: {acknowledgement}"
|
||||
);
|
||||
|
||||
@@ -173,7 +185,7 @@ async fn modern_listen_stream_acknowledges_then_announces_coalesced_changes() {
|
||||
endpoint.commit_tool_only_change().await;
|
||||
|
||||
let mut methods = Vec::new();
|
||||
for _ in 0..2 {
|
||||
for _ in 0..3 {
|
||||
let frame = stream
|
||||
.expect_frame("the coalesced list-change signal")
|
||||
.await;
|
||||
@@ -185,7 +197,8 @@ async fn modern_listen_stream_acknowledges_then_announces_coalesced_changes() {
|
||||
methods,
|
||||
vec![
|
||||
PROMPT_LIST_CHANGED.to_owned(),
|
||||
RESOURCE_LIST_CHANGED.to_owned()
|
||||
RESOURCE_LIST_CHANGED.to_owned(),
|
||||
TOOL_LIST_CHANGED.to_owned()
|
||||
]
|
||||
);
|
||||
stream
|
||||
@@ -228,8 +241,8 @@ async fn modern_listen_delivers_only_the_negotiated_signal_types() {
|
||||
);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "promptsListChanged": true }),
|
||||
"only the implemented and requested type is acknowledged: {acknowledgement}"
|
||||
json!({ "promptsListChanged": true, "toolsListChanged": true }),
|
||||
"only the implemented and requested types are acknowledged: {acknowledgement}"
|
||||
);
|
||||
|
||||
// The watcher takes its baseline before it can report anything.
|
||||
@@ -256,7 +269,7 @@ async fn modern_listen_delivers_only_the_negotiated_signal_types() {
|
||||
json!("prompts-only")
|
||||
);
|
||||
|
||||
// Neither `toolsListChanged` nor `resourceSubscriptions` were honoured, so nothing else follows.
|
||||
// No Tool commit happened, so no Tools frame follows; `resourceSubscriptions` was not honoured.
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
@@ -354,6 +367,11 @@ async fn a_lost_notification_is_recovered_through_the_authoritative_methods() {
|
||||
discover["result"]["capabilities"]["resources"]["listChanged"],
|
||||
json!(true)
|
||||
);
|
||||
assert_eq!(
|
||||
discover["result"]["capabilities"]["tools"]["listChanged"],
|
||||
json!(true),
|
||||
"the Tools list-change capability is advertised: {discover}"
|
||||
);
|
||||
let resources = endpoint.modern("resources/list", json!({})).await;
|
||||
assert!(
|
||||
resource_uris(&resources)
|
||||
@@ -414,3 +432,360 @@ async fn a_lost_notification_is_recovered_through_the_authoritative_methods() {
|
||||
"re-initialization serves the authoritative catalog: {prompts}"
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Story 3.5 (FR-14): Tool catalog change signals. A client that negotiated the Tools list-changed
|
||||
// notification learns about a *committed* Tool catalog change, and `tools/list` stays the
|
||||
// authoritative source for a client without the notification or after a reconnect.
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
/// AT-3.5-01/02 (modern): the Tools signal is delivered only to the stream that negotiated it, once
|
||||
/// per committed change, and `tools/list` serves the same committed snapshot.
|
||||
#[tokio::test]
|
||||
async fn modern_tools_list_changed_is_delivered_only_to_a_client_that_asks_for_it() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
let mut tools = endpoint
|
||||
.listen(json!(51), json!({ "toolsListChanged": true }))
|
||||
.await;
|
||||
let mut prompts = endpoint
|
||||
.listen(json!(52), json!({ "promptsListChanged": true }))
|
||||
.await;
|
||||
|
||||
let acknowledgement = tools.expect_frame("the Tools acknowledgment").await;
|
||||
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "toolsListChanged": true }),
|
||||
"the Tools list-change type must be honoured and acknowledged: {acknowledgement}"
|
||||
);
|
||||
let other = prompts.expect_frame("the Prompts acknowledgment").await;
|
||||
assert_eq!(
|
||||
other["params"]["notifications"],
|
||||
json!({ "promptsListChanged": true }),
|
||||
"a stream that did not ask for Tools is acknowledged without it: {other}"
|
||||
);
|
||||
|
||||
// The watchers take their baseline before they can report anything.
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
|
||||
// A draft-only edit is not a commit: nothing is announced.
|
||||
endpoint.edit_draft_without_publishing().await;
|
||||
tools
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"Tools signal for a draft-only edit",
|
||||
)
|
||||
.await;
|
||||
|
||||
// A committed Resource/Prompt change is not a Tools change: the Tools stream stays silent while
|
||||
// the Prompts stream receives exactly its own type.
|
||||
endpoint.commit_resource_and_prompt_change().await;
|
||||
let prompt_frame = prompts
|
||||
.expect_frame("the committed Prompt list-change signal")
|
||||
.await;
|
||||
assert_eq!(prompt_frame["method"], PROMPT_LIST_CHANGED);
|
||||
tools
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"Tools signal for a Resource/Prompt commit",
|
||||
)
|
||||
.await;
|
||||
|
||||
// One committed Tool catalog change is announced exactly once.
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let frame = tools
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["jsonrpc"], json!("2.0"));
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
assert_eq!(frame["params"]["_meta"][SUBSCRIPTION_ID_KEY], json!(51));
|
||||
tools
|
||||
.expect_silence(Duration::from_millis(2_500), "a repeated Tools signal")
|
||||
.await;
|
||||
|
||||
// The prompts-only stream was never told about Tools, even though the commit happened after it
|
||||
// opened.
|
||||
prompts
|
||||
.expect_silence(Duration::from_millis(1_200), "unrequested Tools signal")
|
||||
.await;
|
||||
|
||||
// `tools/list` is authoritative for the same committed state, notification or not.
|
||||
let list = endpoint.modern("tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()]
|
||||
);
|
||||
}
|
||||
|
||||
/// AT-3.5-01/02 (legacy): the legacy `GET` stream carries the Tools signal after the commit; a
|
||||
/// Resource/Prompt commit produces no Tools frame; a committed change of another Agent is never
|
||||
/// disclosed on this stream, and `tools/list` stays authoritative.
|
||||
#[tokio::test]
|
||||
async fn legacy_stream_announces_a_committed_tools_change_and_never_another_agents() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
let other_key = endpoint.publish_other_agent(OTHER_AGENT_SLUG).await;
|
||||
let session = endpoint.legacy_session().await;
|
||||
|
||||
let mut stream = endpoint.legacy_stream(&session).await;
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
stream
|
||||
.expect_silence(Duration::from_millis(1_200), "signal before any commit")
|
||||
.await;
|
||||
|
||||
// A committed Resource/Prompt change keeps the Tool binding set: no Tools frame.
|
||||
endpoint.commit_resource_and_prompt_change().await;
|
||||
let mut methods = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let frame = stream.expect_frame("the committed catalog signals").await;
|
||||
assert!(
|
||||
frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {frame}"
|
||||
);
|
||||
methods.push(frame["method"].as_str().unwrap().to_owned());
|
||||
}
|
||||
methods.sort();
|
||||
assert_eq!(
|
||||
methods,
|
||||
vec![
|
||||
PROMPT_LIST_CHANGED.to_owned(),
|
||||
RESOURCE_LIST_CHANGED.to_owned()
|
||||
],
|
||||
"a Resource/Prompt commit must not announce Tools"
|
||||
);
|
||||
|
||||
// A committed Tool change of another Agent of the same workspace never reaches this stream or
|
||||
// this Agent's `tools/list`.
|
||||
endpoint
|
||||
.commit_other_agent_tool_change(OTHER_AGENT_SLUG)
|
||||
.await;
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"Tools signal for another Agent's commit",
|
||||
)
|
||||
.await;
|
||||
let isolated = endpoint.legacy(&session, "tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&isolated),
|
||||
vec!["crm_create_lead".to_owned()],
|
||||
"another Agent's commit must not change this Agent's catalog: {isolated}"
|
||||
);
|
||||
// The commit really happened: the other Agent serves its own two-Tool catalog.
|
||||
let other_url = agent_mcp_url(&endpoint.base_url, OTHER_AGENT_SLUG);
|
||||
let other_session = initialize_session(&endpoint.client, &other_url, &other_key).await;
|
||||
let other_list = endpoint
|
||||
.client
|
||||
.post(&other_url)
|
||||
.header(header::ACCEPT, "application/json, text/event-stream")
|
||||
.header(header::AUTHORIZATION, format!("Bearer {other_key}"))
|
||||
.header("MCP-Protocol-Version", LEGACY_REVISION)
|
||||
.header("MCP-Session-Id", &other_session)
|
||||
.json(&json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 7,
|
||||
"method": "tools/list",
|
||||
"params": {}
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let other_list: Value = serde_json::from_str(&other_list.text().await.unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
tool_names(&other_list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
|
||||
"the other Agent's committed catalog changed: {other_list}"
|
||||
);
|
||||
|
||||
// This Agent's committed Tool change is announced after the commit, exactly once.
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let frame = stream
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
assert!(
|
||||
frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {frame}"
|
||||
);
|
||||
stream
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a repeated or unrequested signal",
|
||||
)
|
||||
.await;
|
||||
|
||||
let list = endpoint.legacy(&session, "tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
|
||||
"tools/list must serve the committed catalog without the notification: {list}"
|
||||
);
|
||||
}
|
||||
|
||||
/// AT-3.5-04: the Tools signal follows the same projection `tools/list` serves, so a committed
|
||||
/// publication that only changes the Tool selection policy is announced on both revisions, while a
|
||||
/// committed binding change that leaves `tools/list` unchanged is silent.
|
||||
#[tokio::test]
|
||||
async fn a_policy_only_commit_is_announced_on_both_revisions_and_an_invisible_change_is_silent() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
let direct_projection = vec!["crm_create_lead".to_owned()];
|
||||
// `tool_names` sorts the served page.
|
||||
let search_projection = vec!["call_tool".to_owned(), "search_tools".to_owned()];
|
||||
|
||||
// Both revisions negotiate the Tools signal and take their baseline before anything can arrive.
|
||||
let mut modern = endpoint
|
||||
.listen(json!(61), json!({ "toolsListChanged": true }))
|
||||
.await;
|
||||
let acknowledgement = modern.expect_frame("the Tools acknowledgment").await;
|
||||
assert_eq!(acknowledgement["method"], SUBSCRIPTIONS_ACKNOWLEDGED);
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "toolsListChanged": true })
|
||||
);
|
||||
let session = endpoint.legacy_session().await;
|
||||
let mut legacy = endpoint.legacy_stream(&session).await;
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
|
||||
// The served projection before the commit is the direct-mode binding itself.
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.modern("tools/list", json!({})).await),
|
||||
direct_projection
|
||||
);
|
||||
|
||||
// A policy-only commit: identical Tool/Resource/Prompt bindings, direct -> search mode. The
|
||||
// served list changes, so each negotiated stream receives exactly one Tools frame.
|
||||
endpoint.commit_policy_only_change().await;
|
||||
let modern_frame = modern
|
||||
.expect_frame("the committed Tools list-change signal (modern)")
|
||||
.await;
|
||||
assert_eq!(modern_frame["method"], TOOL_LIST_CHANGED);
|
||||
assert_eq!(
|
||||
modern_frame["params"]["_meta"][SUBSCRIPTION_ID_KEY],
|
||||
json!(61)
|
||||
);
|
||||
let legacy_frame = legacy
|
||||
.expect_frame("the committed Tools list-change signal (legacy)")
|
||||
.await;
|
||||
assert_eq!(legacy_frame["method"], TOOL_LIST_CHANGED);
|
||||
assert!(
|
||||
legacy_frame.get("params").is_none(),
|
||||
"a legacy list-change notification carries no parameters: {legacy_frame}"
|
||||
);
|
||||
modern
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a repeated modern Tools signal",
|
||||
)
|
||||
.await;
|
||||
legacy
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a repeated legacy Tools signal",
|
||||
)
|
||||
.await;
|
||||
|
||||
// `tools/list` is authoritative for the new projection on both revisions.
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.modern("tools/list", json!({})).await),
|
||||
search_projection
|
||||
);
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.legacy(&session, "tools/list", json!({})).await),
|
||||
search_projection
|
||||
);
|
||||
|
||||
// The inverse direction: in search mode a binding-only commit does not change `tools/list`, so
|
||||
// neither stream may claim a visible Tool change.
|
||||
endpoint.commit_search_mode_binding_change().await;
|
||||
modern
|
||||
.expect_silence(
|
||||
Duration::from_millis(3_500),
|
||||
"a Tools signal for an unchanged modern tools/list",
|
||||
)
|
||||
.await;
|
||||
legacy
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a Tools signal for an unchanged legacy tools/list",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
tool_names(&endpoint.modern("tools/list", json!({})).await),
|
||||
search_projection
|
||||
);
|
||||
}
|
||||
|
||||
/// AT-3.5-02/03: a client that missed the notification recovers Tool discovery from the authoritative
|
||||
/// state — the modern stateless client after a reconnect, and the legacy client after re-initializing
|
||||
/// a retired session — and a reconnected stream never replays the missed change.
|
||||
#[tokio::test]
|
||||
async fn a_reconnected_client_recovers_tools_discovery_from_the_authoritative_state() {
|
||||
let endpoint = signals_endpoint().await;
|
||||
|
||||
// Modern: the commit happens while no stream is open at all, then a client reconnects.
|
||||
endpoint.commit_tool_only_change().await;
|
||||
let mut reconnected = endpoint
|
||||
.listen(
|
||||
json!("after-reconnect"),
|
||||
json!({ "toolsListChanged": true }),
|
||||
)
|
||||
.await;
|
||||
let acknowledgement = reconnected.expect_frame("the Tools acknowledgment").await;
|
||||
assert_eq!(
|
||||
acknowledgement["params"]["notifications"],
|
||||
json!({ "toolsListChanged": true })
|
||||
);
|
||||
// The baseline of a new stream is the committed state: the missed change is not replayed.
|
||||
reconnected
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a replayed Tools signal after reconnect",
|
||||
)
|
||||
.await;
|
||||
let list = endpoint.modern("tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned(), "crm_create_second".to_owned()],
|
||||
"a client without the notification recovers the committed catalog: {list}"
|
||||
);
|
||||
|
||||
// Legacy: one more committed Tool change is announced on the live stream, then the session is
|
||||
// retired and the client re-initializes.
|
||||
let session = endpoint.legacy_session().await;
|
||||
let mut stale = endpoint.legacy_stream(&session).await;
|
||||
tokio::time::sleep(Duration::from_millis(1_500)).await;
|
||||
endpoint.commit_tool_only_revert().await;
|
||||
let frame = stale
|
||||
.expect_frame("the committed Tools list-change signal")
|
||||
.await;
|
||||
assert_eq!(frame["method"], TOOL_LIST_CHANGED);
|
||||
|
||||
let deleted = endpoint
|
||||
.client
|
||||
.delete(&endpoint.mcp_url)
|
||||
.header(
|
||||
header::AUTHORIZATION,
|
||||
format!("Bearer {}", endpoint.api_key),
|
||||
)
|
||||
.header("MCP-Protocol-Version", LEGACY_REVISION)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(deleted.status(), reqwest::StatusCode::NO_CONTENT);
|
||||
|
||||
let fresh = endpoint.legacy_session().await;
|
||||
let list = endpoint.legacy(&fresh, "tools/list", json!({})).await;
|
||||
assert_eq!(
|
||||
tool_names(&list),
|
||||
vec!["crm_create_lead".to_owned()],
|
||||
"re-initialization serves the committed catalog: {list}"
|
||||
);
|
||||
// A stream reopened after the reconnect starts from the committed baseline, not from a replay.
|
||||
let mut reopened = endpoint.legacy_stream(&fresh).await;
|
||||
reopened
|
||||
.expect_silence(
|
||||
Duration::from_millis(2_500),
|
||||
"a replayed Tools signal on a reconnected stream",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
FROM node:22-alpine AS build
|
||||
|
||||
# The CI host uplink drops TLS 1.3 ClientHello carrying the hybrid
|
||||
# X25519MLKEM768 key share; pin the npm stage to TLS 1.2 until that network
|
||||
# defect is fixed. Certificates, SRI and the lockfile are unaffected.
|
||||
ENV NODE_OPTIONS=--tls-max-v1.2
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY apps/ui/package.json apps/ui/package-lock.json ./
|
||||
|
||||
@@ -1036,6 +1036,7 @@ var TRANSLATIONS = {
|
||||
'wizard.error.secret_name_required': 'Secret name is required.',
|
||||
'wizard.error.secret_value_required': 'Secret value is required.',
|
||||
'wizard.error.save_upstream_first': 'Save the API host after creating a new authorization profile before continuing.',
|
||||
'wizard.error.execution_policy_invalid': 'Execution policy is invalid: mode, deadline from 1 to 3600000 ms and no-Tasks fallback are required.',
|
||||
'wizard.toast.auth_profile_created_title': 'Authorization profile created',
|
||||
'wizard.toast.auth_profile_created_body': 'New authorization profile is ready and selected.',
|
||||
'wizard.toast.auth_profile_error_title': 'Authorization profile creation failed',
|
||||
@@ -2353,6 +2354,7 @@ var TRANSLATIONS = {
|
||||
'wizard.error.secret_name_required': 'Нужно указать имя секрета.',
|
||||
'wizard.error.secret_value_required': 'Нужно указать значение секрета.',
|
||||
'wizard.error.save_upstream_first': 'Сначала сохраните API-хост после создания нового профиля авторизации.',
|
||||
'wizard.error.execution_policy_invalid': 'Политика выполнения некорректна: нужны режим, deadline от 1 до 3600000 мс и поведение без Tasks.',
|
||||
'wizard.toast.auth_profile_created_title': 'Профиль авторизации создан',
|
||||
'wizard.toast.auth_profile_created_body': 'Новый профиль авторизации готов и уже выбран.',
|
||||
'wizard.toast.auth_profile_error_title': 'Не удалось создать профиль авторизации',
|
||||
|
||||
@@ -214,10 +214,33 @@ function parseExecutionConfig(text) {
|
||||
auth_profile_ref: authProfileRef,
|
||||
headers: headers,
|
||||
};
|
||||
var executionPolicy = parseExecutionPolicy(value.execution_policy);
|
||||
if (executionPolicy) {
|
||||
config.execution_policy = executionPolicy;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
// Story 4.1: the published execution policy must survive a wizard save. The editor is a free-form
|
||||
// document, so unknown/legacy shapes stay untouched: only a fully valid policy is projected, and an
|
||||
// explicit invalid value is reported instead of being silently replaced by defaults.
|
||||
function parseExecutionPolicy(raw) {
|
||||
if (raw === undefined || raw === null) return null;
|
||||
if (typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
throw new Error(tKey('wizard.error.execution_policy_invalid'));
|
||||
}
|
||||
var mode = raw.mode === undefined ? 'synchronous' : raw.mode;
|
||||
var fallback = raw.no_tasks_fallback === undefined ? 'synchronous' : raw.no_tasks_fallback;
|
||||
var deadline = Number(raw.deadline_ms);
|
||||
if (['synchronous', 'background', 'adaptive'].indexOf(mode) < 0
|
||||
|| ['synchronous', 'unsupported'].indexOf(fallback) < 0
|
||||
|| !Number.isInteger(deadline) || deadline < 1 || deadline > 3600000) {
|
||||
throw new Error(tKey('wizard.error.execution_policy_invalid'));
|
||||
}
|
||||
return { mode: mode, deadline_ms: deadline, no_tasks_fallback: fallback };
|
||||
}
|
||||
|
||||
function currentUpstream() {
|
||||
var selected = upstreams.find(function(item) { return item.id === selectedUpstreamId; });
|
||||
if (selected) return selected;
|
||||
@@ -415,6 +438,13 @@ function executionConfigToEditorValue(config) {
|
||||
var value = {
|
||||
timeout_ms: config.timeout_ms,
|
||||
};
|
||||
if (config.execution_policy) {
|
||||
value.execution_policy = {
|
||||
mode: config.execution_policy.mode,
|
||||
deadline_ms: config.execution_policy.deadline_ms,
|
||||
no_tasks_fallback: config.execution_policy.no_tasks_fallback,
|
||||
};
|
||||
}
|
||||
if (config.retry_policy) {
|
||||
value.retry = { max_attempts: config.retry_policy.max_attempts };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
const { test, expect } = require('@playwright/test');
|
||||
const { getCurrentWorkspace, login } = require('./helpers');
|
||||
|
||||
// Story 4.1 (FR-21): the published execution policy is part of the Tool version and must survive a
|
||||
// wizard save. The editor is a free-form document, so a no-op edit must not erase it and an explicit
|
||||
// invalid value must be reported instead of being silently replaced by defaults.
|
||||
function draftSnapshot(operationId) {
|
||||
return {
|
||||
id: operationId,
|
||||
name: operationId,
|
||||
display_name: 'Policy rates',
|
||||
category: 'general',
|
||||
protocol: 'rest',
|
||||
target: {
|
||||
kind: 'rest',
|
||||
base_url: 'https://api.example.test',
|
||||
method: 'GET',
|
||||
path_template: '/rates',
|
||||
static_headers: {},
|
||||
},
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
description: null,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: null,
|
||||
fields: {},
|
||||
items: null,
|
||||
enum_values: [],
|
||||
variants: [],
|
||||
},
|
||||
output_schema: {
|
||||
type: 'object',
|
||||
description: null,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: null,
|
||||
fields: {},
|
||||
items: null,
|
||||
enum_values: [],
|
||||
variants: [],
|
||||
},
|
||||
input_mapping: { rules: [] },
|
||||
output_mapping: { rules: [{ source: '$.response.body.rate', target: '$.output.rate', required: true }] },
|
||||
execution_config: {
|
||||
timeout_ms: 10000,
|
||||
retry_policy: null,
|
||||
auth_profile_ref: null,
|
||||
headers: {},
|
||||
approval_policy: null,
|
||||
execution_policy: {
|
||||
mode: 'adaptive',
|
||||
deadline_ms: 45000,
|
||||
no_tasks_fallback: 'unsupported',
|
||||
},
|
||||
},
|
||||
tool_description: { title: 'Policy rates', description: 'Get rates.', tags: [], examples: [] },
|
||||
wizard_state: { input_sample: {}, output_sample: {}, test_input: {}, import_findings: [] },
|
||||
};
|
||||
}
|
||||
|
||||
test('wizard round-trips the published execution policy and rejects an invalid edit', async ({ page }) => {
|
||||
await login(page);
|
||||
const workspace = await getCurrentWorkspace(page);
|
||||
const operationId = 'policy_rates_41';
|
||||
const operationPath = `/api/admin/workspaces/${encodeURIComponent(workspace.id)}/operations/${operationId}`;
|
||||
const savedPayloads = [];
|
||||
|
||||
await page.route(operationPath, async (route) => {
|
||||
if (route.request().method() === 'PATCH') {
|
||||
savedPayloads.push(route.request().postDataJSON());
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ operation_id: operationId, workspace_id: workspace.id, version: 2, status: 'draft' }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
id: operationId,
|
||||
workspace_id: workspace.id,
|
||||
name: operationId,
|
||||
display_name: 'Policy rates',
|
||||
category: 'general',
|
||||
protocol: 'rest',
|
||||
status: 'draft',
|
||||
current_draft_version: 1,
|
||||
draft_version_ref: { version: 1, status: 'draft' },
|
||||
published_version_ref: null,
|
||||
agent_refs: [],
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route(`${operationPath}/versions/*`, async (route) => {
|
||||
await route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
operation_id: operationId,
|
||||
workspace_id: workspace.id,
|
||||
version: 1,
|
||||
status: 'draft',
|
||||
snapshot: draftSnapshot(operationId),
|
||||
}),
|
||||
});
|
||||
});
|
||||
await page.route('**/operations/analyze-quality', async (route) => {
|
||||
await route.fulfill({ contentType: 'application/json', body: JSON.stringify({ blocking: false, findings: [] }) });
|
||||
});
|
||||
page.on('dialog', (dialog) => dialog.accept());
|
||||
|
||||
await page.goto(`/wizard/?mode=edit&operationId=${operationId}`);
|
||||
await expect(page.locator('#tool-display-name')).toHaveValue('Policy rates');
|
||||
|
||||
const editorValue = await page.locator('#tool-exec-config').inputValue();
|
||||
expect(editorValue).toContain('execution_policy');
|
||||
expect(editorValue).toContain('adaptive');
|
||||
expect(editorValue).toContain('45000');
|
||||
expect(editorValue).toContain('unsupported');
|
||||
|
||||
// A real edit still saves the pinned policy unchanged.
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(4));
|
||||
await page.locator('#tool-display-name').fill('Policy rates edited');
|
||||
await page.locator('.btn-save-draft').click();
|
||||
await expect.poll(() => savedPayloads.length).toBe(1);
|
||||
expect(savedPayloads[0].execution_config.execution_policy).toEqual({
|
||||
mode: 'adaptive',
|
||||
deadline_ms: 45000,
|
||||
no_tasks_fallback: 'unsupported',
|
||||
});
|
||||
|
||||
// An explicit invalid policy is refused instead of being silently rewritten to defaults.
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
|
||||
await expect(page.locator('#tool-exec-config')).toBeVisible();
|
||||
await page.locator('#tool-exec-config').fill('timeout_ms: 10000\nexecution_policy:\n mode: background\n deadline_ms: 0\n');
|
||||
await page.locator('.btn-save-draft').click();
|
||||
await expect(page.locator('#tool-exec-config')).toHaveValue(/deadline_ms: 0/);
|
||||
expect(savedPayloads.length).toBe(1);
|
||||
});
|
||||
@@ -21,6 +21,7 @@ crank-schema = { path = "../crank-schema" }
|
||||
crank-trace = { path = "../crank-trace" }
|
||||
futures-util = "0.3"
|
||||
reqwest.workspace = true
|
||||
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
@@ -32,6 +33,7 @@ tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rmcp = { version = "=3.7.0", features = ["transport-streamable-http-server"] }
|
||||
crank-mapping = { path = "../crank-mapping" }
|
||||
crank-test-support = { path = "../crank-test-support" }
|
||||
metrics-util = "0.20.4"
|
||||
|
||||
@@ -272,7 +272,10 @@ fn record_unknown_approval_access_denial(reason: &'static str) {
|
||||
);
|
||||
}
|
||||
|
||||
fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool {
|
||||
pub(super) fn allows_scope(
|
||||
scopes: &[PlatformApiKeyScope],
|
||||
required_scope: PlatformApiKeyScope,
|
||||
) -> bool {
|
||||
match required_scope {
|
||||
PlatformApiKeyScope::Read => scopes.iter().any(|scope| {
|
||||
matches!(
|
||||
|
||||
@@ -34,6 +34,18 @@ pub(crate) mod lifecycle;
|
||||
mod metrics;
|
||||
mod request;
|
||||
mod response;
|
||||
#[cfg(test)]
|
||||
mod rmcp_http;
|
||||
#[cfg(test)]
|
||||
#[path = "rmcp_http_tests.rs"]
|
||||
mod rmcp_http_tests;
|
||||
#[cfg(test)]
|
||||
mod rmcp_session_tests;
|
||||
#[cfg(test)]
|
||||
mod rmcp_tools;
|
||||
#[cfg(test)]
|
||||
#[path = "rmcp_tools_tests.rs"]
|
||||
mod rmcp_tools_tests;
|
||||
mod stages;
|
||||
mod tool_resolution;
|
||||
pub(crate) use self::approval_policy::{
|
||||
@@ -51,13 +63,15 @@ mod auth_resolution;
|
||||
mod dispatch;
|
||||
mod endpoint;
|
||||
mod legacy_dispatch;
|
||||
|
||||
mod task_worker;
|
||||
mod tasks;
|
||||
use self::approvals::{
|
||||
approve_request, deny_request, get_approval_request, list_pending_approvals,
|
||||
};
|
||||
pub(super) use self::auth_resolution::resolve_operation_auth;
|
||||
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
|
||||
|
||||
pub use self::task_worker::{TaskWorkerHandler, VerifiedIdempotentWrite};
|
||||
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
|
||||
// through `super`, and the plain lib build does not use them (same pattern as
|
||||
// `#[cfg(test)] use invocation_history::observe_invocation_history_outcome;` above).
|
||||
@@ -92,6 +106,9 @@ pub(super) struct AppState {
|
||||
pub(super) signal_streams: Arc<crate::protocol::StreamRegistry>,
|
||||
/// Story 3.4: the bounded table of the `tools/call` requests a legacy client can still cancel.
|
||||
pub(super) in_flight: Arc<self::lifecycle::InFlightCalls>,
|
||||
/// Story 4.2: durable background Tasks over the same registry pool. The public wire stays
|
||||
/// synchronous/unsupported; readiness depends on this mandatory store.
|
||||
tasks: Arc<TaskApplication>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -137,6 +154,7 @@ pub fn build_app(
|
||||
DEFAULT_MAX_CONCURRENT_SESSIONS,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -192,6 +210,7 @@ pub fn build_app_with_background_workers_and_limits(
|
||||
max_concurrent_sessions,
|
||||
true,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -227,6 +246,41 @@ pub fn build_app_with_artifacts(
|
||||
max_concurrent_sessions,
|
||||
true,
|
||||
Some(artifact_store),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
/// Production launcher entry point with the durable Task application entry constructed by the
|
||||
/// process composition root from the same registry pool and admission limits as the runtime.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn build_app_with_artifacts_and_tasks(
|
||||
registry: PostgresRegistry,
|
||||
refresh_interval: Duration,
|
||||
public_base_url: Option<String>,
|
||||
secret_crypto: SecretCrypto,
|
||||
runtime: RuntimeExecutor,
|
||||
api_rate_limiter: RequestRateLimiter,
|
||||
coordination_store: Arc<dyn CoordinationStateStore>,
|
||||
sessions: SharedSessionStore,
|
||||
credential_verifier: SharedMachineCredentialVerifier,
|
||||
max_concurrent_sessions: usize,
|
||||
artifact_store: crank_artifacts::ArtifactStore,
|
||||
tasks: Arc<TaskApplication>,
|
||||
) -> Router {
|
||||
build_app_inner(
|
||||
registry,
|
||||
refresh_interval,
|
||||
public_base_url,
|
||||
secret_crypto,
|
||||
runtime,
|
||||
api_rate_limiter,
|
||||
coordination_store,
|
||||
sessions,
|
||||
credential_verifier,
|
||||
max_concurrent_sessions,
|
||||
true,
|
||||
Some(artifact_store),
|
||||
Some(tasks),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -244,8 +298,23 @@ fn build_app_inner(
|
||||
max_concurrent_sessions: usize,
|
||||
start_background_workers: bool,
|
||||
artifact_store: Option<crank_artifacts::ArtifactStore>,
|
||||
tasks: Option<Arc<TaskApplication>>,
|
||||
) -> Router {
|
||||
let session_metrics = ActiveSessionMetrics::start(Arc::clone(&sessions));
|
||||
let tasks = tasks.unwrap_or_else(|| {
|
||||
let store = crank_registry::PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits::default(),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("default admission limits and scheduler weights are valid");
|
||||
Arc::new(TaskApplication::new(
|
||||
registry.clone(),
|
||||
refresh_interval,
|
||||
Arc::clone(&coordination_store),
|
||||
crank_runtime::TaskService::new(Arc::new(store)),
|
||||
))
|
||||
});
|
||||
let state = Arc::new(AppState {
|
||||
registry: registry.clone(),
|
||||
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
|
||||
@@ -263,6 +332,7 @@ fn build_app_inner(
|
||||
in_flight: Arc::new(lifecycle::InFlightCalls::new(
|
||||
lifecycle::MAX_IN_FLIGHT_CALLS,
|
||||
)),
|
||||
tasks,
|
||||
});
|
||||
if start_background_workers {
|
||||
spawn_approval_recovery(Arc::clone(&state));
|
||||
@@ -311,24 +381,55 @@ async fn health() -> Json<Value> {
|
||||
}
|
||||
|
||||
async fn readiness(State(state): State<Arc<AppState>>) -> Response {
|
||||
match state.registry.ping().await {
|
||||
Ok(()) => Json(json!({
|
||||
let postgres_ready = state.registry.ping().await.is_ok();
|
||||
// Story 4.1: admission state is a mandatory dependency of every execution, so readiness must
|
||||
// reflect it rather than reporting ready on a bare `select 1`.
|
||||
let admission_ready = state.registry.execution_admission_health().await.is_ok();
|
||||
// Story 4.2: the durable Task store is mandatory for a durable acceptance acknowledgement.
|
||||
let durable_tasks_ready = state.tasks.health().await.is_ok();
|
||||
if postgres_ready && admission_ready && durable_tasks_ready {
|
||||
return Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": "ready" }
|
||||
"checks": {
|
||||
"postgres": "ready",
|
||||
"execution_admission": "ready",
|
||||
"durable_tasks": "ready"
|
||||
}
|
||||
}))
|
||||
.into_response(),
|
||||
Err(_) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": "not_ready" },
|
||||
"error": "database is unavailable"
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
.into_response();
|
||||
}
|
||||
let mut checks = serde_json::Map::new();
|
||||
checks.insert(
|
||||
"postgres".to_owned(),
|
||||
json!(if postgres_ready { "ready" } else { "not_ready" }),
|
||||
);
|
||||
checks.insert(
|
||||
"execution_admission".to_owned(),
|
||||
json!(if admission_ready {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
}),
|
||||
);
|
||||
checks.insert(
|
||||
"durable_tasks".to_owned(),
|
||||
json!(if durable_tasks_ready {
|
||||
"ready"
|
||||
} else {
|
||||
"not_ready"
|
||||
}),
|
||||
);
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "not_ready",
|
||||
"checks": checks,
|
||||
"error": "a mandatory dependency is unavailable"
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,18 +19,31 @@ pub(crate) async fn resolve_operation_auth(
|
||||
workspace_id: &crank_core::WorkspaceId,
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<Option<ResolvedAuth>, RuntimeError> {
|
||||
if execution_config.auth_profile_ref.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let span = Stage::AuthResolve.span();
|
||||
let result = resolve_runtime_auth_for_task(
|
||||
resolve_operation_auth_with(
|
||||
&state.registry,
|
||||
&state.secret_crypto,
|
||||
workspace_id,
|
||||
execution_config,
|
||||
)
|
||||
.instrument(span.clone())
|
||||
.await;
|
||||
.await
|
||||
}
|
||||
|
||||
/// Same resolution for callers that hold the authoritative registry/secret-crypto pair without the
|
||||
/// HTTP application state (the durable Task worker).
|
||||
pub(crate) async fn resolve_operation_auth_with(
|
||||
registry: &PostgresRegistry,
|
||||
secret_crypto: &SecretCrypto,
|
||||
workspace_id: &crank_core::WorkspaceId,
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<Option<ResolvedAuth>, RuntimeError> {
|
||||
if execution_config.auth_profile_ref.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let span = Stage::AuthResolve.span();
|
||||
let result =
|
||||
resolve_runtime_auth_for_task(registry, secret_crypto, workspace_id, execution_config)
|
||||
.instrument(span.clone())
|
||||
.await;
|
||||
match &result {
|
||||
Ok(_) => StageOutcome::Success.record(&span),
|
||||
Err(_) => {
|
||||
|
||||
@@ -144,7 +144,7 @@ fn increment(counter: impl Fn(&CounterSet) -> &AtomicUsize) {
|
||||
/// execution-boundary reason as the tests below).
|
||||
#[cfg(test)]
|
||||
#[path = "../dispatch_observer_support.rs"]
|
||||
mod support;
|
||||
pub(crate) mod support;
|
||||
|
||||
/// The behavioural section 7.2 harness: real app, real PostgreSQL, one counted loopback upstream and
|
||||
/// one request-scoped observer per request. Its file lives outside `src/app/` only because the
|
||||
|
||||
@@ -103,7 +103,7 @@ pub(super) async fn mcp_get(
|
||||
};
|
||||
|
||||
// Story 3.19: the pinned legacy transport forbids broadcasting one JSON-RPC message across the
|
||||
// SSE streams of a session, so exactly one admitted `GET` receives the Resource/Prompt
|
||||
// SSE streams of a session, so exactly one admitted `GET` receives the Resource/Prompt/Tool
|
||||
// list-change signals. A newer `GET` takes them over and the older stream simply ends; a
|
||||
// `DELETE` retires the producer, so a stale generation never writes into a reconnect.
|
||||
let Some(generation) = state.signal_streams.admit(&session_id) else {
|
||||
@@ -115,7 +115,7 @@ pub(super) async fn mcp_get(
|
||||
crate::protocol::Watch {
|
||||
workspace_slug: path.workspace_slug.clone(),
|
||||
agent_slug: path.agent_slug.clone(),
|
||||
signals: crate::protocol::Signals::BOTH,
|
||||
signals: crate::protocol::Signals::ALL,
|
||||
wire: crate::protocol::Wire::Legacy,
|
||||
legacy_stream: Some((
|
||||
Arc::clone(&state.signal_streams),
|
||||
|
||||
@@ -133,7 +133,7 @@ pub(crate) struct ProgressEvent {
|
||||
|
||||
/// The pinned `notifications/progress` frame. Both supported revisions declare the same parameters,
|
||||
/// so the projection is shared; the token is echoed verbatim and never validated as a credential.
|
||||
fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
|
||||
pub(super) fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/progress",
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
//! Isolated single-Agent loopback experiment. Not installed in the production router.
|
||||
//! Each instance owns one route and one SDK manager; no durable store is replaced.
|
||||
|
||||
use super::{
|
||||
AgentRoutePath, AppState,
|
||||
lifecycle::InFlightScope,
|
||||
metrics::ActiveStreamGuard,
|
||||
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
|
||||
stages::{enforce_traced_rate_limit, require_traced_machine_access},
|
||||
};
|
||||
use crate::{
|
||||
protocol::{self, ContinuationSupport, RequestEra},
|
||||
transport,
|
||||
};
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
extract::Request,
|
||||
http::{Method, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use crank_core::PlatformApiKeyScope;
|
||||
use futures_util::StreamExt;
|
||||
use rmcp::transport::streamable_http_server::{
|
||||
StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use std::sync::Arc;
|
||||
|
||||
const BODY_LIMIT: usize = 2 * 1024 * 1024;
|
||||
|
||||
pub(super) fn router(
|
||||
state: Arc<AppState>,
|
||||
path: AgentRoutePath,
|
||||
active: Arc<std::sync::atomic::AtomicUsize>,
|
||||
) -> Router {
|
||||
let handler_state = state.clone();
|
||||
let mut config = StreamableHttpServerConfig::default().with_json_response(true);
|
||||
config.max_request_body_bytes = BODY_LIMIT;
|
||||
let shutdown = Arc::new(config.cancellation_token.clone().drop_guard());
|
||||
let mut manager = LocalSessionManager::default();
|
||||
manager.session_config.channel_capacity = 16;
|
||||
let service = StreamableHttpService::new(
|
||||
move || {
|
||||
Ok(RmcpTools {
|
||||
state: handler_state.clone(),
|
||||
active: active.clone(),
|
||||
})
|
||||
},
|
||||
Arc::new(manager),
|
||||
config,
|
||||
);
|
||||
let route = format!("/v1/{}/{}", path.workspace_slug, path.agent_slug);
|
||||
Router::new()
|
||||
.route(
|
||||
&route,
|
||||
axum::routing::any(move |request: Request| {
|
||||
let state = state.clone();
|
||||
let path = path.clone();
|
||||
let service = service.clone();
|
||||
let shutdown = shutdown.clone();
|
||||
async move {
|
||||
let _shutdown = shutdown;
|
||||
let (mut parts, body) = request.into_parts();
|
||||
if let Err(status) =
|
||||
transport::validate_origin(&state.allowed_origins, &parts.headers)
|
||||
{
|
||||
return status.into_response();
|
||||
}
|
||||
let bytes = match to_bytes(body, BODY_LIMIT).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(),
|
||||
};
|
||||
let message: Value = if parts.method == Method::POST {
|
||||
match serde_json::from_slice(&bytes) {
|
||||
Ok(message) => message,
|
||||
Err(_) => return StatusCode::BAD_REQUEST.into_response(),
|
||||
}
|
||||
} else {
|
||||
Value::Null
|
||||
};
|
||||
let method = message["method"].as_str().unwrap_or("");
|
||||
let required = if method == "tools/call" {
|
||||
PlatformApiKeyScope::Write
|
||||
} else {
|
||||
PlatformApiKeyScope::Read
|
||||
};
|
||||
let credential = match require_traced_machine_access(
|
||||
&state,
|
||||
&path,
|
||||
&parts.headers,
|
||||
required,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(credential) => credential,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let session = match transport::session_id_from_headers(&parts.headers) {
|
||||
Ok(session) => session,
|
||||
Err(_) => {
|
||||
return protocol::ambiguous_session_carrier_response(
|
||||
&parts.headers,
|
||||
&message,
|
||||
);
|
||||
}
|
||||
};
|
||||
let era = if parts.method == Method::POST {
|
||||
if let Err(response) = protocol::validate_jsonrpc_envelope(&message) {
|
||||
return *response;
|
||||
}
|
||||
if let Err(status) = transport::negotiate_post_response_mode(&parts.headers)
|
||||
{
|
||||
return status.into_response();
|
||||
}
|
||||
match protocol::resolve_request_identity(&parts.headers, &message) {
|
||||
Ok(identity) => identity.era,
|
||||
Err(response) => return *response,
|
||||
}
|
||||
} else {
|
||||
if let Err(response) = protocol::session_lifecycle_admission(&parts.headers)
|
||||
{
|
||||
return *response;
|
||||
}
|
||||
RequestEra::Legacy
|
||||
};
|
||||
if matches!(era, RequestEra::Modern) && session.is_some() {
|
||||
return protocol::modern_carrier_conflict_response(&message);
|
||||
}
|
||||
if let Err(error) =
|
||||
enforce_traced_rate_limit(&state, &path, &parts.headers).await
|
||||
{
|
||||
return crate::rate_limit::rate_limited_status_response(error);
|
||||
}
|
||||
// SDK cancellation is keyed only by request id. Never forward an unchecked cancel.
|
||||
if method == "notifications/cancelled" {
|
||||
if let Some(session_id) = session {
|
||||
state.in_flight.cancel(
|
||||
&InFlightScope {
|
||||
session_id,
|
||||
owner: credential
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str())
|
||||
.unwrap_or("")
|
||||
.to_owned(),
|
||||
},
|
||||
&message["params"]["requestId"],
|
||||
);
|
||||
}
|
||||
return StatusCode::ACCEPTED.into_response();
|
||||
}
|
||||
let Some(context) = parts
|
||||
.extensions
|
||||
.get::<crate::request_context::RequestContext>()
|
||||
else {
|
||||
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
|
||||
};
|
||||
let correlation = context.correlation.clone();
|
||||
|
||||
parts.extensions.insert(RmcpRequestIdentity {
|
||||
workspace_slug: path.workspace_slug,
|
||||
agent_slug: path.agent_slug,
|
||||
credential,
|
||||
correlation,
|
||||
continuation: ContinuationSupport::from_supported(false),
|
||||
locale: crate::tool_error::execution_locale(&message),
|
||||
session_id: session,
|
||||
progress_meta: message.pointer("/params/_meta").cloned(),
|
||||
sse_acceptable: transport::accepts_event_stream(&parts.headers),
|
||||
});
|
||||
// Legacy SDK calls outlive disconnected bodies: both lifetimes share one slot.
|
||||
let permit = match ActiveStreamGuard::try_acquire(&state.session_slots) {
|
||||
Ok(permit) => Arc::new(permit),
|
||||
Err(_) => return StatusCode::TOO_MANY_REQUESTS.into_response(),
|
||||
};
|
||||
if method == "tools/call" {
|
||||
parts.extensions.insert(permit.clone());
|
||||
}
|
||||
let response = service
|
||||
.handle(Request::from_parts(parts, Body::from(bytes)))
|
||||
.await;
|
||||
let (parts, body) = response.into_parts();
|
||||
let stream = futures_util::stream::unfold(
|
||||
(Box::pin(Body::new(body).into_data_stream()), permit),
|
||||
|(mut stream, permit)| async move {
|
||||
stream.next().await.map(|chunk| (chunk, (stream, permit)))
|
||||
},
|
||||
);
|
||||
Response::from_parts(parts, Body::from_stream(stream))
|
||||
}
|
||||
}),
|
||||
)
|
||||
.layer(axum::middleware::from_fn(
|
||||
crate::request_context::apply_request_context,
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
//! Behavioural ATDD for rmcp 3.7.0 LocalSessionManager reconnect/restart semantics.
|
||||
|
||||
use rmcp::transport::streamable_http_server::session::{
|
||||
SessionManager, local::LocalSessionManager,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_local_session_manager_preserves_access_to_created_session() {
|
||||
let manager = LocalSessionManager::default();
|
||||
|
||||
let (session_id, _transport) = manager
|
||||
.create_session()
|
||||
.await
|
||||
.expect("local manager must create a session");
|
||||
|
||||
assert!(
|
||||
manager
|
||||
.has_session(&session_id)
|
||||
.await
|
||||
.expect("live manager must answer has_session"),
|
||||
"a live manager must keep access to a session it created"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_fresh_session_manager_without_external_store_rejects_known_session_id() {
|
||||
let first_manager = LocalSessionManager::default();
|
||||
|
||||
let (session_id, _transport) = first_manager
|
||||
.create_session()
|
||||
.await
|
||||
.expect("first local manager must create a session");
|
||||
|
||||
let second_manager = LocalSessionManager::default();
|
||||
|
||||
assert!(
|
||||
!second_manager
|
||||
.has_session(&session_id)
|
||||
.await
|
||||
.expect("fresh manager must answer has_session"),
|
||||
"a fresh in-memory manager must not claim ownership of another manager's session"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
//! Isolated rmcp 3.7.0 Tools adapter over Crank's existing typed Tools service.
|
||||
//!
|
||||
//! This module deliberately does not provide a second command engine. It owns only
|
||||
//! the verified request identity (`ServiceContext` data) and delegates both
|
||||
//! `tools/list` and `tools/call` to `handle_tools_command` via rmcp's
|
||||
//! `ServerHandler` trait. HTTP admission must supply identity for each request.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crank_core::{CorrelationContext, ExecutionLocale};
|
||||
|
||||
use crate::app::{AppState, lifecycle::ToolCallLifecycle};
|
||||
use crate::auth::VerifiedMachineCredential;
|
||||
use crate::protocol::{
|
||||
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsListCommand,
|
||||
ToolsOutcome, handle_tools_command,
|
||||
};
|
||||
|
||||
/// Long-lived handler: never stores credentials obtained on initialization.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct RmcpTools {
|
||||
pub(super) state: Arc<AppState>,
|
||||
pub(super) active: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ActiveCall(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ActiveCall {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// Server-created identity attached to the individual admitted HTTP request.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct RmcpRequestIdentity {
|
||||
pub(super) workspace_slug: String,
|
||||
pub(super) agent_slug: String,
|
||||
pub(super) credential: VerifiedMachineCredential,
|
||||
pub(super) correlation: CorrelationContext,
|
||||
pub(super) continuation: ContinuationSupport,
|
||||
pub(super) locale: ExecutionLocale,
|
||||
pub(super) session_id: Option<String>,
|
||||
pub(super) progress_meta: Option<serde_json::Value>,
|
||||
pub(super) sse_acceptable: bool,
|
||||
}
|
||||
|
||||
impl RmcpRequestIdentity {
|
||||
/// Build the neutral `ServiceContext` borrowed from this adapter's owned data.
|
||||
pub(super) fn service_context(&self) -> ServiceContext<'_> {
|
||||
ServiceContext {
|
||||
workspace_slug: &self.workspace_slug,
|
||||
agent_slug: &self.agent_slug,
|
||||
credential: &self.credential,
|
||||
correlation: &self.correlation,
|
||||
continuation: self.continuation,
|
||||
locale: self.locale,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RmcpTools {
|
||||
/// Delegate one neutral Tools command to the existing shared service.
|
||||
pub(super) async fn handle(
|
||||
&self,
|
||||
identity: &RmcpRequestIdentity,
|
||||
command: ToolsCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
handle_tools_command(&self.state, identity.service_context(), command, lifecycle).await
|
||||
}
|
||||
|
||||
pub(super) async fn list_tools(
|
||||
&self,
|
||||
identity: &RmcpRequestIdentity,
|
||||
command: ToolsListCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
self.handle(identity, ToolsCommand::List(command), lifecycle)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn call_tool(
|
||||
&self,
|
||||
identity: &RmcpRequestIdentity,
|
||||
command: ToolCallCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
self.handle(identity, ToolsCommand::Call(command), lifecycle)
|
||||
.await
|
||||
}
|
||||
}
|
||||
use rmcp::ErrorData as McpError;
|
||||
use rmcp::model::{
|
||||
CallToolRequestParams, CallToolResult, ContentBlock, ListToolsResult, PaginatedRequestParams,
|
||||
Tool,
|
||||
};
|
||||
use rmcp::service::RequestContext;
|
||||
use rmcp::{RoleServer, ServerHandler};
|
||||
|
||||
fn request_identity(context: &RequestContext<RoleServer>) -> Result<RmcpRequestIdentity, McpError> {
|
||||
context
|
||||
.extensions
|
||||
.get::<axum::http::request::Parts>()
|
||||
.and_then(|parts| parts.extensions.get::<RmcpRequestIdentity>())
|
||||
.cloned()
|
||||
.ok_or_else(|| McpError::internal_error("missing authenticated request context", None))
|
||||
}
|
||||
|
||||
impl ServerHandler for RmcpTools {
|
||||
fn supported_protocol_versions(
|
||||
&self,
|
||||
) -> std::borrow::Cow<'static, [rmcp::model::ProtocolVersion]> {
|
||||
std::borrow::Cow::Borrowed(&[
|
||||
rmcp::model::ProtocolVersion::V_2025_11_25,
|
||||
rmcp::model::ProtocolVersion::V_2026_07_28,
|
||||
])
|
||||
}
|
||||
|
||||
fn get_info(&self) -> rmcp::model::ServerConfig {
|
||||
let mut info = rmcp::model::ServerConfig::default();
|
||||
info.capabilities = rmcp::model::ServerCapabilities::builder()
|
||||
.enable_tools()
|
||||
.build();
|
||||
info
|
||||
}
|
||||
fn list_tools(
|
||||
&self,
|
||||
request: Option<PaginatedRequestParams>,
|
||||
context: RequestContext<RoleServer>,
|
||||
) -> impl std::future::Future<Output = Result<ListToolsResult, McpError>> + Send + '_ {
|
||||
let request = request.unwrap_or_default();
|
||||
async move {
|
||||
let identity = request_identity(&context)?;
|
||||
let cursor = request.cursor;
|
||||
let command = crate::protocol::ToolsListCommand {
|
||||
cursor,
|
||||
limit: None,
|
||||
};
|
||||
match self.list_tools(&identity, command, None).await {
|
||||
crate::protocol::ToolsOutcome::List { tools, next_cursor } => {
|
||||
let tools = tools
|
||||
.into_iter()
|
||||
.map(serde_json::from_value::<Tool>)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|_| {
|
||||
McpError::internal_error("invalid tool catalog entry", None)
|
||||
})?;
|
||||
Ok(ListToolsResult {
|
||||
tools,
|
||||
next_cursor,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
crate::protocol::ToolsOutcome::InvalidParams(message) => {
|
||||
Err(McpError::invalid_params(message, None))
|
||||
}
|
||||
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn call_tool(
|
||||
&self,
|
||||
request: CallToolRequestParams,
|
||||
context: RequestContext<RoleServer>,
|
||||
) -> Result<rmcp::model::CallToolResponse, McpError> {
|
||||
let identity = request_identity(&context)?;
|
||||
let arguments = request
|
||||
.arguments
|
||||
.map(serde_json::Value::Object)
|
||||
.unwrap_or_else(|| serde_json::json!({}));
|
||||
let command = crate::protocol::ToolCallCommand {
|
||||
tool_name: request.name.to_string(),
|
||||
arguments,
|
||||
confirmation_token: None,
|
||||
};
|
||||
let token = identity
|
||||
.sse_acceptable
|
||||
.then(|| super::lifecycle::token_from_meta(identity.progress_meta.as_ref()))
|
||||
.flatten();
|
||||
// Observe future cleanup directly in the isolated loopback oracle.
|
||||
self.active
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let _active = ActiveCall(self.active.clone());
|
||||
let (mut lifecycle, mut events) = ToolCallLifecycle::new(token.clone());
|
||||
let entry = identity.session_id.as_ref().and_then(|session_id| {
|
||||
self.state.in_flight.register(
|
||||
&super::lifecycle::InFlightScope {
|
||||
session_id: session_id.clone(),
|
||||
owner: identity
|
||||
.credential
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str())
|
||||
.unwrap_or("")
|
||||
.to_owned(),
|
||||
},
|
||||
&serde_json::to_value(&context.id).expect("request id serializes"),
|
||||
)
|
||||
});
|
||||
if let Some(entry) = &entry {
|
||||
lifecycle = lifecycle.with_cancel(entry.cancel_signal());
|
||||
}
|
||||
let call = self.call_tool(&identity, command, Some(lifecycle.clone()));
|
||||
tokio::pin!(call);
|
||||
let outcome = loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
event = async { match events.as_mut() { Some(events) => events.recv().await, None => std::future::pending().await } } => {
|
||||
if let Some(event) = event {
|
||||
if let Some(token) = &token {
|
||||
let params = serde_json::from_value(super::lifecycle::progress_frame(token, &event)["params"].clone()).map_err(|_| McpError::internal_error("invalid progress event", None))?;
|
||||
let _ = context.peer.notify_progress(params).await;
|
||||
}
|
||||
} else { events = None; }
|
||||
}
|
||||
outcome = &mut call => break outcome,
|
||||
_ = lifecycle.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
|
||||
_ = context.ct.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
|
||||
}
|
||||
};
|
||||
match outcome {
|
||||
crate::protocol::ToolsOutcome::Call(outcome) => match outcome {
|
||||
crate::protocol::ToolCallOutcome::Completed(value)
|
||||
| crate::protocol::ToolCallOutcome::ApprovalPending(value) => {
|
||||
let text = value.to_string();
|
||||
let mut result = CallToolResult::default();
|
||||
result.content = vec![ContentBlock::text(text)];
|
||||
result.structured_content = if value.is_null() { None } else { Some(value) };
|
||||
Ok(result.into())
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::ApprovalUnavailable(_) => {
|
||||
let mut result = CallToolResult::default();
|
||||
result.is_error = Some(true);
|
||||
result.content = vec![ContentBlock::text("approval elicitation not supported")];
|
||||
Ok(result.into())
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::InvalidParams(message) => {
|
||||
Err(McpError::invalid_params(message, None))
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::Error(contract) => {
|
||||
let text = serde_json::to_string(&contract).unwrap_or_default();
|
||||
let mut result = CallToolResult::default();
|
||||
result.is_error = Some(true);
|
||||
result.content = vec![ContentBlock::text(text)];
|
||||
Ok(result.into())
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::InternalFailure => {
|
||||
Err(McpError::internal_error("tool execution failed", None))
|
||||
}
|
||||
},
|
||||
crate::protocol::ToolsOutcome::InvalidParams(message) => {
|
||||
Err(McpError::invalid_params(message, None))
|
||||
}
|
||||
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
//! Application-level durable Task handler and attempt authority (Story 4.3/4.4).
|
||||
//!
|
||||
//! The handler owns everything that touches product authority: the exact published pin, the typed
|
||||
//! Platform API key, the approval policy, the encrypted executable input and the resolved upstream
|
||||
//! auth. The runtime owns the single bounded retry loop and the durable attempt checkpoint/resolution;
|
||||
//! this module only revalidates and prepares one attempt at a time.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
|
||||
TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId,
|
||||
current_unix_ms,
|
||||
};
|
||||
use crank_registry::PostgresRegistry;
|
||||
use crank_registry::PublishedAgentTool;
|
||||
use crank_runtime::{
|
||||
ExecutionAuthorization, PrepareAttempt, RuntimeExecutor, RuntimeOperation, SecretCrypto,
|
||||
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
|
||||
TaskAttemptRefusal, TaskHandler, TaskService, TaskWorkerError,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::TaskApplication;
|
||||
use super::auth_resolution::resolve_operation_auth_with;
|
||||
use super::tool_resolution::runtime_operation;
|
||||
use crate::access::{allows_scope, credential_allows_security_level};
|
||||
use crate::catalog::PublishedToolCatalog;
|
||||
|
||||
/// Server-owned evidence that one published operation deduplicates repeated sends of the same stable
|
||||
/// idempotency key.
|
||||
///
|
||||
/// The descriptor is constructed by the application for an integration whose upstream contract was
|
||||
/// actually qualified (key delivery, dedup scope, canonical input binding and a dedup window that
|
||||
/// covers the pinned retry budget). It is never deserialized from Task input, so a caller key or
|
||||
/// header alone can never make a write retryable. Production wiring passes an empty set until such a
|
||||
/// contract exists.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct VerifiedIdempotentWrite {
|
||||
pub operation_id: String,
|
||||
pub operation_version: u32,
|
||||
/// Header through which the stable key is delivered to this exact upstream.
|
||||
pub key_header: String,
|
||||
/// Dedup window of the upstream contract, which must cover the pinned retry budget.
|
||||
pub validity_ms: i64,
|
||||
}
|
||||
|
||||
pub struct TaskWorkerHandler {
|
||||
registry: PostgresRegistry,
|
||||
catalog: PublishedToolCatalog,
|
||||
runtime: RuntimeExecutor,
|
||||
secret_crypto: SecretCrypto,
|
||||
service: TaskService,
|
||||
verified_writes: Vec<VerifiedIdempotentWrite>,
|
||||
}
|
||||
|
||||
impl TaskWorkerHandler {
|
||||
/// Production construction from the same application entry, registry and runtime the MCP
|
||||
/// process serves requests with. The worker needs no HTTP state, only these authoritative
|
||||
/// dependencies.
|
||||
pub fn from_application(
|
||||
tasks: &TaskApplication,
|
||||
registry: PostgresRegistry,
|
||||
runtime: RuntimeExecutor,
|
||||
secret_crypto: SecretCrypto,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
catalog: tasks.catalog(),
|
||||
runtime,
|
||||
secret_crypto,
|
||||
service: tasks.task_service(),
|
||||
verified_writes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers server-owned verified-idempotency evidence of qualified integrations.
|
||||
pub fn with_verified_writes(mut self, verified_writes: Vec<VerifiedIdempotentWrite>) -> Self {
|
||||
self.verified_writes = verified_writes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
|
||||
///
|
||||
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
|
||||
/// policy and revalidates the typed persisted Platform API key identity against the registry:
|
||||
/// the key must still be active, unexpired and bound to this workspace/agent, and its exact
|
||||
/// scopes/security level must still authorize the operation. Any denial is a typed error, so the
|
||||
/// worker finishes the attempt without dispatch (fail closed before any network use).
|
||||
///
|
||||
/// An administrator Task has no authoritative Task revalidation API in the registry, so it stays
|
||||
/// fail-closed here instead of inventing authority.
|
||||
async fn authorize(&self, lease: &TaskLease) -> Result<PublishedAgentTool, TaskError> {
|
||||
if lease.record.owner_kind != TaskOwnerKind::Agent {
|
||||
return Err(TaskError::invalid(
|
||||
"admin task authority revalidation is not implemented",
|
||||
));
|
||||
}
|
||||
let Some(key_id) = lease.record.platform_api_key_id.as_ref() else {
|
||||
return Err(TaskError::invalid("task authority reference is missing"));
|
||||
};
|
||||
|
||||
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
||||
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
||||
let workspace = self
|
||||
.registry
|
||||
.get_workspace(&workspace_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?
|
||||
.ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?;
|
||||
let agents = self
|
||||
.registry
|
||||
.list_agents(&workspace_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
||||
let agent = agents
|
||||
.into_iter()
|
||||
.find(|agent| agent.id.as_str() == lease.record.owner_id.as_str())
|
||||
.ok_or_else(|| TaskError::invalid("task agent no longer exists"))?;
|
||||
|
||||
let catalog = self
|
||||
.catalog
|
||||
.get_catalog(&workspace.workspace.slug, &agent.slug)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
||||
let tool = catalog
|
||||
.tools
|
||||
.into_iter()
|
||||
.find(|tool| {
|
||||
tool.operation.id.as_str() == lease.record.operation_id.as_str()
|
||||
&& tool.operation.version == lease.record.pins.operation_version
|
||||
})
|
||||
.ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?;
|
||||
|
||||
if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() {
|
||||
return Err(TaskError::invalid("task workspace binding mismatch"));
|
||||
}
|
||||
if tool.agent_id.as_str() != lease.record.owner_id.as_str() {
|
||||
return Err(TaskError::invalid("task agent binding mismatch"));
|
||||
}
|
||||
if tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()
|
||||
.is_some_and(|policy| policy.required)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"missing approval proof for task operation",
|
||||
));
|
||||
}
|
||||
|
||||
// Authoritative revalidation of the exact immutable key identity. A revoked, expired,
|
||||
// foreign-workspace or foreign-agent key is absent here and never reaches the network.
|
||||
let key = self
|
||||
.registry
|
||||
.revalidate_platform_api_key(&workspace_id, &agent_id, key_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("task authority store unavailable"))?
|
||||
.ok_or_else(|| TaskError::invalid("task authority is no longer active"))?;
|
||||
let credential = VerifiedMachineCredential {
|
||||
machine_access_mode: MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: OperationSecurityLevel::Standard,
|
||||
scopes: key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(key.api_key.id.clone()),
|
||||
};
|
||||
if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) {
|
||||
return Err(TaskError::invalid("task authority scope is insufficient"));
|
||||
}
|
||||
if !credential_allows_security_level(&credential, tool.operation.security_level) {
|
||||
return Err(TaskError::invalid(
|
||||
"task authority security level is insufficient",
|
||||
));
|
||||
}
|
||||
Ok(tool)
|
||||
}
|
||||
|
||||
/// Safe-retry classification of one attempt and the stable upstream key digest.
|
||||
///
|
||||
/// A read is always safe to repeat. A write is only safe to repeat when this process holds
|
||||
/// server-owned evidence for the exact published operation and that evidence's dedup window still
|
||||
/// covers the accepted retry budget; the presence of a client key or an idempotency header alone
|
||||
/// never qualifies.
|
||||
fn retry_classification(
|
||||
&self,
|
||||
operation: &RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
retry_window_ms: i64,
|
||||
) -> (bool, Option<String>) {
|
||||
if matches!(
|
||||
&operation.target,
|
||||
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
|
||||
) {
|
||||
return (true, None);
|
||||
}
|
||||
let Some(policy) = operation.execution_config.idempotency.as_ref() else {
|
||||
return (false, None);
|
||||
};
|
||||
let Some(field) = policy
|
||||
.input_field
|
||||
.as_deref()
|
||||
.filter(|field| !field.trim().is_empty())
|
||||
else {
|
||||
return (false, None);
|
||||
};
|
||||
let Some(header) = policy
|
||||
.header_name
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return (false, None);
|
||||
};
|
||||
let key = match input.get(field) {
|
||||
Some(serde_json::Value::String(value)) if !value.trim().is_empty() => value.clone(),
|
||||
Some(serde_json::Value::Number(value)) => value.to_string(),
|
||||
_ => return (false, None),
|
||||
};
|
||||
let evidence = self.verified_writes.iter().find(|evidence| {
|
||||
evidence.operation_id == operation.operation_id.as_str()
|
||||
&& evidence.operation_version == operation.operation_version
|
||||
&& evidence.key_header.eq_ignore_ascii_case(header)
|
||||
&& evidence.validity_ms >= retry_window_ms
|
||||
});
|
||||
let Some(evidence) = evidence else {
|
||||
return (false, None);
|
||||
};
|
||||
// Only a digest of the stable key is persisted; the key itself stays in the encrypted input.
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(evidence.key_header.as_bytes());
|
||||
digest.update([0u8]);
|
||||
digest.update(key.as_bytes());
|
||||
(true, Some(format!("{:x}", digest.finalize())))
|
||||
}
|
||||
|
||||
/// Story 4.5 producer: the pinned operation still lacks a required input item, so the Task must be
|
||||
/// suspended before any dispatch. The schema snapshot and its presentation projection are frozen
|
||||
/// here; the TTL never exceeds the accepted absolute execution deadline.
|
||||
fn input_request(
|
||||
&self,
|
||||
tool: &PublishedAgentTool,
|
||||
lease: &TaskLease,
|
||||
missing_path: &str,
|
||||
expires_at_unix_ms: i64,
|
||||
) -> Result<crank_core::TaskInputRequest, TaskAttemptRefusal> {
|
||||
let schema_snapshot = serde_json::to_value(&tool.operation.input_schema).map_err(|_| {
|
||||
TaskAttemptRefusal::Failed {
|
||||
code: "task_input_schema_unavailable".to_owned(),
|
||||
}
|
||||
})?;
|
||||
let request = crank_core::TaskInputRequest {
|
||||
wait_id: format!("wait-{}", uuid::Uuid::now_v7()),
|
||||
expected_version: lease.record.version.saturating_add(1),
|
||||
schema_revision: tool.operation.version,
|
||||
schema_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot,
|
||||
projected_schema: crate::manifest::schema_to_json_schema(&tool.operation.input_schema),
|
||||
prompt_code: "required_input_missing".to_owned(),
|
||||
missing_fields: vec![missing_path.to_owned()],
|
||||
expires_at_unix_ms,
|
||||
fingerprint_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
};
|
||||
request.validate().map_err(|_| TaskAttemptRefusal::Failed {
|
||||
code: "task_input_request_invalid".to_owned(),
|
||||
})?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
/// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store
|
||||
/// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal.
|
||||
fn refusal(error: TaskError) -> TaskAttemptRefusal {
|
||||
match error {
|
||||
TaskError::Unavailable { .. } => TaskAttemptRefusal::Stopped,
|
||||
_ => TaskAttemptRefusal::Failed {
|
||||
code: "task_authority_rejected".to_owned(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskHandler for TaskWorkerHandler {
|
||||
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
self.authorize(lease)
|
||||
.await
|
||||
.map_err(TaskWorkerError::Store)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.runtime.execute_task_outcome(&context, self).await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskAttemptAuthority for TaskWorkerHandler {
|
||||
async fn prepare_attempt(
|
||||
&self,
|
||||
lease: &TaskLease,
|
||||
) -> Result<PrepareAttempt, TaskAttemptRefusal> {
|
||||
let tool = self.authorize(lease).await.map_err(Self::refusal)?;
|
||||
let input = self
|
||||
.service
|
||||
.executable_input(lease)
|
||||
.await
|
||||
.map_err(Self::refusal)?;
|
||||
let operation = runtime_operation(&tool);
|
||||
let now_ms = current_unix_ms();
|
||||
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
|
||||
if deadline_ms <= now_ms {
|
||||
// The accepted absolute deadline already elapsed: no further attempt may be dispatched,
|
||||
// and any previous uncertainty was already recorded as such.
|
||||
return Err(TaskAttemptRefusal::Failed {
|
||||
code: "task_deadline_elapsed".to_owned(),
|
||||
});
|
||||
}
|
||||
// Only a missing required item of the pinned schema becomes a typed input request. Any other
|
||||
// schema violation keeps the existing pre-dispatch attempt validation, and a generation that
|
||||
// already accepted a continuation can never open a second wait for the same Task.
|
||||
if let Some(missing_path) =
|
||||
crank_runtime::missing_required_input(&tool.operation.input_schema, &input)
|
||||
{
|
||||
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
|
||||
return Err(TaskAttemptRefusal::Failed {
|
||||
code: "task_input_still_incomplete".to_owned(),
|
||||
});
|
||||
}
|
||||
return Ok(PrepareAttempt::InputRequired(self.input_request(
|
||||
&tool,
|
||||
lease,
|
||||
&missing_path,
|
||||
deadline_ms,
|
||||
)?));
|
||||
}
|
||||
let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64);
|
||||
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
||||
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
||||
let resolved_auth = resolve_operation_auth_with(
|
||||
&self.registry,
|
||||
&self.secret_crypto,
|
||||
&workspace_id,
|
||||
&operation.execution_config,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| TaskAttemptRefusal::Failed {
|
||||
code: "task_auth_resolution_failed".to_owned(),
|
||||
})?;
|
||||
// Continue the parent trace persisted with the accepted Task; every attempt adds its own
|
||||
// child `adapter_attempt` span carrying the committed durable attempt ID.
|
||||
let trace_context =
|
||||
TraceContext::parse(&lease.record.support.traceparent).map_err(|_| {
|
||||
TaskAttemptRefusal::Failed {
|
||||
code: "task_trace_parent_invalid".to_owned(),
|
||||
}
|
||||
})?;
|
||||
let request_context = crank_runtime::RuntimeRequestContext::new(
|
||||
lease.record.support.request_id.clone(),
|
||||
trace_context,
|
||||
)
|
||||
.with_metering_context(
|
||||
workspace_id.clone(),
|
||||
Some(agent_id.clone()),
|
||||
crank_core::InvocationSource::AgentToolCall,
|
||||
);
|
||||
let (retry_allowed, upstream_key_digest) =
|
||||
self.retry_classification(&operation, &input, deadline_ms - now_ms);
|
||||
Ok(PrepareAttempt::Ready(Box::new(TaskAttemptPlan {
|
||||
workspace_id,
|
||||
origin: ExecutionOrigin::AgentSnapshot,
|
||||
agent_id: Some(agent_id),
|
||||
operation,
|
||||
input,
|
||||
authorization: ExecutionAuthorization::Authorized,
|
||||
resolved_auth,
|
||||
request_context,
|
||||
deadline,
|
||||
retry_allowed,
|
||||
upstream_key_digest,
|
||||
})))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,414 @@
|
||||
//! Application entry of durable background Tasks (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! The entry takes an already authenticated principal (a verified machine credential, or an
|
||||
//! administrator identity verified by the calling admin surface), resolves the exact published Tool,
|
||||
//! repeats the caller's authorization, pins the Published Version/policy/deadline and delegates the
|
||||
//! durable acceptance to the runtime Task service. The repository owns the transaction, so the
|
||||
//! returned Task ID exists only after Task + dispatch intent + queue slot committed.
|
||||
//!
|
||||
//! Deliberately not wired: the public MCP `tools/call`/Tasks wire (Story 4.7) and the worker/leases
|
||||
//! (Story 4.3). The existing synchronous/unsupported `tools/call` path is unchanged, and this entry
|
||||
//! never spawns an ephemeral execution future.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
CorrelationContext, PlatformApiKeyId, TaskAcceptance, TaskAccess, TaskError, TaskId,
|
||||
TaskOwnerKind, TaskRecord, ToolExecutionMode,
|
||||
};
|
||||
use crank_registry::{PostgresRegistry, PublishedAgentTool};
|
||||
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::access::credential_allows_security_level;
|
||||
use crate::catalog::PublishedToolCatalog;
|
||||
|
||||
/// Authenticated principal of one Task operation. The authorization scope identity is derived here
|
||||
/// from the verified credential/admin identity, never from a caller-supplied request string.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum TaskPrincipal {
|
||||
Agent {
|
||||
workspace_slug: String,
|
||||
agent_slug: String,
|
||||
credential: crank_core::VerifiedMachineCredential,
|
||||
},
|
||||
/// Administrator identity that the admin surface already authenticated. It is never a wildcard:
|
||||
/// owner, workspace and authorization scope all participate in the stored predicate.
|
||||
Admin {
|
||||
workspace_slug: String,
|
||||
agent_slug: String,
|
||||
user_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl TaskPrincipal {
|
||||
fn workspace_slug(&self) -> &str {
|
||||
match self {
|
||||
Self::Agent { workspace_slug, .. } | Self::Admin { workspace_slug, .. } => {
|
||||
workspace_slug
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn agent_slug(&self) -> &str {
|
||||
match self {
|
||||
Self::Agent { agent_slug, .. } | Self::Admin { agent_slug, .. } => agent_slug,
|
||||
}
|
||||
}
|
||||
|
||||
fn owner(&self, tool: &PublishedAgentTool) -> (TaskOwnerKind, String) {
|
||||
match self {
|
||||
Self::Agent { .. } => (TaskOwnerKind::Agent, tool.agent_id.as_str().to_owned()),
|
||||
Self::Admin { user_id, .. } => (TaskOwnerKind::Admin, user_id.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed immutable Platform API key identity of this principal. A verified agent credential
|
||||
/// without a key ID cannot be revalidated by a worker, so it is refused instead of accepted;
|
||||
/// an administrator has no machine key and is stored as `None` (the worker fails closed).
|
||||
fn platform_api_key_id(&self) -> Result<Option<PlatformApiKeyId>, TaskApplicationError> {
|
||||
match self {
|
||||
Self::Agent { credential, .. } => credential
|
||||
.platform_api_key_id
|
||||
.clone()
|
||||
.map(Some)
|
||||
.ok_or(TaskApplicationError::Denied),
|
||||
Self::Admin { .. } => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded server-derived authorization scope identity of this principal.
|
||||
fn authorization_scope(&self) -> String {
|
||||
let identity = match self {
|
||||
Self::Agent { credential, .. } => {
|
||||
let mut scopes = credential
|
||||
.scopes
|
||||
.iter()
|
||||
.map(|scope| format!("{scope:?}"))
|
||||
.collect::<Vec<_>>();
|
||||
scopes.sort();
|
||||
format!(
|
||||
"key:{}|mode:{:?}|level:{:?}|scopes:{}",
|
||||
credential
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str().to_owned())
|
||||
.unwrap_or_else(|| "-".to_owned()),
|
||||
credential.machine_access_mode,
|
||||
credential.max_security_level,
|
||||
scopes.join(",")
|
||||
)
|
||||
}
|
||||
Self::Admin { user_id, .. } => format!("admin:{user_id}"),
|
||||
};
|
||||
identity.chars().take(160).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed refusal of the application entry. `NotFound`/`Denied` never reveal whether a foreign Task
|
||||
/// exists, and no variant carries Task payload or fingerprint.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum TaskApplicationError {
|
||||
#[error("task request was not found")]
|
||||
NotFound,
|
||||
#[error("task request is not authorized")]
|
||||
Denied,
|
||||
#[error("the published Tool policy does not accept durable background work")]
|
||||
NotBackground,
|
||||
#[error("the requested published Tool version is not available")]
|
||||
VersionUnavailable,
|
||||
#[error("this Tool requires a verified approval before acceptance")]
|
||||
ApprovalRequired,
|
||||
#[error(transparent)]
|
||||
Task(#[from] TaskError),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct TaskApplication {
|
||||
catalog: PublishedToolCatalog,
|
||||
service: TaskService,
|
||||
}
|
||||
|
||||
impl TaskApplication {
|
||||
/// Production construction of the application entry over the process registry, the published
|
||||
/// catalog and the durable Task service.
|
||||
pub fn new(
|
||||
registry: PostgresRegistry,
|
||||
refresh_interval: Duration,
|
||||
coordination_store: Arc<dyn crank_core::CoordinationStateStore>,
|
||||
service: TaskService,
|
||||
) -> Self {
|
||||
Self {
|
||||
catalog: PublishedToolCatalog::new(registry, refresh_interval, coordination_store),
|
||||
service,
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime task service carrying encrypted executable input.
|
||||
pub fn task_service(&self) -> TaskService {
|
||||
self.service.clone()
|
||||
}
|
||||
|
||||
/// Published catalog of the same application entry, for the durable Task worker's pre-dispatch
|
||||
/// revalidation.
|
||||
pub(super) fn catalog(&self) -> PublishedToolCatalog {
|
||||
self.catalog.clone()
|
||||
}
|
||||
|
||||
/// Readiness of the durable Task store this entry depends on.
|
||||
pub async fn health(&self) -> Result<(), TaskError> {
|
||||
self.service.health().await
|
||||
}
|
||||
|
||||
/// Accepts one background request for an already authenticated principal.
|
||||
///
|
||||
/// `expected_version` is the caller's explicit Published Version pin. Without it the current
|
||||
/// publication is pinned for new work and a retry of the same scoped key returns the stored Task
|
||||
/// even after a newer publication. An explicit pin may only retry the Task accepted with that
|
||||
/// exact version: a different pin is a typed conflict, never a rebind.
|
||||
pub async fn accept_background(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
expected_version: Option<u32>,
|
||||
scoped_key: &str,
|
||||
input: &Value,
|
||||
correlation: &CorrelationContext,
|
||||
) -> Result<TaskAcceptance, TaskApplicationError> {
|
||||
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
|
||||
let current_version = tool.operation.version;
|
||||
let Some(policy) = tool.operation.execution_config.execution_policy else {
|
||||
return Err(TaskApplicationError::NotBackground);
|
||||
};
|
||||
if policy.mode == ToolExecutionMode::Synchronous {
|
||||
return Err(TaskApplicationError::NotBackground);
|
||||
}
|
||||
if tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()
|
||||
.is_some_and(|approval| approval.required)
|
||||
{
|
||||
// Without a verified, non-consuming approval reference this entry fails closed instead
|
||||
// of accepting work a worker could never safely execute.
|
||||
return Err(TaskApplicationError::ApprovalRequired);
|
||||
}
|
||||
let platform_api_key_id = principal.platform_api_key_id()?;
|
||||
let draft = |operation_version: u32| {
|
||||
let (owner_kind, owner_id) = principal.owner(&tool);
|
||||
TaskAcceptDraft {
|
||||
workspace_id: tool.workspace_id.as_str().to_owned(),
|
||||
owner_kind,
|
||||
owner_id,
|
||||
authorization_scope: principal.authorization_scope(),
|
||||
platform_api_key_id: platform_api_key_id.clone(),
|
||||
operation_id: tool.operation.id.as_str().to_owned(),
|
||||
operation_version,
|
||||
policy_mode: policy.mode,
|
||||
scoped_key: scoped_key.to_owned(),
|
||||
input: input.clone(),
|
||||
correlation: correlation.clone(),
|
||||
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
|
||||
retention_ms: TaskService::default_retention_ms(),
|
||||
approval_reference: None,
|
||||
retry: task_retry_policy(&tool),
|
||||
}
|
||||
};
|
||||
|
||||
let Some(requested) = expected_version else {
|
||||
return Ok(self.service.accept(draft(current_version)).await?);
|
||||
};
|
||||
let draft = draft(requested);
|
||||
match self.service.lookup(&draft).await? {
|
||||
Some(record) => {
|
||||
if record.input_fingerprint != TaskService::input_fingerprint(input)? {
|
||||
return Err(TaskApplicationError::Task(TaskError::FingerprintConflict));
|
||||
}
|
||||
if record.pins.operation_version != requested {
|
||||
return Err(TaskApplicationError::VersionUnavailable);
|
||||
}
|
||||
Ok(TaskAcceptance {
|
||||
task_id: record.task_id,
|
||||
state: record.state,
|
||||
operation_version: record.pins.operation_version,
|
||||
support: record.support,
|
||||
created_at_unix_ms: record.created_at_unix_ms,
|
||||
retain_until_unix_ms: record.pins.retain_until_unix_ms,
|
||||
replayed: true,
|
||||
})
|
||||
}
|
||||
None if requested == current_version => Ok(self.service.accept(draft).await?),
|
||||
// A new Task can only be accepted against the currently published version.
|
||||
None => Err(TaskApplicationError::VersionUnavailable),
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped status read of one Task of the addressed published Tool.
|
||||
pub async fn status(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskRecord, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self.service.status(&access, task_id).await {
|
||||
Ok(record) => Ok(record),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped result projection of one Task of the addressed published Tool.
|
||||
pub async fn result(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskOutcome, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self.service.result(&access, task_id).await {
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped cancellation of one Task of the addressed published Tool (Story 4.4).
|
||||
///
|
||||
/// Cancellation is decided by the durable store under a row lock: before the dispatch checkpoint
|
||||
/// it is an honest cancellation with zero outbound calls, after it the Task becomes the
|
||||
/// non-terminal uncertain outcome instead of a plain cancel or a retry.
|
||||
pub async fn cancel(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
expected_version: Option<u32>,
|
||||
) -> Result<crank_core::TaskCancelOutcome, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self
|
||||
.service
|
||||
.cancel(&access, task_id, expected_version, "user_cancelled")
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies one additional validated input to a waiting Task (Story 4.5).
|
||||
///
|
||||
/// This is the server-internal application entry: the public Task MCP API stays Story 4.7. The
|
||||
/// principal is already authenticated here, the owner scope is derived from the verified
|
||||
/// credential and the durable transaction revalidates the same key before any receipt lookup.
|
||||
/// An identical authorized replay returns the committed outcome; a conflicting or stale answer is
|
||||
/// typed, and an elapsed TTL never revives the Task.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn update_input(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
wait_id: &str,
|
||||
expected_version: u32,
|
||||
update_key: &str,
|
||||
input: &Value,
|
||||
) -> Result<crank_core::TaskInputUpdateOutcome, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self
|
||||
.service
|
||||
.update_input(
|
||||
&access,
|
||||
task_id,
|
||||
wait_id,
|
||||
expected_version,
|
||||
update_key,
|
||||
input,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped attempt history of one Task of the addressed published Tool.
|
||||
pub async fn attempts(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
) -> Result<Vec<crank_core::TaskAttempt>, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self.service.attempts(&access, task_id).await {
|
||||
Ok(attempts) => Ok(attempts),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn task_access(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
) -> Result<TaskAccess, TaskApplicationError> {
|
||||
let tool = self.resolve_authorized_tool(principal, tool_name).await?;
|
||||
let (owner_kind, owner_id) = principal.owner(&tool);
|
||||
Ok(TaskAccess {
|
||||
workspace_id: tool.workspace_id.as_str().to_owned(),
|
||||
owner_kind,
|
||||
owner_id,
|
||||
authorization_scope: principal.authorization_scope(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves the addressed Tool of the principal's published catalog and repeats its authorization
|
||||
/// check. A foreign workspace, an unknown Tool and an insufficient security level are typed
|
||||
/// refusals without metadata.
|
||||
async fn resolve_authorized_tool(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
) -> Result<PublishedAgentTool, TaskApplicationError> {
|
||||
let catalog = self
|
||||
.catalog
|
||||
.get_catalog(principal.workspace_slug(), principal.agent_slug())
|
||||
.await
|
||||
.map_err(|_| TaskApplicationError::NotFound)?;
|
||||
let tool = catalog
|
||||
.tools
|
||||
.into_iter()
|
||||
.find(|tool| tool.tool_name == tool_name)
|
||||
.ok_or(TaskApplicationError::NotFound)?;
|
||||
if let TaskPrincipal::Admin { workspace_slug, .. } = principal
|
||||
&& tool.workspace_slug != *workspace_slug
|
||||
{
|
||||
return Err(TaskApplicationError::Denied);
|
||||
}
|
||||
if let TaskPrincipal::Agent { credential, .. } = principal
|
||||
&& !credential_allows_security_level(credential, tool.operation.security_level)
|
||||
{
|
||||
return Err(TaskApplicationError::Denied);
|
||||
}
|
||||
Ok(tool)
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded retry snapshot pinned from the exact published operation at acceptance. A later
|
||||
/// publication, a restart or a new binary never changes the numbers an accepted Task was admitted
|
||||
/// with; a missing or oversized policy is clamped to the closed story bound.
|
||||
fn task_retry_policy(tool: &PublishedAgentTool) -> crank_core::TaskRetryPolicy {
|
||||
let max_attempts = tool
|
||||
.operation
|
||||
.execution_config
|
||||
.retry_policy
|
||||
.as_ref()
|
||||
.map_or(1, |policy| {
|
||||
policy.max_attempts.clamp(1, crank_core::TASK_MAX_ATTEMPTS)
|
||||
});
|
||||
crank_core::TaskRetryPolicy::new(max_attempts, 1)
|
||||
}
|
||||
@@ -415,7 +415,7 @@ pub(super) fn agent_id() -> AgentId {
|
||||
AgentId::new(format!("agent_{AGENT_SLUG}"))
|
||||
}
|
||||
|
||||
pub(super) async fn publish_agent(
|
||||
pub(crate) async fn publish_agent(
|
||||
registry: &PostgresRegistry,
|
||||
operation: &Operation<Schema, MappingSet>,
|
||||
) {
|
||||
@@ -470,7 +470,7 @@ pub(super) async fn publish_agent(
|
||||
.expect("the agent must be published");
|
||||
}
|
||||
|
||||
pub(super) async fn create_api_key(registry: &PostgresRegistry) -> String {
|
||||
pub(crate) async fn create_api_key(registry: &PostgresRegistry) -> String {
|
||||
let secret = format!("crk_dispatch_observer_{}", uuid::Uuid::now_v7().simple());
|
||||
let api_key = PlatformApiKey {
|
||||
id: PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
@@ -500,7 +500,7 @@ pub(super) fn hash_access_secret(secret: &str) -> String {
|
||||
URL_SAFE_NO_PAD.encode(Sha256::digest(secret.as_bytes()))
|
||||
}
|
||||
|
||||
pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
|
||||
pub(crate) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
|
||||
Operation {
|
||||
id: OperationId::new(format!("op_{TOOL_NAME}")),
|
||||
name: TOOL_NAME.to_owned(),
|
||||
@@ -550,6 +550,7 @@ pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -16,6 +16,8 @@ mod tool_search;
|
||||
mod transport;
|
||||
|
||||
pub use app::{
|
||||
build_app, build_app_with_artifacts, build_app_with_background_workers,
|
||||
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler,
|
||||
VerifiedIdempotentWrite, build_app, build_app_with_artifacts,
|
||||
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
|
||||
build_app_with_background_workers_and_limits,
|
||||
};
|
||||
@@ -373,7 +373,9 @@ pub(crate) const CAPABILITY_DESCRIPTORS: &[CapabilityIdDescriptor] = &[
|
||||
extension: None,
|
||||
deprecated: None,
|
||||
sub_features: FeatureSet(&["listChanged"]),
|
||||
advertised: Some(("tools", &[("listChanged", false)])),
|
||||
// Story 3.5: the adapter emits `notifications/tools/list_changed` from the committed
|
||||
// published Tool bindings of the Agent snapshot, so the declaration is now true.
|
||||
advertised: Some(("tools", &[("listChanged", true)])),
|
||||
revisions: MODERN_AND_LEGACY,
|
||||
},
|
||||
CapabilityIdDescriptor {
|
||||
|
||||
@@ -39,7 +39,7 @@ fn static_registration_is_unique_and_advertises_only_the_registered_capability()
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
}),
|
||||
"the registered advertisement must be the same reviewed object in {revision}"
|
||||
);
|
||||
@@ -50,7 +50,7 @@ fn static_registration_is_unique_and_advertises_only_the_registered_capability()
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
}),
|
||||
"the production advertisement entry point must serve {revision}"
|
||||
);
|
||||
@@ -264,7 +264,7 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -275,7 +275,7 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -286,7 +286,7 @@ fn an_inert_test_descriptor_changes_both_projections_and_is_never_registered_in_
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
|
||||
@@ -470,7 +470,7 @@ fn a_single_era_descriptor_never_leaks_into_the_other_era_advertisement() {
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
}),
|
||||
"the other era advertisement stays the reviewed object"
|
||||
);
|
||||
@@ -761,7 +761,7 @@ fn adding_and_removing_a_method_descriptor_changes_the_dispatch_decision() {
|
||||
"logging": {},
|
||||
"prompts": {"listChanged": true},
|
||||
"resources": {"listChanged": true, "subscribe": false},
|
||||
"tools": {"listChanged": false}
|
||||
"tools": {"listChanged": true}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
//! Story 3.19: Resource and Prompt list-change signals (FR-19).
|
||||
//! Story 3.19: Resource and Prompt list-change signals (FR-19); Story 3.5 adds the Tools signal of
|
||||
//! the same seam (FR-14).
|
||||
//!
|
||||
//! The positive result of the story is that a client which asked for the list-change signal of its
|
||||
//! revision learns that the *committed* Resource or Prompt catalog changed, and that the ordinary
|
||||
//! `resources/list`, `resources/read`, `resources/templates/list`, `prompts/list` and `prompts/get`
|
||||
//! stay authoritative for a client that misses the notification.
|
||||
//! revision learns that the *committed* Resource, Prompt or Tool catalog changed, and that the
|
||||
//! ordinary `resources/list`, `resources/read`, `resources/templates/list`, `prompts/list`,
|
||||
//! `prompts/get` and `tools/list` stay authoritative for a client that misses the notification.
|
||||
//!
|
||||
//! Shape of the chosen seam (ARC decision, kanban t_c12863fb): a **stream-local watcher**. One open
|
||||
//! notification stream owns one bounded periodic read of the published Agent catalog the stream is
|
||||
//! scoped to; a change of the visible Resource/Prompt binding projection produces the revision's
|
||||
//! scoped to; a change of the visible Resource/Prompt/Tool projection of that snapshot (the Resource
|
||||
//! and Prompt bindings plus the exact Tool definitions `tools/list` serves) produces the revision's
|
||||
//! list-change notification. There is no post-commit hook, no cross-process bus and no durable
|
||||
//! queue: the committed catalog is the source of truth, so a signal can never be produced before its
|
||||
//! commit, and a failed read can never roll the commit back — it only skips that tick and stays
|
||||
@@ -55,6 +57,8 @@ pub(crate) const SUBSCRIPTION_ID_KEY: &str = "io.modelcontextprotocol/subscripti
|
||||
pub(crate) const RESOURCE_LIST_CHANGED: &str = "notifications/resources/list_changed";
|
||||
/// `notifications/prompts/list_changed` (both approved revisions).
|
||||
pub(crate) const PROMPT_LIST_CHANGED: &str = "notifications/prompts/list_changed";
|
||||
/// `notifications/tools/list_changed` (both approved revisions, Story 3.5).
|
||||
pub(crate) const TOOL_LIST_CHANGED: &str = "notifications/tools/list_changed";
|
||||
/// `notifications/subscriptions/acknowledged` (modern revision only).
|
||||
pub(crate) const SUBSCRIPTIONS_ACKNOWLEDGED: &str = "notifications/subscriptions/acknowledged";
|
||||
|
||||
@@ -79,29 +83,32 @@ pub(crate) const MAX_FILTER_URI_BYTES: usize = 2_048;
|
||||
/// The one safe detail of a malformed or oversized `subscriptions/listen` filter.
|
||||
pub(crate) const INVALID_LISTEN_FILTER: &str = "invalid subscriptions/listen notification filter";
|
||||
|
||||
/// The two list-change signal types this server implements.
|
||||
/// The three list-change signal types this server implements.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct Signals {
|
||||
pub(crate) resources: bool,
|
||||
pub(crate) prompts: bool,
|
||||
pub(crate) tools: bool,
|
||||
}
|
||||
|
||||
impl Signals {
|
||||
/// Both types: the legacy revision defines no client opt-in, so its stream carries both.
|
||||
pub(crate) const BOTH: Self = Self {
|
||||
/// All types: the legacy revision defines no client opt-in, so its stream carries every type.
|
||||
pub(crate) const ALL: Self = Self {
|
||||
resources: true,
|
||||
prompts: true,
|
||||
tools: true,
|
||||
};
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))] // read by the unit oracle of the projection
|
||||
pub(crate) fn any(self) -> bool {
|
||||
self.resources || self.prompts
|
||||
self.resources || self.prompts || self.tools
|
||||
}
|
||||
|
||||
fn intersect(self, other: Self) -> Self {
|
||||
Self {
|
||||
resources: self.resources && other.resources,
|
||||
prompts: self.prompts && other.prompts,
|
||||
tools: self.tools && other.tools,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,8 +121,6 @@ impl Signals {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct ListenFilter {
|
||||
#[serde(default)]
|
||||
/// `toolsListChanged` is not implemented in this scope and is therefore never acknowledged.
|
||||
#[allow(dead_code)]
|
||||
tools_list_changed: bool,
|
||||
#[serde(default)]
|
||||
prompts_list_changed: bool,
|
||||
@@ -145,9 +150,9 @@ pub(crate) enum FilterOutcome {
|
||||
impl ListenFilter {
|
||||
/// Validate the bounded inputs and keep only the honoured types.
|
||||
///
|
||||
/// `toolsListChanged` and `resourceSubscriptions` are not implemented in the current scope, which
|
||||
/// is what the acknowledgment reports: they are simply not honoured, and the pinned schema
|
||||
/// forbids the server from sending a type it did not agree to.
|
||||
/// `resourceSubscriptions` is not implemented in the current scope, which is what the
|
||||
/// acknowledgment reports: it is simply not honoured, and the pinned schema forbids the server
|
||||
/// from sending a type it did not agree to.
|
||||
pub(crate) fn accepted(&self) -> FilterOutcome {
|
||||
if self.resource_subscriptions.len() > MAX_FILTER_URIS
|
||||
|| self
|
||||
@@ -160,6 +165,7 @@ impl ListenFilter {
|
||||
FilterOutcome::Accepted(Signals {
|
||||
resources: self.resources_list_changed,
|
||||
prompts: self.prompts_list_changed,
|
||||
tools: self.tools_list_changed,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -173,16 +179,23 @@ fn acknowledged_filter_wire(accepted: Signals) -> Value {
|
||||
if accepted.resources {
|
||||
filter.insert("resourcesListChanged".to_owned(), Value::Bool(true));
|
||||
}
|
||||
if accepted.tools {
|
||||
filter.insert("toolsListChanged".to_owned(), Value::Bool(true));
|
||||
}
|
||||
Value::Object(filter)
|
||||
}
|
||||
|
||||
/// The exact bound Resource/Prompt projection of one published Agent catalog.
|
||||
/// The exact bound Resource/Prompt/Tool projection of one published Agent catalog.
|
||||
///
|
||||
/// The projection is derived from the same committed catalog read that already feeds
|
||||
/// `resources/list`, `resources/templates/list`, `prompts/list` and `prompts/get`: the exact Resource
|
||||
/// and Prompt bindings of the published Agent snapshot, in deterministic binding order.
|
||||
/// `catalog_revision` alone is *not* the comparison key — a commit that only changes Tools must not
|
||||
/// produce a Resource/Prompt signal — so the two binding vectors are compared on their own.
|
||||
/// `resources/list`, `resources/templates/list`, `prompts/list`, `prompts/get` and `tools/list`: the
|
||||
/// exact Resource and Prompt bindings of the published Agent snapshot, in deterministic binding
|
||||
/// order, and the exact Tool definitions that `tools/list` would serve. The Tool part is taken from
|
||||
/// [`crate::manifest::catalog_tool_definitions`] rather than from the bindings, because that is the
|
||||
/// projection the client sees: it also depends on the Tool selection policy (mode, groups, search
|
||||
/// settings), and in `Search` mode it does not depend on the bindings at all. `catalog_revision`
|
||||
/// alone is *not* the comparison key — a commit that only changes Tools must not produce a Resource
|
||||
/// or Prompt signal, and vice versa — so the three projections are compared on their own.
|
||||
///
|
||||
/// shortcut: a Resource revocation invalidates a bound publication without a new Agent snapshot, so
|
||||
/// it does not change this projection and produces no list-change signal; the revocation gate at
|
||||
@@ -192,40 +205,45 @@ fn acknowledged_filter_wire(accepted: Signals) -> Value {
|
||||
pub(crate) struct CatalogFingerprint {
|
||||
resources: Vec<(String, u32)>,
|
||||
prompts: Vec<(String, u32)>,
|
||||
/// Canonical JSON of every definition `tools/list` serves, sorted so a re-ordered read is not a
|
||||
/// false change.
|
||||
tools: Vec<String>,
|
||||
}
|
||||
|
||||
impl CatalogFingerprint {
|
||||
pub(crate) fn of(catalog: &PublishedAgentCatalog) -> Self {
|
||||
let mut resources = catalog
|
||||
.resources
|
||||
.iter()
|
||||
.map(|binding| {
|
||||
fn sorted<T: Ord>(values: impl Iterator<Item = T>) -> Vec<T> {
|
||||
let mut sorted = values.collect::<Vec<_>>();
|
||||
sorted.sort_unstable();
|
||||
sorted
|
||||
}
|
||||
Self {
|
||||
resources: sorted(catalog.resources.iter().map(|binding| {
|
||||
(
|
||||
binding.resource_id.as_str().to_owned(),
|
||||
binding.resource_version,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut prompts = catalog
|
||||
.prompts
|
||||
.iter()
|
||||
.map(|binding| {
|
||||
})),
|
||||
prompts: sorted(catalog.prompts.iter().map(|binding| {
|
||||
(
|
||||
binding.prompt_id.as_str().to_owned(),
|
||||
binding.prompt_version,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
resources.sort();
|
||||
prompts.sort();
|
||||
Self { resources, prompts }
|
||||
})),
|
||||
tools: sorted(
|
||||
crate::manifest::catalog_tool_definitions(catalog)
|
||||
.into_iter()
|
||||
.map(|definition| definition.to_string()),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// The list-change types whose visible binding projection differs between `previous` and `self`.
|
||||
/// The list-change types whose visible projection differs between `previous` and `self`.
|
||||
pub(crate) fn changes_since(&self, previous: &Self) -> Signals {
|
||||
Signals {
|
||||
resources: self.resources != previous.resources,
|
||||
prompts: self.prompts != previous.prompts,
|
||||
tools: self.tools != previous.tools,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,10 +259,11 @@ pub(crate) enum Wire {
|
||||
|
||||
impl Wire {
|
||||
fn signal_frames(&self, signals: Signals) -> Vec<String> {
|
||||
let mut frames = Vec::with_capacity(2);
|
||||
let mut frames = Vec::with_capacity(3);
|
||||
for (changed, method) in [
|
||||
(signals.resources, RESOURCE_LIST_CHANGED),
|
||||
(signals.prompts, PROMPT_LIST_CHANGED),
|
||||
(signals.tools, TOOL_LIST_CHANGED),
|
||||
] {
|
||||
if !changed {
|
||||
continue;
|
||||
|
||||
@@ -2,35 +2,59 @@
|
||||
//! wire frames of both approved revisions and the bounded legacy stream ownership.
|
||||
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use crank_core::ToolSelectionPolicy;
|
||||
use crank_core::ids::{PromptId, ResourceId};
|
||||
use crank_core::{
|
||||
ExecutionConfig, HttpMethod, Operation, OperationId, OperationSecurityLevel, OperationStatus,
|
||||
Protocol, RestTarget, Target, ToolAccessMode, ToolDescription, ToolGroup, ToolSearchSettings,
|
||||
ToolSelectionPolicy, WorkspaceId,
|
||||
};
|
||||
use crank_mapping::MappingSet;
|
||||
use crank_registry::PublishedAgentCatalog;
|
||||
use crank_registry::model::{AgentPromptBinding, AgentResourceBinding};
|
||||
use crank_registry::model::{AgentPromptBinding, AgentResourceBinding, PublishedAgentTool};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use serde_json::{Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::{
|
||||
CatalogFingerprint, FilterOutcome, INVALID_LISTEN_FILTER, ListenFilter, MAX_FILTER_URIS,
|
||||
MAX_SUBSCRIPTION_ID_BYTES, PROMPT_LIST_CHANGED, RESOURCE_LIST_CHANGED, ReadError, Reader,
|
||||
SIGNAL_QUEUE_CAPACITY, SUBSCRIPTION_ID_KEY, SUBSCRIPTIONS_ACKNOWLEDGED, Signals,
|
||||
StreamRegistry, Watch, Wire, acknowledged_frame, run, subscription_id,
|
||||
StreamRegistry, TOOL_LIST_CHANGED, Watch, Wire, acknowledged_frame, run, subscription_id,
|
||||
};
|
||||
|
||||
fn catalog(
|
||||
resources: &[(&str, u32)],
|
||||
prompts: &[(&str, u32)],
|
||||
catalog_revision: &str,
|
||||
) -> PublishedAgentCatalog {
|
||||
catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
Vec::new(),
|
||||
resources,
|
||||
prompts,
|
||||
catalog_revision,
|
||||
)
|
||||
}
|
||||
|
||||
fn catalog_with(
|
||||
tool_selection_policy: ToolSelectionPolicy,
|
||||
tools: Vec<PublishedAgentTool>,
|
||||
resources: &[(&str, u32)],
|
||||
prompts: &[(&str, u32)],
|
||||
catalog_revision: &str,
|
||||
) -> PublishedAgentCatalog {
|
||||
PublishedAgentCatalog {
|
||||
agent_version: 1,
|
||||
catalog_revision: catalog_revision.to_owned(),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
tools: Vec::new(),
|
||||
tool_selection_policy,
|
||||
tools,
|
||||
resources: resources
|
||||
.iter()
|
||||
.map(|(id, version)| AgentResourceBinding {
|
||||
@@ -48,6 +72,94 @@ fn catalog(
|
||||
}
|
||||
}
|
||||
|
||||
fn search_policy(tool_names: &[&str]) -> ToolSelectionPolicy {
|
||||
ToolSelectionPolicy {
|
||||
mode: ToolAccessMode::Search,
|
||||
groups: vec![ToolGroup {
|
||||
id: "crm".to_owned(),
|
||||
name: "CRM".to_owned(),
|
||||
description: "CRM operations".to_owned(),
|
||||
tool_names: tool_names.iter().map(|name| (*name).to_owned()).collect(),
|
||||
}],
|
||||
search: ToolSearchSettings::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_object_schema() -> Schema {
|
||||
Schema {
|
||||
kind: SchemaKind::Object,
|
||||
format: None,
|
||||
description: None,
|
||||
required: false,
|
||||
nullable: false,
|
||||
default_value: None,
|
||||
fields: BTreeMap::new(),
|
||||
items: None,
|
||||
enum_values: Vec::new(),
|
||||
variants: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// One published Tool binding whose served definition is exactly `title`/`description`.
|
||||
fn published_tool(name: &str, title: &str, description: &str, version: u32) -> PublishedAgentTool {
|
||||
PublishedAgentTool {
|
||||
workspace_id: WorkspaceId::new("ws_test"),
|
||||
workspace_slug: "ws-test".to_owned(),
|
||||
agent_id: "agent_test".into(),
|
||||
agent_slug: "agent".to_owned(),
|
||||
operation: Operation {
|
||||
id: OperationId::new(format!("op_{name}")),
|
||||
name: name.to_owned(),
|
||||
display_name: title.to_owned(),
|
||||
category: "test".to_owned(),
|
||||
protocol: Protocol::Rest,
|
||||
security_level: OperationSecurityLevel::Standard,
|
||||
status: OperationStatus::Published,
|
||||
version,
|
||||
target: Target::Rest(RestTarget {
|
||||
base_url: "https://example.test".to_owned(),
|
||||
method: HttpMethod::Get,
|
||||
path_template: "/leads".to_owned(),
|
||||
static_headers: BTreeMap::new(),
|
||||
request_contract: None,
|
||||
response_contract: Vec::new(),
|
||||
multipart_field_mime_types: BTreeMap::new(),
|
||||
}),
|
||||
input_schema: empty_object_schema(),
|
||||
output_schema: empty_object_schema(),
|
||||
input_mapping: MappingSet { rules: Vec::new() },
|
||||
output_mapping: MappingSet { rules: Vec::new() },
|
||||
execution_config: ExecutionConfig {
|
||||
timeout_ms: 10_000,
|
||||
retry_policy: None,
|
||||
response_cache: None,
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
tool_description: ToolDescription {
|
||||
title: title.to_owned(),
|
||||
description: description.to_owned(),
|
||||
tags: Vec::new(),
|
||||
examples: Vec::new(),
|
||||
},
|
||||
samples: None,
|
||||
generated_draft: None,
|
||||
config_export: None,
|
||||
wizard_state: None,
|
||||
created_at: OffsetDateTime::UNIX_EPOCH,
|
||||
updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
published_at: None,
|
||||
},
|
||||
tool_name: name.to_owned(),
|
||||
tool_title: title.to_owned(),
|
||||
tool_description: description.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_committed_resource_binding_is_a_resource_change_only() {
|
||||
let before = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-1"));
|
||||
@@ -61,7 +173,8 @@ fn a_committed_resource_binding_is_a_resource_change_only() {
|
||||
after.changes_since(&before),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false
|
||||
prompts: false,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
assert!(!before.changes_since(&after).prompts);
|
||||
@@ -69,7 +182,8 @@ fn a_committed_resource_binding_is_a_resource_change_only() {
|
||||
before.changes_since(&after),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false
|
||||
prompts: false,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -83,7 +197,8 @@ fn a_committed_prompt_binding_is_a_prompt_change_only() {
|
||||
after.changes_since(&before),
|
||||
Signals {
|
||||
resources: false,
|
||||
prompts: true
|
||||
prompts: true,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -97,15 +212,219 @@ fn a_pinned_version_bump_is_a_change_even_when_the_binding_names_are_stable() {
|
||||
after.changes_since(&before),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false
|
||||
prompts: false,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tool_only_catalog_commit_is_silent() {
|
||||
// `catalog_revision` moved, but the visible Resource/Prompt projection did not: a Tool-only
|
||||
// commit must not produce a Resource or Prompt list-change signal.
|
||||
fn a_committed_tool_definition_change_is_a_tool_change_only() {
|
||||
let lead = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![lead()],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let added = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![
|
||||
lead(),
|
||||
published_tool(
|
||||
"crm_create_second",
|
||||
"Create Second",
|
||||
"Creates a second lead",
|
||||
1,
|
||||
),
|
||||
],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
let retitled = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create a lead",
|
||||
"Creates a lead",
|
||||
1,
|
||||
)],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
let tools_only = Signals {
|
||||
resources: false,
|
||||
prompts: false,
|
||||
tools: true,
|
||||
};
|
||||
assert_eq!(added.changes_since(&before), tools_only);
|
||||
assert_eq!(
|
||||
retitled.changes_since(&before),
|
||||
tools_only,
|
||||
"a retitled binding changes the visible Tool projection without a new Operation version"
|
||||
);
|
||||
// The reverse direction is a Tool change as well: a removed binding is a real change.
|
||||
assert!(before.changes_since(&added).tools);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_republished_operation_version_with_an_identical_definition_is_silent() {
|
||||
// `tools/list` serves the definition, not the Operation version: a version bump that leaves the
|
||||
// served definition unchanged is not a client-visible Tool catalog change.
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create Lead",
|
||||
"Creates a lead",
|
||||
1,
|
||||
)],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let after = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create Lead",
|
||||
"Creates a lead",
|
||||
2,
|
||||
)],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
assert!(!after.changes_since(&before).any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_policy_only_commit_that_changes_tools_list_is_a_tool_change_only() {
|
||||
// The Tool selection policy is part of the served `tools/list` projection: switching the same
|
||||
// bindings from direct to search mode replaces the served tools with `search_tools`/`call_tool`.
|
||||
let tool = published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let direct = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool.clone()],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let search = CatalogFingerprint::of(&catalog_with(
|
||||
search_policy(&["crm_create_lead"]),
|
||||
vec![tool],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
let tools_only = Signals {
|
||||
resources: false,
|
||||
prompts: false,
|
||||
tools: true,
|
||||
};
|
||||
assert_eq!(search.changes_since(&direct), tools_only);
|
||||
assert_eq!(direct.changes_since(&search), tools_only);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_policy_field_invisible_to_tools_list_is_silent() {
|
||||
// `search.max_results` is only served inside the search-mode definitions: in direct mode it
|
||||
// changes nothing the client receives.
|
||||
let tool = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool()],
|
||||
&[],
|
||||
&[],
|
||||
"rev-1",
|
||||
));
|
||||
let after = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy {
|
||||
search: ToolSearchSettings { max_results: 5 },
|
||||
..ToolSelectionPolicy::default()
|
||||
},
|
||||
vec![tool()],
|
||||
&[],
|
||||
&[],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
assert!(!after.changes_since(&before).any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_search_mode_binding_change_is_silent_when_tools_list_is_unchanged() {
|
||||
// In search mode the served list is `search_tools`/`call_tool`; the bindings behind it are not
|
||||
// part of `tools/list`, so a binding-only commit must not claim a visible Tool change.
|
||||
let policy = search_policy(&["crm_create_lead"]);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
policy.clone(),
|
||||
vec![published_tool(
|
||||
"crm_create_lead",
|
||||
"Create Lead",
|
||||
"Creates a lead",
|
||||
1,
|
||||
)],
|
||||
&[],
|
||||
&[],
|
||||
"rev-1",
|
||||
));
|
||||
let after = CatalogFingerprint::of(&catalog_with(
|
||||
policy,
|
||||
vec![
|
||||
published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1),
|
||||
published_tool(
|
||||
"crm_create_second",
|
||||
"Create Second",
|
||||
"Creates a second lead",
|
||||
1,
|
||||
),
|
||||
],
|
||||
&[],
|
||||
&[],
|
||||
"rev-2",
|
||||
));
|
||||
|
||||
assert!(!after.changes_since(&before).any());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resource_or_prompt_change_is_not_a_tool_change() {
|
||||
let tool = || published_tool("crm_create_lead", "Create Lead", "Creates a lead", 1);
|
||||
let before = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool()],
|
||||
&[("res_a", 1)],
|
||||
&[("prm_a", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
let changed = CatalogFingerprint::of(&catalog_with(
|
||||
ToolSelectionPolicy::default(),
|
||||
vec![tool()],
|
||||
&[("res_a", 2)],
|
||||
&[("prm_a", 1), ("prm_b", 1)],
|
||||
"rev-1",
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
changed.changes_since(&before),
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: true,
|
||||
tools: false
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_catalog_revision_bump_without_a_visible_change_is_silent() {
|
||||
// `catalog_revision` moved, but no visible Resource/Prompt/Tool projection did: no signal.
|
||||
let before = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-1"));
|
||||
let after = CatalogFingerprint::of(&catalog(&[("res_a", 1)], &[("prm_a", 1)], "rev-2"));
|
||||
|
||||
@@ -130,13 +449,14 @@ fn the_projection_is_order_independent_and_has_no_duplicate_growth() {
|
||||
|
||||
#[test]
|
||||
fn the_legacy_frames_carry_a_closed_method_name_and_no_parameters() {
|
||||
let frames = Wire::Legacy.signal_frames(Signals::BOTH);
|
||||
let frames = Wire::Legacy.signal_frames(Signals::ALL);
|
||||
|
||||
assert_eq!(
|
||||
frames,
|
||||
vec![
|
||||
json!({"jsonrpc": "2.0", "method": RESOURCE_LIST_CHANGED}).to_string(),
|
||||
json!({"jsonrpc": "2.0", "method": PROMPT_LIST_CHANGED}).to_string(),
|
||||
json!({"jsonrpc": "2.0", "method": TOOL_LIST_CHANGED}).to_string(),
|
||||
]
|
||||
);
|
||||
// No URI, name or payload of a changed object is part of a list-change notification.
|
||||
@@ -146,7 +466,7 @@ fn the_legacy_frames_carry_a_closed_method_name_and_no_parameters() {
|
||||
#[test]
|
||||
fn the_modern_frames_carry_the_subscription_id_of_their_listen_request() {
|
||||
let id = json!(41);
|
||||
let frames = Wire::Modern(id.clone()).signal_frames(Signals::BOTH);
|
||||
let frames = Wire::Modern(id.clone()).signal_frames(Signals::ALL);
|
||||
|
||||
assert_eq!(
|
||||
frames,
|
||||
@@ -163,6 +483,12 @@ fn the_modern_frames_carry_the_subscription_id_of_their_listen_request() {
|
||||
"params": { "_meta": { SUBSCRIPTION_ID_KEY: 41 } }
|
||||
})
|
||||
.to_string(),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": TOOL_LIST_CHANGED,
|
||||
"params": { "_meta": { SUBSCRIPTION_ID_KEY: 41 } }
|
||||
})
|
||||
.to_string(),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -176,6 +502,7 @@ fn only_the_requested_and_honoured_types_are_encoded() {
|
||||
let frames = Wire::Modern(json!(1)).signal_frames(Signals {
|
||||
resources: false,
|
||||
prompts: true,
|
||||
tools: false,
|
||||
});
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
@@ -189,6 +516,7 @@ fn the_acknowledgment_reports_exactly_the_honoured_subset() {
|
||||
Signals {
|
||||
resources: true,
|
||||
prompts: false,
|
||||
tools: true,
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
@@ -197,7 +525,7 @@ fn the_acknowledgment_reports_exactly_the_honoured_subset() {
|
||||
assert_eq!(frame["params"]["_meta"][SUBSCRIPTION_ID_KEY], json!(7));
|
||||
assert_eq!(
|
||||
frame["params"]["notifications"],
|
||||
json!({ "resourcesListChanged": true })
|
||||
json!({ "resourcesListChanged": true, "toolsListChanged": true })
|
||||
);
|
||||
}
|
||||
|
||||
@@ -224,8 +552,10 @@ fn the_filter_keeps_only_the_implemented_list_change_types() {
|
||||
signals,
|
||||
Signals {
|
||||
resources: false,
|
||||
prompts: true
|
||||
}
|
||||
prompts: true,
|
||||
tools: true
|
||||
},
|
||||
"the honoured set is the requested list-change subset: {signals:?}"
|
||||
),
|
||||
FilterOutcome::Invalid(detail) => panic!("a bounded filter must be accepted: {detail}"),
|
||||
}
|
||||
@@ -372,7 +702,7 @@ async fn a_failed_catalog_read_is_observable_and_loses_no_committed_change() {
|
||||
Watch {
|
||||
workspace_slug: "workspace-canary".to_owned(),
|
||||
agent_slug: "agent-canary".to_owned(),
|
||||
signals: Signals::BOTH,
|
||||
signals: Signals::ALL,
|
||||
wire: Wire::Legacy,
|
||||
legacy_stream: None,
|
||||
deadline: Some(Instant::now() + Duration::from_millis(2_100)),
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_catalog_follows_new_publication_without_reinitializing() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let list = json!({"jsonrpc":"2.0","id":60,"method":"tools/list"});
|
||||
let listed = frames(
|
||||
h.request("2025-11-25", Some(&session), &list)
|
||||
.send()
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
listed.last().unwrap()["result"]["tools"][0]["name"],
|
||||
"create_lead"
|
||||
);
|
||||
let workspace = crank_core::WorkspaceId::new("ws_default");
|
||||
let agent = crank_core::AgentId::new("agent_dispatch-observer");
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let version = crank_core::AgentVersion {
|
||||
agent_id: agent.clone(),
|
||||
version: 2,
|
||||
status: crank_core::AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: crank_core::ToolSelectionPolicy::default(),
|
||||
created_at: now,
|
||||
};
|
||||
let bindings = [crank_core::AgentOperationBinding {
|
||||
operation_id: crank_core::OperationId::new("op_create_lead"),
|
||||
operation_version: 1,
|
||||
tool_name: "new_publication_tool".into(),
|
||||
tool_title: "New Tool".into(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}];
|
||||
h.state
|
||||
.registry
|
||||
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
|
||||
workspace_id: &workspace,
|
||||
agent_id: &agent,
|
||||
version: &version,
|
||||
bindings: &bindings,
|
||||
updated_at: &now,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
h.state
|
||||
.registry
|
||||
.publish_agent(crank_registry::PublishAgentRequest {
|
||||
workspace_id: &workspace,
|
||||
agent_id: &agent,
|
||||
version: 2,
|
||||
published_at: &now,
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let listed = frames(
|
||||
h.request("2025-11-25", Some(&session), &list)
|
||||
.send()
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
listed.last().unwrap()["result"]["tools"][0]["name"],
|
||||
"new_publication_tool"
|
||||
);
|
||||
let mut message = call(61, "new-publication@example.test", false, false);
|
||||
message["params"]["name"] = json!("new_publication_tool");
|
||||
let result = frames(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
result.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_new-publication@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_identical_request_ids_in_two_sessions_cancel_independently() {
|
||||
let h = Harness::new().await;
|
||||
let a = h.initialize().await;
|
||||
let b = h.initialize().await;
|
||||
let message = call(62, "slow-same-id@example.test", false, true);
|
||||
let first = h
|
||||
.request("2025-11-25", Some(&a), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let second = h
|
||||
.request("2025-11-25", Some(&b), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
wait_count(&h.calls, 2).await;
|
||||
let cancel =
|
||||
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":62}});
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&a), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
wait_count(&h.active, 1).await;
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&b), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
wait_count(&h.active, 0).await;
|
||||
for response in [first, second] {
|
||||
assert_eq!(
|
||||
frames(response).await.last().unwrap()["error"]["message"],
|
||||
"request cancelled"
|
||||
);
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
// Independent QA: dropped legacy response bodies must not bypass the stream ceiling.
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_qa_legacy_disconnect_does_not_bypass_admission() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let mut admitted = Vec::new();
|
||||
for id in 100..117 {
|
||||
let response = h
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&call(id, "slow-qa-limit@example.test", false, true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
if response.status() == 429 {
|
||||
break;
|
||||
}
|
||||
assert_eq!(response.status(), 200);
|
||||
admitted.push(id);
|
||||
wait_count(&h.calls, admitted.len()).await;
|
||||
drop(response);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
let active_before_cleanup = h.active.load(Ordering::SeqCst);
|
||||
let free_slots = h.state.session_slots.available_permits();
|
||||
let calls_before_cleanup = h.calls.load(Ordering::SeqCst);
|
||||
for id in &admitted {
|
||||
let cancel =
|
||||
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":id}});
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
}
|
||||
wait_count(&h.active, 0).await;
|
||||
assert!(
|
||||
admitted.len() <= 16 && calls_before_cleanup <= 16 && active_before_cleanup <= 16,
|
||||
"legacy disconnect bypassed admission: admitted={}, calls={calls_before_cleanup}, active={active_before_cleanup}, free_slots={free_slots}",
|
||||
admitted.len()
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while h.state.session_slots.available_permits() != 16 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned cancellation cleanup must release all admission permits");
|
||||
assert_eq!(h.state.session_slots.available_permits(), 16);
|
||||
|
||||
// After cleanup, normal admission and REST execution must work again.
|
||||
let calls_before = h.calls.load(Ordering::SeqCst);
|
||||
let message = call(199, "recovery-after-cleanup@example.test", false, false);
|
||||
let response = h
|
||||
.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let result = frames(response).await;
|
||||
assert_eq!(
|
||||
result.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_recovery-after-cleanup@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), calls_before + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_correlation_headers_preserve_each_concurrent_request() {
|
||||
let h = Harness::new().await;
|
||||
let first = call(50, "correlation-first@example.test", true, false);
|
||||
let second = call(51, "correlation-second@example.test", true, false);
|
||||
let (a, b) = tokio::join!(
|
||||
h.request("2026-07-28", None, &first)
|
||||
.header("x-request-id", "rmcp-oracle-first")
|
||||
.send(),
|
||||
h.request("2026-07-28", None, &second)
|
||||
.header("x-request-id", "rmcp-oracle-second")
|
||||
.send()
|
||||
);
|
||||
let a = a.unwrap();
|
||||
let b = b.unwrap();
|
||||
assert_eq!(a.headers()["x-request-id"], "rmcp-oracle-first");
|
||||
assert_eq!(b.headers()["x-request-id"], "rmcp-oracle-second");
|
||||
assert_eq!(
|
||||
frames(a).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_correlation-first@example.test"
|
||||
);
|
||||
assert_eq!(
|
||||
frames(b).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_correlation-second@example.test"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_stream_limit_rejects_without_dispatch_then_recovers() {
|
||||
let h = Harness::new().await;
|
||||
let permits = h
|
||||
.state
|
||||
.session_slots
|
||||
.clone()
|
||||
.acquire_many_owned(16)
|
||||
.await
|
||||
.unwrap();
|
||||
let message = call(40, "limit@example.test", true, false);
|
||||
assert_eq!(
|
||||
h.request("2026-07-28", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
429
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
drop(permits);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
frames(response).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_limit@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_rate_limit_is_existing_admission_not_sdk_default() {
|
||||
let h = Harness::with_rate(1).await;
|
||||
let list = json!({"jsonrpc":"2.0","id":41,"method":"tools/list","params":{"_meta":meta()}});
|
||||
assert_eq!(
|
||||
h.request("2026-07-28", None, &list)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
200
|
||||
);
|
||||
assert_eq!(
|
||||
h.request(
|
||||
"2026-07-28",
|
||||
None,
|
||||
&call(42, "rate@example.test", true, false)
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
429
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_current_durable_store_survives_recreation_but_is_not_sdk_store() {
|
||||
use crate::session::{PostgresTransportSessionStore, TransportSessionStore};
|
||||
let h = Harness::new().await;
|
||||
let store = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let id = store
|
||||
.create(
|
||||
"2025-11-25",
|
||||
"default",
|
||||
"dispatch-observer",
|
||||
false,
|
||||
now,
|
||||
Some(now + time::Duration::hours(24)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(store.mark_initialized(&id, now).await.unwrap());
|
||||
drop(store);
|
||||
let restarted = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let persisted = restarted.get(&id).await.unwrap().unwrap();
|
||||
assert!(persisted.initialized);
|
||||
assert_eq!(persisted.workspace_slug, "default");
|
||||
assert_eq!(persisted.protocol_version, "2025-11-25");
|
||||
assert_eq!(
|
||||
h.request(
|
||||
"2025-11-25",
|
||||
Some(&id),
|
||||
&call(43, "durable@example.test", false, false)
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
404
|
||||
);
|
||||
assert!(
|
||||
restarted.get(&id).await.unwrap().is_some(),
|
||||
"SDK experiment never mutates existing storage"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_json_modern_and_legacy_sse_only_gap_are_observed() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let response = h
|
||||
.request(
|
||||
"2026-07-28",
|
||||
None,
|
||||
&call(44, "json@example.test", true, false),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("application/json")
|
||||
);
|
||||
assert_eq!(
|
||||
frames(response).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_json@example.test"
|
||||
);
|
||||
// Build a JSON-only request (not two contradictory Accept carriers).
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/dispatch-observer", h.server.1))
|
||||
.bearer_auth(&h.key)
|
||||
.header("Accept", "application/json")
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("MCP-Session-Id", &session)
|
||||
.json(&call(45, "legacy-json@example.test", false, false))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
406,
|
||||
"bundled SDK legacy mode requires SSE Accept; not transport parity"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_concurrent_keys_keep_outputs_and_cancel_ownership_separate() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
|
||||
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
|
||||
let first = call(46, "first@example.test", false, false);
|
||||
let second = call(47, "second@example.test", false, false);
|
||||
let (a, b) = tokio::join!(
|
||||
h.request("2025-11-25", Some(&session), &first).send(),
|
||||
h.request_as(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&second,
|
||||
"test-second-valid-key"
|
||||
)
|
||||
.send()
|
||||
);
|
||||
assert_eq!(
|
||||
frames(a.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_first@example.test"
|
||||
);
|
||||
assert_eq!(
|
||||
frames(b.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_second@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
//! Wire oracle for the isolated SDK adapter, with real registry and REST execution.
|
||||
#[path = "rmcp_http_limits.rs"]
|
||||
mod limits;
|
||||
use super::{AgentRoutePath, dispatch_observer::support, rmcp_http, rmcp_tools_tests::test_state};
|
||||
use axum::{Json, Router, extract::State, routing::post};
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
struct Server(tokio::task::JoinHandle<()>, String);
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
async fn serve(router: Router) -> Server {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
Server(
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
}),
|
||||
url,
|
||||
)
|
||||
}
|
||||
async fn upstream(State(count): State<Arc<AtomicUsize>>, Json(body): Json<Value>) -> Json<Value> {
|
||||
count.fetch_add(1, Ordering::SeqCst);
|
||||
if body["email"].as_str().unwrap().starts_with("slow") {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
|
||||
}
|
||||
Json(json!({"id": format!("lead_for_{}", body["email"].as_str().unwrap())}))
|
||||
}
|
||||
struct Harness {
|
||||
server: Server,
|
||||
_upstream: Server,
|
||||
key: String,
|
||||
calls: Arc<AtomicUsize>,
|
||||
state: Arc<super::AppState>,
|
||||
active: Arc<AtomicUsize>,
|
||||
}
|
||||
impl Harness {
|
||||
async fn new() -> Self {
|
||||
Self::with_rate(10_000).await
|
||||
}
|
||||
async fn with_rate(burst: u32) -> Self {
|
||||
let mut state = test_state().await;
|
||||
Arc::get_mut(&mut state).unwrap().api_rate_limiter = crank_runtime::RequestRateLimiter::new(
|
||||
crank_runtime::RequestRateLimitConfig::new(burst, burst).unwrap(),
|
||||
);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream = serve(
|
||||
Router::new()
|
||||
.route("/crm/leads", post(upstream))
|
||||
.with_state(calls.clone()),
|
||||
)
|
||||
.await;
|
||||
let mut operation = support::test_operation(&upstream.1);
|
||||
operation.execution_config.timeout_ms = 10_000;
|
||||
state
|
||||
.registry
|
||||
.create_operation(
|
||||
&crank_core::WorkspaceId::new("ws_default"),
|
||||
&operation,
|
||||
Some("alice"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
state
|
||||
.registry
|
||||
.publish_operation(crank_registry::PublishRequest {
|
||||
workspace_id: &crank_core::WorkspaceId::new("ws_default"),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &operation.created_at,
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
support::publish_agent(&state.registry, &operation).await;
|
||||
let key = support::create_api_key(&state.registry).await;
|
||||
let active = Arc::new(AtomicUsize::new(0));
|
||||
let server = serve(rmcp_http::router(
|
||||
state.clone(),
|
||||
AgentRoutePath {
|
||||
workspace_slug: "default".into(),
|
||||
agent_slug: "dispatch-observer".into(),
|
||||
},
|
||||
active.clone(),
|
||||
))
|
||||
.await;
|
||||
Self {
|
||||
server,
|
||||
_upstream: upstream,
|
||||
key,
|
||||
calls,
|
||||
state,
|
||||
active,
|
||||
}
|
||||
}
|
||||
fn request(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
message: &Value,
|
||||
) -> reqwest::RequestBuilder {
|
||||
self.request_as(revision, session, message, &self.key)
|
||||
}
|
||||
fn request_as(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
message: &Value,
|
||||
key: &str,
|
||||
) -> reqwest::RequestBuilder {
|
||||
let mut request = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/dispatch-observer", self.server.1))
|
||||
.bearer_auth(key)
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.header("MCP-Protocol-Version", revision);
|
||||
if let Some(session) = session {
|
||||
request = request.header("MCP-Session-Id", session);
|
||||
}
|
||||
if revision == "2026-07-28" {
|
||||
request = request.header("Mcp-Method", message["method"].as_str().unwrap());
|
||||
if let Some(name) = message.pointer("/params/name").and_then(Value::as_str) {
|
||||
request = request.header("Mcp-Name", name);
|
||||
}
|
||||
}
|
||||
request
|
||||
.json(message)
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
}
|
||||
async fn initialize(&self) -> String {
|
||||
let message = json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"oracle","version":"1"}}});
|
||||
let response = self
|
||||
.request("2025-11-25", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let session = response.headers()["MCP-Session-Id"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let body = frames(response).await;
|
||||
assert_eq!(
|
||||
body.last().unwrap()["result"]["protocolVersion"],
|
||||
"2025-11-25"
|
||||
);
|
||||
let response = self
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&json!({"jsonrpc":"2.0","method":"notifications/initialized"}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 202);
|
||||
session
|
||||
}
|
||||
}
|
||||
fn meta() -> Value {
|
||||
json!({"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{},"io.modelcontextprotocol/clientInfo":{"name":"oracle","version":"1"}})
|
||||
}
|
||||
async fn frames(response: reqwest::Response) -> Vec<Value> {
|
||||
let status = response.status();
|
||||
let text = response.text().await.unwrap();
|
||||
if let Ok(json) = serde_json::from_str(&text) {
|
||||
return vec![json];
|
||||
}
|
||||
let values: Vec<Value> = text
|
||||
.lines()
|
||||
.filter_map(|line| line.strip_prefix("data:"))
|
||||
.filter_map(|data| serde_json::from_str(data.trim()).ok())
|
||||
.collect();
|
||||
assert!(!values.is_empty(), "{status}: {text}");
|
||||
values
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_positive_tools_both_revisions() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
|
||||
if session.is_none() {
|
||||
let discover = json!({"jsonrpc":"2.0","id":2,"method":"server/discover","params":{"_meta":meta()}});
|
||||
let response = h.request(revision, None, &discover).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(!frames(response).await[0]["result"].is_null());
|
||||
}
|
||||
let list = json!({"jsonrpc":"2.0","id":3,"method":"tools/list","params":{"_meta": if session.is_none() { meta() } else { json!({}) }}});
|
||||
let response = h.request(revision, session, &list).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(response.headers().contains_key("x-request-id"));
|
||||
let listed = frames(response).await;
|
||||
assert_eq!(
|
||||
listed.last().unwrap()["result"]["tools"][0]["name"],
|
||||
"create_lead"
|
||||
);
|
||||
let call = json!({"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":"rmcp@example.test"},"_meta":if session.is_none() { meta() } else { json!({}) }}});
|
||||
let response = h.request(revision, session, &call).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
if session.is_none() {
|
||||
assert!(!response.headers().contains_key("MCP-Session-Id"));
|
||||
}
|
||||
let result = frames(response).await;
|
||||
assert_eq!(
|
||||
result.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_rmcp@example.test",
|
||||
"{result:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
fn call(id: i64, email: &str, modern: bool, progress: bool) -> Value {
|
||||
let mut metadata = if modern { meta() } else { json!({}) };
|
||||
if progress {
|
||||
metadata["progressToken"] = json!("wire-progress");
|
||||
}
|
||||
json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":email},"_meta":metadata}})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_credentials_revalidated_after_initialize_and_no_outbound() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let message = call(10, "denied@example.test", false, false);
|
||||
assert_eq!(
|
||||
h.request_as("2025-11-25", Some(&session), &message, "invalid-test-key")
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
// A scope change and expiry are authoritative registry changes, not session credentials.
|
||||
sqlx::query("update platform_api_keys set scopes_json = '[\"read\"]'::jsonb where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
403
|
||||
);
|
||||
sqlx::query("update platform_api_keys set scopes_json = '[\"read\",\"write\"]'::jsonb, expires_at = now() - interval '1 second' where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
sqlx::query("update platform_api_keys set expires_at = null where id = 'pk_dispatch_observer'")
|
||||
.execute(h.state.registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
h.state
|
||||
.registry
|
||||
.revoke_platform_api_key(
|
||||
&crank_core::WorkspaceId::new("ws_default"),
|
||||
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
&time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
for method in [reqwest::Method::GET, reqwest::Method::DELETE] {
|
||||
assert_eq!(
|
||||
reqwest::Client::new()
|
||||
.request(
|
||||
method,
|
||||
format!("{}/v1/default/dispatch-observer", h.server.1)
|
||||
)
|
||||
.bearer_auth(&h.key)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("Accept", "text/event-stream")
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_admission_rejects_foreign_route_revision_headers_and_body_overflow() {
|
||||
let h = Harness::new().await;
|
||||
let message = call(11, "denied@example.test", true, false);
|
||||
let response = h
|
||||
.request("2026-07-28", Some("foreign"), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 400);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.header("Mcp-Name", "foreign")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 400);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.header("Origin", "https://evil.example")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 403);
|
||||
assert_eq!(
|
||||
h.request("2030-01-01", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
400
|
||||
);
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/foreign-agent", h.server.1))
|
||||
.bearer_auth(&h.key)
|
||||
.json(&message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 404);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.body(vec![b' '; 2 * 1024 * 1024 + 1])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 413);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_progress_is_real_and_tool_error_is_not_protocol_error() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
|
||||
let response = h
|
||||
.request(
|
||||
revision,
|
||||
session,
|
||||
&call(20, "progress@example.test", session.is_none(), true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(
|
||||
response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("text/event-stream")
|
||||
);
|
||||
let messages = frames(response).await;
|
||||
let progress: Vec<_> = messages
|
||||
.iter()
|
||||
.filter(|m| m["method"] == "notifications/progress")
|
||||
.collect();
|
||||
assert_eq!(progress.len(), 3, "{messages:?}");
|
||||
for (index, phase) in [
|
||||
"tool_resolved",
|
||||
"authorization_resolved",
|
||||
"execution_dispatched",
|
||||
]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
assert_eq!(progress[index]["params"]["message"], *phase);
|
||||
assert_eq!(
|
||||
progress[index]["params"]["progress"].as_f64(),
|
||||
Some((index + 1) as f64)
|
||||
);
|
||||
assert_eq!(progress[index]["params"]["progressToken"], "wire-progress");
|
||||
}
|
||||
let mut invalid = call(21, "unused", session.is_none(), false);
|
||||
invalid["params"]["arguments"] = json!({"email":123});
|
||||
let response = h.request(revision, session, &invalid).send().await.unwrap();
|
||||
let messages = frames(response).await;
|
||||
let result = messages.last().unwrap();
|
||||
assert_eq!(result["result"]["isError"], true, "{messages:?}");
|
||||
assert!(result.get("error").is_none());
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_reconnect_live_manager_and_restart_loses_sdk_session() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
for _ in 0..2 {
|
||||
let response = reqwest::Client::new()
|
||||
.get(format!("{}/v1/default/dispatch-observer", h.server.1))
|
||||
.bearer_auth(&h.key)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("Accept", "text/event-stream")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
drop(response);
|
||||
}
|
||||
let response = h
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&call(22, "reconnect@example.test", false, false),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
frames(response).await.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_reconnect@example.test"
|
||||
);
|
||||
let fresh = serve(rmcp_http::router(
|
||||
h.state.clone(),
|
||||
AgentRoutePath {
|
||||
workspace_slug: "default".into(),
|
||||
agent_slug: "dispatch-observer".into(),
|
||||
},
|
||||
h.active.clone(),
|
||||
))
|
||||
.await;
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/dispatch-observer", fresh.1))
|
||||
.bearer_auth(&h.key)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.json(&call(23, "restart@example.test", false, false))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
404,
|
||||
"new SDK manager cannot restore from Crank session storage"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
async fn wait_count(counter: &AtomicUsize, expected: usize) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(3), async {
|
||||
while counter.load(Ordering::SeqCst) != expected {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded wait for the real execution/cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_legacy_cancel_is_owned_by_session_and_current_key() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let foreign_session = h.initialize().await;
|
||||
// A second valid credential can use this Agent/session but does not own the first call.
|
||||
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
|
||||
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
|
||||
let response = h
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&call(30, "slow-cancel@example.test", false, true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
wait_count(&h.calls, 1).await;
|
||||
let cancel =
|
||||
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":30}});
|
||||
assert_eq!(
|
||||
h.request_as(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&cancel,
|
||||
"test-second-valid-key"
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&foreign_session), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(75)).await;
|
||||
assert_eq!(
|
||||
h.active.load(Ordering::SeqCst),
|
||||
1,
|
||||
"foreign cancel must not stop call"
|
||||
);
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
wait_count(&h.active, 0).await;
|
||||
// Gap: SDK emits an error frame where the existing transport suppresses all terminal frames.
|
||||
let messages = frames(response).await;
|
||||
assert_eq!(
|
||||
messages.last().unwrap()["error"]["message"],
|
||||
"request cancelled"
|
||||
);
|
||||
assert!(!messages.iter().any(|m| m.get("result").is_some()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_modern_disconnect_drops_execution_and_releases_stream_slot() {
|
||||
let h = Harness::new().await;
|
||||
let response = h
|
||||
.request(
|
||||
"2026-07-28",
|
||||
None,
|
||||
&call(31, "slow-disconnect@example.test", true, true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
wait_count(&h.calls, 1).await;
|
||||
assert_eq!(h.active.load(Ordering::SeqCst), 1);
|
||||
drop(response);
|
||||
wait_count(&h.active, 0).await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while h.state.session_slots.available_permits() != 16 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
h.calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"disconnect never retries a possible external effect"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! Behavioural ATDD for the rmcp 3.7.0 Tools adapter over the existing typed Tools service.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crank_core::{
|
||||
AdmissionSchedulerConfig, CoordinationStateStore, CorrelationContext, ExecutionLocale,
|
||||
MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, MigrationAuthority, PostgresRegistry, PostgresTaskStore};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
|
||||
TaskService,
|
||||
};
|
||||
|
||||
use rmcp::model::{CallToolRequestParams, PaginatedRequestParams, RequestId};
|
||||
use rmcp::service::{RequestContext, serve_directly};
|
||||
use rmcp::{RoleServer, ServerHandler};
|
||||
|
||||
use super::{
|
||||
AppState,
|
||||
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
|
||||
};
|
||||
use crate::app::TaskApplication;
|
||||
use crate::auth::{CommunityMachineCredentialVerifier, VerifiedMachineCredential};
|
||||
use crate::catalog::PublishedToolCatalog;
|
||||
use crate::session::{ActiveSessionMetrics, InMemorySessionStore};
|
||||
use crate::transport::AllowedOrigins;
|
||||
|
||||
pub(super) async fn test_state() -> Arc<AppState> {
|
||||
let database_url =
|
||||
crank_test_support::postgres_schema_url_for_task("rmcp_tools_tests", "t_fdcc7df8").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
||||
|
||||
let coordination: Arc<dyn CoordinationStateStore> =
|
||||
Arc::new(InMemoryCoordinationStateStore::default());
|
||||
let sessions = Arc::new(InMemorySessionStore::default());
|
||||
let session_metrics = ActiveSessionMetrics::start(sessions.clone());
|
||||
let catalog = PublishedToolCatalog::new(registry.clone(), Duration::ZERO, coordination.clone());
|
||||
|
||||
let tasks_store = PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits::default(),
|
||||
AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("default admission limits and scheduler weights are valid");
|
||||
let tasks = Arc::new(TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_secs(60),
|
||||
coordination.clone(),
|
||||
TaskService::new(Arc::new(tasks_store)),
|
||||
));
|
||||
|
||||
Arc::new(AppState {
|
||||
registry,
|
||||
catalog,
|
||||
runtime: crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
api_rate_limiter: RequestRateLimiter::new(RequestRateLimitConfig {
|
||||
requests_per_second: 10_000,
|
||||
burst: 10_000,
|
||||
}),
|
||||
artifact_store: None,
|
||||
secret_crypto: SecretCrypto::new("01234567890123456789012345678901")
|
||||
.expect("test master key"),
|
||||
sessions,
|
||||
session_metrics,
|
||||
session_slots: Arc::new(tokio::sync::Semaphore::new(16)),
|
||||
credential_verifier: Arc::new(CommunityMachineCredentialVerifier),
|
||||
allowed_origins: AllowedOrigins::new(None),
|
||||
logging_levels: Arc::new(crate::protocol::LogLevelStore::new()),
|
||||
signal_streams: Arc::new(crate::protocol::StreamRegistry::new()),
|
||||
in_flight: Arc::new(super::lifecycle::InFlightCalls::new(
|
||||
super::lifecycle::MAX_IN_FLIGHT_CALLS,
|
||||
)),
|
||||
tasks,
|
||||
})
|
||||
}
|
||||
|
||||
fn verified_credential() -> VerifiedMachineCredential {
|
||||
VerifiedMachineCredential {
|
||||
machine_access_mode: MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: OperationSecurityLevel::Standard,
|
||||
scopes: vec![PlatformApiKeyScope::Write],
|
||||
platform_api_key_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn rmcp_tools(state: Arc<AppState>) -> RmcpTools {
|
||||
RmcpTools {
|
||||
state,
|
||||
active: Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn request_context(adapter: &RmcpTools) -> RequestContext<RoleServer> {
|
||||
let (client_io, server_io) = tokio::io::duplex(1024);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_io);
|
||||
let running = serve_directly::<RoleServer, _, _, _, _>(
|
||||
adapter.clone(),
|
||||
(server_reader, server_writer),
|
||||
None,
|
||||
);
|
||||
// keep client side alive for transport lifetime
|
||||
let _ = client_io;
|
||||
let mut context = RequestContext::new(RequestId::Number(1), running.peer().clone());
|
||||
let (mut parts, _) = axum::http::Request::new(()).into_parts();
|
||||
parts.extensions.insert(RmcpRequestIdentity {
|
||||
workspace_slug: "missing-workspace".into(),
|
||||
agent_slug: "missing-agent".into(),
|
||||
credential: verified_credential(),
|
||||
correlation: CorrelationContext::generate(),
|
||||
continuation: crate::protocol::ContinuationSupport::from_supported(false),
|
||||
locale: ExecutionLocale::En,
|
||||
session_id: None,
|
||||
progress_meta: None,
|
||||
sse_acceptable: false,
|
||||
});
|
||||
context.extensions.insert(parts);
|
||||
context
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_adapter_rejects_missing_request_identity() {
|
||||
let adapter = rmcp_tools(test_state().await);
|
||||
let mut context = request_context(&adapter).await;
|
||||
context.extensions.remove::<axum::http::request::Parts>();
|
||||
let result = <RmcpTools as ServerHandler>::list_tools(&adapter, None, context).await;
|
||||
assert!(
|
||||
result
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("missing authenticated request context")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_adapter_list_tools_reports_catalog_unavailable_for_unknown_agent() {
|
||||
let state = test_state().await;
|
||||
let adapter = rmcp_tools(state);
|
||||
let ctx = request_context(&adapter).await;
|
||||
|
||||
match <RmcpTools as ServerHandler>::list_tools(
|
||||
&adapter,
|
||||
Some(PaginatedRequestParams::default()),
|
||||
ctx,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(err) => assert!(
|
||||
err.message.contains("crate catalog unavailable"),
|
||||
"unexpected error: {}",
|
||||
err.message
|
||||
),
|
||||
Ok(_) => panic!("expected error outcome for unknown agent"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_adapter_call_tool_reports_catalog_unavailable_for_unknown_agent() {
|
||||
let state = test_state().await;
|
||||
let adapter = rmcp_tools(state);
|
||||
let ctx = request_context(&adapter).await;
|
||||
|
||||
let request = CallToolRequestParams::new("missing_tool");
|
||||
match <RmcpTools as ServerHandler>::call_tool(&adapter, request, ctx).await {
|
||||
Err(err) => assert!(
|
||||
err.message.contains("crate catalog unavailable"),
|
||||
"unexpected error: {}",
|
||||
err.message
|
||||
),
|
||||
Ok(_) => panic!("expected error outcome for unknown agent"),
|
||||
}
|
||||
}
|
||||
@@ -284,6 +284,7 @@ fn operation() -> RegistryOperation {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
//! Typed execution-admission seam (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Admission is the single gate in front of every real Tool execution: it decides whether the
|
||||
//! caller receives a scoped permit or a durable enqueue decision, and it owns the deployment-wide
|
||||
//! global/per-Agent/per-Tool capacity and the bounded waiting queue. The authoritative state lives
|
||||
//! in PostgreSQL (`crank-registry`), so several processes share one budget.
|
||||
//!
|
||||
//! Deliberately absent from this contract: credentials, connection or protocol revision (they must
|
||||
//! never widen a quota), request bodies and prepared headers (payload ownership is Story 4.2).
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{NoTasksFallback, ToolExecutionMode};
|
||||
|
||||
pub const ADMISSION_FINGERPRINT_MAX_BYTES: usize = 128;
|
||||
pub const ADMISSION_OWNER_TOKEN_MAX_BYTES: usize = 64;
|
||||
pub const MAX_RETRY_AFTER_MS: u64 = 60_000;
|
||||
/// Bounded retry guidance for an admission refusal (capacity or queue wait exhausted). One value for
|
||||
/// the store and every runtime so the client contract stays stable.
|
||||
pub const ADMISSION_OVERLOAD_RETRY_AFTER_MS: u64 = 250;
|
||||
|
||||
/// Current wall clock in unix milliseconds (absolute deadlines are persisted as unix ms).
|
||||
pub fn current_unix_ms() -> i64 {
|
||||
i64::try_from(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|elapsed| elapsed.as_millis())
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
.unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Bucket used for an execution that has no Agent (admin draft/test-run): an explicit admin origin
|
||||
/// bucket that is still bounded by the global limit.
|
||||
pub const ADMIN_ORIGIN_KEY: &str = "admin";
|
||||
|
||||
/// Exact execution identity pinned at admission time.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AdmissionScope {
|
||||
pub workspace_id: String,
|
||||
/// Agent id, or [`ADMIN_ORIGIN_KEY`] for an Agent-less administrative execution.
|
||||
pub agent_key: String,
|
||||
pub operation_id: String,
|
||||
/// Quota keys exclude the version; the version is pinned for execution semantics only.
|
||||
pub operation_version: u32,
|
||||
}
|
||||
|
||||
impl AdmissionScope {
|
||||
/// Quota key of the Agent bucket: `(workspace, agent)`.
|
||||
pub fn agent_bucket(&self) -> String {
|
||||
format!("{}:{}", self.workspace_id, self.agent_key)
|
||||
}
|
||||
|
||||
/// Quota key of the Tool bucket: `(workspace, operation)` without version.
|
||||
pub fn tool_bucket(&self) -> String {
|
||||
format!("{}:{}", self.workspace_id, self.operation_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AdmissionMode {
|
||||
/// The caller waits for a scoped permit and dispatches inside it.
|
||||
Synchronous,
|
||||
/// The store commits a durable enqueue decision and reserves queue capacity.
|
||||
Background,
|
||||
}
|
||||
|
||||
impl AdmissionMode {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Synchronous => "synchronous",
|
||||
Self::Background => "background",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-consuming approval evidence gathered by the runtime before admission.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ApprovalEvidence {
|
||||
/// The operation does not require confirmation.
|
||||
NotRequired,
|
||||
/// A verified approval exists for this exact scope and input.
|
||||
Granted,
|
||||
/// Confirmation is required and no verified approval exists yet.
|
||||
NotGranted,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionRequest {
|
||||
pub scope: AdmissionScope,
|
||||
pub mode: AdmissionMode,
|
||||
/// Published policy mode that produced this request.
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
pub no_tasks_fallback: NoTasksFallback,
|
||||
pub approval: ApprovalEvidence,
|
||||
/// Bounded request identity used only to correlate one admission entry.
|
||||
pub fingerprint: String,
|
||||
/// Absolute deadline (unix milliseconds), already including queue waiting time.
|
||||
pub expires_at_unix_ms: i64,
|
||||
}
|
||||
|
||||
impl AdmissionRequest {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
if self.fingerprint.is_empty()
|
||||
|| self.fingerprint.len() > ADMISSION_FINGERPRINT_MAX_BYTES
|
||||
|| self
|
||||
.fingerprint
|
||||
.bytes()
|
||||
.any(|byte| byte.is_ascii_control() || byte == b' ')
|
||||
{
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission fingerprint is outside its allowed bounds",
|
||||
));
|
||||
}
|
||||
if self.scope.workspace_id.is_empty() || self.scope.workspace_id.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission workspace is invalid",
|
||||
));
|
||||
}
|
||||
if self.scope.agent_key.is_empty() || self.scope.agent_key.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid("admission agent is invalid"));
|
||||
}
|
||||
if self.scope.operation_id.is_empty() || self.scope.operation_id.len() > 160 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission operation is invalid",
|
||||
));
|
||||
}
|
||||
if self.scope.operation_version == 0 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission operation version must be positive",
|
||||
));
|
||||
}
|
||||
if self.expires_at_unix_ms <= 0 {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission deadline is invalid",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Scoped permit. Held for the whole local execution future (including retries) and released
|
||||
/// explicitly; a dropped permit is cleaned up by a bounded owner-token CAS, never reused by TTL.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionPermit {
|
||||
pub ticket_id: u64,
|
||||
pub owner_token: String,
|
||||
pub scope: AdmissionScope,
|
||||
pub mode: AdmissionMode,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
}
|
||||
|
||||
/// Durable enqueue decision: a reserved queue slot with exact Published Version semantics.
|
||||
/// It is not a public Task id and it never authorises a worker dispatch (Story 4.2).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionDecision {
|
||||
pub ticket_id: u64,
|
||||
pub scope: AdmissionScope,
|
||||
pub expires_at_unix_ms: i64,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AdmissionOutcome {
|
||||
Permitted(AdmissionPermit),
|
||||
Enqueued(AdmissionDecision),
|
||||
/// Capacity or queue bounds were exhausted before any unbounded allocation.
|
||||
Overloaded {
|
||||
retry_after_ms: u64,
|
||||
},
|
||||
/// The queue wait reached the absolute deadline; nothing was dispatched.
|
||||
DeadlineElapsed,
|
||||
/// Confirmation is mandatory for this operation and no verified approval exists.
|
||||
ApprovalRequired,
|
||||
/// The published policy does not advertise durable Tasks and refuses the fallback.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct AdmissionQueueSnapshot {
|
||||
pub waiting: u64,
|
||||
pub executing: u64,
|
||||
/// Age of the oldest waiting request in the authoritative queue, in milliseconds.
|
||||
pub oldest_waiting_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum AdmissionStoreError {
|
||||
/// The mandatory safety/capacity store cannot confirm state; every dependent path fails closed.
|
||||
#[error("execution admission store is unavailable: {message}")]
|
||||
Unavailable { message: String },
|
||||
/// The store rejected a malformed admission request.
|
||||
#[error("execution admission request is invalid: {message}")]
|
||||
Invalid { message: String },
|
||||
}
|
||||
|
||||
impl AdmissionStoreError {
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::Unavailable {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn invalid(message: impl Into<String>) -> Self {
|
||||
Self::Invalid {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ExecutionAdmission: Send + Sync {
|
||||
/// Proves the mandatory admission state is usable (schema and operations reachable).
|
||||
async fn health(&self) -> Result<(), AdmissionStoreError>;
|
||||
|
||||
/// Waits (bounded by `expires_at_unix_ms`) for a scoped permit, or commits a durable enqueue
|
||||
/// decision for a background request, or returns a typed refusal.
|
||||
async fn admit(
|
||||
&self,
|
||||
request: AdmissionRequest,
|
||||
) -> Result<AdmissionOutcome, AdmissionStoreError>;
|
||||
|
||||
/// Releases the caller's own permit. Idempotent: a repeated release returns `false`.
|
||||
async fn release(&self, permit: &AdmissionPermit) -> Result<bool, AdmissionStoreError>;
|
||||
|
||||
/// Removes the caller's own waiting ticket. Idempotent and never touches a foreign ticket.
|
||||
async fn cancel(&self, ticket_id: u64, owner_token: &str) -> Result<bool, AdmissionStoreError>;
|
||||
|
||||
/// Bounded authoritative queue readings for low-cardinality metrics.
|
||||
async fn snapshot(&self) -> Result<AdmissionQueueSnapshot, AdmissionStoreError>;
|
||||
}
|
||||
|
||||
/// One waiting entry as seen by the fairness scheduler.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionQueueEntry {
|
||||
pub ticket_id: u64,
|
||||
/// Authoritative insertion order inside `(Agent, Tool)`; never rewritten.
|
||||
pub sequence: u64,
|
||||
pub agent_key: String,
|
||||
pub operation_id: String,
|
||||
}
|
||||
|
||||
/// Weighted round-robin state persisted with the authoritative queue.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct AdmissionCursor {
|
||||
pub agent: Option<String>,
|
||||
pub agent_credit: u32,
|
||||
pub tool: Option<String>,
|
||||
pub tool_credit: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionSchedulerConfig {
|
||||
pub agent_weight: u32,
|
||||
pub tool_weight: u32,
|
||||
}
|
||||
|
||||
impl Default for AdmissionSchedulerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
agent_weight: 1,
|
||||
tool_weight: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdmissionSchedulerConfig {
|
||||
pub fn validate(&self) -> Result<(), AdmissionStoreError> {
|
||||
if !(1..=64).contains(&self.agent_weight) || !(1..=64).contains(&self.tool_weight) {
|
||||
return Err(AdmissionStoreError::invalid(
|
||||
"admission weights must be between 1 and 64",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of one fair-selection step: the queue index to grant and the cursor to persist when the
|
||||
/// grant actually happens. The cursor is not mutated on a blocked head, so a blocked head keeps its
|
||||
/// turn instead of being skipped by newer arrivals.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdmissionSelection {
|
||||
pub index: usize,
|
||||
pub cursor: AdmissionCursor,
|
||||
}
|
||||
|
||||
/// Hierarchical weighted round-robin: Agents first, then the Tools of the chosen Agent, FIFO inside
|
||||
/// `(Agent, Tool)`. A new waiter never resets the cursor and never inherits idle credit.
|
||||
pub fn select_next_grant(
|
||||
queue: &[AdmissionQueueEntry],
|
||||
cursor: &AdmissionCursor,
|
||||
config: AdmissionSchedulerConfig,
|
||||
) -> Option<AdmissionSelection> {
|
||||
if queue.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let agent_weight = config.agent_weight.max(1);
|
||||
let tool_weight = config.tool_weight.max(1);
|
||||
let mut agents: Vec<&str> = queue.iter().map(|entry| entry.agent_key.as_str()).collect();
|
||||
agents.sort_unstable();
|
||||
agents.dedup();
|
||||
|
||||
let mut next = cursor.clone();
|
||||
let agent = match cursor.agent.as_deref() {
|
||||
Some(current) if cursor.agent_credit > 0 && agents.contains(¤t) => current.to_owned(),
|
||||
_ => {
|
||||
let chosen = match cursor.agent.as_deref() {
|
||||
None => agents[0].to_owned(),
|
||||
Some(current) => agents
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| *candidate > current)
|
||||
.unwrap_or(agents[0])
|
||||
.to_owned(),
|
||||
};
|
||||
if cursor.agent.as_deref() != Some(chosen.as_str()) {
|
||||
next.tool = None;
|
||||
next.tool_credit = 0;
|
||||
}
|
||||
next.agent = Some(chosen.clone());
|
||||
next.agent_credit = agent_weight;
|
||||
chosen
|
||||
}
|
||||
};
|
||||
|
||||
let mut tools: Vec<&str> = queue
|
||||
.iter()
|
||||
.filter(|entry| entry.agent_key == agent)
|
||||
.map(|entry| entry.operation_id.as_str())
|
||||
.collect();
|
||||
tools.sort_unstable();
|
||||
tools.dedup();
|
||||
|
||||
let tool = match cursor.tool.as_deref() {
|
||||
Some(current) if cursor.tool_credit > 0 && tools.contains(¤t) => current.to_owned(),
|
||||
_ => {
|
||||
let chosen = match cursor.tool.as_deref() {
|
||||
None => tools[0].to_owned(),
|
||||
Some(current) => tools
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| *candidate > current)
|
||||
.unwrap_or(tools[0])
|
||||
.to_owned(),
|
||||
};
|
||||
next.tool = Some(chosen.clone());
|
||||
next.tool_credit = tool_weight;
|
||||
chosen
|
||||
}
|
||||
};
|
||||
|
||||
let index = queue
|
||||
.iter()
|
||||
.position(|entry| entry.agent_key == agent && entry.operation_id == tool)?;
|
||||
next.agent_credit = next.agent_credit.saturating_sub(1);
|
||||
next.tool_credit = next.tool_credit.saturating_sub(1);
|
||||
Some(AdmissionSelection {
|
||||
index,
|
||||
cursor: next,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ADMIN_ORIGIN_KEY, AdmissionCursor, AdmissionMode, AdmissionRequest,
|
||||
AdmissionSchedulerConfig, AdmissionScope, AdmissionStoreError, ApprovalEvidence,
|
||||
select_next_grant,
|
||||
};
|
||||
use crate::{NoTasksFallback, ToolExecutionMode};
|
||||
|
||||
fn entry(ticket_id: u64, agent: &str, tool: &str) -> super::AdmissionQueueEntry {
|
||||
super::AdmissionQueueEntry {
|
||||
ticket_id,
|
||||
sequence: ticket_id,
|
||||
agent_key: agent.to_owned(),
|
||||
operation_id: tool.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn request() -> AdmissionRequest {
|
||||
AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
agent_key: ADMIN_ORIGIN_KEY.to_owned(),
|
||||
operation_id: "op_leads".to_owned(),
|
||||
operation_version: 3,
|
||||
},
|
||||
mode: AdmissionMode::Synchronous,
|
||||
policy_mode: ToolExecutionMode::Synchronous,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: "fp_0123456789".to_owned(),
|
||||
expires_at_unix_ms: 1_790_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_keys_exclude_version_and_keep_agent_bucket() {
|
||||
let scope = request().scope;
|
||||
assert_eq!(scope.agent_bucket(), "ws_default:admin");
|
||||
assert_eq!(scope.tool_bucket(), "ws_default:op_leads");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_validation_rejects_zero_deadline_and_oversized_fingerprint() {
|
||||
let mut invalid = request();
|
||||
invalid.expires_at_unix_ms = 0;
|
||||
assert!(matches!(
|
||||
invalid.validate(),
|
||||
Err(AdmissionStoreError::Invalid { .. })
|
||||
));
|
||||
|
||||
let mut invalid = request();
|
||||
invalid.fingerprint = "x".repeat(super::ADMISSION_FINGERPRINT_MAX_BYTES + 1);
|
||||
assert!(invalid.validate().is_err());
|
||||
|
||||
let mut invalid = request();
|
||||
invalid.scope.operation_version = 0;
|
||||
assert!(invalid.validate().is_err());
|
||||
|
||||
assert!(request().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weights_are_bounded_positive_configuration() {
|
||||
assert!(
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 0,
|
||||
tool_weight: 1
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
AdmissionSchedulerConfig {
|
||||
agent_weight: 1,
|
||||
tool_weight: 65
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(AdmissionSchedulerConfig::default().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weighted_round_robin_serves_each_agent_its_quantum() {
|
||||
let mut queue = vec![
|
||||
entry(1, "agent_a", "op_1"),
|
||||
entry(2, "agent_a", "op_1"),
|
||||
entry(3, "agent_a", "op_1"),
|
||||
entry(4, "agent_b", "op_1"),
|
||||
entry(5, "agent_b", "op_1"),
|
||||
];
|
||||
let config = AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
};
|
||||
let mut cursor = AdmissionCursor::default();
|
||||
let mut order = Vec::new();
|
||||
while !queue.is_empty() {
|
||||
let selection = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
order.push(queue.remove(selection.index).agent_key.clone());
|
||||
cursor = selection.cursor;
|
||||
}
|
||||
assert_eq!(
|
||||
order,
|
||||
vec!["agent_a", "agent_a", "agent_b", "agent_b", "agent_a"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fifo_inside_one_tool_and_round_robin_between_tools() {
|
||||
let mut queue = vec![
|
||||
entry(1, "agent_a", "op_1"),
|
||||
entry(2, "agent_a", "op_1"),
|
||||
entry(3, "agent_a", "op_2"),
|
||||
];
|
||||
let config = AdmissionSchedulerConfig::default();
|
||||
let mut cursor = AdmissionCursor::default();
|
||||
let mut order = Vec::new();
|
||||
while !queue.is_empty() {
|
||||
let selection = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
// A granted ticket leaves the waiting queue: granted entries are never re-granted.
|
||||
order.push(queue.remove(selection.index).ticket_id);
|
||||
cursor = selection.cursor;
|
||||
}
|
||||
// One grant per Tool per round; FIFO is preserved inside each `(Agent, Tool)` queue.
|
||||
assert_eq!(order, vec![1, 3, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_arrival_does_not_reset_cursor_or_receive_idle_credit() {
|
||||
let queue = vec![entry(1, "agent_a", "op_1")];
|
||||
let config = AdmissionSchedulerConfig {
|
||||
agent_weight: 2,
|
||||
tool_weight: 1,
|
||||
};
|
||||
let cursor = AdmissionCursor::default();
|
||||
let first = select_next_grant(&queue, &cursor, config).unwrap();
|
||||
assert_eq!(first.cursor.agent.as_deref(), Some("agent_a"));
|
||||
assert_eq!(first.cursor.agent_credit, 1);
|
||||
|
||||
// A later Agent joins; the cursor keeps its remaining quantum for `agent_a`.
|
||||
let queue = vec![entry(1, "agent_a", "op_1"), entry(2, "agent_b", "op_9")];
|
||||
let second = select_next_grant(&queue, &first.cursor, config).unwrap();
|
||||
assert_eq!(queue[second.index].agent_key, "agent_a");
|
||||
assert_eq!(second.cursor.agent_credit, 0);
|
||||
|
||||
let third = select_next_grant(&queue, &second.cursor, config).unwrap();
|
||||
assert_eq!(queue[third.index].agent_key, "agent_b");
|
||||
assert_eq!(third.cursor.agent_credit, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_queue_keeps_cursor_untouched() {
|
||||
let cursor = AdmissionCursor {
|
||||
agent: Some("agent_b".to_owned()),
|
||||
agent_credit: 1,
|
||||
tool: Some("op_1".to_owned()),
|
||||
tool_credit: 0,
|
||||
};
|
||||
assert_eq!(
|
||||
select_next_grant(&[], &cursor, AdmissionSchedulerConfig::default()),
|
||||
None
|
||||
);
|
||||
assert_eq!(cursor.agent.as_deref(), Some("agent_b"));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod access;
|
||||
pub mod admission;
|
||||
pub mod agent;
|
||||
pub mod approval;
|
||||
pub mod auth;
|
||||
@@ -17,6 +18,7 @@ pub mod prompt_composition;
|
||||
pub mod protocol;
|
||||
pub mod resource;
|
||||
pub mod secret;
|
||||
pub mod task;
|
||||
pub mod tool_catalog;
|
||||
pub mod tool_quality;
|
||||
pub mod tool_search;
|
||||
@@ -60,10 +62,11 @@ pub mod domain {
|
||||
pub use crate::onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
|
||||
pub use crate::operation::{
|
||||
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
|
||||
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
|
||||
IdempotencyMode, IdempotencyPolicy, NoTasksFallback, Operation, OperationApprovalMode,
|
||||
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
|
||||
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, ResponseCachePolicy,
|
||||
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, WizardState,
|
||||
RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
|
||||
ToolExecutionPolicy, WizardState,
|
||||
};
|
||||
pub use crate::product_event::{
|
||||
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
|
||||
@@ -71,6 +74,18 @@ pub mod domain {
|
||||
};
|
||||
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
|
||||
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use crate::task::{
|
||||
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
|
||||
AttemptResolution, BeginAttempt, TASK_APPROVAL_REFERENCE_MAX_BYTES,
|
||||
TASK_CANCEL_REASON_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
|
||||
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES,
|
||||
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
|
||||
TASK_IDENTITY_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
|
||||
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
|
||||
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind,
|
||||
TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskSupportIds,
|
||||
};
|
||||
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
|
||||
pub use crate::tool_quality::{
|
||||
ToolQualityCatalogTool, ToolQualityFinding, ToolQualityMappingRule, ToolQualityMappingSet,
|
||||
@@ -108,12 +123,20 @@ pub mod ports {
|
||||
PreparedRequest, ProtocolAdapter, ProtocolAdapterError, ResponseCacheScope,
|
||||
RuntimeRequestContext, SharedProtocolAdapter,
|
||||
};
|
||||
pub use crate::task::TaskStore;
|
||||
}
|
||||
|
||||
pub use access::{
|
||||
InvitationStatus, InvitationToken, Membership, MembershipRole, PlatformApiKey,
|
||||
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, User, UserStatus,
|
||||
};
|
||||
pub use admission::{
|
||||
ADMIN_ORIGIN_KEY, ADMISSION_FINGERPRINT_MAX_BYTES, ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
AdmissionCursor, AdmissionDecision, AdmissionMode, AdmissionOutcome, AdmissionPermit,
|
||||
AdmissionQueueEntry, AdmissionQueueSnapshot, AdmissionRequest, AdmissionSchedulerConfig,
|
||||
AdmissionScope, AdmissionSelection, AdmissionStoreError, ApprovalEvidence, ExecutionAdmission,
|
||||
MAX_RETRY_AFTER_MS, current_unix_ms, select_next_grant,
|
||||
};
|
||||
pub use agent::{
|
||||
Agent, AgentOperationBinding, AgentStatus, AgentVersion, ToolAccessMode, ToolGroup,
|
||||
ToolSearchSettings, ToolSelectionPolicy, ToolSelectionPolicyError,
|
||||
@@ -168,13 +191,14 @@ pub use observability::{
|
||||
pub use onboarding::{OnboardingProjection, OnboardingStep, OnboardingStepId};
|
||||
pub use operation::{
|
||||
ConfigExport, ConfirmationPolicy, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus,
|
||||
IdempotencyMode, IdempotencyPolicy, Operation, OperationApprovalMode,
|
||||
OperationApprovalPayloadPreviewMode, OperationApprovalPolicy, OperationApprovalRiskLevel,
|
||||
OperationAvailability, OperationLifecycle, OperationLifecycleAction, OperationLifecycleError,
|
||||
OperationSafetyClass, OperationSafetyPolicy, OperationStatus, OperationVersionState,
|
||||
ResponseCachePolicy, RestRequestContract, RestRequestContractVersion, RestRequestDescriptor,
|
||||
RestResponseDescriptor, RestTarget, RetryPolicy, Samples, Target, ToolDescription, ToolExample,
|
||||
WizardState,
|
||||
IdempotencyMode, IdempotencyPolicy, MAX_TOOL_DEADLINE_MS, NoTasksFallback, Operation,
|
||||
OperationApprovalMode, OperationApprovalPayloadPreviewMode, OperationApprovalPolicy,
|
||||
OperationApprovalRiskLevel, OperationAvailability, OperationLifecycle,
|
||||
OperationLifecycleAction, OperationLifecycleError, OperationSafetyClass, OperationSafetyPolicy,
|
||||
OperationStatus, OperationVersionState, ResponseCachePolicy, RestRequestContract,
|
||||
RestRequestContractVersion, RestRequestDescriptor, RestResponseDescriptor, RestTarget,
|
||||
RetryPolicy, Samples, Target, ToolDescription, ToolExample, ToolExecutionMode,
|
||||
ToolExecutionPolicy, ToolExecutionPolicyError, WizardState,
|
||||
};
|
||||
pub use product_event::{
|
||||
OnboardingMilestone, PRODUCT_EVENT_IDEMPOTENCY_KEY_MAX_BYTES, PRODUCT_EVENT_SCHEMA_VERSION,
|
||||
@@ -182,6 +206,19 @@ pub use product_event::{
|
||||
};
|
||||
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
|
||||
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use task::{
|
||||
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
|
||||
AttemptResolution, BeginAttempt, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
|
||||
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
|
||||
TASK_INPUT_PROMPT_CODE_MAX_BYTES, TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
TASK_INPUT_WAIT_ID_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
|
||||
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest,
|
||||
TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
|
||||
TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
|
||||
TaskRetryPolicy, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
|
||||
redact_task_envelope, sensitive_task_key, validate_task_envelope,
|
||||
};
|
||||
pub use tool_catalog::{
|
||||
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
|
||||
ToolCatalogBudget, analyze_tool_catalog,
|
||||
|
||||
@@ -270,6 +270,72 @@ pub struct OperationApprovalPolicy {
|
||||
pub elicitation_message: Option<String>,
|
||||
}
|
||||
|
||||
/// Published execution mode of one Tool (Story 4.1, FR-21).
|
||||
///
|
||||
/// This is a scheduling policy of the Published Tool version. It is deliberately separate from
|
||||
/// `ext::protocol::ExecutionMode`, which describes the protocol adapter (streaming vs unary).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolExecutionMode {
|
||||
#[default]
|
||||
Synchronous,
|
||||
Background,
|
||||
Adaptive,
|
||||
}
|
||||
|
||||
/// Behaviour promised to a client that does not negotiate durable Tasks (Story 4.1).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NoTasksFallback {
|
||||
/// Execute through the same synchronous admission path.
|
||||
#[default]
|
||||
Synchronous,
|
||||
/// Reject with a typed unsupported-execution-mode failure; never a Task ID.
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
pub const MAX_TOOL_DEADLINE_MS: u64 = 3_600_000;
|
||||
|
||||
/// Execution policy pinned by a Published Tool version.
|
||||
///
|
||||
/// Absence of the policy keeps the pre-4.1 behaviour: synchronous execution bounded by
|
||||
/// `ExecutionConfig::timeout_ms`, with no background path.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ToolExecutionPolicy {
|
||||
#[serde(default)]
|
||||
pub mode: ToolExecutionMode,
|
||||
/// Absolute budget of one execution, counted from the start of admission.
|
||||
pub deadline_ms: u64,
|
||||
#[serde(default)]
|
||||
pub no_tasks_fallback: NoTasksFallback,
|
||||
}
|
||||
|
||||
impl Default for ToolExecutionPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
mode: ToolExecutionMode::Synchronous,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolExecutionPolicy {
|
||||
pub fn validate(&self) -> Result<(), ToolExecutionPolicyError> {
|
||||
if !(1..=MAX_TOOL_DEADLINE_MS).contains(&self.deadline_ms) {
|
||||
return Err(ToolExecutionPolicyError::DeadlineOutOfRange);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ToolExecutionPolicyError {
|
||||
#[error("tool execution deadline must be between 1 and {MAX_TOOL_DEADLINE_MS} ms")]
|
||||
DeadlineOutOfRange,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ExecutionConfig {
|
||||
@@ -285,6 +351,8 @@ pub struct ExecutionConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub approval_policy: Option<OperationApprovalPolicy>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub execution_policy: Option<ToolExecutionPolicy>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub auth_profile_ref: Option<AuthProfileId>,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub headers: BTreeMap<String, String>,
|
||||
@@ -444,9 +512,10 @@ mod tests {
|
||||
edition::OperationSecurityLevel,
|
||||
ids::{AuthProfileId, OperationId},
|
||||
operation::{
|
||||
ConfigExport, ExecutionConfig, Operation, OperationAvailability, OperationLifecycle,
|
||||
OperationLifecycleAction, OperationLifecycleError, OperationStatus,
|
||||
ConfigExport, ExecutionConfig, NoTasksFallback, Operation, OperationAvailability,
|
||||
OperationLifecycle, OperationLifecycleAction, OperationLifecycleError, OperationStatus,
|
||||
OperationVersionState, RestTarget, Samples, Target, ToolDescription, ToolExample,
|
||||
ToolExecutionMode, ToolExecutionPolicy,
|
||||
},
|
||||
protocol::{AuthKind, ExportMode, HttpMethod, Protocol},
|
||||
};
|
||||
@@ -664,6 +733,7 @@ updated_at: 2026-03-25T08:10:00Z
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: Some(AuthProfileId::new("auth_01")),
|
||||
headers: BTreeMap::new(),
|
||||
},
|
||||
@@ -684,4 +754,76 @@ updated_at: 2026-03-25T08:10:00Z
|
||||
published_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_round_trips_all_modes_and_fallbacks() {
|
||||
for (mode, fallback) in [
|
||||
(ToolExecutionMode::Synchronous, NoTasksFallback::Synchronous),
|
||||
(ToolExecutionMode::Background, NoTasksFallback::Synchronous),
|
||||
(ToolExecutionMode::Background, NoTasksFallback::Unsupported),
|
||||
(ToolExecutionMode::Adaptive, NoTasksFallback::Unsupported),
|
||||
] {
|
||||
let config = ExecutionConfig {
|
||||
execution_policy: Some(ToolExecutionPolicy {
|
||||
mode,
|
||||
deadline_ms: 45_000,
|
||||
no_tasks_fallback: fallback,
|
||||
}),
|
||||
..ExecutionConfig::default()
|
||||
};
|
||||
let encoded = serde_json::to_value(&config).unwrap();
|
||||
assert_eq!(encoded["execution_policy"]["deadline_ms"], json!(45_000));
|
||||
let decoded: ExecutionConfig = serde_json::from_value(encoded).unwrap();
|
||||
assert_eq!(decoded, config);
|
||||
assert_eq!(
|
||||
decoded.execution_policy.unwrap().mode,
|
||||
mode,
|
||||
"the published mode must survive the JSON round-trip"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_execution_config_without_policy_stays_synchronous() {
|
||||
let legacy: ExecutionConfig =
|
||||
serde_json::from_value(json!({ "timeout_ms": 12_000 })).unwrap();
|
||||
assert_eq!(legacy.execution_policy, None);
|
||||
assert_eq!(legacy.timeout_ms, 12_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_rejects_explicit_invalid_deadline() {
|
||||
assert!(
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 0,
|
||||
no_tasks_fallback: NoTasksFallback::Synchronous,
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Adaptive,
|
||||
deadline_ms: crate::operation::MAX_TOOL_DEADLINE_MS + 1,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(ToolExecutionPolicy::default().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_policy_rejects_unknown_fields() {
|
||||
assert!(
|
||||
serde_json::from_value::<ToolExecutionPolicy>(json!({
|
||||
"mode": "background",
|
||||
"deadline_ms": 1000,
|
||||
"no_tasks_fallback": "synchronous",
|
||||
"task_queue": "default"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
//! Durable attempt, retry-snapshot and cancellation vocabulary of a Task (Story 4.4).
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{TASK_IDENTITY_MAX_BYTES, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskRecord};
|
||||
|
||||
/// Upper bound of the retry budget a Task may pin at acceptance (Story 4.4).
|
||||
pub const TASK_MAX_ATTEMPTS: u32 = 5;
|
||||
/// Upper bound of the pinned backoff step between two durable attempts (Story 4.4).
|
||||
pub const TASK_MAX_BACKOFF_STEP_MS: i64 = 1_000;
|
||||
/// Bound of a stored cancellation reason.
|
||||
pub const TASK_CANCEL_REASON_MAX_BYTES: usize = 160;
|
||||
|
||||
/// Immutable retry snapshot of one accepted Task.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct TaskRetryPolicy {
|
||||
/// Total durable outbound attempts (1 = a single attempt, never retried).
|
||||
pub max_attempts: u32,
|
||||
/// Pinned base backoff between two attempts; the delay grows by this step per attempt.
|
||||
pub backoff_step_ms: i64,
|
||||
}
|
||||
|
||||
impl TaskRetryPolicy {
|
||||
/// A Task that may be dispatched exactly once (legacy Tasks and unverified writes).
|
||||
pub const fn single_attempt() -> Self {
|
||||
Self {
|
||||
max_attempts: 1,
|
||||
backoff_step_ms: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(max_attempts: u32, backoff_step_ms: i64) -> Self {
|
||||
Self {
|
||||
max_attempts,
|
||||
backoff_step_ms,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if !(1..=TASK_MAX_ATTEMPTS).contains(&self.max_attempts) {
|
||||
return Err(TaskError::invalid("task retry budget is invalid"));
|
||||
}
|
||||
if !(0..=TASK_MAX_BACKOFF_STEP_MS).contains(&self.backoff_step_ms) {
|
||||
return Err(TaskError::invalid("task retry backoff is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delay before attempt number `next_ordinal` (2..=max_attempts), capped so a pinned snapshot can
|
||||
/// never stretch the accepted execution deadline on its own.
|
||||
pub fn backoff_before(&self, next_ordinal: u32) -> i64 {
|
||||
let steps = i64::from(next_ordinal.saturating_sub(1)).max(1);
|
||||
self.backoff_step_ms
|
||||
.saturating_mul(steps)
|
||||
.min(self.backoff_step_ms.saturating_mul(4))
|
||||
}
|
||||
}
|
||||
|
||||
/// Certainty of one durable attempt outcome.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AttemptCertainty {
|
||||
/// The outcome of the outbound operation is proven (success or a definite failure).
|
||||
Known,
|
||||
/// The operation may have been applied upstream; the effect cannot be proven locally.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AttemptCertainty {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Known => "known",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"known" => Ok(Self::Known),
|
||||
"unknown" => Ok(Self::Unknown),
|
||||
_ => Err(TaskError::invalid("unknown attempt certainty")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Phase of one durable attempt at the moment it was resolved.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AttemptPhase {
|
||||
/// The durable checkpoint exists but no outbound send was observed.
|
||||
BeforeDispatch,
|
||||
/// The request was handed to the transport; its effect may exist upstream.
|
||||
Dispatched,
|
||||
}
|
||||
|
||||
impl AttemptPhase {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::BeforeDispatch => "before_dispatch",
|
||||
Self::Dispatched => "dispatched",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"before_dispatch" => Ok(Self::BeforeDispatch),
|
||||
"dispatched" => Ok(Self::Dispatched),
|
||||
_ => Err(TaskError::invalid("unknown attempt phase")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Closed outcome vocabulary of one resolved attempt.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AttemptOutcomeClass {
|
||||
Completed,
|
||||
Failed,
|
||||
/// A bounded further attempt was scheduled instead of a terminal outcome.
|
||||
RetryScheduled,
|
||||
OutcomeUnknown,
|
||||
}
|
||||
|
||||
impl AttemptOutcomeClass {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::RetryScheduled => "retry_scheduled",
|
||||
Self::OutcomeUnknown => "outcome_unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"retry_scheduled" => Ok(Self::RetryScheduled),
|
||||
"outcome_unknown" => Ok(Self::OutcomeUnknown),
|
||||
_ => Err(TaskError::invalid("unknown attempt outcome")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One durable outbound attempt. The row is created by the dispatch checkpoint before any network
|
||||
/// use and becomes immutable once resolved, so a stale worker can never append to the history.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAttempt {
|
||||
pub attempt_id: String,
|
||||
pub task_id: TaskId,
|
||||
pub ordinal: u32,
|
||||
pub owner: String,
|
||||
pub fence: u64,
|
||||
/// Canonical fingerprint of the sealed executable input this attempt was admitted for.
|
||||
pub input_fingerprint: String,
|
||||
/// Digest of the stable upstream idempotency key (never the raw key or headers).
|
||||
pub upstream_key_digest: Option<String>,
|
||||
/// Whether a retry of this attempt is safe (read or server-verified idempotent write).
|
||||
pub retry_allowed: bool,
|
||||
pub checkpoint_at_unix_ms: i64,
|
||||
pub resolved_at_unix_ms: Option<i64>,
|
||||
pub outcome: Option<AttemptOutcomeClass>,
|
||||
pub certainty: Option<AttemptCertainty>,
|
||||
pub phase: Option<AttemptPhase>,
|
||||
}
|
||||
|
||||
/// Committed dispatch intent of one durable attempt (Story 4.4). The store owns the transaction: the
|
||||
/// attempt row, the incremented budget and the new Task version commit together before the network is
|
||||
/// touched, so an uncommitted checkpoint can never produce an outbound call.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct BeginAttempt {
|
||||
pub task_id: TaskId,
|
||||
pub owner: String,
|
||||
pub fence: u64,
|
||||
/// Version the worker read from the lease; a concurrent change rejects the checkpoint.
|
||||
pub expected_version: u32,
|
||||
pub attempt_id: String,
|
||||
pub input_fingerprint: String,
|
||||
pub upstream_key_digest: Option<String>,
|
||||
/// The runtime's decision that a retry of this attempt is safe; the store still enforces the
|
||||
/// pinned budget and the absolute deadline.
|
||||
pub retry_allowed: bool,
|
||||
}
|
||||
|
||||
impl BeginAttempt {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.owner.is_empty() || self.owner.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task attempt owner is invalid"));
|
||||
}
|
||||
if self.attempt_id.is_empty() || self.attempt_id.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task attempt identity is invalid"));
|
||||
}
|
||||
if self.input_fingerprint.len() != 64 {
|
||||
return Err(TaskError::invalid("task attempt fingerprint is invalid"));
|
||||
}
|
||||
if let Some(digest) = self.upstream_key_digest.as_deref()
|
||||
&& (digest.is_empty() || digest.len() > TASK_IDENTITY_MAX_BYTES)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task attempt upstream key digest is invalid",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Leased, committed view of one in-flight attempt and the Task version it was admitted under.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct AttemptLease {
|
||||
pub attempt: TaskAttempt,
|
||||
pub task: TaskRecord,
|
||||
}
|
||||
|
||||
/// Typed outcome of one attempt, handed to the store for a single atomic resolution.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AttemptOutcome {
|
||||
/// Proven success; the Task becomes `completed` with this result.
|
||||
Completed(TaskEnvelope),
|
||||
/// Proven failure. `retryable` only schedules another attempt when the store's pinned budget,
|
||||
/// the attempt's safe-retry decision and the absolute deadline all still allow it; otherwise the
|
||||
/// Task fails with this error.
|
||||
Failed {
|
||||
error: TaskEnvelope,
|
||||
retryable: bool,
|
||||
phase: AttemptPhase,
|
||||
},
|
||||
/// The outbound effect cannot be proven. `repeatable` only schedules another bounded attempt when
|
||||
/// repeating it is safe and the pinned budget/deadline still allow it; otherwise the Task becomes
|
||||
/// the non-terminal `outcome_unknown`, never a plain failure.
|
||||
Uncertain {
|
||||
error: TaskEnvelope,
|
||||
repeatable: bool,
|
||||
phase: AttemptPhase,
|
||||
},
|
||||
}
|
||||
|
||||
/// Single-winner result of resolving one attempt.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum AttemptResolution {
|
||||
/// A terminal transition committed (completed or failed).
|
||||
Finished(TaskRecord),
|
||||
/// The attempt failed in a known way and another bounded attempt was armed.
|
||||
RetryAt {
|
||||
task: TaskRecord,
|
||||
next_attempt_at_unix_ms: i64,
|
||||
},
|
||||
/// The attempt resolved to the non-terminal uncertain state.
|
||||
Unknown(TaskRecord),
|
||||
}
|
||||
|
||||
/// Owner-side cancellation request. `expected_version` is optional so a repeated cancel of the same
|
||||
/// accepted Task stays idempotent while a stale version is reported as a conflict.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskCancelRequest {
|
||||
pub access: TaskAccess,
|
||||
pub task_id: TaskId,
|
||||
pub expected_version: Option<u32>,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
impl TaskCancelRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
self.access.validate()?;
|
||||
if self.reason.is_empty()
|
||||
|| self.reason.len() > TASK_CANCEL_REASON_MAX_BYTES
|
||||
|| self.reason.bytes().any(|byte| byte.is_ascii_control())
|
||||
{
|
||||
return Err(TaskError::invalid("task cancellation reason is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed result of an owner-scoped cancellation.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum TaskCancelOutcome {
|
||||
/// Cancelled before the dispatch checkpoint: no outbound call can have happened.
|
||||
Cancelled(TaskRecord),
|
||||
/// The Task already had a checkpointed attempt: non-terminal uncertain, never a plain cancel.
|
||||
OutcomeUnknown(TaskRecord),
|
||||
/// The Task was already in a terminal state; its accepted result is unchanged.
|
||||
AlreadyTerminal(TaskRecord),
|
||||
/// The caller's expected version is stale; `current` is the authorized current state.
|
||||
Conflict(TaskRecord),
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
//! Continuation of an accepted Task through a validated additional input (Story 4.5, FR-25).
|
||||
//!
|
||||
//! A Task that cannot run because the pinned operation still lacks a required input item is
|
||||
//! suspended as `input_required` before any outbound dispatch. The durable request keeps the exact
|
||||
//! immutable schema snapshot, its JSON projection, safe prompt metadata, the absolute TTL and the
|
||||
//! update fingerprint contract. A valid update moves the Task back to `working` exactly once; an
|
||||
//! identical replay returns the committed outcome and a conflicting or stale reply is typed.
|
||||
//!
|
||||
//! Deliberately absent: raw input in the request/receipt/audit read models, a second cipher store and
|
||||
//! any public transport. The regular executable-input ciphertext carries the continuation binding.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::task::{TaskAccess, TaskError, TaskId};
|
||||
|
||||
/// Version of the persisted input-continuation contract. A stored snapshot always declares the
|
||||
/// version it was written with, so a later binary never reinterprets it silently.
|
||||
pub const TASK_INPUT_SCHEMA_FORMAT_VERSION: u32 = 1;
|
||||
/// Bound of a wait identity before it is persisted.
|
||||
pub const TASK_INPUT_WAIT_ID_MAX_BYTES: usize = 64;
|
||||
/// Bound of the safe prompt code (never the prompt text or an input value).
|
||||
pub const TASK_INPUT_PROMPT_CODE_MAX_BYTES: usize = 64;
|
||||
/// Bound of the missing-field path metadata.
|
||||
pub const TASK_INPUT_MAX_MISSING_FIELDS: usize = 32;
|
||||
/// Bound of one missing-field path.
|
||||
pub const TASK_INPUT_FIELD_PATH_MAX_BYTES: usize = 160;
|
||||
|
||||
/// Durable request of additional input for one suspended Task.
|
||||
///
|
||||
/// `schema_snapshot` is the native serialized schema subset the server validates against; the JSON
|
||||
/// projection is presentation metadata only and never the validation authority. `expected_version`
|
||||
/// is the Task version a caller must present with the answer.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TaskInputRequest {
|
||||
pub wait_id: String,
|
||||
pub expected_version: u32,
|
||||
pub schema_revision: u32,
|
||||
pub schema_format_version: u32,
|
||||
pub schema_snapshot: Value,
|
||||
pub projected_schema: Value,
|
||||
pub prompt_code: String,
|
||||
pub missing_fields: Vec<String>,
|
||||
pub expires_at_unix_ms: i64,
|
||||
pub fingerprint_format_version: u32,
|
||||
}
|
||||
|
||||
impl TaskInputRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.wait_id.is_empty()
|
||||
|| self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES
|
||||
|| self.wait_id.bytes().any(|byte| byte.is_ascii_control())
|
||||
{
|
||||
return Err(TaskError::invalid("task input wait identity is invalid"));
|
||||
}
|
||||
if self.expected_version == 0 || self.schema_revision == 0 {
|
||||
return Err(TaskError::invalid(
|
||||
"task input request generation is invalid",
|
||||
));
|
||||
}
|
||||
if self.schema_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
|
||||
|| self.fingerprint_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task input request format version is invalid",
|
||||
));
|
||||
}
|
||||
if self.schema_snapshot.as_object().is_none() || self.projected_schema.as_object().is_none()
|
||||
{
|
||||
return Err(TaskError::invalid("task input schema snapshot is invalid"));
|
||||
}
|
||||
if self.prompt_code.is_empty() || self.prompt_code.len() > TASK_INPUT_PROMPT_CODE_MAX_BYTES
|
||||
{
|
||||
return Err(TaskError::invalid("task input prompt code is invalid"));
|
||||
}
|
||||
if self.missing_fields.len() > TASK_INPUT_MAX_MISSING_FIELDS
|
||||
|| self
|
||||
.missing_fields
|
||||
.iter()
|
||||
.any(|field| field.is_empty() || field.len() > TASK_INPUT_FIELD_PATH_MAX_BYTES)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task input missing-field metadata is invalid",
|
||||
));
|
||||
}
|
||||
if self.expires_at_unix_ms <= 0 {
|
||||
return Err(TaskError::invalid("task input TTL is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Authoritative write of one validated answer. The transaction owns schema-generation, TTL,
|
||||
/// fingerprint and CAS checks; this request only carries the committed values.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TaskInputUpdateRequest {
|
||||
pub access: TaskAccess,
|
||||
pub task_id: TaskId,
|
||||
pub wait_id: String,
|
||||
pub expected_version: u32,
|
||||
/// Digest of the scoped update key; the caller key is never persisted.
|
||||
pub update_key_digest: String,
|
||||
/// Canonical digest of the validated full executable input.
|
||||
pub canonical_fingerprint: String,
|
||||
/// Effective execution fingerprint of the resumed payload (never the acceptance fingerprint).
|
||||
pub effective_input_fingerprint: String,
|
||||
pub executable_input_ciphertext: String,
|
||||
pub executable_input_key_version: String,
|
||||
pub executable_input_master_key_epoch: i64,
|
||||
}
|
||||
|
||||
impl TaskInputUpdateRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
self.access.validate()?;
|
||||
if self.wait_id.is_empty() || self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task input wait identity is invalid"));
|
||||
}
|
||||
if self.expected_version == 0 {
|
||||
return Err(TaskError::invalid("task input version is invalid"));
|
||||
}
|
||||
if !valid_digest(&self.update_key_digest)
|
||||
|| !valid_digest(&self.canonical_fingerprint)
|
||||
|| !valid_digest(&self.effective_input_fingerprint)
|
||||
{
|
||||
return Err(TaskError::invalid("task input digest is invalid"));
|
||||
}
|
||||
if self.executable_input_ciphertext.is_empty()
|
||||
|| self.executable_input_ciphertext.len()
|
||||
> crate::task::TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task continuation ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
if self.executable_input_key_version.is_empty()
|
||||
|| self.executable_input_key_version.len() > 64
|
||||
|| self.executable_input_master_key_epoch < 1
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task continuation ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Committed outcome of one input update. An identical replay returns the same stored outcome with
|
||||
/// `replayed = true` instead of repeating the CAS, audit or dispatch.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskInputUpdateOutcome {
|
||||
pub task_id: TaskId,
|
||||
pub wait_id: String,
|
||||
pub state: crate::task::TaskState,
|
||||
pub version: u32,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
fn valid_digest(value: &str) -> bool {
|
||||
value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
/// Durable input-continuation operations of the Task store.
|
||||
///
|
||||
/// Split out of the general [`crate::task::TaskStore`] port so the lifecycle store stays readable and
|
||||
/// the continuation contract (suspend, answer, receipt, expiry) is implemented and audited as one
|
||||
/// cohesive surface. [`crate::task::TaskStore`] extends this trait, so an owned `Arc<dyn TaskStore>`
|
||||
/// exposes both halves through one object.
|
||||
#[async_trait::async_trait]
|
||||
pub trait TaskInputStore: Send + Sync {
|
||||
/// Suspends a leased Task that still lacks a required input of its pinned operation. The
|
||||
/// transaction persists the immutable request and its audit row, moves `working` to
|
||||
/// `input_required` exactly once, releases the worker lease and keeps the existing admission
|
||||
/// permit. No outbound dispatch may have happened in this generation.
|
||||
async fn suspend_for_input(
|
||||
&self,
|
||||
lease: &crate::task::TaskLease,
|
||||
request: TaskInputRequest,
|
||||
) -> Result<crate::task::TaskRecord, TaskError>;
|
||||
|
||||
/// Owner-scoped read of the durable input request of one Task.
|
||||
async fn input_request(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskInputRequest, TaskError>;
|
||||
|
||||
/// Owner-scoped read of the executable-input binding that a committed continuation may extend.
|
||||
/// It is not lease-authorized: it only serves the update transaction and never dispatch.
|
||||
async fn read_executable_input_for_update(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<(String, String, i64), TaskError>;
|
||||
|
||||
/// Applies exactly one validated answer under a row lock: authority recheck, receipt lookup,
|
||||
/// generation/TTL check, atomic ciphertext/effective-fingerprint/receipt/`working` transition and
|
||||
/// audit. An identical replay returns the stored outcome without a second transition.
|
||||
async fn apply_input(
|
||||
&self,
|
||||
request: TaskInputUpdateRequest,
|
||||
) -> Result<TaskInputUpdateOutcome, TaskError>;
|
||||
|
||||
/// Bounded sweep of `input_required` Tasks whose TTL elapsed: `failed` with an audit reason and
|
||||
/// the release of the admission permit, without any dispatch.
|
||||
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn request() -> TaskInputRequest {
|
||||
TaskInputRequest {
|
||||
wait_id: "wait-1".to_owned(),
|
||||
expected_version: 2,
|
||||
schema_revision: 1,
|
||||
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot: json!({"type": "object", "fields": {}}),
|
||||
projected_schema: json!({"type": "object", "properties": {}}),
|
||||
prompt_code: "required_input_missing".to_owned(),
|
||||
missing_fields: vec!["$.email".to_owned()],
|
||||
expires_at_unix_ms: 1_900_000_000_000,
|
||||
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
}
|
||||
}
|
||||
|
||||
fn update() -> TaskInputUpdateRequest {
|
||||
TaskInputUpdateRequest {
|
||||
access: crate::task::TaskAccess {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
owner_kind: crate::task::TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
},
|
||||
task_id: crate::task::TaskId::generate(),
|
||||
wait_id: "wait-1".to_owned(),
|
||||
expected_version: 2,
|
||||
update_key_digest: "a".repeat(64),
|
||||
canonical_fingerprint: "b".repeat(64),
|
||||
effective_input_fingerprint: "c".repeat(64),
|
||||
executable_input_ciphertext: "sealed".to_owned(),
|
||||
executable_input_key_version: "key-v1".to_owned(),
|
||||
executable_input_master_key_epoch: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_request_holds_generation_schema_ttl_and_bounds() {
|
||||
assert!(request().validate().is_ok());
|
||||
|
||||
let mut no_wait = request();
|
||||
no_wait.wait_id.clear();
|
||||
assert!(no_wait.validate().is_err());
|
||||
|
||||
let mut no_generation = request();
|
||||
no_generation.expected_version = 0;
|
||||
assert!(no_generation.validate().is_err());
|
||||
|
||||
let mut bad_schema = request();
|
||||
bad_schema.schema_snapshot = json!("not-an-object");
|
||||
assert!(bad_schema.validate().is_err());
|
||||
|
||||
let mut no_ttl = request();
|
||||
no_ttl.expires_at_unix_ms = 0;
|
||||
assert!(no_ttl.validate().is_err());
|
||||
|
||||
let mut too_many_fields = request();
|
||||
too_many_fields.missing_fields = vec!["$.x".to_owned(); TASK_INPUT_MAX_MISSING_FIELDS + 1];
|
||||
assert!(too_many_fields.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_request_rejects_invalid_digests_and_ciphertext_bounds() {
|
||||
assert!(update().validate().is_ok());
|
||||
|
||||
let mut short_digest = update();
|
||||
short_digest.canonical_fingerprint = "abc".to_owned();
|
||||
assert!(short_digest.validate().is_err());
|
||||
|
||||
let mut upper_digest = update();
|
||||
upper_digest.update_key_digest = "A".repeat(64);
|
||||
assert!(upper_digest.validate().is_err());
|
||||
|
||||
let mut no_ciphertext = update();
|
||||
no_ciphertext.executable_input_ciphertext.clear();
|
||||
assert!(no_ciphertext.validate().is_err());
|
||||
|
||||
let mut no_epoch = update();
|
||||
no_epoch.executable_input_master_key_epoch = 0;
|
||||
assert!(no_epoch.validate().is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,964 @@
|
||||
//! Durable background Task seam (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! A Task is the internal, protocol-independent record of one accepted background request. This
|
||||
//! module owns the domain vocabulary only: the lifecycle states, the accepted identity/pins, the
|
||||
//! bounded persisted envelopes and the `TaskStore` port. The protocol adapter (Story 4.7) and the
|
||||
//! worker (Story 4.3) do not define these semantics; the authoritative store lives in PostgreSQL
|
||||
//! (`crank-registry`), so an accepted Task survives a crash or restart.
|
||||
//!
|
||||
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
|
||||
//! Task was executed. A terminal state is written by a conditional transition only.
|
||||
|
||||
mod attempt;
|
||||
mod input;
|
||||
use async_trait::async_trait;
|
||||
pub use attempt::*;
|
||||
pub use input::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::correlation::{RequestId, TraceContext, TraceId};
|
||||
use crate::ids::PlatformApiKeyId;
|
||||
use crate::operation::ToolExecutionMode;
|
||||
|
||||
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
|
||||
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
|
||||
/// Maximum byte budget of the decrypted executable Task input.
|
||||
pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES;
|
||||
/// Maximum byte budget of the stored encrypted executable Task input ciphertext.
|
||||
pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304;
|
||||
/// Maximum nesting depth of a persisted Task envelope.
|
||||
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
|
||||
/// Maximum number of JSON members/nodes of a persisted Task envelope.
|
||||
pub const TASK_ENVELOPE_MAX_NODES: usize = 256;
|
||||
/// Bound of every scope/owner/operation identity column.
|
||||
pub const TASK_IDENTITY_MAX_BYTES: usize = 160;
|
||||
/// Bound of the caller-supplied idempotency key before it becomes a scoped digest.
|
||||
pub const TASK_SCOPED_KEY_MAX_BYTES: usize = 128;
|
||||
/// Bound of a saved approval reference (an identifier, never a bearer token).
|
||||
pub const TASK_APPROVAL_REFERENCE_MAX_BYTES: usize = 160;
|
||||
/// Default retention of a terminal Task after its execution deadline (internal bounded default).
|
||||
pub const TASK_DEFAULT_RETENTION_MS: i64 = 86_400_000;
|
||||
/// Upper bound on the retention an accepted Task may pin.
|
||||
pub const TASK_MAX_RETENTION_MS: i64 = 30 * 86_400_000;
|
||||
/// Upper bound of one retention/housekeeping batch.
|
||||
pub const TASK_PURGE_BATCH_LIMIT: u32 = 128;
|
||||
|
||||
/// Stable identity of one accepted Task. Generated as UUIDv7 and returned only after the accepting
|
||||
/// transaction committed, so the ID never exists without its durable record.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct TaskId(Uuid);
|
||||
|
||||
impl TaskId {
|
||||
pub fn generate() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
Uuid::parse_str(value)
|
||||
.map(Self)
|
||||
.map_err(|_| TaskError::invalid("task identity is not a UUID"))
|
||||
}
|
||||
|
||||
pub fn as_uuid(self) -> Uuid {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TaskId {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
self.0.fmt(formatter)
|
||||
}
|
||||
}
|
||||
|
||||
/// Closed Task lifecycle vocabulary. `outcome_unknown` is a non-terminal state: a value commit that
|
||||
/// failed to confirm its dispatch is never collapsed into a plain failure.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TaskState {
|
||||
Queued,
|
||||
Working,
|
||||
InputRequired,
|
||||
OutcomeUnknown,
|
||||
Completed,
|
||||
Failed,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl TaskState {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Queued => "queued",
|
||||
Self::Working => "working",
|
||||
Self::InputRequired => "input_required",
|
||||
Self::OutcomeUnknown => "outcome_unknown",
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::Cancelled => "cancelled",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"queued" => Ok(Self::Queued),
|
||||
"working" => Ok(Self::Working),
|
||||
"input_required" => Ok(Self::InputRequired),
|
||||
"outcome_unknown" => Ok(Self::OutcomeUnknown),
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"cancelled" => Ok(Self::Cancelled),
|
||||
_ => Err(TaskError::invalid("unknown task state")),
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn is_terminal(self) -> bool {
|
||||
matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
|
||||
}
|
||||
|
||||
/// One conditional transition may only move along this table. A terminal state has no successor,
|
||||
/// so a repeated terminal write can never rewrite an accepted result.
|
||||
pub const fn allows_transition_to(self, next: Self) -> bool {
|
||||
match self {
|
||||
Self::Queued => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
|
||||
Self::Working => matches!(
|
||||
next,
|
||||
Self::InputRequired
|
||||
| Self::OutcomeUnknown
|
||||
| Self::Completed
|
||||
| Self::Failed
|
||||
| Self::Cancelled
|
||||
),
|
||||
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
|
||||
// Story 4.4: an uncertain outbound outcome is never resolved by a generic transition.
|
||||
// Reconciliation is Story 4.6, so no ordinary finish/cancel/retry may leave this state.
|
||||
Self::OutcomeUnknown => false,
|
||||
Self::Completed | Self::Failed | Self::Cancelled => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Who owns the accepted Task. An administrator is never a wildcard owner of every Task: the owner
|
||||
/// identity and the authorization scope are separate, both-pinned columns.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TaskOwnerKind {
|
||||
Agent,
|
||||
Admin,
|
||||
}
|
||||
|
||||
impl TaskOwnerKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Agent => "agent",
|
||||
Self::Admin => "admin",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"agent" => Ok(Self::Agent),
|
||||
"admin" => Ok(Self::Admin),
|
||||
_ => Err(TaskError::invalid("unknown task owner kind")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Support identities captured at acceptance. They are stored and returned independently of
|
||||
/// sampling, and a replay never rewrites them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskSupportIds {
|
||||
pub request_id: RequestId,
|
||||
pub trace_id: TraceId,
|
||||
pub traceparent: String,
|
||||
pub trace_sampled: bool,
|
||||
}
|
||||
|
||||
impl TaskSupportIds {
|
||||
pub fn new(request_id: RequestId, trace_context: &TraceContext) -> Self {
|
||||
Self {
|
||||
request_id,
|
||||
trace_id: trace_context.trace_id().clone(),
|
||||
traceparent: trace_context.traceparent().to_owned(),
|
||||
trace_sampled: trace_context.is_sampled(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-validates both the trace identity and the parent context: a Task must never persist an
|
||||
/// inconsistent or malformed support pair, and an unsampled context is still a valid parent.
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.trace_id.as_str().is_empty() {
|
||||
return Err(TaskError::invalid("task trace identity is missing"));
|
||||
}
|
||||
let context = TraceContext::parse(&self.traceparent)
|
||||
.map_err(|_| TaskError::invalid("task trace parent is invalid"))?;
|
||||
if context.trace_id() != &self.trace_id || context.is_sampled() != self.trace_sampled {
|
||||
return Err(TaskError::invalid("task trace parent is inconsistent"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Execution semantics pinned at acceptance: a later publication never rewrites them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskPins {
|
||||
pub operation_version: u32,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
pub execution_deadline_unix_ms: i64,
|
||||
/// Retention deadline of the record; never the same thing as the execution deadline.
|
||||
pub retain_until_unix_ms: i64,
|
||||
/// Verified approval evidence reference (identifier only, never a bearer token).
|
||||
pub approval_reference: Option<String>,
|
||||
/// Bounded retry budget pinned at acceptance (Story 4.4). A later publication, a restart or a
|
||||
/// new binary never changes the numbers an accepted Task was admitted with.
|
||||
pub retry: TaskRetryPolicy,
|
||||
}
|
||||
|
||||
impl TaskPins {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.operation_version == 0 {
|
||||
return Err(TaskError::invalid(
|
||||
"task operation version must be positive",
|
||||
));
|
||||
}
|
||||
if self.execution_deadline_unix_ms <= 0 {
|
||||
return Err(TaskError::invalid("task execution deadline is invalid"));
|
||||
}
|
||||
if self.retain_until_unix_ms < self.execution_deadline_unix_ms {
|
||||
return Err(TaskError::invalid(
|
||||
"task retention deadline precedes execution",
|
||||
));
|
||||
}
|
||||
if self.retain_until_unix_ms - self.execution_deadline_unix_ms > TASK_MAX_RETENTION_MS {
|
||||
return Err(TaskError::invalid("task retention exceeds its bound"));
|
||||
}
|
||||
if let Some(reference) = self.approval_reference.as_deref()
|
||||
&& (reference.is_empty()
|
||||
|| reference.len() > TASK_APPROVAL_REFERENCE_MAX_BYTES
|
||||
|| reference.bytes().any(|byte| byte.is_ascii_control()))
|
||||
{
|
||||
return Err(TaskError::invalid("task approval reference is invalid"));
|
||||
}
|
||||
self.retry.validate()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded, classified payload of a Task. Construction validates the byte/depth/node budget and
|
||||
/// redacts secret-bearing fields before anything is hashed or persisted.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskEnvelope {
|
||||
value: Value,
|
||||
}
|
||||
|
||||
impl TaskEnvelope {
|
||||
/// Validates the caller-provided value, then stores a redacted copy of it. The caller hashes the
|
||||
/// original value separately, so redaction never collapses two different requests into one.
|
||||
pub fn try_redacted(input: &Value) -> Result<Self, TaskError> {
|
||||
validate_task_envelope(input)?;
|
||||
Ok(Self {
|
||||
value: redact_task_envelope(input),
|
||||
})
|
||||
}
|
||||
|
||||
/// Accepts a value that was already validated and redacted at a trust boundary.
|
||||
pub fn from_persisted(value: Value) -> Result<Self, TaskError> {
|
||||
validate_task_envelope(&value)?;
|
||||
Ok(Self { value })
|
||||
}
|
||||
|
||||
pub fn value(&self) -> &Value {
|
||||
&self.value
|
||||
}
|
||||
|
||||
pub fn into_value(self) -> Value {
|
||||
self.value
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the size, nesting depth and node count of one Task envelope. Every byte, collection and
|
||||
/// node is counted before the value is cloned, hashed or persisted.
|
||||
pub fn validate_task_envelope(value: &Value) -> Result<(), TaskError> {
|
||||
let encoded = serde_json::to_vec(value)
|
||||
.map_err(|_| TaskError::invalid("task envelope is not serializable"))?;
|
||||
if encoded.len() > TASK_ENVELOPE_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task envelope exceeds its byte budget"));
|
||||
}
|
||||
let mut nodes = 0usize;
|
||||
check_budget(value, 1, &mut nodes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_budget(value: &Value, depth: usize, nodes: &mut usize) -> Result<(), TaskError> {
|
||||
if depth > TASK_ENVELOPE_MAX_DEPTH {
|
||||
return Err(TaskError::invalid("task envelope exceeds its depth budget"));
|
||||
}
|
||||
*nodes += 1;
|
||||
if *nodes > TASK_ENVELOPE_MAX_NODES {
|
||||
return Err(TaskError::invalid("task envelope exceeds its node budget"));
|
||||
}
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
for (key, child) in object {
|
||||
check_budget_name(key)?;
|
||||
check_budget(child, depth + 1, nodes)?;
|
||||
}
|
||||
}
|
||||
Value::Array(values) => {
|
||||
for child in values {
|
||||
check_budget(child, depth + 1, nodes)?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_budget_name(key: &str) -> Result<(), TaskError> {
|
||||
if key.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task envelope key exceeds its budget"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Field-name classification of secret-bearing values. A matching field keeps its position but never
|
||||
/// its plaintext in the persisted envelope.
|
||||
pub fn sensitive_task_key(name: &str) -> bool {
|
||||
let lowered = name.to_ascii_lowercase();
|
||||
[
|
||||
"password",
|
||||
"passwd",
|
||||
"secret",
|
||||
"token",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"authorization",
|
||||
"credential",
|
||||
"private_key",
|
||||
"cookie",
|
||||
"signature",
|
||||
]
|
||||
.iter()
|
||||
.any(|marker| lowered.contains(marker))
|
||||
}
|
||||
|
||||
/// Redacted projection of an already validated envelope.
|
||||
pub fn redact_task_envelope(value: &Value) -> Value {
|
||||
match value {
|
||||
Value::Object(object) => Value::Object(
|
||||
object
|
||||
.iter()
|
||||
.map(|(key, child)| {
|
||||
let value = if sensitive_task_key(key) {
|
||||
Value::String("[redacted]".to_owned())
|
||||
} else {
|
||||
redact_task_envelope(child)
|
||||
};
|
||||
(key.clone(), value)
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
Value::Array(values) => Value::Array(values.iter().map(redact_task_envelope).collect()),
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Recursive check used to fail closed on sensitive fields before encryption.
|
||||
pub fn contains_sensitive_task_keys(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Object(object) => object
|
||||
.iter()
|
||||
.any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)),
|
||||
Value::Array(values) => values.iter().any(contains_sensitive_task_keys),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
|
||||
/// queue capacity commit together, and no acknowledgement exists before that commit.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAcceptRequest {
|
||||
pub task_id: TaskId,
|
||||
pub workspace_id: String,
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
/// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the
|
||||
/// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively.
|
||||
/// `None` means the accepting principal has no machine key (administrator) and fail-closes.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
pub pins: TaskPins,
|
||||
/// Digest of the caller key over the full owner/authorization/Tool scope.
|
||||
pub scoped_key: String,
|
||||
/// Digest of the canonical validated executable input.
|
||||
pub input_fingerprint: String,
|
||||
pub input: TaskEnvelope,
|
||||
pub support: TaskSupportIds,
|
||||
/// Bounded encrypted executable input, persisted alongside the redacted input projection.
|
||||
pub executable_input_ciphertext: Option<String>,
|
||||
pub executable_input_key_version: Option<String>,
|
||||
pub executable_input_master_key_epoch: Option<i64>,
|
||||
}
|
||||
|
||||
impl TaskAcceptRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
for (field, value) in [
|
||||
("workspace", self.workspace_id.as_str()),
|
||||
("owner", self.owner_id.as_str()),
|
||||
("authorization scope", self.authorization_scope.as_str()),
|
||||
("operation", self.operation_id.as_str()),
|
||||
] {
|
||||
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
||||
}
|
||||
}
|
||||
if let Some(key_id) = self.platform_api_key_id.as_ref()
|
||||
&& (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task platform api key reference is invalid",
|
||||
));
|
||||
}
|
||||
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task scoped key digest is invalid"));
|
||||
}
|
||||
if self.input_fingerprint.len() != 64
|
||||
|| !self
|
||||
.input_fingerprint
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err(TaskError::invalid("task input fingerprint is invalid"));
|
||||
}
|
||||
self.pins.validate()?;
|
||||
self.support.validate()?;
|
||||
let has_ciphertext = self.executable_input_ciphertext.is_some();
|
||||
let has_key = self.executable_input_key_version.is_some();
|
||||
let has_epoch = self.executable_input_master_key_epoch.is_some();
|
||||
match (has_ciphertext, has_key, has_epoch) {
|
||||
(false, false, false) => {}
|
||||
(true, true, true) => {
|
||||
let ciphertext = self.executable_input_ciphertext.as_deref().unwrap();
|
||||
if ciphertext.is_empty()
|
||||
|| ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
let key_version = self.executable_input_key_version.as_deref().unwrap();
|
||||
if key_version.is_empty() || key_version.len() > 64 {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
if self.executable_input_master_key_epoch.unwrap_or_default() < 1 {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Successful acceptance, returned only after the accepting transaction committed. A replay returns
|
||||
/// the stored identity and support IDs of the original acceptance.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAcceptance {
|
||||
pub task_id: TaskId,
|
||||
pub state: TaskState,
|
||||
/// Exact Published Version pinned by this acceptance; a replay returns the stored pin.
|
||||
pub operation_version: u32,
|
||||
pub support: TaskSupportIds,
|
||||
pub created_at_unix_ms: i64,
|
||||
pub retain_until_unix_ms: i64,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
/// Owner-side access proof of a read or a terminal write. It is built by the application layer from
|
||||
/// the verified principal, never from a caller-supplied marker, and every store query carries the
|
||||
/// whole predicate so a foreign row is never fetched.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAccess {
|
||||
pub workspace_id: String,
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
}
|
||||
|
||||
impl TaskAccess {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
for (field, value) in [
|
||||
("workspace", self.workspace_id.as_str()),
|
||||
("owner", self.owner_id.as_str()),
|
||||
("authorization scope", self.authorization_scope.as_str()),
|
||||
] {
|
||||
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Read model of one Task. Lifecycle fields, pins and support IDs stay readable together; the
|
||||
/// protocol adapter only maps them.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskRecord {
|
||||
pub task_id: TaskId,
|
||||
pub state: TaskState,
|
||||
pub workspace_id: String,
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
/// Typed immutable identity of the Platform API key that authorized the acceptance; the worker
|
||||
/// revalidates this exact key against the registry before any outbound dispatch.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
|
||||
pub input_fingerprint: String,
|
||||
/// Effective fingerprint of the payload the next attempt will execute. It equals the acceptance
|
||||
/// fingerprint until a committed input continuation replaces the effective payload; the
|
||||
/// acceptance fingerprint above is never overwritten, so an original accept replay stays replay.
|
||||
pub effective_input_fingerprint: String,
|
||||
pub pins: TaskPins,
|
||||
pub support: TaskSupportIds,
|
||||
pub input: TaskEnvelope,
|
||||
pub progress: Option<TaskEnvelope>,
|
||||
pub result: Option<TaskEnvelope>,
|
||||
pub error: Option<TaskEnvelope>,
|
||||
pub attempts: u32,
|
||||
/// Monotonic lease/fencing token. It never resets and protects Task state mutations.
|
||||
pub fence: u64,
|
||||
/// Random process-instance lease owner, if a worker currently owns the Task.
|
||||
pub lease_owner: Option<String>,
|
||||
/// Lease expiry epoch milliseconds, if the Task is claimed.
|
||||
pub lease_until_unix_ms: Option<i64>,
|
||||
/// Current dispatch attempt identity, committed before any outbound adapter call.
|
||||
pub current_attempt_id: Option<String>,
|
||||
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
|
||||
pub dispatch_started_at_unix_ms: Option<i64>,
|
||||
/// Earliest moment the next durable attempt may be dispatched (pinned backoff).
|
||||
pub next_attempt_at_unix_ms: Option<i64>,
|
||||
/// Durable cancellation request that arrived while an attempt was already checkpointed.
|
||||
pub cancel_requested_at_unix_ms: Option<i64>,
|
||||
/// Bounded reason of the cancellation request.
|
||||
pub cancel_reason: Option<String>,
|
||||
/// Monotonic lifecycle version used by the terminal CAS.
|
||||
pub version: u32,
|
||||
pub created_at_unix_ms: i64,
|
||||
pub updated_at_unix_ms: i64,
|
||||
}
|
||||
|
||||
/// One conditional terminal transition. Exactly one concurrent writer wins; the losers observe the
|
||||
/// accepted state and never rewrite the result.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskFinishRequest {
|
||||
pub access: TaskAccess,
|
||||
pub task_id: TaskId,
|
||||
pub expected_state: TaskState,
|
||||
pub expected_version: u32,
|
||||
pub target_state: TaskState,
|
||||
pub result: Option<TaskEnvelope>,
|
||||
pub error: Option<TaskEnvelope>,
|
||||
}
|
||||
|
||||
impl TaskFinishRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
self.access.validate()?;
|
||||
if !self.target_state.is_terminal() {
|
||||
return Err(TaskError::invalid("task finish target is not terminal"));
|
||||
}
|
||||
if self.expected_state.is_terminal() {
|
||||
return Err(TaskError::invalid("task finish source is already terminal"));
|
||||
}
|
||||
if !self.expected_state.allows_transition_to(self.target_state) {
|
||||
return Err(TaskError::invalid("task state transition is not allowed"));
|
||||
}
|
||||
if self.target_state == TaskState::Completed && self.result.is_none() {
|
||||
return Err(TaskError::invalid("a completed task requires a result"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded retention outcome.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct TaskPurgeReport {
|
||||
/// Queued Tasks that passed their execution deadline and were failed with their intent removed.
|
||||
pub expired_queued: u64,
|
||||
/// Terminal Tasks that passed their retention deadline.
|
||||
pub expired_terminal: u64,
|
||||
}
|
||||
|
||||
/// Typed Task failure. A denial and a missing Task share one outward shape, so a foreign caller can
|
||||
/// never distinguish "not yours" from "does not exist".
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum TaskError {
|
||||
#[error("task request is invalid: {message}")]
|
||||
Invalid { message: String },
|
||||
#[error("task store is unavailable: {message}")]
|
||||
Unavailable { message: String },
|
||||
#[error("task was not found")]
|
||||
NotFound,
|
||||
#[error("task request conflicts with the accepted request")]
|
||||
FingerprintConflict,
|
||||
#[error("task lifecycle transition was rejected")]
|
||||
StateConflict,
|
||||
#[error("task input request has expired")]
|
||||
InputExpired,
|
||||
#[error("background task capacity is exhausted")]
|
||||
Overloaded { retry_after_ms: u64 },
|
||||
}
|
||||
|
||||
impl TaskError {
|
||||
pub fn invalid(message: impl Into<String>) -> Self {
|
||||
Self::Invalid {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::Unavailable {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the caller may tell this failure apart from an absent Task.
|
||||
pub fn is_not_found(&self) -> bool {
|
||||
matches!(self, Self::NotFound)
|
||||
}
|
||||
}
|
||||
|
||||
/// Leased view of a claimed Task: its durable record plus lease fencing identity.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TaskLease {
|
||||
pub record: TaskRecord,
|
||||
pub owner: String,
|
||||
pub fence: u64,
|
||||
pub lease_until: i64,
|
||||
}
|
||||
|
||||
/// Authoritative durable Task port.
|
||||
#[async_trait]
|
||||
pub trait TaskStore: TaskInputStore + Send + Sync {
|
||||
/// Proves the durable Task state is usable (schema and operations reachable).
|
||||
async fn health(&self) -> Result<(), TaskError>;
|
||||
|
||||
/// Atomically accepts a Task: Task record, dispatch intent and one queue slot commit together.
|
||||
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError>;
|
||||
|
||||
/// Reads one Task inside the caller's owner scope.
|
||||
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Scoped lookup of an already accepted Task by its full-scope idempotency digest. Used to keep a
|
||||
/// retry of an explicitly pinned version from becoming new work; `None` means "not accepted yet".
|
||||
async fn find(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
operation_id: &str,
|
||||
scoped_key: &str,
|
||||
) -> Result<Option<TaskRecord>, TaskError>;
|
||||
|
||||
/// Applies exactly one conditional terminal transition and releases its intent atomically.
|
||||
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Reads encrypted executable input only under a valid, unexpired current lease.
|
||||
async fn read_executable_input(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
) -> Result<(String, String, i64), TaskError>;
|
||||
|
||||
/// Bounded retention...
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
|
||||
|
||||
/// Claims a bounded batch of queued or safely recoverable Tasks under leases.
|
||||
async fn claim_batch(
|
||||
&self,
|
||||
owner: &str,
|
||||
batch_limit: u32,
|
||||
lease_ms: i64,
|
||||
) -> Result<Vec<TaskLease>, TaskError>;
|
||||
|
||||
/// Renews a lease owned by `owner` when fence and expiry still match.
|
||||
async fn renew_lease(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
lease_ms: i64,
|
||||
) -> Result<TaskLease, TaskError>;
|
||||
|
||||
/// Persists redacted progress under a valid lease.
|
||||
async fn write_progress(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
progress: TaskEnvelope,
|
||||
) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Commits one durable outbound attempt before network use: attempt row, incremented budget and
|
||||
/// the new Task version commit in one transaction. A returned lease is the only authority to
|
||||
/// dispatch that attempt; an unknown commit means zero outbound calls.
|
||||
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError>;
|
||||
|
||||
/// Resolves exactly one committed attempt under its lease/fence/version. Only the current attempt
|
||||
/// of the current owner may be resolved, and the attempt becomes immutable afterwards.
|
||||
async fn resolve_attempt(
|
||||
&self,
|
||||
lease: &AttemptLease,
|
||||
outcome: AttemptOutcome,
|
||||
) -> Result<AttemptResolution, TaskError>;
|
||||
|
||||
/// Owner-scoped atomic cancellation. It serializes with claim and checkpoint on the same Task.
|
||||
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError>;
|
||||
|
||||
/// Owner-scoped bounded history of the durable attempts of one Task.
|
||||
async fn attempts(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<Vec<TaskAttempt>, TaskError>;
|
||||
|
||||
/// Applies a terminal transition only under a valid lease and releases intent atomically.
|
||||
async fn finish_leased(
|
||||
&self,
|
||||
request: TaskFinishRequest,
|
||||
owner: &str,
|
||||
fence: u64,
|
||||
) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Resolves a leased Task whose next attempt was refused before its dispatch checkpoint (elapsed
|
||||
/// deadline, rejected authority). A Task that still carries a proven ambiguity is never downgraded
|
||||
/// to a plain failure: it ends non-terminal `outcome_unknown` with `code`; otherwise `failed`.
|
||||
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn request() -> TaskAcceptRequest {
|
||||
TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 3,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_790_000_000_000,
|
||||
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
|
||||
approval_reference: Some("appr_1".to_owned()),
|
||||
retry: TaskRetryPolicy::new(3, 1),
|
||||
},
|
||||
scoped_key: "a".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uuid_v7_task_ids_are_unique_and_round_trip() {
|
||||
let first = TaskId::generate();
|
||||
let second = TaskId::generate();
|
||||
assert_ne!(first, second);
|
||||
assert_eq!(TaskId::parse(&first.to_string()).unwrap(), first);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_states_have_no_successor() {
|
||||
for terminal in [
|
||||
TaskState::Completed,
|
||||
TaskState::Failed,
|
||||
TaskState::Cancelled,
|
||||
] {
|
||||
assert!(terminal.is_terminal());
|
||||
for next in [
|
||||
TaskState::Queued,
|
||||
TaskState::Working,
|
||||
TaskState::Completed,
|
||||
TaskState::Failed,
|
||||
] {
|
||||
assert!(!terminal.allows_transition_to(next));
|
||||
}
|
||||
}
|
||||
assert!(TaskState::Queued.allows_transition_to(TaskState::Working));
|
||||
assert!(TaskState::Working.allows_transition_to(TaskState::OutcomeUnknown));
|
||||
assert!(!TaskState::Queued.allows_transition_to(TaskState::Completed));
|
||||
assert!(!TaskState::InputRequired.allows_transition_to(TaskState::Completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn envelope_bounds_reject_bytes_depth_and_nodes() {
|
||||
assert!(TaskEnvelope::try_redacted(&json!({"a": 1})).is_ok());
|
||||
assert!(
|
||||
TaskEnvelope::try_redacted(&json!({"a": "x".repeat(TASK_ENVELOPE_MAX_BYTES)})).is_err()
|
||||
);
|
||||
|
||||
let mut deep = json!(1);
|
||||
for _ in 0..=TASK_ENVELOPE_MAX_DEPTH {
|
||||
deep = json!({"n": deep});
|
||||
}
|
||||
assert!(TaskEnvelope::try_redacted(&deep).is_err());
|
||||
|
||||
let wide = Value::Array((0..=TASK_ENVELOPE_MAX_NODES).map(|_| json!(1)).collect());
|
||||
assert!(TaskEnvelope::try_redacted(&wide).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_bearing_fields_are_redacted_and_keep_their_shape() {
|
||||
let envelope = TaskEnvelope::try_redacted(&json!({
|
||||
"query": "leads",
|
||||
"auth": {"api_key": "sk_live_1", "nested": [{"password": "p"}]}
|
||||
}))
|
||||
.unwrap();
|
||||
let value = envelope.value();
|
||||
assert_eq!(value["query"], json!("leads"));
|
||||
assert_eq!(value["auth"]["api_key"], json!("[redacted]"));
|
||||
assert_eq!(value["auth"]["nested"][0]["password"], json!("[redacted]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsampled_support_ids_are_valid_and_consistent() {
|
||||
let support = TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
);
|
||||
assert!(!support.trace_sampled);
|
||||
assert!(support.traceparent.ends_with("-00"));
|
||||
support.validate().unwrap();
|
||||
|
||||
let mut inconsistent = support.clone();
|
||||
inconsistent.traceparent = inconsistent.traceparent.replace("-00", "-01");
|
||||
assert!(inconsistent.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accept_request_validation_holds_the_identity_and_pin_bounds() {
|
||||
assert!(request().validate().is_ok());
|
||||
|
||||
let mut oversized = request();
|
||||
oversized.workspace_id = "w".repeat(TASK_IDENTITY_MAX_BYTES + 1);
|
||||
assert!(oversized.validate().is_err());
|
||||
|
||||
let mut bad_fingerprint = request();
|
||||
bad_fingerprint.input_fingerprint = "XYZ".to_owned();
|
||||
assert!(bad_fingerprint.validate().is_err());
|
||||
|
||||
let mut bad_retention = request();
|
||||
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
|
||||
assert!(bad_retention.validate().is_err());
|
||||
|
||||
// The persisted authority reference is bounded and never empty.
|
||||
let mut empty_key_reference = request();
|
||||
empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new(""));
|
||||
assert!(empty_key_reference.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_unknown_has_no_generic_successor() {
|
||||
// Story 4.4: reconciliation of an uncertain outcome is Story 4.6. Until then no ordinary
|
||||
// finish/cancel/retry may leave the non-terminal uncertain state.
|
||||
for next in [
|
||||
TaskState::Queued,
|
||||
TaskState::Working,
|
||||
TaskState::InputRequired,
|
||||
TaskState::OutcomeUnknown,
|
||||
TaskState::Completed,
|
||||
TaskState::Failed,
|
||||
TaskState::Cancelled,
|
||||
] {
|
||||
assert!(
|
||||
!TaskState::OutcomeUnknown.allows_transition_to(next),
|
||||
"outcome_unknown must not transition to {next:?}"
|
||||
);
|
||||
}
|
||||
assert!(!TaskState::OutcomeUnknown.is_terminal());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_policy_bounds_are_pinned_and_bounded() {
|
||||
assert!(TaskRetryPolicy::single_attempt().validate().is_ok());
|
||||
assert!(
|
||||
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS)
|
||||
.validate()
|
||||
.is_ok()
|
||||
);
|
||||
assert!(TaskRetryPolicy::new(0, 0).validate().is_err());
|
||||
assert!(
|
||||
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS + 1, 0)
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
TaskRetryPolicy::new(1, TASK_MAX_BACKOFF_STEP_MS + 1)
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
// The pinned step grows the delay but never beyond four steps.
|
||||
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(2), 2);
|
||||
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(5), 8);
|
||||
assert_eq!(TaskRetryPolicy::single_attempt().backoff_before(2), 0);
|
||||
// A pin whose retry snapshot is invalid is rejected at acceptance.
|
||||
let mut invalid = request();
|
||||
invalid.pins.retry = TaskRetryPolicy::new(0, 0);
|
||||
assert!(invalid.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_requires_a_terminal_reachable_transition() {
|
||||
let access = TaskAccess {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
};
|
||||
let base = TaskFinishRequest {
|
||||
access,
|
||||
task_id: TaskId::generate(),
|
||||
expected_state: TaskState::Working,
|
||||
expected_version: 1,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
error: None,
|
||||
};
|
||||
assert!(base.validate().is_ok());
|
||||
|
||||
let mut non_terminal = base.clone();
|
||||
non_terminal.target_state = TaskState::Working;
|
||||
assert!(non_terminal.validate().is_err());
|
||||
|
||||
let mut terminal_source = base.clone();
|
||||
terminal_source.expected_state = TaskState::Completed;
|
||||
assert!(terminal_source.validate().is_err());
|
||||
|
||||
let mut completed_without_result = base;
|
||||
completed_without_result.result = None;
|
||||
assert!(completed_without_result.validate().is_err());
|
||||
}
|
||||
}
|
||||
@@ -231,6 +231,7 @@ fn default_execution_config() -> ExecutionConfig {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
|
||||
@@ -564,6 +564,7 @@ fn default_execution_config() -> ExecutionConfig {
|
||||
idempotency: None,
|
||||
safety: None,
|
||||
approval_policy: None,
|
||||
execution_policy: None,
|
||||
auth_profile_ref: None,
|
||||
headers: BTreeMap::new(),
|
||||
}
|
||||
|
||||
@@ -217,6 +217,34 @@ string_enum! {
|
||||
Concurrency => "concurrency",
|
||||
RateLimit => "rate_limit",
|
||||
McpStream => "mcp_stream",
|
||||
Admission => "admission",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum RejectionReason {
|
||||
Limit => "limit",
|
||||
Capacity => "capacity",
|
||||
QueueFull => "queue_full",
|
||||
Deadline => "deadline",
|
||||
Unsupported => "unsupported",
|
||||
DependencyUnavailable => "dependency_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum ScopeKind {
|
||||
Process => "process",
|
||||
Global => "global",
|
||||
Agent => "agent",
|
||||
Tool => "tool",
|
||||
}
|
||||
}
|
||||
|
||||
string_enum! {
|
||||
pub enum AdmissionModeLabel {
|
||||
Synchronous => "synchronous",
|
||||
Background => "background",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,14 +16,15 @@ mod schema;
|
||||
|
||||
pub use exemplar::{ExemplarObservation, ExemplarTraceId, exemplar_snapshot};
|
||||
pub use labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind,
|
||||
UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
pub use record::{
|
||||
InFlightGuard, ToolInvocationMetrics, UpstreamRequestMetrics, bind_metric_service,
|
||||
bound_metric_service, initialize_gauges, normalize_http_route, record_cache_outcome,
|
||||
bound_metric_service, initialize_gauges, normalize_http_route, record_admission_oldest_waiting,
|
||||
record_admission_queue_wait, record_admission_rejection, record_cache_outcome,
|
||||
record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_http_request_with_exemplar, record_idempotency_outcome, record_invocation_history_lost,
|
||||
record_limit_rejection, record_mcp_request, record_mcp_request_with_exemplar,
|
||||
|
||||
@@ -5,10 +5,10 @@ use metrics::Gauge;
|
||||
|
||||
use crate::schema::{MetricDefinition, metric_schema};
|
||||
use crate::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, MetricBindingError, MetricService, SignalType, ToolErrorKind,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, ExemplarTraceId, Exporter,
|
||||
HttpMethod, HttpRoute, HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage,
|
||||
McpMethod, McpOutcome, McpResponseMode, MetricBindingError, MetricService, RejectionReason,
|
||||
ScopeKind, SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
|
||||
static BINDING: OnceLock<MetricService> = OnceLock::new();
|
||||
@@ -352,11 +352,47 @@ pub fn record_limit_rejection(stage: LimitStage) {
|
||||
}
|
||||
metrics::counter!(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
"stage" => stage.as_str()
|
||||
"stage" => stage.as_str(),
|
||||
"reason" => RejectionReason::Limit.as_str(),
|
||||
"scope_kind" => ScopeKind::Process.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Typed admission refusal with finite, low-cardinality labels only: no workspace, Agent, Tool,
|
||||
/// request id, fingerprint or arbitrary error text ever becomes a label.
|
||||
pub fn record_admission_rejection(reason: RejectionReason, scope_kind: ScopeKind) {
|
||||
if bound_family("crank_runtime_limit_rejections_total").is_none() {
|
||||
return;
|
||||
}
|
||||
metrics::counter!(
|
||||
"crank_runtime_limit_rejections_total",
|
||||
"stage" => LimitStage::Admission.as_str(),
|
||||
"reason" => reason.as_str(),
|
||||
"scope_kind" => scope_kind.as_str()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_admission_queue_wait(mode: AdmissionModeLabel, waited_ms: u64) {
|
||||
if bound_family("crank_runtime_admission_queue_wait_seconds").is_none() {
|
||||
return;
|
||||
}
|
||||
metrics::gauge!(
|
||||
"crank_runtime_admission_queue_wait_seconds",
|
||||
"mode" => mode.as_str()
|
||||
)
|
||||
.set(waited_ms as f64 / 1000.0);
|
||||
}
|
||||
|
||||
pub fn record_admission_oldest_waiting(oldest_waiting_ms: Option<u64>) {
|
||||
if bound_family("crank_runtime_admission_oldest_waiting_seconds").is_none() {
|
||||
return;
|
||||
}
|
||||
let seconds = oldest_waiting_ms.unwrap_or_default() as f64 / 1000.0;
|
||||
metrics::gauge!("crank_runtime_admission_oldest_waiting_seconds").set(seconds);
|
||||
}
|
||||
|
||||
pub fn record_cache_outcome(outcome: CacheOutcome) {
|
||||
if bound_family("crank_runtime_cache_total").is_none() {
|
||||
return;
|
||||
|
||||
@@ -3,16 +3,17 @@
|
||||
|
||||
use super::routes::{BOTH, HTTP_ROUTE_DOMAIN, MCP};
|
||||
use crate::labels::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpStatusClass,
|
||||
IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome, McpResponseMode,
|
||||
SignalType, ToolErrorKind, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpStatusClass, IdempotencyOutcome, InvocationSource, LimitStage, McpMethod, McpOutcome,
|
||||
McpResponseMode, RejectionReason, ScopeKind, SignalType, ToolErrorKind, ToolOutcome,
|
||||
UpstreamOperationKind, UpstreamOutcome,
|
||||
};
|
||||
|
||||
pub const METRIC_SCHEMA_VERSION: u32 = 2;
|
||||
pub const MAX_METRIC_FAMILIES: usize = 128;
|
||||
pub const MAX_PRODUCT_LABELS_PER_FAMILY: usize = 4;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_285;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_515;
|
||||
pub const MAX_LOGICAL_SERIES_PER_PROCESS: usize = 6_381;
|
||||
pub const MAX_RENDERED_SERIES_PER_PROCESS: usize = 19_611;
|
||||
pub const MAX_EXEMPLAR_SLOTS_PER_PROCESS: usize = 12_348;
|
||||
pub const MAX_EXPOSITION_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
@@ -179,6 +180,9 @@ const TOOL_ERRORS: &[&str] = ToolErrorKind::VALUES;
|
||||
const OPERATION_KINDS: &[&str] = UpstreamOperationKind::VALUES;
|
||||
const UPSTREAM_OUTCOMES: &[&str] = UpstreamOutcome::VALUES;
|
||||
const LIMIT_STAGES: &[&str] = LimitStage::VALUES;
|
||||
const REJECTION_REASONS: &[&str] = RejectionReason::VALUES;
|
||||
const SCOPE_KINDS: &[&str] = ScopeKind::VALUES;
|
||||
const ADMISSION_MODES: &[&str] = AdmissionModeLabel::VALUES;
|
||||
const CACHE_OUTCOMES: &[&str] = CacheOutcome::VALUES;
|
||||
const IDEMPOTENCY_OUTCOMES: &[&str] = IdempotencyOutcome::VALUES;
|
||||
const CONFIRMATION_OUTCOMES: &[&str] = ConfirmationOutcome::VALUES;
|
||||
@@ -403,13 +407,39 @@ pub(crate) const METRIC_SCHEMA: &[MetricDefinition] = &[
|
||||
"crank_runtime_limit_rejections_total",
|
||||
MetricKind::Counter,
|
||||
MetricUnit::Count,
|
||||
&["stage"],
|
||||
&[domain("stage", LIMIT_STAGES)],
|
||||
&["stage", "reason", "scope_kind"],
|
||||
&[
|
||||
domain("stage", LIMIT_STAGES),
|
||||
domain("reason", REJECTION_REASONS),
|
||||
domain("scope_kind", SCOPE_KINDS),
|
||||
],
|
||||
&[],
|
||||
BOTH,
|
||||
3,
|
||||
96,
|
||||
"Runtime executions rejected by a bounded limit.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_admission_queue_wait_seconds",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Seconds,
|
||||
&["mode"],
|
||||
&[domain("mode", ADMISSION_MODES)],
|
||||
&[],
|
||||
BOTH,
|
||||
2,
|
||||
"Wait of the most recently admitted execution before it received a scoped permit.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_admission_oldest_waiting_seconds",
|
||||
MetricKind::Gauge,
|
||||
MetricUnit::Seconds,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
BOTH,
|
||||
1,
|
||||
"Age of the oldest waiting admission request.",
|
||||
),
|
||||
metric(
|
||||
"crank_runtime_cache_total",
|
||||
MetricKind::Counter,
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use std::{collections::BTreeSet, time::Duration};
|
||||
|
||||
use crank_metrics::{
|
||||
CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod, HttpRoute,
|
||||
HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage, McpMethod,
|
||||
McpOutcome, McpResponseMode, MetricService, SignalType, ToolErrorKind, ToolInvocationMetrics,
|
||||
ToolOutcome, UpstreamOperationKind, UpstreamOutcome, UpstreamRequestMetrics,
|
||||
bind_metric_service, initialize_gauges, metric_schema, record_cache_outcome,
|
||||
record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
AdmissionModeLabel, CacheOutcome, ConfirmationOutcome, DbPoolState, Exporter, HttpMethod,
|
||||
HttpRoute, HttpStatusClass, IdempotencyOutcome, InFlightGuard, InvocationSource, LimitStage,
|
||||
McpMethod, McpOutcome, McpResponseMode, MetricService, RejectionReason, ScopeKind, SignalType,
|
||||
ToolErrorKind, ToolInvocationMetrics, ToolOutcome, UpstreamOperationKind, UpstreamOutcome,
|
||||
UpstreamRequestMetrics, bind_metric_service, initialize_gauges, metric_schema,
|
||||
record_admission_oldest_waiting, record_admission_queue_wait, record_admission_rejection,
|
||||
record_cache_outcome, record_confirmation_outcome, record_export_failure, record_http_request,
|
||||
record_idempotency_outcome, record_invocation_history_lost, record_limit_rejection,
|
||||
record_mcp_request, record_tool_invocation, record_upstream_request, set_catalog,
|
||||
set_db_pool_connections, set_mcp_active_sessions, set_mcp_session_metrics_fresh,
|
||||
@@ -71,6 +72,9 @@ fn schema_names_are_unique_and_labels_are_closed() {
|
||||
"state",
|
||||
"signal_type",
|
||||
"exporter",
|
||||
"reason",
|
||||
"scope_kind",
|
||||
"mode",
|
||||
];
|
||||
for metric in schema {
|
||||
assert!(
|
||||
@@ -265,6 +269,9 @@ fn every_recording_api_matches_the_declared_schema() {
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
record_limit_rejection(LimitStage::Concurrency);
|
||||
record_admission_rejection(RejectionReason::Capacity, ScopeKind::Global);
|
||||
record_admission_queue_wait(AdmissionModeLabel::Synchronous, 5);
|
||||
record_admission_oldest_waiting(Some(5));
|
||||
record_cache_outcome(CacheOutcome::Hit);
|
||||
record_idempotency_outcome(IdempotencyOutcome::Replay);
|
||||
record_confirmation_outcome(ConfirmationOutcome::Required);
|
||||
|
||||
@@ -48,10 +48,10 @@ fn schema_v1_is_complete_deterministic_and_inside_the_frozen_budget() {
|
||||
|
||||
#[test]
|
||||
fn frozen_budget_edges_are_exact() {
|
||||
assert!(validate_budget(6_284, 19_514).is_ok());
|
||||
assert!(validate_budget(6_285, 19_515).is_ok());
|
||||
assert!(validate_budget(6_286, 16_000).is_err());
|
||||
assert!(validate_budget(6_000, 19_516).is_err());
|
||||
assert!(validate_budget(6_380, 19_610).is_ok());
|
||||
assert!(validate_budget(6_381, 19_611).is_ok());
|
||||
assert!(validate_budget(6_382, 19_000).is_err());
|
||||
assert!(validate_budget(6_000, 19_612).is_err());
|
||||
assert_eq!(
|
||||
crank_metrics::max_exemplar_slots(MetricService::AdminApi),
|
||||
12_348
|
||||
@@ -79,15 +79,15 @@ fn canonical_schema_validation_reports_its_category() {
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::AdminApi),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 6_285,
|
||||
rendered_series: 19_515
|
||||
logical_series: 6_381,
|
||||
rendered_series: 19_611
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::McpServer),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 1_083,
|
||||
rendered_series: 3_558
|
||||
logical_series: 1_179,
|
||||
rendered_series: 3_654
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -543,22 +543,22 @@ fn root_budgets_are_per_process_only() {
|
||||
"root обязан объявлять ровно один ключ budgets"
|
||||
);
|
||||
assert!(rendered.ends_with(
|
||||
",\"budgets\":[{\"process\":\"admin_api\",\"families\":15,\"logical_series\":6285,\
|
||||
\"rendered_series\":19515,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
|
||||
\"families\":23,\"logical_series\":1083,\"rendered_series\":3558,\"exemplar_slots\":2310}]}\n"
|
||||
",\"budgets\":[{\"process\":\"admin_api\",\"families\":17,\"logical_series\":6381,\
|
||||
\"rendered_series\":19611,\"exemplar_slots\":12348},{\"process\":\"mcp_server\",\
|
||||
\"families\":25,\"logical_series\":1179,\"rendered_series\":3654,\"exemplar_slots\":2310}]}\n"
|
||||
));
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::AdminApi),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 6_285,
|
||||
rendered_series: 19_515
|
||||
logical_series: 6_381,
|
||||
rendered_series: 19_611
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
schema_budget(MetricService::McpServer),
|
||||
Ok(crank_metrics::SchemaBudget {
|
||||
logical_series: 1_083,
|
||||
rendered_series: 3_558
|
||||
logical_series: 1_179,
|
||||
rendered_series: 3_654
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -566,11 +566,11 @@ fn root_budgets_are_per_process_only() {
|
||||
/// TD-M3 — frozen caps и границы `validate_exemplar_budget` точны.
|
||||
#[test]
|
||||
fn frozen_caps_and_edges_are_exact() {
|
||||
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_285);
|
||||
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_515);
|
||||
assert_eq!(MAX_LOGICAL_SERIES_PER_PROCESS, 6_381);
|
||||
assert_eq!(MAX_RENDERED_SERIES_PER_PROCESS, 19_611);
|
||||
assert_eq!(MAX_EXEMPLAR_SLOTS_PER_PROCESS, 12_348);
|
||||
assert!(validate_budget(6_285, 19_515).is_ok());
|
||||
assert!(validate_budget(6_286, 19_515).is_err());
|
||||
assert!(validate_budget(6_381, 19_611).is_ok());
|
||||
assert!(validate_budget(6_382, 19_611).is_err());
|
||||
assert!(validate_exemplar_budget(12_347).is_ok());
|
||||
assert!(validate_exemplar_budget(12_348).is_ok());
|
||||
assert!(validate_exemplar_budget(12_349).is_err());
|
||||
|
||||
@@ -13,7 +13,7 @@ mod schema;
|
||||
pub use config::{ObservabilityConfig, ObservabilityConfigError, ServiceIdentity};
|
||||
pub use crank_metrics::{
|
||||
DURATION_BUCKETS_SECONDS, MetricDefinition, MetricKind, MetricService, MetricUnit,
|
||||
bind_metric_service, bound_metric_service, metric_schema,
|
||||
bind_metric_service, bound_metric_service, metric_schema, record_admission_oldest_waiting,
|
||||
};
|
||||
pub use error_reporting::{
|
||||
CriticalErrorCategory, SentryConfig, SentryConfigError, capture_critical_error,
|
||||
|
||||
@@ -7,6 +7,7 @@ publish.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1"
|
||||
crank-artifacts = { path = "../crank-artifacts" }
|
||||
crank-core = { path = "../crank-core" }
|
||||
crank-mapping = { path = "../crank-mapping" }
|
||||
|
||||
@@ -116,6 +116,7 @@ pub use model::{
|
||||
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
|
||||
};
|
||||
pub use postgres::{
|
||||
AuthorizedResourceRead, BoundResourceAdmission, PostgresPoolConfig, PostgresPoolConfigError,
|
||||
PostgresRegistry,
|
||||
AdmissionLimits, AuthorizedResourceRead, BoundResourceAdmission, PostgresExecutionAdmission,
|
||||
PostgresPoolConfig, PostgresPoolConfigError, PostgresRegistry, PostgresTaskStore,
|
||||
admission_health, admission_snapshot, tasks_health,
|
||||
};
|
||||
@@ -6,6 +6,13 @@ mod artifact_cleanup_indexes_v13;
|
||||
mod artifact_metadata_v12;
|
||||
mod authority;
|
||||
mod baseline_v1;
|
||||
mod durable_task_input_v30;
|
||||
mod durable_tasks_v25;
|
||||
mod durable_tasks_v26;
|
||||
mod durable_tasks_v27;
|
||||
mod durable_tasks_v28;
|
||||
mod durable_tasks_v29;
|
||||
mod execution_admission_v24;
|
||||
mod execution_outcome_v5;
|
||||
mod import_plan_v14;
|
||||
mod master_key_identity_v7;
|
||||
@@ -35,6 +42,13 @@ mod schema_guard_v19;
|
||||
mod schema_guard_v20;
|
||||
mod schema_guard_v22;
|
||||
mod schema_guard_v23;
|
||||
mod schema_guard_v24;
|
||||
mod schema_guard_v25;
|
||||
mod schema_guard_v26;
|
||||
mod schema_guard_v27;
|
||||
mod schema_guard_v28;
|
||||
mod schema_guard_v29;
|
||||
mod schema_guard_v30;
|
||||
mod schema_guard_v7;
|
||||
mod schema_guard_v8;
|
||||
mod schema_guard_v9;
|
||||
|
||||
@@ -4,6 +4,13 @@ use super::agent_resource_bindings_v18;
|
||||
use super::approval_side_effects_v10;
|
||||
use super::artifact_cleanup_indexes_v13;
|
||||
use super::artifact_metadata_v12;
|
||||
use super::durable_task_input_v30;
|
||||
use super::durable_tasks_v25;
|
||||
use super::durable_tasks_v26;
|
||||
use super::durable_tasks_v27;
|
||||
use super::durable_tasks_v28;
|
||||
use super::durable_tasks_v29;
|
||||
use super::execution_admission_v24;
|
||||
use super::execution_outcome_v5;
|
||||
use super::import_plan_v14;
|
||||
use super::master_key_identity_v7;
|
||||
@@ -39,9 +46,10 @@ use preflight::*;
|
||||
use contract::baseline_source_digest;
|
||||
use contract::validate_descriptors;
|
||||
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
|
||||
const CURRENT_VERSION: i64 = 23;
|
||||
const CURRENT_VERSION: i64 = 30;
|
||||
const IMPLEMENTED_VERSIONS: &[i64] = &[
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
|
||||
27, 28, 29, 30,
|
||||
];
|
||||
const BASELINE_SOURCE_SHA256: &str =
|
||||
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
|
||||
@@ -330,6 +338,48 @@ impl MigrationAuthority {
|
||||
prompt_dependencies_v23::SOURCE_SHA256,
|
||||
22,
|
||||
),
|
||||
expand_descriptor(
|
||||
24,
|
||||
execution_admission_v24::NAME,
|
||||
execution_admission_v24::SOURCE_SHA256,
|
||||
23,
|
||||
),
|
||||
expand_descriptor(
|
||||
25,
|
||||
durable_tasks_v25::NAME,
|
||||
durable_tasks_v25::SOURCE_SHA256,
|
||||
24,
|
||||
),
|
||||
expand_descriptor(
|
||||
26,
|
||||
durable_tasks_v26::NAME,
|
||||
durable_tasks_v26::SOURCE_SHA256,
|
||||
25,
|
||||
),
|
||||
expand_descriptor(
|
||||
27,
|
||||
durable_tasks_v27::NAME,
|
||||
durable_tasks_v27::SOURCE_SHA256,
|
||||
26,
|
||||
),
|
||||
expand_descriptor(
|
||||
28,
|
||||
durable_tasks_v28::NAME,
|
||||
durable_tasks_v28::SOURCE_SHA256,
|
||||
27,
|
||||
),
|
||||
expand_descriptor(
|
||||
29,
|
||||
durable_tasks_v29::NAME,
|
||||
durable_tasks_v29::SOURCE_SHA256,
|
||||
28,
|
||||
),
|
||||
expand_descriptor(
|
||||
30,
|
||||
durable_task_input_v30::NAME,
|
||||
durable_task_input_v30::SOURCE_SHA256,
|
||||
29,
|
||||
),
|
||||
]
|
||||
}
|
||||
pub fn validate_sequence() -> Result<(), MigrationError> {
|
||||
@@ -489,6 +539,27 @@ impl MigrationAuthority {
|
||||
if from < 23 {
|
||||
prompt_dependencies_v23::apply(&mut transaction, &Self::sequence()[22]).await?;
|
||||
}
|
||||
if from < 24 {
|
||||
execution_admission_v24::apply(&mut transaction, &Self::sequence()[23]).await?;
|
||||
}
|
||||
if from < 25 {
|
||||
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
|
||||
}
|
||||
if from < 26 {
|
||||
durable_tasks_v26::apply(&mut transaction, &Self::sequence()[25]).await?;
|
||||
}
|
||||
if from < 27 {
|
||||
durable_tasks_v27::apply(&mut transaction, &Self::sequence()[26]).await?;
|
||||
}
|
||||
if from < 28 {
|
||||
durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?;
|
||||
}
|
||||
if from < 29 {
|
||||
durable_tasks_v29::apply(&mut transaction, &Self::sequence()[28]).await?;
|
||||
}
|
||||
if from < 30 {
|
||||
durable_task_input_v30::apply(&mut transaction, &Self::sequence()[29]).await?;
|
||||
}
|
||||
match inspect(&mut transaction).await? {
|
||||
MigrationPreflight::Current {
|
||||
version: CURRENT_VERSION,
|
||||
|
||||
@@ -140,6 +140,8 @@ pub(super) fn validate_descriptors(
|
||||
!= agent_resource_bindings_v18::SOURCE_SHA256
|
||||
|| sha256_hex(resource_security_revocations_v19::SOURCE.as_bytes())
|
||||
!= resource_security_revocations_v19::SOURCE_SHA256
|
||||
|| sha256_hex(execution_admission_v24::SOURCE.as_bytes())
|
||||
!= execution_admission_v24::SOURCE_SHA256
|
||||
{
|
||||
return Err(MigrationError::new(
|
||||
"invalid_contract",
|
||||
|
||||
@@ -148,6 +148,7 @@ pub(super) async fn inspect(
|
||||
owned_relations::PROMPTS,
|
||||
owned_relations::PROMPT_PUBLICATIONS,
|
||||
owned_relations::PROMPT_DEPENDENCIES,
|
||||
owned_relations::EXECUTION_ADMISSION,
|
||||
] {
|
||||
validate_required_relations(connection, relations, CURRENT_VERSION).await?;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-task-input-v30";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_task_input_v30.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"84a51f712fd0bc439acba2ab3849232777c29e3b181586bbb4b06fd129843a38";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_task_input_v30",
|
||||
Some(30),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_task_input_v30",
|
||||
Some(30),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
-- V30: continuation through validated input (Story 4.5).
|
||||
alter table tasks add column if not exists effective_input_fingerprint text;
|
||||
update tasks set effective_input_fingerprint = input_fingerprint where effective_input_fingerprint is null;
|
||||
alter table tasks alter column effective_input_fingerprint set not null;
|
||||
alter table tasks add constraint tasks_effective_input_fingerprint_check check (
|
||||
length(effective_input_fingerprint) = 64 and effective_input_fingerprint ~ '^[0-9a-f]{64}$'
|
||||
);
|
||||
|
||||
create table if not exists task_input_requests (
|
||||
task_id uuid primary key references tasks (task_id) on delete cascade,
|
||||
wait_id text not null,
|
||||
expected_version integer not null,
|
||||
schema_revision integer not null,
|
||||
schema_format_version integer not null,
|
||||
fingerprint_format_version integer not null,
|
||||
schema_snapshot jsonb not null,
|
||||
projected_schema jsonb not null,
|
||||
prompt_code text not null,
|
||||
missing_fields jsonb not null default '[]'::jsonb,
|
||||
expires_at timestamptz not null,
|
||||
created_at timestamptz not null default now(),
|
||||
accepted_at timestamptz,
|
||||
update_key_digest text,
|
||||
canonical_fingerprint text,
|
||||
effective_input_fingerprint text,
|
||||
committed_state text,
|
||||
committed_version integer,
|
||||
constraint task_input_requests_wait_unique unique (wait_id),
|
||||
constraint task_input_requests_prompt_check check (octet_length(prompt_code) between 1 and 64),
|
||||
constraint task_input_requests_receipt_check check (
|
||||
(accepted_at is null and update_key_digest is null and canonical_fingerprint is null
|
||||
and effective_input_fingerprint is null and committed_state is null and committed_version is null)
|
||||
or (accepted_at is not null and update_key_digest is not null and canonical_fingerprint is not null
|
||||
and effective_input_fingerprint is not null and committed_state is not null and committed_version is not null)
|
||||
)
|
||||
);
|
||||
-- Not unique on purpose: several waiting Tasks may share one absolute TTL value (the same pinned
|
||||
-- policy deadline), and an expired/cancelled wait keeps its row. Per-Task uniqueness comes from the
|
||||
-- primary key and task_input_requests_wait_unique.
|
||||
create index if not exists task_input_requests_active_ttl_idx
|
||||
on task_input_requests (expires_at) where accepted_at is null;
|
||||
|
||||
create table if not exists task_input_audit (
|
||||
id bigint generated always as identity primary key,
|
||||
task_id uuid not null references tasks (task_id) on delete cascade,
|
||||
wait_id text not null,
|
||||
event text not null,
|
||||
from_state text,
|
||||
to_state text,
|
||||
from_version integer,
|
||||
to_version integer,
|
||||
actor text,
|
||||
reason_code text,
|
||||
created_at timestamptz not null default now(),
|
||||
constraint task_input_audit_event_check check (event in ('input_required','input_accepted','input_expired')),
|
||||
constraint task_input_audit_actor_check check (actor is null or octet_length(actor) between 1 and 160),
|
||||
constraint task_input_audit_reason_check check (reason_code is null or octet_length(reason_code) between 1 and 64)
|
||||
);
|
||||
create index if not exists task_input_audit_task_idx on task_input_audit (task_id, id);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v25";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v25.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"ada48bfb7d7130c84ab38217795608cd368ccb28af66faafbdfc4fc7609f67cd";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v25",
|
||||
Some(25),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.canonical_ledger",
|
||||
Some(25),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
-- V25: durable background Tasks (Story 4.2, FR-22, FR-23).
|
||||
--
|
||||
-- Одна строка `tasks` — принятая durable работа. Идентификатор Task (UUIDv7), owner scope,
|
||||
-- authorization scope, запрошенный Tool и точная published-версия, canonical fingerprint входа,
|
||||
-- support IDs (Request/Trace/parent TraceContext независимо от sampling) и bounded envelopes
|
||||
-- закрепляются в той же PostgreSQL transaction, что и dispatch intent
|
||||
-- (`execution_admission_entries.task_id`) и его queue slot. До commit наружу не выдаётся ни ID, ни ack.
|
||||
--
|
||||
-- `scoped_key` — digest caller-ключа по полному (workspace, owner kind/id, authorization scope,
|
||||
-- operation) scope; UNIQUE по этому набору и есть защита идемпотентности. Повтор того же ключа с тем
|
||||
-- же canonical fingerprint возвращает сохранённую Task, другой fingerprint — typed conflict.
|
||||
--
|
||||
-- `retain_until` — срок хранения записи, а не execution deadline: пока запись в retention, replay
|
||||
-- находит тот же Task. Удаление terminal-записей возможно только после retain_until; queued/working/
|
||||
-- input_required/outcome_unknown не удаляются никогда ради лимита.
|
||||
--
|
||||
-- Строка не хранит credentials, bearer/prepared headers и confirmation tokens: секретные поля входа
|
||||
-- редактируются до записи, а approval хранится безопасной ссылкой.
|
||||
|
||||
create table tasks (
|
||||
task_id uuid not null,
|
||||
workspace_id text not null,
|
||||
owner_kind text not null,
|
||||
owner_id text not null,
|
||||
authorization_scope text not null,
|
||||
operation_id text not null,
|
||||
operation_version integer not null,
|
||||
scoped_key text not null,
|
||||
input_fingerprint text not null,
|
||||
request_id text not null,
|
||||
trace_id text not null,
|
||||
traceparent text not null,
|
||||
trace_sampled boolean not null,
|
||||
policy_mode text not null,
|
||||
approval_reference text,
|
||||
state text not null,
|
||||
version integer not null default 1,
|
||||
attempts integer not null default 0,
|
||||
input_envelope jsonb not null,
|
||||
progress_envelope jsonb,
|
||||
result_envelope jsonb,
|
||||
error_envelope jsonb,
|
||||
execution_deadline timestamptz not null,
|
||||
retain_until timestamptz not null,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now(),
|
||||
constraint tasks_pkey primary key (task_id),
|
||||
constraint tasks_workspace_check check (octet_length(workspace_id) between 1 and 160),
|
||||
constraint tasks_owner_kind_check check (owner_kind in ('agent', 'admin')),
|
||||
constraint tasks_owner_check check (octet_length(owner_id) between 1 and 160),
|
||||
constraint tasks_authorization_scope_check check (
|
||||
octet_length(authorization_scope) between 1 and 160
|
||||
),
|
||||
constraint tasks_operation_check check (octet_length(operation_id) between 1 and 160),
|
||||
constraint tasks_operation_version_check check (operation_version >= 1),
|
||||
constraint tasks_scoped_key_check check (octet_length(scoped_key) between 16 and 128),
|
||||
constraint tasks_fingerprint_check check (octet_length(input_fingerprint) = 64),
|
||||
constraint tasks_request_check check (octet_length(request_id) between 1 and 128),
|
||||
constraint tasks_trace_check check (octet_length(trace_id) = 32),
|
||||
constraint tasks_traceparent_check check (octet_length(traceparent) = 55),
|
||||
constraint tasks_policy_mode_check check (
|
||||
policy_mode in ('synchronous', 'background', 'adaptive')
|
||||
),
|
||||
constraint tasks_state_check check (
|
||||
state in ('queued', 'working', 'input_required', 'outcome_unknown',
|
||||
'completed', 'failed', 'cancelled')
|
||||
),
|
||||
constraint tasks_lifecycle_check check (version >= 1 and attempts >= 0),
|
||||
constraint tasks_retention_check check (retain_until >= execution_deadline),
|
||||
constraint tasks_approval_check check (
|
||||
approval_reference is null or octet_length(approval_reference) between 1 and 160
|
||||
)
|
||||
);
|
||||
|
||||
create unique index tasks_scoped_key_uidx
|
||||
on tasks (workspace_id, owner_kind, owner_id, authorization_scope, operation_id, scoped_key);
|
||||
|
||||
create index tasks_retention_idx on tasks (state, retain_until, task_id);
|
||||
|
||||
create index tasks_deadline_idx on tasks (state, execution_deadline, task_id);
|
||||
|
||||
-- Dispatch intent: прежняя reserved-строка admission становится точкой связи Task и capacity.
|
||||
-- FK без cascade: удаление принятой Task не маскируется housekeeping-удалением intent.
|
||||
alter table execution_admission_entries
|
||||
add column task_id uuid,
|
||||
add constraint execution_admission_entries_task_fkey
|
||||
foreign key (task_id) references tasks (task_id);
|
||||
|
||||
create unique index execution_admission_entries_task_uidx
|
||||
on execution_admission_entries (task_id) where task_id is not null;
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v26";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v26.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v26",
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v26",
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
-- V26: PostgreSQL lease/fencing/CAS additions for durable background Tasks (Story 4.3).
|
||||
alter table tasks add column if not exists fence bigint not null default 0;
|
||||
alter table tasks add column if not exists lease_owner text;
|
||||
alter table tasks add column if not exists lease_until timestamptz;
|
||||
alter table tasks add column if not exists current_attempt_id text;
|
||||
alter table tasks add column if not exists dispatch_started_at timestamptz;
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v27";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v27.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v27",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v27",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
-- V27: encrypted executable Task input (Story 4.3).
|
||||
alter table tasks add column if not exists executable_input_ciphertext text;
|
||||
alter table tasks add column if not exists executable_input_key_version text;
|
||||
alter table tasks add column if not exists executable_input_master_key_epoch bigint;
|
||||
|
||||
alter table tasks add constraint tasks_executable_input_key_version_check check (
|
||||
executable_input_ciphertext is null
|
||||
or (executable_input_key_version is not null and octet_length(executable_input_key_version) between 1 and 64)
|
||||
);
|
||||
|
||||
alter table tasks add column if not exists target_executable_input_ciphertext text;
|
||||
alter table tasks add column if not exists target_executable_input_key_version text;
|
||||
alter table tasks add column if not exists target_executable_input_master_key_epoch bigint;
|
||||
|
||||
alter table tasks add constraint tasks_executable_input_epoch_check check (
|
||||
executable_input_ciphertext is null
|
||||
or (executable_input_master_key_epoch is not null
|
||||
and executable_input_master_key_epoch >= 1)
|
||||
);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v28";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v28.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v28",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v28",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
-- V28: typed Platform API key authority reference of a durable Task (Story 4.3).
|
||||
-- Only the immutable key identifier is stored, never the bearer secret.
|
||||
alter table tasks add column if not exists platform_api_key_id text;
|
||||
|
||||
alter table tasks add constraint tasks_platform_api_key_id_check check (
|
||||
platform_api_key_id is null
|
||||
or octet_length(platform_api_key_id) between 1 and 160
|
||||
);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v29";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v29.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"0666c7d2aa440e6aae2170ceaf2fce2549118612e1505ef78817be60436a75dd";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v29",
|
||||
Some(29),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v29",
|
||||
Some(29),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
-- V29: durable per-outbound attempt history, pinned retry policy and cancellation (Story 4.4).
|
||||
-- An accepted Task keeps the retry snapshot it was admitted with; legacy Tasks get a single attempt.
|
||||
alter table tasks add column if not exists max_attempts integer not null default 1;
|
||||
alter table tasks add column if not exists retry_backoff_step_ms bigint not null default 0;
|
||||
alter table tasks add column if not exists next_attempt_at timestamptz;
|
||||
alter table tasks add column if not exists cancel_requested_at timestamptz;
|
||||
alter table tasks add column if not exists cancel_reason text;
|
||||
|
||||
alter table tasks add constraint tasks_max_attempts_check check (max_attempts between 1 and 5);
|
||||
alter table tasks add constraint tasks_retry_backoff_step_check
|
||||
check (retry_backoff_step_ms between 0 and 1000);
|
||||
alter table tasks add constraint tasks_cancel_reason_check check (
|
||||
cancel_reason is null or octet_length(cancel_reason) between 1 and 160
|
||||
);
|
||||
|
||||
-- One row per committed outbound attempt. The authoritative outcome of a Task stays on `tasks`;
|
||||
-- this history only records what was attempted and how each attempt resolved.
|
||||
create table if not exists task_attempts (
|
||||
attempt_id text primary key,
|
||||
task_id uuid not null references tasks (task_id) on delete cascade,
|
||||
ordinal integer not null,
|
||||
owner text not null,
|
||||
fence bigint not null default 0,
|
||||
input_fingerprint text not null,
|
||||
upstream_key_digest text,
|
||||
retry_allowed boolean not null default false,
|
||||
checkpoint_at timestamptz not null default now(),
|
||||
resolved_at timestamptz,
|
||||
outcome text,
|
||||
certainty text,
|
||||
phase text,
|
||||
constraint task_attempts_task_ordinal_unique unique (task_id, ordinal),
|
||||
constraint task_attempts_ordinal_check check (ordinal between 1 and 5),
|
||||
constraint task_attempts_outcome_check check (
|
||||
outcome is null
|
||||
or outcome in ('completed', 'failed', 'retry_scheduled', 'outcome_unknown')
|
||||
),
|
||||
constraint task_attempts_certainty_check check (
|
||||
certainty is null or certainty in ('known', 'unknown')
|
||||
),
|
||||
constraint task_attempts_phase_check check (
|
||||
phase is null or phase in ('before_dispatch', 'dispatched')
|
||||
),
|
||||
-- A resolved attempt is complete: an unresolved row carries no resolution at all, so a stale
|
||||
-- writer can never append a second outcome to the same attempt.
|
||||
constraint task_attempts_resolution_check check (
|
||||
(resolved_at is null and outcome is null and certainty is null and phase is null)
|
||||
or (resolved_at is not null and outcome is not null and certainty is not null)
|
||||
)
|
||||
);
|
||||
|
||||
create index if not exists task_attempts_task_id_idx on task_attempts (task_id, ordinal);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "execution-admission-v24";
|
||||
pub(super) const SOURCE: &str = include_str!("execution_admission_v24.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"efe021ebcd68bbed64edaa1e90c3a851f7b167b94ba65d88f525d58160f02681";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.execution_admission_v24",
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.canonical_ledger",
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
-- V24: authoritative execution admission state (Story 4.1, FR-21, FR-24).
|
||||
--
|
||||
-- Один deployment-wide admission: PostgreSQL хранит cursor/fairness state, ожидающие тикеты,
|
||||
-- выданные permits и durable enqueue decisions, поэтому несколько процессов делят общий бюджет.
|
||||
--
|
||||
-- `execution_admission_entries` — bounded набор: waiting (ожидает capacity), executing (scoped
|
||||
-- permit выдан, держится до явного release) и reserved (durable enqueue decision с зарезервированным
|
||||
-- queue slot; не исполняется и не рекламируется как Task до Story 4.2).
|
||||
--
|
||||
-- Строка не хранит ни credentials, ни connection/session id, ни prepared headers, ни тело запроса:
|
||||
-- quota key — (workspace, agent) и (workspace, operation) без version. `operation_version` пишется
|
||||
-- только как pin исполняемых semantics и не участвует в квотах.
|
||||
--
|
||||
-- `expires_at` — абсолютный deadline. Queue decisions без dispatch можно expire по deadline; активный
|
||||
-- permit НЕ освобождается по TTL (иначе paused/crashed owner обойдёт global limit): его снимает только
|
||||
-- явный release/cancel владельца.
|
||||
|
||||
create table execution_admission_controllers (
|
||||
workspace_id text not null,
|
||||
cursor_agent text,
|
||||
cursor_agent_credit integer not null default 0,
|
||||
cursor_tool text,
|
||||
cursor_tool_credit integer not null default 0,
|
||||
updated_at timestamptz not null default now(),
|
||||
constraint execution_admission_controllers_pkey primary key (workspace_id),
|
||||
constraint execution_admission_controllers_workspace_check check (
|
||||
octet_length(workspace_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_controllers_agent_credit_check check (
|
||||
cursor_agent_credit between 0 and 64
|
||||
),
|
||||
constraint execution_admission_controllers_tool_credit_check check (
|
||||
cursor_tool_credit between 0 and 64
|
||||
)
|
||||
);
|
||||
|
||||
create table execution_admission_entries (
|
||||
ticket_id bigint generated always as identity,
|
||||
workspace_id text not null,
|
||||
agent_key text not null,
|
||||
operation_id text not null,
|
||||
operation_version integer not null,
|
||||
mode text not null,
|
||||
policy_mode text not null,
|
||||
state text not null,
|
||||
owner_token text not null,
|
||||
fingerprint text not null,
|
||||
created_at timestamptz not null default now(),
|
||||
expires_at timestamptz not null,
|
||||
granted_at timestamptz,
|
||||
constraint execution_admission_entries_pkey primary key (ticket_id),
|
||||
constraint execution_admission_entries_workspace_check check (
|
||||
octet_length(workspace_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_agent_check check (
|
||||
octet_length(agent_key) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_operation_check check (
|
||||
octet_length(operation_id) between 1 and 160
|
||||
),
|
||||
constraint execution_admission_entries_version_check check (operation_version >= 1),
|
||||
constraint execution_admission_entries_mode_check check (
|
||||
mode in ('synchronous', 'background')
|
||||
),
|
||||
constraint execution_admission_entries_policy_mode_check check (
|
||||
policy_mode in ('synchronous', 'background', 'adaptive')
|
||||
),
|
||||
constraint execution_admission_entries_state_check check (
|
||||
state in ('waiting', 'executing', 'reserved')
|
||||
),
|
||||
constraint execution_admission_entries_owner_check check (
|
||||
octet_length(owner_token) between 1 and 64
|
||||
),
|
||||
constraint execution_admission_entries_fingerprint_check check (
|
||||
octet_length(fingerprint) between 1 and 128
|
||||
),
|
||||
constraint execution_admission_entries_reserved_mode_check check (
|
||||
state <> 'reserved' or mode = 'background'
|
||||
)
|
||||
);
|
||||
|
||||
create unique index execution_admission_entries_owner_uidx
|
||||
on execution_admission_entries (owner_token);
|
||||
|
||||
create index execution_admission_entries_state_idx
|
||||
on execution_admission_entries (state, workspace_id, ticket_id);
|
||||
|
||||
create index execution_admission_entries_agent_idx
|
||||
on execution_admission_entries (state, workspace_id, agent_key);
|
||||
|
||||
create index execution_admission_entries_operation_idx
|
||||
on execution_admission_entries (state, workspace_id, operation_id);
|
||||
@@ -55,6 +55,11 @@ pub(super) const PROMPT_PUBLICATIONS: &[&str] = &["prompt_publications", "agent_
|
||||
pub(super) const PROMPT_DEPENDENCIES: &[&str] =
|
||||
&["prompt_dependencies", "prompt_message_resource_refs"];
|
||||
|
||||
pub(super) const EXECUTION_ADMISSION: &[&str] = &[
|
||||
"execution_admission_controllers",
|
||||
"execution_admission_entries",
|
||||
];
|
||||
|
||||
pub(super) const ARTIFACT_METADATA: &[&str] = &["artifact_blobs", "artifact_sources"];
|
||||
|
||||
pub(super) const IMPORT_PLAN: &[&str] = &["import_plans", "import_plan_applications"];
|
||||
|
||||
@@ -468,6 +468,92 @@ pub(super) async fn validate_schema_fingerprint(
|
||||
} else {
|
||||
super::schema_guard_v23::validate_v23_prompt_dependencies(connection).await?;
|
||||
}
|
||||
if current_version < 24 {
|
||||
if !super::schema_guard_v24::validate_v24_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v24::validate_v24_execution_admission(connection, current_version)
|
||||
.await?;
|
||||
}
|
||||
if current_version < 25 {
|
||||
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 26 {
|
||||
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 27 {
|
||||
super::schema_guard_v26::validate_v26_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 28 {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v27::validate_v27_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 29 {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v28::validate_v28_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 30 {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v29::validate_v29_durable_attempts(connection).await?;
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v30::validate_v30_input_continuation(connection).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,8 @@ pub(crate) const OWNED_RELATIONS: &[&str] = &[
|
||||
"prompt_messages",
|
||||
"prompt_dependencies",
|
||||
"prompt_message_resource_refs",
|
||||
"execution_admission_controllers",
|
||||
"execution_admission_entries",
|
||||
];
|
||||
|
||||
pub(crate) const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
//! V24 schema guard: authoritative execution admission state (Story 4.1, FR-21/FR-24).
|
||||
//!
|
||||
//! Схема V24 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
|
||||
//! ожидаемой форме: controller-row с weighted round-robin cursor и bounded entries с закрытым
|
||||
//! словарём mode/policy_mode/state, уникальным owner token и индексами квот.
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const CONTROLLERS: &str = "execution_admission_controllers";
|
||||
const ENTRIES: &str = "execution_admission_entries";
|
||||
|
||||
const OWNED_RELATIONS: &[&str] = &[CONTROLLERS, ENTRIES];
|
||||
|
||||
const CONTROLLER_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("workspace_id", "text", false),
|
||||
("cursor_agent", "text", true),
|
||||
("cursor_agent_credit", "integer", false),
|
||||
("cursor_tool", "text", true),
|
||||
("cursor_tool_credit", "integer", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
];
|
||||
|
||||
const ENTRY_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("ticket_id", "bigint", false),
|
||||
("workspace_id", "text", false),
|
||||
("agent_key", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("mode", "text", false),
|
||||
("policy_mode", "text", false),
|
||||
("state", "text", false),
|
||||
("owner_token", "text", false),
|
||||
("fingerprint", "text", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("expires_at", "timestamp with time zone", false),
|
||||
("granted_at", "timestamp with time zone", true),
|
||||
];
|
||||
|
||||
/// V25 adds the nullable Task dispatch-intent link to the same reserved row. At version 24 the
|
||||
/// column must not exist yet, so the guard validates the shape of the version it describes.
|
||||
fn entry_columns(current_version: i64) -> Vec<(&'static str, &'static str, bool)> {
|
||||
let mut columns = ENTRY_COLUMNS.to_vec();
|
||||
if current_version >= 25 {
|
||||
columns.push(("task_id", "uuid", true));
|
||||
}
|
||||
columns
|
||||
}
|
||||
|
||||
fn constraint_count(table: &str, current_version: i64) -> Option<usize> {
|
||||
match table {
|
||||
CONTROLLERS => Some(4),
|
||||
ENTRIES if current_version >= 25 => Some(12),
|
||||
ENTRIES => Some(11),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const UNIQUE_INDEXES: &[&str] = &["execution_admission_entries_owner_uidx"];
|
||||
|
||||
fn v24_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(24),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
/// Схема V24 отсутствует целиком (для current_version < 24).
|
||||
pub(super) async fn validate_v24_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
for relation in OWNED_RELATIONS {
|
||||
if relation_exists(connection, relation).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
for index in UNIQUE_INDEXES {
|
||||
if index_exists(connection, index).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Схема V24 присутствует в ожидаемой форме (для current_version >= 24).
|
||||
pub(super) async fn validate_v24_execution_admission(
|
||||
connection: &mut PgConnection,
|
||||
current_version: i64,
|
||||
) -> Result<(), MigrationError> {
|
||||
for relation in OWNED_RELATIONS {
|
||||
if !relation_exists(connection, relation).await? {
|
||||
return Err(v24_error("v24.relations"));
|
||||
}
|
||||
}
|
||||
let tables = [
|
||||
(CONTROLLERS, CONTROLLER_COLUMNS.to_vec()),
|
||||
(ENTRIES, entry_columns(current_version)),
|
||||
];
|
||||
for (table, columns) in tables {
|
||||
validate_columns(connection, table, &columns).await?;
|
||||
}
|
||||
for table in [CONTROLLERS, ENTRIES] {
|
||||
let expected = constraint_count(table, current_version).unwrap_or_default();
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1",
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(expected).unwrap_or_default() {
|
||||
return Err(v24_error("v24.constraints"));
|
||||
}
|
||||
}
|
||||
for (table, constraint) in [
|
||||
(CONTROLLERS, "execution_admission_controllers_pkey"),
|
||||
(ENTRIES, "execution_admission_entries_pkey"),
|
||||
(ENTRIES, "execution_admission_entries_state_check"),
|
||||
] {
|
||||
if !constraint_exists(connection, table, constraint).await? {
|
||||
return Err(v24_error("v24.constraint_missing"));
|
||||
}
|
||||
}
|
||||
if !normalized_expression(
|
||||
connection,
|
||||
ENTRIES,
|
||||
"execution_admission_entries_state_check",
|
||||
)
|
||||
.await?
|
||||
.is_some_and(|expression| {
|
||||
super::schema_guard::enum_constraint_matches(
|
||||
&expression,
|
||||
"state",
|
||||
&["waiting", "executing", "reserved"],
|
||||
)
|
||||
}) {
|
||||
return Err(v24_error("v24.state_vocabulary"));
|
||||
}
|
||||
for index in UNIQUE_INDEXES {
|
||||
let unique = index_unique(connection, index).await?;
|
||||
if unique != Some(true) {
|
||||
return Err(v24_error("v24.indexes"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_columns(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
columns: &[(&str, &str, bool)],
|
||||
) -> Result<(), MigrationError> {
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(table)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(columns.len()).unwrap_or_default() {
|
||||
return Err(v24_error("v24.columns"));
|
||||
}
|
||||
for (column, data_type, nullable) in columns {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v24_error("v24.columns"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn constraint_exists(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
) as present",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn normalized_expression(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<Option<String>, MigrationError> {
|
||||
let row = query(
|
||||
"select pg_get_expr(c.conbin, c.conrelid) as expression
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
and c.contype = 'c'",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
|
||||
}
|
||||
|
||||
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_class idx
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1
|
||||
) as present",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn index_unique(
|
||||
connection: &mut PgConnection,
|
||||
index: &str,
|
||||
) -> Result<Option<bool>, MigrationError> {
|
||||
let row = query(
|
||||
"select i.indisunique, i.indisvalid
|
||||
from pg_catalog.pg_index i
|
||||
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| {
|
||||
let unique = row.try_get::<bool, _>("indisunique").ok()?;
|
||||
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
|
||||
Some(unique && valid)
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
//! V25 schema guard: durable background Tasks (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! Схема V25 либо полностью отсутствует (миграция ещё не применена), либо присутствует ровно в
|
||||
//! ожидаемой форме: таблица `tasks` с закрытым словарём state/policy_mode и scoped uniqueness, плюс
|
||||
//! nullable `task_id` на существующей reserved-строке admission с FK и UNIQUE-индексом.
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
const ENTRIES: &str = "execution_admission_entries";
|
||||
const SCOPED_INDEX: &str = "tasks_scoped_key_uidx";
|
||||
const TASK_INDEX: &str = "execution_admission_entries_task_uidx";
|
||||
const TASK_FK: &str = "execution_admission_entries_task_fkey";
|
||||
|
||||
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("task_id", "uuid", false),
|
||||
("workspace_id", "text", false),
|
||||
("owner_kind", "text", false),
|
||||
("owner_id", "text", false),
|
||||
("authorization_scope", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("scoped_key", "text", false),
|
||||
("input_fingerprint", "text", false),
|
||||
("request_id", "text", false),
|
||||
("trace_id", "text", false),
|
||||
("traceparent", "text", false),
|
||||
("trace_sampled", "boolean", false),
|
||||
("policy_mode", "text", false),
|
||||
("approval_reference", "text", true),
|
||||
("state", "text", false),
|
||||
("version", "integer", false),
|
||||
("attempts", "integer", false),
|
||||
("input_envelope", "jsonb", false),
|
||||
("progress_envelope", "jsonb", true),
|
||||
("result_envelope", "jsonb", true),
|
||||
("error_envelope", "jsonb", true),
|
||||
("execution_deadline", "timestamp with time zone", false),
|
||||
("retain_until", "timestamp with time zone", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
];
|
||||
|
||||
fn v25_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(25),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
/// Схема V25 отсутствует целиком (для current_version < 25).
|
||||
pub(super) async fn validate_v25_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
if super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
|
||||
return Ok(false);
|
||||
}
|
||||
for index in [SCOPED_INDEX, TASK_INDEX] {
|
||||
if index_exists(connection, index).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Схема V25 присутствует в ожидаемой форме (для current_version >= 25).
|
||||
pub(super) async fn validate_v25_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
if !relation_exists(connection, TASKS).await? {
|
||||
return Err(v25_error("v25.relations"));
|
||||
}
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
|
||||
return Err(v25_error("v25.columns"));
|
||||
}
|
||||
for (column, data_type, nullable) in TASK_COLUMNS {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v25_error("v25.columns"));
|
||||
}
|
||||
}
|
||||
if !super::schema_guard::column_exists(connection, ENTRIES, "task_id").await? {
|
||||
return Err(v25_error("v25.intent_column"));
|
||||
}
|
||||
for (constraint, vocabulary) in [
|
||||
(
|
||||
"tasks_state_check",
|
||||
[
|
||||
"queued",
|
||||
"working",
|
||||
"input_required",
|
||||
"outcome_unknown",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
]
|
||||
.as_slice(),
|
||||
),
|
||||
(
|
||||
"tasks_policy_mode_check",
|
||||
["synchronous", "background", "adaptive"].as_slice(),
|
||||
),
|
||||
] {
|
||||
let expression = normalized_expression(connection, TASKS, constraint).await?;
|
||||
if !expression.is_some_and(|expression| {
|
||||
super::schema_guard::enum_constraint_matches(
|
||||
&expression,
|
||||
column_of(constraint),
|
||||
vocabulary,
|
||||
)
|
||||
}) {
|
||||
return Err(v25_error("v25.vocabulary"));
|
||||
}
|
||||
}
|
||||
for (index, table) in [(SCOPED_INDEX, TASKS), (TASK_INDEX, ENTRIES)] {
|
||||
if index_unique(connection, table, index).await? != Some(true) {
|
||||
return Err(v25_error("v25.indexes"));
|
||||
}
|
||||
}
|
||||
if !constraint_exists(connection, ENTRIES, TASK_FK).await? {
|
||||
return Err(v25_error("v25.intent_fk"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn column_of(constraint: &str) -> &'static str {
|
||||
if constraint.contains("policy_mode") {
|
||||
"policy_mode"
|
||||
} else {
|
||||
"state"
|
||||
}
|
||||
}
|
||||
|
||||
async fn constraint_exists(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
) as present",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn normalized_expression(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<Option<String>, MigrationError> {
|
||||
let row = query(
|
||||
"select pg_get_expr(c.conbin, c.conrelid) as expression
|
||||
from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema() and t.relname = $1 and c.conname = $2
|
||||
and c.contype = 'c'",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| row.try_get::<String, _>("expression").ok()))
|
||||
}
|
||||
|
||||
async fn index_exists(connection: &mut PgConnection, index: &str) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_class idx
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1
|
||||
) as present",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
|
||||
async fn index_unique(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
index: &str,
|
||||
) -> Result<Option<bool>, MigrationError> {
|
||||
let row = query(
|
||||
"select i.indisunique, i.indisvalid
|
||||
from pg_catalog.pg_index i
|
||||
join pg_catalog.pg_class idx on idx.oid = i.indexrelid
|
||||
join pg_catalog.pg_class t on t.oid = i.indrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = idx.relnamespace
|
||||
where n.nspname = current_schema() and idx.relname = $1 and t.relname = $2",
|
||||
)
|
||||
.bind(index)
|
||||
.bind(table)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
Ok(row.and_then(|row| {
|
||||
let unique = row.try_get::<bool, _>("indisunique").ok()?;
|
||||
let valid = row.try_get::<bool, _>("indisvalid").ok()?;
|
||||
Some(unique && valid)
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//! V26 schema guard: lease/fencing columns for durable background Tasks (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("task_id", "uuid", false),
|
||||
("workspace_id", "text", false),
|
||||
("owner_kind", "text", false),
|
||||
("owner_id", "text", false),
|
||||
("authorization_scope", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("scoped_key", "text", false),
|
||||
("input_fingerprint", "text", false),
|
||||
("request_id", "text", false),
|
||||
("trace_id", "text", false),
|
||||
("traceparent", "text", false),
|
||||
("trace_sampled", "boolean", false),
|
||||
("policy_mode", "text", false),
|
||||
("approval_reference", "text", true),
|
||||
("state", "text", false),
|
||||
("version", "integer", false),
|
||||
("attempts", "integer", false),
|
||||
("input_envelope", "jsonb", false),
|
||||
("progress_envelope", "jsonb", true),
|
||||
("result_envelope", "jsonb", true),
|
||||
("error_envelope", "jsonb", true),
|
||||
("execution_deadline", "timestamp with time zone", false),
|
||||
("retain_until", "timestamp with time zone", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
("fence", "bigint", false),
|
||||
("lease_owner", "text", true),
|
||||
("lease_until", "timestamp with time zone", true),
|
||||
("current_attempt_id", "text", true),
|
||||
("dispatch_started_at", "timestamp with time zone", true),
|
||||
];
|
||||
|
||||
fn v26_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
// delegate to v25 absent for relation mostly; V26 columns must not exist.
|
||||
for column in [
|
||||
"fence",
|
||||
"lease_owner",
|
||||
"lease_until",
|
||||
"current_attempt_id",
|
||||
"dispatch_started_at",
|
||||
] {
|
||||
if super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_base_columns(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
if !relation_exists(connection, TASKS).await? {
|
||||
return Err(v26_error("v26.relations"));
|
||||
}
|
||||
for (column, data_type, nullable) in TASK_COLUMNS {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v26_error("v26.columns"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
validate_v26_base_columns(connection).await?;
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
|
||||
return Err(v26_error("v26.columns"));
|
||||
}
|
||||
// Constraints/index checks inherited conceptually from V25; V26 adds columns only.
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
//! V27 schema guard: encrypted executable Task input columns (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
use super::schema_guard_v26::validate_v26_base_columns;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
pub(super) async fn validate_v27_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
for column in [
|
||||
"executable_input_ciphertext",
|
||||
"executable_input_key_version",
|
||||
"executable_input_master_key_epoch",
|
||||
] {
|
||||
if super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v27_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
validate_v26_base_columns(connection).await?;
|
||||
// Additional V27 columns.
|
||||
let extra = [
|
||||
("executable_input_ciphertext", "text", true),
|
||||
("executable_input_key_version", "text", true),
|
||||
("executable_input_master_key_epoch", "bigint", true),
|
||||
("target_executable_input_ciphertext", "text", true),
|
||||
("target_executable_input_key_version", "text", true),
|
||||
("target_executable_input_master_key_epoch", "bigint", true),
|
||||
];
|
||||
for (column, data_type, nullable) in extra {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v27.columns",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
}
|
||||
// Exact total column count for version 27: 43 V26 columns + 3 V27 columns.
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != 37_i64 {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v27.columns",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Loaded 100 of 171 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user