Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4832878468 | ||
|
|
5bf4e3e9bf | ||
|
|
1ac5b10c67 | ||
|
|
07a9ef0a4a | ||
|
|
4f83d8d24c | ||
|
|
2a57800f63 | ||
|
|
e359077f6e | ||
|
|
c0117ff6b8 | ||
|
|
24bba2cf33 | ||
|
|
f56553c4d6 | ||
|
|
027e2cb911 | ||
|
|
adabb2065a | ||
|
|
bf773d61b8 | ||
|
|
74a98d9362 | ||
|
|
7f18bce679 | ||
|
|
4ec9d9201b | ||
|
|
cf2909273b | ||
|
|
4a7b230f35 | ||
|
|
cfe11a0a04 | ||
|
|
35a21f8ab5 | ||
|
|
2d0deb8963 | ||
|
|
f0a79362eb | ||
|
|
27f86d4916 | ||
|
|
f728d31dbf | ||
|
|
29ee01b6ef | ||
|
|
bcf8b43516 | ||
|
|
1c256e6c75 | ||
|
|
e4953abcd2 | ||
|
|
90e5357a08 | ||
|
|
d80eda7329 | ||
|
|
a3057de51e | ||
|
|
8c02380762 | ||
|
|
3b5fcc51cc | ||
|
|
433987413e | ||
|
|
5488eb73ec | ||
|
|
f9f8c625d1 |
No files matched your search
+34
-9
@@ -40,6 +40,15 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect release-tooling-only changes
|
||||
id: release_tooling
|
||||
env:
|
||||
GITEA_EVENT_NAME: ${{ gitea.event_name }}
|
||||
GITEA_EVENT_BEFORE: ${{ gitea.event.before }}
|
||||
GITEA_PR_BASE_SHA: ${{ gitea.event.pull_request.base.sha }}
|
||||
FORCE_FULL: ${{ inputs.validation_level == 'feature' || contains(gitea.event.pull_request.labels.*.name, 'feature-ready') }}
|
||||
run: python3 scripts/ci-release-tooling-scope.py
|
||||
|
||||
- name: Install Rust toolchain
|
||||
run: |
|
||||
set -eu
|
||||
@@ -48,9 +57,13 @@ jobs:
|
||||
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
|
||||
exit 1
|
||||
fi
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
rustup default "$toolchain"
|
||||
host="$(rustc -vV | sed -n 's/^host: //p')"
|
||||
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
fi
|
||||
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
|
||||
toolchain_bin="$toolchain_dir/bin"
|
||||
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
|
||||
@@ -74,15 +87,18 @@ jobs:
|
||||
docker info
|
||||
|
||||
- name: Check typed runtime configuration contract
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: |
|
||||
cargo run -p crank-config --bin crank-config-contract -- --check
|
||||
python3 scripts/check-runtime-config.py --root .
|
||||
python3 scripts/check-config-boundaries.py --root .
|
||||
|
||||
- name: Check canonical migration contract
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: cargo run -p admin-api --bin crank-migrate -- plan --check
|
||||
|
||||
- name: Check typed metrics contract
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: |
|
||||
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
|
||||
python3 scripts/check-metrics-boundaries.py --root .
|
||||
@@ -138,6 +154,7 @@ jobs:
|
||||
python3 scripts/check-openapi-matrix-compatibility.py --base "$base_ref"
|
||||
|
||||
- name: Run Crank story ATDD
|
||||
if: steps.release_tooling.outputs.ci_only != 'true'
|
||||
run: scripts/ci-story-checks.sh
|
||||
ui:
|
||||
name: UI Checks
|
||||
@@ -187,9 +204,13 @@ jobs:
|
||||
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
|
||||
exit 1
|
||||
fi
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
rustup default "$toolchain"
|
||||
host="$(rustc -vV | sed -n 's/^host: //p')"
|
||||
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
fi
|
||||
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
|
||||
toolchain_bin="$toolchain_dir/bin"
|
||||
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
|
||||
@@ -266,9 +287,13 @@ jobs:
|
||||
echo "Unable to read Rust toolchain channel from rust-toolchain.toml" >&2
|
||||
exit 1
|
||||
fi
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
rustup default "$toolchain"
|
||||
host="$(rustc -vV | sed -n 's/^host: //p')"
|
||||
if ! rustup run "$toolchain" rustc --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" rustfmt --version >/dev/null 2>&1 || \
|
||||
! rustup run "$toolchain" cargo-clippy --version >/dev/null 2>&1; then
|
||||
rustup toolchain install "$toolchain" --profile minimal --component clippy --component rustfmt
|
||||
fi
|
||||
host="$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')"
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/${toolchain}-${host}"
|
||||
toolchain_bin="$toolchain_dir/bin"
|
||||
if [ ! -x "$toolchain_bin/rustc" ] || [ ! -x "$toolchain_bin/cargo" ]; then
|
||||
|
||||
@@ -12,10 +12,20 @@ on:
|
||||
options: [story, feature]
|
||||
default: story
|
||||
required: true
|
||||
resume_tag:
|
||||
description: Existing draft release tag to resume with image-only steps
|
||||
type: string
|
||||
default: ""
|
||||
required: false
|
||||
resume_source_sha:
|
||||
description: Exact accepted source commit SHA for the resume tag
|
||||
type: string
|
||||
default: ""
|
||||
required: false
|
||||
|
||||
env:
|
||||
REGISTRY: git.itexp.me
|
||||
IMAGE_TAG: ${{ gitea.ref_name }}
|
||||
IMAGE_TAG: ${{ inputs.resume_tag || gitea.ref_name }}
|
||||
ADMIN_API_IMAGE: git.itexp.me/bsodfather/crank-community-admin-api
|
||||
MCP_SERVER_IMAGE: git.itexp.me/bsodfather/crank-community-mcp-server
|
||||
UI_IMAGE: git.itexp.me/bsodfather/crank-community-ui
|
||||
@@ -31,12 +41,47 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Verify existing release before resume
|
||||
id: resume
|
||||
if: inputs.resume_tag != ''
|
||||
env:
|
||||
GITEA_SERVER_URL: https://git.itexp.me
|
||||
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
RELEASE_TAG: ${{ inputs.resume_tag }}
|
||||
RESUME_SOURCE_SHA: ${{ inputs.resume_source_sha }}
|
||||
run: python3 scripts/ci-release-resume.py
|
||||
- name: Preserve updated image scanner for resume
|
||||
id: pin_resume_scanner
|
||||
if: inputs.resume_tag != ''
|
||||
run: |
|
||||
set -eu
|
||||
scanner_dir="$(mktemp -d "${RUNNER_TEMP:?RUNNER_TEMP is unavailable}/crank-release-scanner.XXXXXX")"
|
||||
cp scripts/scan-images.sh "$scanner_dir/scan-images.sh"
|
||||
cp scripts/check-image-scan.py "$scanner_dir/check-image-scan.py"
|
||||
chmod +x "$scanner_dir/scan-images.sh"
|
||||
printf 'path=%s\n' "$scanner_dir/scan-images.sh" >> "$GITHUB_OUTPUT"
|
||||
- name: Checkout validated release source
|
||||
if: inputs.resume_tag != ''
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
with:
|
||||
ref: ${{ steps.resume.outputs.release_tag }}
|
||||
fetch-depth: 0
|
||||
- name: Assert exact release source checkout
|
||||
if: inputs.resume_tag != ''
|
||||
env:
|
||||
EXPECTED_SOURCE_SHA: ${{ steps.resume.outputs.source_sha }}
|
||||
run: test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"
|
||||
- name: Clear optional release evidence
|
||||
if: inputs.resume_tag == ''
|
||||
run: rm -f .tmp/openapi-playwright.json .tmp/openapi-ui-evidence.json
|
||||
|
||||
|
||||
|
||||
- name: Use preinstalled Rust toolchain
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
set -eu
|
||||
toolchain_dir="${RUSTUP_HOME:-$HOME/.rustup}/toolchains/1.96.1-x86_64-unknown-linux-gnu"
|
||||
@@ -54,6 +99,7 @@ jobs:
|
||||
"$toolchain_bin/cargo-clippy" --version
|
||||
|
||||
- name: Verify runner toolchain
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
python3 --version
|
||||
rustc --version
|
||||
@@ -65,19 +111,23 @@ jobs:
|
||||
bao version
|
||||
|
||||
- name: Install dependency policy tool
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo install cargo-deny --version 0.20.2 --locked
|
||||
|
||||
- name: Check formatting
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo fmt --all --check
|
||||
|
||||
- name: Audit Rust dependencies
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo deny --locked check advisories bans licenses sources
|
||||
|
||||
- name: Run tooling unit tests
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
run: python3 -m unittest discover -s tests/unit
|
||||
|
||||
- name: Check typed runtime configuration contract
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
cargo run -p crank-config --bin crank-config-contract -- --check
|
||||
python3 scripts/check-runtime-config.py --root .
|
||||
@@ -85,14 +135,17 @@ jobs:
|
||||
scripts/check-rust-boundaries.sh
|
||||
|
||||
- name: Check canonical migration contract
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo run -p admin-api --bin crank-migrate -- plan --check
|
||||
|
||||
- name: Check typed metrics contract
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
cargo run -p crank-metrics --bin crank-metrics-contract -- --check
|
||||
python3 scripts/check-metrics-boundaries.py --root .
|
||||
|
||||
- name: Check Capability Inventory
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
required_args=""
|
||||
for number in $(seq 1 54); do
|
||||
@@ -105,6 +158,7 @@ jobs:
|
||||
$required_args
|
||||
|
||||
- name: Check Capability Baseline
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
python3 scripts/validate-capability-baseline.py \
|
||||
--root . \
|
||||
@@ -112,40 +166,46 @@ jobs:
|
||||
--schema docs/schemas/capability-baseline.schema.json
|
||||
|
||||
- name: Check Community scope
|
||||
if: inputs.resume_tag == ''
|
||||
run: scripts/check-community-scope.sh
|
||||
|
||||
- name: Run Crank story ATDD
|
||||
if: inputs.resume_tag == ''
|
||||
run: scripts/ci-story-checks.sh
|
||||
|
||||
- name: Run release quality gates
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
run: |
|
||||
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
cargo test --workspace --all-targets -- --test-threads=1
|
||||
|
||||
- name: Build release binaries
|
||||
if: inputs.resume_tag == ''
|
||||
run: cargo build --release -p admin-api -p mcp-server
|
||||
|
||||
|
||||
- name: Install UI dependencies
|
||||
if: inputs.resume_tag == ''
|
||||
working-directory: apps/ui
|
||||
run: npm ci
|
||||
|
||||
- name: Audit UI dependencies
|
||||
if: inputs.resume_tag == ''
|
||||
working-directory: apps/ui
|
||||
run: npm audit --audit-level=high
|
||||
|
||||
- name: Build UI dist
|
||||
if: inputs.resume_tag == ''
|
||||
working-directory: apps/ui
|
||||
run: npm run build
|
||||
|
||||
- name: Install Playwright browser
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
working-directory: apps/ui
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run release end-to-end tests
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
working-directory: apps/ui
|
||||
run: |
|
||||
mkdir -p ../../.tmp
|
||||
@@ -153,7 +213,7 @@ jobs:
|
||||
PLAYWRIGHT_JSON_OUTPUT=../../.tmp/openapi-playwright.json npm run e2e
|
||||
|
||||
- name: Collect sanitized OpenAPI UI release evidence
|
||||
if: inputs.validation_level == 'feature'
|
||||
if: inputs.resume_tag == '' && inputs.validation_level == 'feature'
|
||||
working-directory: apps/ui
|
||||
run: |
|
||||
trap 'rm -f ../../.tmp/openapi-playwright.json' EXIT
|
||||
@@ -175,12 +235,14 @@ jobs:
|
||||
--require-accepted
|
||||
|
||||
- name: Validate deployment manifests
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
docker compose -f docker-compose.yml --env-file .env.example config -q
|
||||
docker compose -f deploy/community/docker-compose.yml --env-file deploy/community/.env.example config -q
|
||||
docker compose -f deploy/community/docker-compose.images.yml --env-file deploy/community/.env.images.example --profile local-db config -q
|
||||
|
||||
- name: Package release artifacts
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p dist/release
|
||||
@@ -204,13 +266,14 @@ jobs:
|
||||
sha256sum "${files[@]}" > "dist/crank-community-${IMAGE_TAG}-checksums.txt"
|
||||
|
||||
- name: Generate SBOM
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
docker run --rm -v "$PWD:/workspace" anchore/syft:latest \
|
||||
dir:/workspace -o spdx-json \
|
||||
> dist/crank-community-${IMAGE_TAG}-sbom.spdx.json
|
||||
|
||||
- name: Publish release bundles, checksums, evidence, and SBOM to Gitea
|
||||
if: startsWith(gitea.ref, 'refs/tags/v')
|
||||
if: inputs.resume_tag == '' && startsWith(gitea.ref, 'refs/tags/v')
|
||||
env:
|
||||
# gitea.server_url inside the job container is the internal http://gitea:3000 origin;
|
||||
# the publisher only forwards its token to the public HTTPS origin.
|
||||
@@ -249,6 +312,8 @@ jobs:
|
||||
docker login '${{ env.REGISTRY }}' -u "$DEPLOY_REGISTRY_USER" --password-stdin
|
||||
|
||||
- name: Build and push release images
|
||||
env:
|
||||
RELEASE_IMAGE_SCANNER: ${{ steps.pin_resume_scanner.outputs.path || 'scripts/scan-images.sh' }}
|
||||
run: |
|
||||
docker build -f apps/admin-api/Dockerfile \
|
||||
-t '${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
|
||||
@@ -285,7 +350,7 @@ jobs:
|
||||
--env-file .tmp/release-migration-smoke.env --profile local-db up -d --wait
|
||||
docker compose -f deploy/community/docker-compose.images.yml \
|
||||
--env-file .tmp/release-migration-smoke.env --profile local-db logs migrate | grep '"status":"applied"'
|
||||
scripts/scan-images.sh \
|
||||
"$RELEASE_IMAGE_SCANNER" \
|
||||
'${{ env.ADMIN_API_IMAGE }}:${{ env.IMAGE_TAG }}' \
|
||||
'${{ env.MCP_SERVER_IMAGE }}:${{ env.IMAGE_TAG }}' \
|
||||
'${{ env.UI_IMAGE }}:${{ env.IMAGE_TAG }}'
|
||||
@@ -297,5 +362,6 @@ jobs:
|
||||
docker push '${{ env.UI_IMAGE }}:latest'
|
||||
|
||||
- name: Show release bundle summary
|
||||
if: inputs.resume_tag == ''
|
||||
run: |
|
||||
ls -lh dist
|
||||
Generated
+215
-65
@@ -25,7 +25,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"axum-extra",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-artifacts",
|
||||
"crank-community-auth",
|
||||
"crank-community-mcp",
|
||||
@@ -186,7 +186,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -197,7 +197,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -349,6 +349,12 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
@@ -398,7 +404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bitflags",
|
||||
"bollard-buildkit-proto",
|
||||
"bollard-stubs",
|
||||
@@ -455,7 +461,7 @@ version = "1.52.1-rc.29.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bollard-buildkit-proto",
|
||||
"bytes",
|
||||
"prost",
|
||||
@@ -584,6 +590,12 @@ dependencies = [
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a"
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.18.1"
|
||||
@@ -653,7 +665,7 @@ version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-core",
|
||||
"crank-metrics",
|
||||
"crank-trace",
|
||||
@@ -688,7 +700,7 @@ dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"axum-extra",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-core",
|
||||
"crank-registry",
|
||||
"rand 0.10.2",
|
||||
@@ -705,7 +717,7 @@ version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-adapter-rest",
|
||||
"crank-artifacts",
|
||||
"crank-core",
|
||||
@@ -722,6 +734,7 @@ dependencies = [
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"reqwest 0.12.28",
|
||||
"rmcp",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
@@ -850,7 +863,7 @@ dependencies = [
|
||||
"aes-gcm",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-adapter-rest",
|
||||
"crank-core",
|
||||
"crank-import",
|
||||
@@ -992,8 +1005,18 @@ version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
"darling_core 0.23.0",
|
||||
"darling_macro 0.23.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"darling_macro 0.24.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1006,7 +1029,20 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff"
|
||||
dependencies = [
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1015,9 +1051,20 @@ version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_core 0.23.0",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1057,6 +1104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
||||
dependencies = [
|
||||
"block-buffer 0.12.1",
|
||||
"const-oid",
|
||||
"crypto-common 0.2.2",
|
||||
"ctutils",
|
||||
]
|
||||
@@ -1069,7 +1117,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1078,7 +1126,7 @@ version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
@@ -1147,7 +1195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1309,7 +1357,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1661,7 +1709,7 @@ version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
@@ -1672,7 +1720,7 @@ dependencies = [
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
@@ -1920,7 +1968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2056,7 +2104,7 @@ version = "0.3.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"crank-artifacts",
|
||||
"crank-community-mcp",
|
||||
"crank-config",
|
||||
@@ -2121,7 +2169,7 @@ version = "0.18.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"evmap",
|
||||
"indexmap 2.14.0",
|
||||
"metrics",
|
||||
@@ -2457,7 +2505,7 @@ dependencies = [
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"structmeta",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2471,6 +2519,12 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pastey"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
@@ -2494,7 +2548,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2580,7 +2634,7 @@ dependencies = [
|
||||
"itertools",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2636,7 +2690,7 @@ dependencies = [
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -2675,9 +2729,9 @@ dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2852,7 +2906,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2890,7 +2944,7 @@ version = "0.12.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
@@ -2934,7 +2988,7 @@ version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -2981,6 +3035,53 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmcp"
|
||||
version = "3.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d24a7e92f4bcaa36b97fa93c4fa7ca8f3e669eced949345b3462dba333c7f909"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"indexmap 2.14.0",
|
||||
"libc",
|
||||
"pastey",
|
||||
"pin-project-lite",
|
||||
"rand 0.10.2",
|
||||
"rmcp-macros",
|
||||
"schemars 1.2.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"sse-stream",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmcp-macros"
|
||||
version = "3.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3140f24a766f80f9ec1607bde85680959a69b83d46fa9ce87dadab7e67ab029"
|
||||
dependencies = [
|
||||
"darling 0.24.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_json",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.28"
|
||||
@@ -3008,15 +3109,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
version = "1.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3075,7 +3176,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3144,12 +3245,26 @@ version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scoped-tls"
|
||||
version = "1.0.1"
|
||||
@@ -3293,7 +3408,18 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3328,7 +3454,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3349,7 +3475,7 @@ version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bs58",
|
||||
"chrono",
|
||||
"hex",
|
||||
@@ -3369,10 +3495,10 @@ version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3410,7 +3536,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3544,7 +3670,7 @@ dependencies = [
|
||||
name = "sqlx-core"
|
||||
version = "0.9.0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"crc",
|
||||
@@ -3586,7 +3712,7 @@ dependencies = [
|
||||
"quote",
|
||||
"sqlx-core",
|
||||
"sqlx-macros-core",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3609,7 +3735,7 @@ dependencies = [
|
||||
"sqlx-mysql",
|
||||
"sqlx-postgres",
|
||||
"sqlx-sqlite",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"url",
|
||||
@@ -3650,7 +3776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e"
|
||||
dependencies = [
|
||||
"atoi",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"crc",
|
||||
@@ -3706,6 +3832,19 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sse-stream"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
@@ -3738,7 +3877,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"structmeta-derive",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3749,7 +3888,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3769,6 +3908,17 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sync_wrapper"
|
||||
version = "1.0.2"
|
||||
@@ -3786,7 +3936,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3855,7 +4005,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3866,7 +4016,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3957,7 +4107,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4002,7 +4152,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"h2",
|
||||
"http",
|
||||
@@ -4103,7 +4253,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4232,7 +4382,7 @@ version = "3.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
@@ -4248,7 +4398,7 @@ version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
@@ -4385,7 +4535,7 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -4477,7 +4627,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4507,7 +4657,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4518,7 +4668,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4712,7 +4862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rustix 1.1.4",
|
||||
"rustix 1.1.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4734,7 +4884,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4755,7 +4905,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4775,7 +4925,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
@@ -4815,7 +4965,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
mod admin_auth_command;
|
||||
#[path = "crank_migrate/db_connect.rs"]
|
||||
mod db_connect;
|
||||
#[path = "crank_migrate/reporting.rs"]
|
||||
mod reporting;
|
||||
|
||||
use reporting::{active_rotation, ensure_no_options, read_required_key_file, rotation_json};
|
||||
|
||||
use crank_config::{ConfigSource, parse_migrator};
|
||||
use crank_registry::{
|
||||
@@ -11,7 +15,7 @@ use crank_registry::{
|
||||
};
|
||||
use crank_runtime::SecretCrypto;
|
||||
use serde_json::json;
|
||||
use std::{path::Path, process::ExitCode};
|
||||
use std::process::ExitCode;
|
||||
use time::OffsetDateTime;
|
||||
const MASTER_KEY_ROTATION_PAGE_SIZE: i64 = 1_000;
|
||||
#[tokio::main]
|
||||
@@ -765,6 +769,59 @@ async fn process_rotation_batch(
|
||||
processed += 1;
|
||||
}
|
||||
}
|
||||
let mut after_task_id: Option<String> = None;
|
||||
loop {
|
||||
let tasks = registry
|
||||
.list_tasks_for_master_key_epoch_page(
|
||||
rotation.source_epoch,
|
||||
after_task_id.as_deref(),
|
||||
MASTER_KEY_ROTATION_PAGE_SIZE,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
if tasks.is_empty() {
|
||||
break;
|
||||
}
|
||||
for task in tasks {
|
||||
after_task_id = Some(task.task_id.clone());
|
||||
if task.target_epoch == Some(rotation.target_epoch) {
|
||||
continue;
|
||||
}
|
||||
if max_versions.is_some_and(|limit| processed as usize >= limit) {
|
||||
return Ok(processed);
|
||||
}
|
||||
let plaintext = current_crypto
|
||||
.decrypt_for_epoch(&task.key_version, task.epoch, &task.ciphertext)
|
||||
.map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let target_ciphertext = target_crypto.encrypt(&plaintext).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.stage_master_key_rotation_task_ciphertext(
|
||||
&rotation.id,
|
||||
&task.task_id,
|
||||
rotation.source_epoch,
|
||||
&target_ciphertext,
|
||||
target_crypto.key_version(),
|
||||
rotation.target_epoch,
|
||||
&now,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
processed += 1;
|
||||
}
|
||||
}
|
||||
Ok(processed)
|
||||
}
|
||||
|
||||
@@ -818,6 +875,47 @@ async fn verify_staged_targets(
|
||||
verified += 1;
|
||||
}
|
||||
}
|
||||
let mut after_task_id: Option<String> = None;
|
||||
loop {
|
||||
let tasks = registry
|
||||
.list_target_tasks_for_master_key_rotation_page(
|
||||
rotation.target_epoch,
|
||||
after_task_id.as_deref(),
|
||||
MASTER_KEY_ROTATION_PAGE_SIZE,
|
||||
)
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
if tasks.is_empty() {
|
||||
break;
|
||||
}
|
||||
for task in tasks {
|
||||
after_task_id = Some(task.task_id.clone());
|
||||
let ciphertext = task.target_ciphertext.as_deref().ok_or_else(|| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
let key_version = task.target_key_version.as_deref().ok_or_else(|| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
target_crypto
|
||||
.decrypt_for_epoch(key_version, rotation.target_epoch, ciphertext)
|
||||
.map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
"master_key.rotation",
|
||||
"rerun_rotation_or_abort",
|
||||
)
|
||||
})?;
|
||||
verified += 1;
|
||||
}
|
||||
}
|
||||
if verified != rotation.total_secret_versions {
|
||||
return Err(CliError::new(
|
||||
"master_key_rotation_verification_failed",
|
||||
@@ -877,89 +975,3 @@ fn decrypt_current(
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn active_rotation(
|
||||
registry: &PostgresRegistry,
|
||||
allowed_states: &[&str],
|
||||
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
|
||||
let status = registry
|
||||
.master_key_rotation_status()
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
Ok(status
|
||||
.rotations
|
||||
.into_iter()
|
||||
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
|
||||
}
|
||||
|
||||
fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
|
||||
json!({
|
||||
"id": rotation.id,
|
||||
"source_epoch": rotation.source_epoch,
|
||||
"target_epoch": rotation.target_epoch,
|
||||
"state": rotation.state,
|
||||
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
|
||||
"checkpoint_secret_id": rotation.checkpoint_secret_id,
|
||||
"total_secret_versions": rotation.total_secret_versions,
|
||||
"processed_secret_versions": rotation.processed_secret_versions,
|
||||
"verified_secret_versions": rotation.verified_secret_versions,
|
||||
"failure_code": rotation.failure_code,
|
||||
})
|
||||
}
|
||||
|
||||
fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
|
||||
let path = path.ok_or_else(|| {
|
||||
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
|
||||
})?;
|
||||
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if !metadata.is_file() || metadata.len() > 16_384 {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if value.trim().is_empty() {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
|
||||
if options.current_key_file.is_some()
|
||||
|| options.target_key_file.is_some()
|
||||
|| options.backup_ref.is_some()
|
||||
|| options.rotation_id.is_some()
|
||||
|| options.max_versions.is_some()
|
||||
{
|
||||
return Err(CliError::new(
|
||||
"invalid_command",
|
||||
"cli.arguments",
|
||||
"run_master_key_status",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Leaf/reporting helpers of the migrator CLI.
|
||||
//!
|
||||
//! Extracted from `crank-migrate.rs` without behaviour changes so the binary entry stays within
|
||||
//! the code-health line limit.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crank_registry::{MasterKeyRotationRecord, PostgresRegistry};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{CliError, MasterKeyOptions};
|
||||
|
||||
pub(super) async fn active_rotation(
|
||||
registry: &PostgresRegistry,
|
||||
allowed_states: &[&str],
|
||||
) -> Result<Option<MasterKeyRotationRecord>, CliError> {
|
||||
let status = registry
|
||||
.master_key_rotation_status()
|
||||
.await
|
||||
.map_err(CliError::from_registry)?;
|
||||
Ok(status
|
||||
.rotations
|
||||
.into_iter()
|
||||
.find(|rotation| allowed_states.contains(&rotation.state.as_str())))
|
||||
}
|
||||
|
||||
pub(super) fn rotation_json(rotation: &MasterKeyRotationRecord) -> serde_json::Value {
|
||||
json!({
|
||||
"id": rotation.id,
|
||||
"source_epoch": rotation.source_epoch,
|
||||
"target_epoch": rotation.target_epoch,
|
||||
"state": rotation.state,
|
||||
"backup_ref": rotation.backup_ref.as_ref().map(|_| "configured"),
|
||||
"checkpoint_secret_id": rotation.checkpoint_secret_id,
|
||||
"total_secret_versions": rotation.total_secret_versions,
|
||||
"processed_secret_versions": rotation.processed_secret_versions,
|
||||
"verified_secret_versions": rotation.verified_secret_versions,
|
||||
"failure_code": rotation.failure_code,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn read_required_key_file(path: Option<&str>) -> Result<String, CliError> {
|
||||
let path = path.ok_or_else(|| {
|
||||
CliError::new("invalid_command", "cli.arguments", "run_master_key_status")
|
||||
})?;
|
||||
if path.len() > 512 || path.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let metadata = std::fs::metadata(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if !metadata.is_file() || metadata.len() > 16_384 {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
let value = std::fs::read_to_string(Path::new(path)).map_err(|_| {
|
||||
CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
)
|
||||
})?;
|
||||
if value.trim().is_empty() {
|
||||
return Err(CliError::new(
|
||||
"master_key_input_invalid",
|
||||
"master_key.input",
|
||||
"verify_operator_input",
|
||||
));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub(super) fn ensure_no_options(options: &MasterKeyOptions) -> Result<(), CliError> {
|
||||
if options.current_key_file.is_some()
|
||||
|| options.target_key_file.is_some()
|
||||
|| options.backup_ref.is_some()
|
||||
|| options.rotation_id.is_some()
|
||||
|| options.max_versions.is_some()
|
||||
{
|
||||
return Err(CliError::new(
|
||||
"invalid_command",
|
||||
"cli.arguments",
|
||||
"run_master_key_status",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -29,7 +29,7 @@ sha2.workspace = true
|
||||
sqlx.workspace = true
|
||||
thiserror.workspace = true
|
||||
time.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio = { workspace = true, features = ["sync", "signal"] }
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
@@ -45,5 +45,6 @@ opentelemetry_sdk.workspace = true
|
||||
prost.workspace = true
|
||||
reqwest.workspace = true
|
||||
tower.workspace = true
|
||||
tokio = { workspace = true, features = ["net", "time"] }
|
||||
tracing-opentelemetry.workspace = true
|
||||
metrics-util = "0.20.4"
|
||||
+149
-13
@@ -201,20 +201,45 @@ async fn run(
|
||||
// admin-api publishes into. The compatibility constructor keeps an unprovisioned/unsafe root
|
||||
// fail-closed per read instead of preventing startup; nothing is ever written from this process.
|
||||
let artifact_store = crank_artifacts::ArtifactStore::new(config.storage_root.clone());
|
||||
// Story 4.2: the durable Task application entry shares the registry pool, the published catalog
|
||||
// refresh interval and the exact admission limits of the runtime; the public wire stays
|
||||
// synchronous/unsupported and no worker is started.
|
||||
let task_store = PostgresTaskStore::new(
|
||||
// Story 4.2/4.3: the durable Task application entry shares the registry pool, the published
|
||||
// catalog refresh interval and the exact admission limits of the runtime; the public wire stays
|
||||
// synchronous/unsupported while a fenced in-process worker executes accepted Tasks.
|
||||
let task_store = std::sync::Arc::new(PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
admission_limits(runtime_limits),
|
||||
crank_core::AdmissionSchedulerConfig::default(),
|
||||
)?;
|
||||
)?);
|
||||
let tasks = std::sync::Arc::new(crank_community_mcp::TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
cache_stores.coordination.clone(),
|
||||
crank_runtime::TaskService::new(std::sync::Arc::new(task_store)),
|
||||
crank_runtime::TaskService::with_secret_crypto(
|
||||
task_store.clone(),
|
||||
std::sync::Arc::new(secret_crypto.clone()),
|
||||
),
|
||||
));
|
||||
// Story 4.3: the durable Task worker runs in this process over the same PostgreSQL store,
|
||||
// published catalog, executor and encrypted-input authority as the accepting application.
|
||||
let task_handler =
|
||||
std::sync::Arc::new(crank_community_mcp::TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime.clone(),
|
||||
secret_crypto.clone(),
|
||||
));
|
||||
let task_worker = std::sync::Arc::new(crank_runtime::TaskWorker::new(
|
||||
task_store,
|
||||
task_handler,
|
||||
crank_runtime::TaskWorkerConfig {
|
||||
owner: format!("mcp-server-{}", std::process::id()),
|
||||
..crank_runtime::TaskWorkerConfig::default()
|
||||
},
|
||||
)?);
|
||||
let (task_shutdown_tx, task_shutdown_rx) = tokio::sync::watch::channel(false);
|
||||
let task_worker_handle = tokio::spawn({
|
||||
let worker = task_worker.clone();
|
||||
async move { worker.run_until_shutdown(task_shutdown_rx).await }
|
||||
});
|
||||
let app = build_app_with_artifacts_and_tasks(
|
||||
registry,
|
||||
Duration::from_millis(config.refresh_ms),
|
||||
@@ -252,15 +277,77 @@ async fn run(
|
||||
info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening");
|
||||
*startup_completed = true;
|
||||
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = axum::serve(listener, app) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
let serve_result: Result<(), Box<dyn std::error::Error>> = async {
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone()) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
}
|
||||
} else {
|
||||
serve_with_worker_shutdown(app, listener, shutdown_signal(), task_shutdown_tx.clone())
|
||||
.await?;
|
||||
}
|
||||
} else {
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
// Stop new claims; active attempts checkpoint/drain or are cancelled inside the bounded worker
|
||||
// shutdown timeout (<=30s), and queued PostgreSQL work is left for recovery.
|
||||
task_shutdown_tx.send(true).ok();
|
||||
if tokio::time::timeout(Duration::from_secs(30), task_worker_handle)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!(
|
||||
name: "mcp.task_worker.shutdown_timeout",
|
||||
"durable Task worker did not stop within the story shutdown budget"
|
||||
);
|
||||
}
|
||||
serve_result
|
||||
}
|
||||
|
||||
/// Gracefully stop the durable Task worker as soon as shutdown is requested, while still
|
||||
/// draining in-flight HTTP connections before returning.
|
||||
async fn serve_with_worker_shutdown<S>(
|
||||
app: axum::Router,
|
||||
listener: tokio::net::TcpListener,
|
||||
signal: S,
|
||||
worker_stop: tokio::sync::watch::Sender<bool>,
|
||||
) -> std::io::Result<()>
|
||||
where
|
||||
S: std::future::Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let worker_stop_graceful = worker_stop.clone();
|
||||
let graceful = async move {
|
||||
signal.await;
|
||||
let _ = worker_stop_graceful.send(true);
|
||||
};
|
||||
let result = axum::serve(listener, app)
|
||||
.with_graceful_shutdown(graceful)
|
||||
.await;
|
||||
let _ = worker_stop.send(true);
|
||||
result
|
||||
}
|
||||
|
||||
/// Process shutdown trigger for the HTTP surface and the durable Task worker.
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
if let Ok(mut signal) =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
{
|
||||
signal.recv().await;
|
||||
}
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let terminate = std::future::pending::<()>();
|
||||
tokio::select! {
|
||||
_ = ctrl_c => {}
|
||||
_ = terminate => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn init_observability(
|
||||
@@ -506,3 +593,52 @@ fn runtime_cache_config(
|
||||
.map(|value| value.expose_secret().to_owned()),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{Router, routing::get};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{oneshot, watch};
|
||||
|
||||
async fn slow_handler() -> &'static str {
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
"ok"
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_stops_before_http_drain_completes() {
|
||||
let app = Router::new().route("/slow", get(slow_handler));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (stop_tx, mut stop_rx) = watch::channel(false);
|
||||
let (signal_tx, signal_rx) = oneshot::channel::<()>();
|
||||
|
||||
let handle = tokio::spawn(serve_with_worker_shutdown(
|
||||
app,
|
||||
listener,
|
||||
async {
|
||||
let _ = signal_rx.await;
|
||||
},
|
||||
stop_tx,
|
||||
));
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let request = client.get(format!("http://{}/slow", addr));
|
||||
let in_flight = tokio::spawn(async move { request.send().await });
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
let _ = signal_tx.send(());
|
||||
|
||||
tokio::time::timeout(Duration::from_millis(250), stop_rx.changed())
|
||||
.await
|
||||
.expect("worker stop signal should arrive before HTTP drain")
|
||||
.expect("watch changed");
|
||||
assert!(*stop_rx.borrow());
|
||||
|
||||
let response = in_flight.await.unwrap().unwrap();
|
||||
assert!(response.status().is_success());
|
||||
let serve_result = handle.await.unwrap();
|
||||
assert!(serve_result.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,555 @@
|
||||
//! Story 4.4 ATDD: no outbound send exists without a committed durable attempt, and an elapsed
|
||||
//! accepted deadline fails before dispatch.
|
||||
//!
|
||||
//! Both oracles drive the production composition (application entry → PG store → runtime executor →
|
||||
//! real local HTTP upstream); only the upstream and the checkpoint failure are test-controlled.
|
||||
|
||||
use super::*;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{
|
||||
AgentOperationBinding, PlatformApiKeyScope, TaskState, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
async fn composition(
|
||||
registry: &PostgresRegistry,
|
||||
upstream: &str,
|
||||
policy: ToolExecutionPolicy,
|
||||
caller_name: &str,
|
||||
scoped_key: &str,
|
||||
) -> (
|
||||
Arc<PostgresTaskStore>,
|
||||
crank_community_mcp::TaskApplication,
|
||||
SecretCrypto,
|
||||
crank_community_mcp::TaskPrincipal,
|
||||
) {
|
||||
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(registry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(
|
||||
registry,
|
||||
caller_name,
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let _ = scoped_key;
|
||||
(store, tasks, crypto, caller)
|
||||
}
|
||||
|
||||
fn worker_for(
|
||||
registry: &PostgresRegistry,
|
||||
store: &Arc<PostgresTaskStore>,
|
||||
tasks: &TaskApplication,
|
||||
crypto: &SecretCrypto,
|
||||
owner: &str,
|
||||
) -> TaskWorker {
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto.clone(),
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: owner.to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
TaskWorker::new(store.clone(), handler, config).expect("valid worker")
|
||||
}
|
||||
|
||||
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// AC1: a dispatch checkpoint that cannot commit produces zero outbound calls and no attempt row.
|
||||
#[tokio::test]
|
||||
async fn checkpoint_commit_failure_never_reaches_upstream() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let (store, tasks, crypto, caller) = composition(
|
||||
®istry,
|
||||
&upstream,
|
||||
background_policy(),
|
||||
"attempt-guards-checkpoint",
|
||||
"guards-checkpoint-key",
|
||||
)
|
||||
.await;
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"guards-checkpoint-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
// The checkpoint is the only statement that commits an attempt; it is refused here.
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"create function refuse_task_checkpoint() returns trigger language plpgsql as $$
|
||||
begin
|
||||
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
||||
raise exception 'durable checkpoint refused for test';
|
||||
end if;
|
||||
return new;
|
||||
end $$;
|
||||
create trigger refuse_task_checkpoint before update on tasks
|
||||
for each row execute function refuse_task_checkpoint();",
|
||||
))
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.expect("checkpoint refusal trigger");
|
||||
|
||||
let worker = worker_for(
|
||||
®istry,
|
||||
&store,
|
||||
&tasks,
|
||||
&crypto,
|
||||
"worker-guards-checkpoint",
|
||||
);
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"an uncommitted durable checkpoint must never produce an outbound call"
|
||||
);
|
||||
let attempts: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
attempts, 0,
|
||||
"no attempt history without a committed checkpoint"
|
||||
);
|
||||
assert_ne!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("completed"),
|
||||
"the refused attempt cannot complete the Task"
|
||||
);
|
||||
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"drop trigger refuse_task_checkpoint on tasks; drop function refuse_task_checkpoint();",
|
||||
))
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.expect("checkpoint refusal trigger removed");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
/// AC6: an accepted absolute deadline that already elapsed fails the Task before any dispatch.
|
||||
#[tokio::test]
|
||||
async fn elapsed_deadline_fails_before_dispatch_without_upstream_call() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let short_deadline = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 300,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
};
|
||||
let (store, tasks, crypto, caller) = composition(
|
||||
®istry,
|
||||
&upstream,
|
||||
short_deadline,
|
||||
"attempt-guards-deadline",
|
||||
"guards-deadline-key",
|
||||
)
|
||||
.await;
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"guards-deadline-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
// The accepted deadline elapses before any worker claims the Task.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
let worker = worker_for(®istry, &store, &tasks, &crypto, "worker-guards-deadline");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
state = state_of(®istry, acceptance.task_id).await;
|
||||
if state.as_deref() == Some("failed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state.as_deref(), Some("failed"));
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"an elapsed deadline must fail before dispatch"
|
||||
);
|
||||
let attempts: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts, 0, "no attempt is checkpointed after the deadline");
|
||||
let code: Option<String> =
|
||||
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(code.as_deref(), Some("task_deadline_elapsed"));
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, String>("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap(),
|
||||
TaskState::Failed.as_str()
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
/// AC2: a write whose upstream idempotency is backed by server-owned evidence is retried with the
|
||||
/// same stable key, in a bounded number of distinct durable attempts, and applies exactly one effect.
|
||||
#[tokio::test]
|
||||
async fn verified_idempotent_write_retries_with_stable_key_and_one_effect() {
|
||||
use crank_community_mcp::VerifiedIdempotentWrite;
|
||||
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let effects = Arc::new(AtomicUsize::new(0));
|
||||
let keys = Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
let seen = Arc::new(Mutex::new(HashSet::<String>::new()));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let requests = Arc::clone(&requests);
|
||||
let effects = Arc::clone(&effects);
|
||||
let keys = Arc::clone(&keys);
|
||||
let seen = Arc::clone(&seen);
|
||||
move |headers: axum::http::HeaderMap| {
|
||||
let requests = Arc::clone(&requests);
|
||||
let effects = Arc::clone(&effects);
|
||||
let keys = Arc::clone(&keys);
|
||||
let seen = Arc::clone(&seen);
|
||||
async move {
|
||||
let key = headers
|
||||
.get("idempotency-key")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
keys.lock().expect("keys").push(key.clone());
|
||||
let first_application = seen.lock().expect("seen").insert(key);
|
||||
if first_application {
|
||||
effects.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
||||
if attempt == 0 {
|
||||
// The first application is stored but its acknowledgement is lost.
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(json!({"error": "acknowledgement lost"})),
|
||||
);
|
||||
}
|
||||
(
|
||||
StatusCode::OK,
|
||||
axum::Json(json!({"id": "lead-1", "deduped": !first_application})),
|
||||
)
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
||||
mode: IdempotencyMode::Required,
|
||||
ttl_ms: 60_000,
|
||||
input_field: Some("idempotency_key".to_owned()),
|
||||
header_name: Some("Idempotency-Key".to_owned()),
|
||||
});
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"attempt-guards-verified",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"guards-verified-key",
|
||||
&json!({"email": "user@example.com", "idempotency_key": "k-1"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
||||
.build();
|
||||
let handler = Arc::new(
|
||||
TaskWorkerHandler::from_application(&tasks, registry.clone(), runtime, crypto.clone())
|
||||
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
||||
operation_id: operation.id.as_str().to_owned(),
|
||||
operation_version: 1,
|
||||
key_header: "Idempotency-Key".to_owned(),
|
||||
validity_ms: 60_000,
|
||||
}]),
|
||||
);
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "worker-guards-verified".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
state = state_of(®istry, acceptance.task_id).await;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
2,
|
||||
"exactly one retry of the write"
|
||||
);
|
||||
assert_eq!(
|
||||
effects.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the upstream applied the write exactly once"
|
||||
);
|
||||
assert_eq!(
|
||||
keys.lock().expect("keys").as_slice(),
|
||||
["k-1".to_owned(), "k-1".to_owned()],
|
||||
"the same stable key is delivered on both attempts"
|
||||
);
|
||||
|
||||
let attempts = store
|
||||
.attempts(
|
||||
&crank_core::TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: crank_core::TaskOwnerKind::Agent,
|
||||
owner_id: "agent_attempt_guards".to_owned(),
|
||||
authorization_scope: String::new(),
|
||||
},
|
||||
acceptance.task_id,
|
||||
)
|
||||
.await;
|
||||
// Owner scope of this principal is the agent of the published binding; read the history through
|
||||
// the application entry instead of guessing the authorization scope string.
|
||||
assert!(attempts.is_err(), "a foreign scope never reads the history");
|
||||
let attempts = tasks
|
||||
.attempts(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.expect("owner-scoped attempt history");
|
||||
assert_eq!(attempts.len(), 2, "two distinct durable attempts");
|
||||
assert_eq!(attempts[0].ordinal, 1);
|
||||
assert_eq!(attempts[1].ordinal, 2);
|
||||
assert!(attempts.iter().all(|attempt| attempt.retry_allowed));
|
||||
assert_eq!(
|
||||
attempts[0].outcome,
|
||||
Some(crank_core::AttemptOutcomeClass::RetryScheduled)
|
||||
);
|
||||
assert_eq!(
|
||||
attempts[1].outcome,
|
||||
Some(crank_core::AttemptOutcomeClass::Completed)
|
||||
);
|
||||
assert_eq!(
|
||||
attempts[0].upstream_key_digest, attempts[1].upstream_key_digest,
|
||||
"the same stable key digest is recorded on both attempts"
|
||||
);
|
||||
assert!(attempts[0].upstream_key_digest.is_some());
|
||||
let record = tasks
|
||||
.status(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.expect("status");
|
||||
assert_eq!(record.pins.retry.max_attempts, 2);
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
//! Story 4.3 ATDD: durable Attempt ID reaches the adapter attempt span (PF-1).
|
||||
//!
|
||||
//! End-to-end: a real durable Task worker dispatches a retrying GET through a real local
|
||||
//! upstream, and the `adapter_attempt` spans carry the same durable Attempt ID that is
|
||||
//! persisted in `tasks.current_attempt_id` on both attempts.
|
||||
|
||||
use super::super::common;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::get;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{
|
||||
AgentOperationBinding, CorrelationContext, RetryPolicy, Target, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_runtime::{
|
||||
OutboundHttpPolicy, SecretCrypto, TaskWorker, TaskWorkerConfig, community_with_outbound_policy,
|
||||
};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{Id, Subscriber, field::Visit};
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
const TOOL_NAME: &str = "attempt_trace";
|
||||
const KEY: &str = "attempt-trace-key-1";
|
||||
|
||||
async fn publish(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
operation_id: &crank_core::OperationId,
|
||||
version: u32,
|
||||
) {
|
||||
registry
|
||||
.publish_operation(crank_registry::PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id,
|
||||
version,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn application_with_store(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
) -> (
|
||||
crank_community_mcp::TaskApplication,
|
||||
std::sync::Arc<crank_registry::PostgresTaskStore>,
|
||||
SecretCrypto,
|
||||
) {
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = std::sync::Arc::new(
|
||||
crank_registry::PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
crank_registry::AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = crank_community_mcp::TaskApplication::new(
|
||||
registry.clone(),
|
||||
std::time::Duration::ZERO,
|
||||
std::sync::Arc::new(crank_runtime::InMemoryCoordinationStateStore::default()),
|
||||
crank_runtime::TaskService::with_secret_crypto(
|
||||
store.clone(),
|
||||
std::sync::Arc::new(crypto.clone()),
|
||||
),
|
||||
);
|
||||
(tasks, store, crypto)
|
||||
}
|
||||
|
||||
async fn principal(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
name: &str,
|
||||
scopes: &[crank_core::PlatformApiKeyScope],
|
||||
) -> crank_community_mcp::TaskPrincipal {
|
||||
let token = common::create_platform_api_key(registry, AGENT_SLUG, name, scopes).await;
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("an active platform key resolves");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crank_community_mcp::TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: crank_core::VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn adapter_attempt_span_carries_durable_attempt_id() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
get({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
let n = calls.fetch_add(1, Ordering::SeqCst);
|
||||
if n == 0 {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(serde_json::json!({"error":"boom"})),
|
||||
);
|
||||
}
|
||||
(
|
||||
StatusCode::OK,
|
||||
axum::Json(serde_json::json!({"id":"lead-1"})),
|
||||
)
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
let Target::Rest(target) = &mut operation.target;
|
||||
target.method = crank_core::HttpMethod::Get;
|
||||
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: crank_core::ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
|
||||
});
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"attempt-trace",
|
||||
&[
|
||||
crank_core::PlatformApiKeyScope::Read,
|
||||
crank_core::PlatformApiKeyScope::Write,
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let (tasks, store, crypto) = application_with_store(®istry).await;
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
KEY,
|
||||
&serde_json::json!({"email": "user@example.com"}),
|
||||
&CorrelationContext::generate(),
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance");
|
||||
|
||||
let runtime =
|
||||
community_with_outbound_policy(OutboundHttpPolicy::allowing_hosts(["127.0.0.1"])).build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-attempt-trace".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: std::time::Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: std::time::Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
|
||||
let capture = TraceCapture::default();
|
||||
let subscriber = tracing_subscriber::registry().with(capture.clone());
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
server.abort();
|
||||
|
||||
let durable_attempt_id: Option<String> = sqlx::query_scalar::<_, Option<String>>(
|
||||
"select current_attempt_id from tasks where task_id = $1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
|
||||
// Story 4.4: every outbound send is its own durable attempt, so the two retries of one lease carry
|
||||
// two distinct attempt identities, both recorded in the immutable attempt history.
|
||||
let attempt_rows: Vec<(String, i32)> = sqlx::query_as(
|
||||
"select attempt_id, ordinal from task_attempts where task_id = $1 order by ordinal",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_all(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempt_rows.len(), 2, "two durable attempts were committed");
|
||||
assert_eq!(attempt_rows[0].1, 1);
|
||||
assert_eq!(attempt_rows[1].1, 2);
|
||||
assert_eq!(
|
||||
Some(attempt_rows[1].0.as_str()),
|
||||
durable_attempt_id.as_deref(),
|
||||
"the Task points at its latest durable attempt"
|
||||
);
|
||||
|
||||
let spans = capture.snapshot();
|
||||
let attempt_spans: Vec<_> = spans
|
||||
.iter()
|
||||
.filter(|s| s.name == "adapter_attempt")
|
||||
.collect();
|
||||
assert_eq!(attempt_spans.len(), 2, "two adapter_attempt spans");
|
||||
for span in &attempt_spans {
|
||||
let aid = span.fields.get("attempt_id").cloned().unwrap_or_default();
|
||||
assert!(!aid.is_empty(), "attempt_id must not be empty");
|
||||
assert!(
|
||||
!aid.contains("traceparent"),
|
||||
"attempt_id must not contain traceparent"
|
||||
);
|
||||
assert!(
|
||||
attempt_rows.iter().any(|(id, _)| id == &aid),
|
||||
"every attempt span names an attempt that was durably committed"
|
||||
);
|
||||
}
|
||||
let ids: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert_ne!(ids[0], ids[1], "each retry has its own durable attempt id");
|
||||
assert_eq!(
|
||||
spans
|
||||
.iter()
|
||||
.filter(|span| span.name == "durable_task_attempt")
|
||||
.count(),
|
||||
1,
|
||||
"one durable attempt span per dispatch"
|
||||
);
|
||||
let runtime_span = spans
|
||||
.iter()
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime.execute span");
|
||||
assert_eq!(
|
||||
runtime_span.parent_name,
|
||||
Some("durable_task_attempt"),
|
||||
"the execution continues the durable attempt trace"
|
||||
);
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(
|
||||
span.parent_name,
|
||||
Some("runtime.execute"),
|
||||
"each retry is a child attempt span under the durable execution"
|
||||
);
|
||||
}
|
||||
|
||||
let attempts: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|s| s.fields.get("attempt").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert!(attempts.contains(&"1".to_string()));
|
||||
assert!(attempts.contains(&"2".to_string()));
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(
|
||||
span.fields.get("max_attempts").cloned().unwrap_or_default(),
|
||||
"2"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// TraceCapture minimal port from crates/crank-runtime/tests/stages.rs
|
||||
#[derive(Clone, Default)]
|
||||
struct TraceCapture {
|
||||
spans: Arc<Mutex<Vec<CapturedSpan>>>,
|
||||
}
|
||||
|
||||
impl TraceCapture {
|
||||
fn snapshot(&self) -> Vec<CapturedSpan> {
|
||||
self.spans.lock().expect("span lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturedSpan {
|
||||
name: &'static str,
|
||||
parent_name: Option<&'static str>,
|
||||
fields: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for TraceCapture
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
{
|
||||
fn on_new_span(
|
||||
&self,
|
||||
attributes: &tracing::span::Attributes<'_>,
|
||||
id: &Id,
|
||||
context: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let parent = attributes
|
||||
.parent()
|
||||
.and_then(|parent| context.span(parent))
|
||||
.or_else(|| {
|
||||
attributes
|
||||
.is_contextual()
|
||||
.then(|| context.lookup_current())
|
||||
.flatten()
|
||||
});
|
||||
let mut visitor = FieldVisitor::default();
|
||||
attributes.record(&mut visitor);
|
||||
let mut spans = self.spans.lock().expect("span lock");
|
||||
let index = spans.len();
|
||||
spans.push(CapturedSpan {
|
||||
name: attributes.metadata().name(),
|
||||
parent_name: parent.map(|span| span.metadata().name()),
|
||||
fields: visitor.fields,
|
||||
});
|
||||
context
|
||||
.span(id)
|
||||
.expect("span exists")
|
||||
.extensions_mut()
|
||||
.insert(index);
|
||||
}
|
||||
|
||||
fn on_record(
|
||||
&self,
|
||||
id: &Id,
|
||||
values: &tracing::span::Record<'_>,
|
||||
context: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
let mut visitor = FieldVisitor::default();
|
||||
values.record(&mut visitor);
|
||||
let span = context.span(id).expect("span exists");
|
||||
let index = *span.extensions().get::<usize>().expect("capture index");
|
||||
self.spans.lock().expect("span lock")[index]
|
||||
.fields
|
||||
.extend(visitor.fields);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FieldVisitor {
|
||||
fields: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl Visit for FieldVisitor {
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), value.to_owned());
|
||||
}
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), format!("{value:?}"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,846 @@
|
||||
//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream.
|
||||
|
||||
use super::common;
|
||||
use super::unsampled_correlation;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
|
||||
use crank_core::{
|
||||
AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState,
|
||||
TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
const TOOL_NAME: &str = "report_gen";
|
||||
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
||||
|
||||
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) {
|
||||
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal {
|
||||
let token = common::create_platform_api_key(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
name,
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("active key");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crank_community_mcp::TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store, Arc::new(crypto)),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let signal = Arc::new(std::sync::Mutex::new(Some(tx)));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
let signal = Arc::clone(&signal);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
let signal = Arc::clone(&signal);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(tx) = signal.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &upstream, policy).await;
|
||||
let caller = principal(®istry, "cancel-s1").await;
|
||||
let foreign = principal(®istry, "cancel-s1-foreign").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s1-key",
|
||||
&json!({"email":"u"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&app,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto.clone(),
|
||||
));
|
||||
let worker = TaskWorker::new(
|
||||
store,
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-s1".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
renewal_interval: Duration::from_millis(100),
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let _ = tokio::time::timeout(Duration::from_secs(5), rx).await;
|
||||
|
||||
let cancel = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
match cancel {
|
||||
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
|
||||
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
||||
}
|
||||
other => panic!("expected OutcomeUnknown got {other:?}"),
|
||||
}
|
||||
|
||||
let foreign_cancel = app
|
||||
.cancel(&foreign, TOOL_NAME, acceptance.task_id, None)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
foreign_cancel,
|
||||
Err(crank_community_mcp::TaskApplicationError::NotFound)
|
||||
));
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(1600)).await;
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("outcome_unknown")
|
||||
);
|
||||
let certainty: Option<String> = sqlx::query_scalar(
|
||||
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(certainty.as_deref(), Some("unknown"));
|
||||
let code: Option<String> =
|
||||
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
|
||||
let attempts_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts_count, 1);
|
||||
|
||||
let cancel2 = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel2");
|
||||
assert!(matches!(
|
||||
cancel2,
|
||||
crank_core::TaskCancelOutcome::OutcomeUnknown(_)
|
||||
));
|
||||
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &upstream, policy).await;
|
||||
let caller = principal(®istry, "cancel-s3").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s3-key",
|
||||
&json!({"email":"u"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let cancel = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
match cancel {
|
||||
crank_core::TaskCancelOutcome::Cancelled(record) => {
|
||||
assert_eq!(record.state, TaskState::Cancelled)
|
||||
}
|
||||
other => panic!("expected Cancelled got {other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
let attempts = app
|
||||
.attempts(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
.expect("attempts");
|
||||
assert!(attempts.is_empty());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() {
|
||||
use crank_community_mcp::VerifiedIdempotentWrite;
|
||||
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let requests = Arc::clone(&requests);
|
||||
move |headers: axum::http::HeaderMap| {
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
let _ = headers.get("idempotency-key");
|
||||
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
||||
if attempt == 0 {
|
||||
// Applied upstream, acknowledgement lost after the caller had time to observe that
|
||||
// the request really left the worker.
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(json!({"error":"acknowledgement lost"})),
|
||||
);
|
||||
}
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
});
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
||||
mode: IdempotencyMode::Required,
|
||||
ttl_ms: 60_000,
|
||||
input_field: Some("idempotency_key".to_owned()),
|
||||
header_name: Some("Idempotency-Key".to_owned()),
|
||||
});
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(®istry, "cancel-s2").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s2-key",
|
||||
&json!({"email":"u","idempotency_key":"k-2"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
||||
.build();
|
||||
let handler = Arc::new(
|
||||
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
|
||||
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
||||
operation_id: operation.id.as_str().to_owned(),
|
||||
operation_version: 1,
|
||||
key_header: "Idempotency-Key".to_owned(),
|
||||
validity_ms: 60_000,
|
||||
}]),
|
||||
);
|
||||
// Reproduce deterministically the durable state the executor commits after an ambiguous attempt
|
||||
// whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and
|
||||
// `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a
|
||||
// running executor against the cancel would not be a repeatable oracle.
|
||||
let lease = store
|
||||
.claim_batch("worker-s2-prep", 1, 10_000)
|
||||
.await
|
||||
.expect("claim")
|
||||
.pop()
|
||||
.expect("one claimed lease");
|
||||
let attempt = store
|
||||
.checkpoint_dispatch(crank_core::BeginAttempt {
|
||||
task_id: acceptance.task_id,
|
||||
owner: lease.owner.clone(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: true,
|
||||
})
|
||||
.await
|
||||
.expect("durable checkpoint");
|
||||
let resolution = store
|
||||
.resolve_attempt(
|
||||
&attempt,
|
||||
crank_core::AttemptOutcome::Uncertain {
|
||||
error: crank_core::TaskEnvelope::try_redacted(
|
||||
&json!({"code": "upstream_ack_lost", "phase": "dispatched"}),
|
||||
)
|
||||
.expect("bounded attempt error"),
|
||||
repeatable: true,
|
||||
phase: crank_core::AttemptPhase::Dispatched,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("attempt resolution");
|
||||
assert!(
|
||||
matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }),
|
||||
"an ambiguous but safely repeatable attempt arms the pinned retry"
|
||||
);
|
||||
|
||||
let cancel = app
|
||||
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
match cancel {
|
||||
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
|
||||
assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown)
|
||||
}
|
||||
other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"),
|
||||
}
|
||||
|
||||
// A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not
|
||||
// claimable and no second terminal write is possible.
|
||||
let worker = TaskWorker::new(
|
||||
store.clone(),
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-s2".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
0,
|
||||
"no outbound send after the cancel"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, acceptance.task_id).await.as_deref(),
|
||||
Some("outcome_unknown")
|
||||
);
|
||||
let attempts_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts_count, 1, "no second durable attempt");
|
||||
let certainty: Option<String> = sqlx::query_scalar(
|
||||
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(certainty.as_deref(), Some("unknown"));
|
||||
let code: Option<String> =
|
||||
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() {
|
||||
use crank_community_mcp::VerifiedIdempotentWrite;
|
||||
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let requests = Arc::clone(&requests);
|
||||
move |headers: axum::http::HeaderMap| {
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
let _ = headers.get("idempotency-key");
|
||||
let attempt = requests.fetch_add(1, Ordering::SeqCst);
|
||||
if attempt == 0 {
|
||||
// Applied upstream, acknowledgement lost after the caller had time to observe that
|
||||
// the request really left the worker.
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
axum::Json(json!({"error":"acknowledgement lost"})),
|
||||
);
|
||||
}
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 30_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
});
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
operation.execution_config.idempotency = Some(IdempotencyPolicy {
|
||||
mode: IdempotencyMode::Required,
|
||||
ttl_ms: 60_000,
|
||||
input_field: Some("idempotency_key".to_owned()),
|
||||
header_name: Some("Idempotency-Key".to_owned()),
|
||||
});
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
let caller = principal(®istry, "cancel-s4").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"s4-key",
|
||||
&json!({"email":"u","idempotency_key":"k-4"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
|
||||
.build();
|
||||
let handler = Arc::new(
|
||||
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
|
||||
.with_verified_writes(vec![VerifiedIdempotentWrite {
|
||||
operation_id: operation.id.as_str().to_owned(),
|
||||
operation_version: 1,
|
||||
key_header: "Idempotency-Key".to_owned(),
|
||||
validity_ms: 60_000,
|
||||
}]),
|
||||
);
|
||||
let worker = TaskWorker::new(
|
||||
store.clone(),
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-s4".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
// Revoke the key the Task was accepted with while the first outbound write is provably in flight:
|
||||
// that attempt still resolves as an uncertain write, and the next attempt is refused by the
|
||||
// authority before its checkpoint.
|
||||
for _ in 0..200 {
|
||||
if requests.load(Ordering::SeqCst) >= 1 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the first attempt is dispatched before the refusal"
|
||||
);
|
||||
registry
|
||||
.revoke_platform_api_key_for_agent(
|
||||
&common::test_workspace_id(),
|
||||
&common::test_agent_id(AGENT_SLUG),
|
||||
&PlatformApiKeyId::new("pk_cancel-s4".to_owned()),
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.expect("revoke");
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
state = state_of(®istry, acceptance.task_id).await;
|
||||
if state.as_deref() == Some("outcome_unknown") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(
|
||||
state.as_deref(),
|
||||
Some("outcome_unknown"),
|
||||
"a refused attempt after a lost acknowledgement must stay uncertain"
|
||||
);
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the refused attempt is never dispatched"
|
||||
);
|
||||
let attempts_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts_count, 1, "no second durable attempt");
|
||||
let certainty: Option<String> = sqlx::query_scalar(
|
||||
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(certainty.as_deref(), Some("unknown"));
|
||||
let phase: Option<String> = sqlx::query_scalar(
|
||||
"select phase from task_attempts where task_id = $1 order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(phase.as_deref(), Some("dispatched"));
|
||||
let error: Option<(Option<String>, Option<String>)> = sqlx::query_as(
|
||||
"select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1",
|
||||
)
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let (code, error_phase) = error.expect("the refused Task carries its bounded error");
|
||||
assert_eq!(code.as_deref(), Some("task_authority_rejected"));
|
||||
assert_eq!(
|
||||
error_phase.as_deref(),
|
||||
Some("dispatched"),
|
||||
"the error keeps the phase of the attempt that may have reached the upstream"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,965 @@
|
||||
//! ATDD Story 4.5 (Beam): durable input continuation through TaskApplication -> PG -> worker -> upstream.
|
||||
|
||||
use super::common;
|
||||
use super::unsampled_correlation;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
|
||||
use crank_core::{
|
||||
NoTasksFallback, PlatformApiKeyScope, TaskId, TaskState, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
|
||||
};
|
||||
use serde_json::json;
|
||||
use sqlx::Row;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const AGENT_SLUG: &str = "reports";
|
||||
pub(crate) const TOOL_NAME: &str = "report_gen";
|
||||
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
||||
|
||||
pub(crate) async fn state_of(registry: &PostgresRegistry, task_id: TaskId) -> Option<String> {
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_state(
|
||||
registry: &PostgresRegistry,
|
||||
task_id: TaskId,
|
||||
desired: &str,
|
||||
limit: Duration,
|
||||
) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + limit;
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
if state_of(registry, task_id).await.as_deref() == Some(desired) {
|
||||
return true;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
state_of(registry, task_id).await.as_deref() == Some(desired)
|
||||
}
|
||||
|
||||
pub(crate) async fn seed_tool(
|
||||
registry: &PostgresRegistry,
|
||||
upstream: &str,
|
||||
policy: ToolExecutionPolicy,
|
||||
) {
|
||||
let mut operation = common::test_operation(upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(policy);
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &common::test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse(
|
||||
"2026-03-26T10:00:00Z",
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.unwrap(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
common::publish_agent_with_bindings(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
vec![crank_core::AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub(crate) async fn principal(
|
||||
registry: &PostgresRegistry,
|
||||
name: &str,
|
||||
) -> crank_community_mcp::TaskPrincipal {
|
||||
let token = common::create_platform_api_key(
|
||||
registry,
|
||||
AGENT_SLUG,
|
||||
name,
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let secret_hash = common::hash_access_secret(&token);
|
||||
let api_key = registry
|
||||
.get_platform_api_key_by_secret_for_agent_slug(
|
||||
common::test_workspace_slug(),
|
||||
AGENT_SLUG,
|
||||
&secret_hash,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("active key");
|
||||
registry
|
||||
.touch_platform_api_key(
|
||||
&api_key.api_key.workspace_id,
|
||||
&api_key.api_key.id,
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
crank_community_mcp::TaskPrincipal::Agent {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
credential: crank_core::VerifiedMachineCredential {
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(api_key.api_key.id.clone()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("master key");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store, Arc::new(crypto)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_worker(
|
||||
registry: &PostgresRegistry,
|
||||
app: &TaskApplication,
|
||||
) -> (
|
||||
Arc<std::sync::atomic::AtomicBool>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
) {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
|
||||
let pool = registry.pool().clone();
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
app,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let worker = TaskWorker::new(
|
||||
store,
|
||||
handler,
|
||||
TaskWorkerConfig {
|
||||
owner: "worker-input".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
renewal_interval: Duration::from_millis(100),
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
},
|
||||
)
|
||||
.expect("worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let jh = tokio::spawn(async move {
|
||||
let _ = worker.run().await;
|
||||
});
|
||||
(stop_flag, jh)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn input_continuation_happy_path_accept_update_resume_replay() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &upstream, policy).await;
|
||||
let caller = principal(®istry, "input-happy").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"input-key-1",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("acceptance");
|
||||
assert_eq!(acceptance.state, TaskState::Queued);
|
||||
let task_id = acceptance.task_id;
|
||||
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(
|
||||
®istry,
|
||||
task_id,
|
||||
"input_required",
|
||||
Duration::from_secs(10)
|
||||
)
|
||||
.await,
|
||||
"worker must suspend for missing email"
|
||||
);
|
||||
// positive control: before update no outbound request
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
// persisted request fields
|
||||
let row = sqlx::query(
|
||||
"select wait_id, expected_version, schema_snapshot, projected_schema, prompt_code, expires_at from task_input_requests where task_id = $1",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let wait_id: String = row.get("wait_id");
|
||||
let expected_version: i32 = row.get("expected_version");
|
||||
let schema_snapshot: serde_json::Value = row.get("schema_snapshot");
|
||||
let projected_schema: serde_json::Value = row.get("projected_schema");
|
||||
let prompt_code: String = row.get("prompt_code");
|
||||
let _expires_at: OffsetDateTime = row.get("expires_at");
|
||||
assert!(!wait_id.is_empty());
|
||||
assert!(expected_version > 0);
|
||||
assert!(schema_snapshot.is_object());
|
||||
assert!(projected_schema.is_object());
|
||||
assert_eq!(prompt_code, "required_input_missing");
|
||||
let audit_count_before: i64 = sqlx::query_scalar(
|
||||
"select count(*)::bigint from task_input_audit where task_id = $1 and event = 'input_required'",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(audit_count_before >= 1);
|
||||
|
||||
// stop worker before update to avoid races
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
let outcome = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version as u32,
|
||||
"update-key-1",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("valid update");
|
||||
assert_eq!(outcome.state, TaskState::Working);
|
||||
assert!(!outcome.replayed);
|
||||
assert_eq!(outcome.task_id, task_id);
|
||||
assert_eq!(outcome.wait_id, wait_id);
|
||||
|
||||
let (stop2, worker_task2) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
||||
"worker must complete after update"
|
||||
);
|
||||
stop2.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let attempts = app
|
||||
.attempts(&caller, TOOL_NAME, task_id)
|
||||
.await
|
||||
.expect("attempts");
|
||||
assert_eq!(attempts.len(), 1);
|
||||
let effective_fp: String =
|
||||
sqlx::query_scalar("select effective_input_fingerprint from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts[0].input_fingerprint, effective_fp);
|
||||
let input_fp: String =
|
||||
sqlx::query_scalar("select input_fingerprint from tasks where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_ne!(effective_fp, input_fp);
|
||||
|
||||
// An identical answer replay still returns the committed outcome after the Task completed:
|
||||
// no second transition, audit row or outbound call.
|
||||
let completed_replay = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version as u32,
|
||||
"update-key-1",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("identical replay after completion");
|
||||
assert!(completed_replay.replayed);
|
||||
assert_eq!(completed_replay.version, outcome.version);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
// replay with original scoped key/partial input: same Task, no fingerprint conflict
|
||||
let replay = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"input-key-1",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("replay accepted");
|
||||
assert_eq!(replay.task_id, task_id);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
pub(crate) async fn start_and_suspend(
|
||||
registry: &PostgresRegistry,
|
||||
app: &TaskApplication,
|
||||
caller: &crank_community_mcp::TaskPrincipal,
|
||||
scoped_key: &str,
|
||||
) -> (TaskId, String, u32) {
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
scoped_key,
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("acceptance");
|
||||
let (stop, worker_task) = spawn_worker(registry, app);
|
||||
assert!(
|
||||
wait_for_state(
|
||||
registry,
|
||||
acceptance.task_id,
|
||||
"input_required",
|
||||
Duration::from_secs(10)
|
||||
)
|
||||
.await,
|
||||
"worker must suspend for missing email"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
let (wait_id, expected_version) = wait_row(registry, acceptance.task_id).await;
|
||||
(acceptance.task_id, wait_id, expected_version)
|
||||
}
|
||||
|
||||
async fn wait_row(registry: &PostgresRegistry, task_id: TaskId) -> (String, u32) {
|
||||
let row =
|
||||
sqlx::query("select wait_id, expected_version from task_input_requests where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
(
|
||||
row.get::<String, _>("wait_id"),
|
||||
u32::try_from(row.get::<i32, _>("expected_version")).unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn audit_events(
|
||||
registry: &PostgresRegistry,
|
||||
task_id: TaskId,
|
||||
) -> Vec<(String, Option<String>)> {
|
||||
sqlx::query("select event, reason_code from task_input_audit where task_id = $1 order by id")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_all(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|row| (row.get("event"), row.get("reason_code")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn admission_entries(
|
||||
registry: &PostgresRegistry,
|
||||
task_id: TaskId,
|
||||
) -> Vec<(String, i64)> {
|
||||
sqlx::query(
|
||||
"select state, count(*)::bigint as count from execution_admission_entries
|
||||
where task_id = $1 group by state order by state",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_all(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|row| (row.get("state"), row.get("count")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn key_id(principal: &crank_community_mcp::TaskPrincipal) -> crank_core::PlatformApiKeyId {
|
||||
match principal {
|
||||
crank_community_mcp::TaskPrincipal::Agent { credential, .. } => credential
|
||||
.platform_api_key_id
|
||||
.clone()
|
||||
.expect("agent principal carries a key id"),
|
||||
crank_community_mcp::TaskPrincipal::Admin { .. } => panic!("agent principal expected"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_schema_foreign_scope_and_revoked_key_never_transition_or_dispatch() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "input-scope").await;
|
||||
let app = application(®istry).await;
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-scope").await;
|
||||
|
||||
// Invalid answer: server-side schema validation refuses it, nothing transitions and nothing leaks.
|
||||
let invalid = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-invalid",
|
||||
&json!({}),
|
||||
)
|
||||
.await
|
||||
.expect_err("the pinned schema requires email");
|
||||
assert!(
|
||||
matches!(
|
||||
invalid,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::Invalid { .. })
|
||||
),
|
||||
"unexpected error: {invalid:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
assert!(
|
||||
audit_events(®istry, task_id)
|
||||
.await
|
||||
.iter()
|
||||
.all(|(event, _)| event == "input_required"),
|
||||
"a rejected answer must not be audited as accepted"
|
||||
);
|
||||
let stored_snapshot: serde_json::Value =
|
||||
sqlx::query_scalar("select schema_snapshot from task_input_requests where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!stored_snapshot.to_string().contains("user@example.com"));
|
||||
|
||||
// A different key of the same agent carries a different authorization scope: opaque not-found.
|
||||
let foreign = principal(®istry, "input-foreign").await;
|
||||
let foreign_result = app
|
||||
.update_input(
|
||||
&foreign,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-foreign",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a foreign scope cannot continue the Task");
|
||||
assert_eq!(
|
||||
foreign_result,
|
||||
crank_community_mcp::TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
|
||||
// Positive control: the accepting principal (same key/scope) completes the transition.
|
||||
let accepted = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-valid",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("valid answer for the accepting scope");
|
||||
assert_eq!(accepted.state, TaskState::Working);
|
||||
assert!(!accepted.replayed);
|
||||
// Finish the positive-control Task through a real worker so the revocation case starts from a
|
||||
// known dispatch count instead of leaving resumable work behind.
|
||||
let (finish_stop, finish_worker) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
||||
"the positive-control Task completes"
|
||||
);
|
||||
finish_stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), finish_worker).await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
// Revocation race: the accepting key is revoked after suspension. The same waiting scope must
|
||||
// fail closed inside the update transaction and never revive or dispatch the Task.
|
||||
let revocable = principal(®istry, "input-revoked").await;
|
||||
let (revoked_task, revoked_wait, revoked_version) =
|
||||
start_and_suspend(®istry, &app, &revocable, "input-key-revoked").await;
|
||||
registry
|
||||
.revoke_platform_api_key(
|
||||
&common::test_workspace_id(),
|
||||
&key_id(&revocable),
|
||||
&OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let revoked_result = app
|
||||
.update_input(
|
||||
&revocable,
|
||||
TOOL_NAME,
|
||||
revoked_task,
|
||||
&revoked_wait,
|
||||
revoked_version,
|
||||
"update-revoked",
|
||||
&json!({"email": "revoked@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a revoked key cannot continue the Task");
|
||||
assert_eq!(
|
||||
revoked_result,
|
||||
crank_community_mcp::TaskApplicationError::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, revoked_task).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"the revoked answer must not add an outbound call"
|
||||
);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replay_conflict_and_stale_answers_have_one_typed_outcome() {
|
||||
let registry = common::test_registry().await;
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, "http://127.0.0.1:9", policy).await;
|
||||
let caller = principal(®istry, "input-replay").await;
|
||||
let app = application(®istry).await;
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-replay").await;
|
||||
|
||||
let first = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("first valid answer");
|
||||
assert!(!first.replayed);
|
||||
assert_eq!(first.state, TaskState::Working);
|
||||
|
||||
// Identical replay returns the stored outcome without a second CAS/audit.
|
||||
let replay = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("identical replay");
|
||||
assert!(replay.replayed);
|
||||
assert_eq!(replay.version, first.version);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("working")
|
||||
);
|
||||
let accepted_audit = audit_events(®istry, task_id)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|(event, _)| event == "input_accepted")
|
||||
.count();
|
||||
assert_eq!(accepted_audit, 1);
|
||||
|
||||
// Same key with another payload is a fingerprint conflict.
|
||||
let conflict = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-key",
|
||||
&json!({"email": "other@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("same key with another payload conflicts");
|
||||
assert_eq!(
|
||||
conflict,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::FingerprintConflict)
|
||||
);
|
||||
|
||||
// Another key after a committed answer, and a foreign wait id, are state conflicts.
|
||||
let other_key = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
first.version,
|
||||
"update-key-2",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("another key after commit conflicts");
|
||||
assert_eq!(
|
||||
other_key,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
||||
);
|
||||
let foreign_wait = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
"wait-does-not-exist",
|
||||
first.version,
|
||||
"update-key-3",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a foreign wait id conflicts");
|
||||
assert_eq!(
|
||||
foreign_wait,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
||||
);
|
||||
|
||||
// A stale generation on a fresh wait is refused; the valid generation still works.
|
||||
let (stale_task, stale_wait, stale_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-stale").await;
|
||||
let stale = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
stale_task,
|
||||
&stale_wait,
|
||||
stale_version.saturating_sub(1).max(1),
|
||||
"update-stale",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("a stale generation cannot be continued");
|
||||
assert_eq!(
|
||||
stale,
|
||||
crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict)
|
||||
);
|
||||
let repaired = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
stale_task,
|
||||
&stale_wait,
|
||||
stale_version,
|
||||
"update-stale",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect("the current generation still accepts the answer");
|
||||
assert!(!repaired.replayed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ttl_expiry_fails_with_audit_and_releases_capacity_without_dispatch() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
// The wait TTL is the remaining pinned absolute execution deadline.
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 1_200,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "input-ttl").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"input-key-ttl",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("acceptance");
|
||||
let task_id = acceptance.task_id;
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "input_required", Duration::from_secs(5)).await,
|
||||
"worker must suspend"
|
||||
);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "failed", Duration::from_secs(6)).await,
|
||||
"the per-poll sweeper must fail the expired wait"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
let (wait_id, expected_version) = wait_row(®istry, task_id).await;
|
||||
let events = audit_events(®istry, task_id).await;
|
||||
assert!(
|
||||
events.iter().any(|(event, reason)| event == "input_expired"
|
||||
&& reason.as_deref() == Some("input_ttl_expired")),
|
||||
"expiry reason must be audited: {events:?}"
|
||||
);
|
||||
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
// A late answer cannot revive the failed Task.
|
||||
let late = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-late",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("an expired wait never revives");
|
||||
assert!(
|
||||
matches!(
|
||||
late,
|
||||
crank_community_mcp::TaskApplicationError::Task(
|
||||
crank_core::TaskError::InputExpired | crank_core::TaskError::StateConflict
|
||||
)
|
||||
),
|
||||
"unexpected late answer error: {late:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("failed")
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn competing_identical_updates_commit_once_and_keep_one_permit() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "input-race").await;
|
||||
let app = application(®istry).await;
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-race").await;
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)],
|
||||
"the waiting Task keeps exactly one executing permit"
|
||||
);
|
||||
|
||||
let payload = json!({"email": "user@example.com"});
|
||||
let first = app.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-race",
|
||||
&payload,
|
||||
);
|
||||
let second = app.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-race",
|
||||
&payload,
|
||||
);
|
||||
let (first, second) = tokio::join!(first, second);
|
||||
let mut outcomes = [
|
||||
first.expect("competing answer"),
|
||||
second.expect("competing answer"),
|
||||
];
|
||||
outcomes.sort_by_key(|outcome| outcome.replayed);
|
||||
assert!(!outcomes[0].replayed);
|
||||
assert!(outcomes[1].replayed);
|
||||
assert_eq!(outcomes[0].version, outcomes[1].version);
|
||||
assert_eq!(
|
||||
audit_events(®istry, task_id)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|(event, _)| event == "input_accepted")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)],
|
||||
"the update must not grant a second permit"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await,
|
||||
"the resumed Task completes"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
||||
server.abort();
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
//! ATDD Story 4.5 QA1 guards (F1/F3): one pinned deadline for several waiting Tasks, cancellation
|
||||
//! while the Task waits for input and refusal of a stale lease/fence around the continuation seam.
|
||||
|
||||
use super::common;
|
||||
use super::input_continuation::{
|
||||
TOOL_NAME, admission_entries, application, audit_events, principal, seed_tool, spawn_worker,
|
||||
start_and_suspend, state_of, wait_for_state,
|
||||
};
|
||||
use super::unsampled_correlation;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_core::{
|
||||
BeginAttempt, NoTasksFallback, TASK_INPUT_SCHEMA_FORMAT_VERSION, TaskInputRequest,
|
||||
TaskInputStore, TaskState, TaskStore, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresTaskStore};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_waits_with_one_pinned_deadline_and_cancelled_leftover_do_not_block_suspension() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "deadline-cancel").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acc_a = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"deadline-key-a",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept a");
|
||||
let acc_b = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"deadline-key-b",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept b");
|
||||
let task_a = acc_a.task_id;
|
||||
let task_b = acc_b.task_id;
|
||||
|
||||
let pinned_ms = (OffsetDateTime::now_utc().unix_timestamp() as f64 * 1000.0) + 60_000.0;
|
||||
for tid in [task_a, task_b] {
|
||||
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
|
||||
.bind(pinned_ms)
|
||||
.bind(tid.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let (stop, worker_task) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_a, "input_required", Duration::from_secs(10)).await,
|
||||
"a must suspend"
|
||||
);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_b, "input_required", Duration::from_secs(10)).await,
|
||||
"b must suspend"
|
||||
);
|
||||
stop.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
// Both active waits carry the same absolute TTL value: the per-Task uniqueness above is the only
|
||||
// uniqueness, so this must not be a UNIQUE index on expires_at.
|
||||
let same_deadline_wait: i64 = sqlx::query_scalar(
|
||||
"select count(*)::bigint from task_input_requests a
|
||||
join task_input_requests b on b.task_id = $2
|
||||
where a.task_id = $1 and a.accepted_at is null and b.accepted_at is null
|
||||
and a.expires_at = b.expires_at",
|
||||
)
|
||||
.bind(task_a.as_uuid())
|
||||
.bind(task_b.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
same_deadline_wait, 1,
|
||||
"both waits share the pinned deadline"
|
||||
);
|
||||
|
||||
let outcome = app
|
||||
.cancel(&caller, TOOL_NAME, task_a, None)
|
||||
.await
|
||||
.expect("cancel a");
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
crank_core::TaskCancelOutcome::Cancelled(_)
|
||||
));
|
||||
assert_eq!(
|
||||
state_of(®istry, task_a).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
assert_eq!(admission_entries(®istry, task_a).await, Vec::new());
|
||||
|
||||
// accept C with same pinned deadline
|
||||
let acc_c = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"deadline-key-c",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept c");
|
||||
let task_c = acc_c.task_id;
|
||||
sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2")
|
||||
.bind(pinned_ms)
|
||||
.bind(task_c.as_uuid())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (stop2, worker_task2) = spawn_worker(®istry, &app);
|
||||
assert!(
|
||||
wait_for_state(®istry, task_c, "input_required", Duration::from_secs(10)).await,
|
||||
"c must suspend despite leftover a"
|
||||
);
|
||||
stop2.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await;
|
||||
|
||||
assert_eq!(
|
||||
state_of(®istry, task_b).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
let c_deadline_ms: i64 = sqlx::query_scalar(
|
||||
"select (extract(epoch from expires_at) * 1000)::bigint
|
||||
from task_input_requests where task_id = $1",
|
||||
)
|
||||
.bind(task_c.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
c_deadline_ms, pinned_ms as i64,
|
||||
"the third wait reuses the same pinned deadline value"
|
||||
);
|
||||
let a_accepted: Option<OffsetDateTime> =
|
||||
sqlx::query_scalar("select accepted_at from task_input_requests where task_id = $1")
|
||||
.bind(task_a.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
a_accepted.is_none(),
|
||||
"cancellation leaves the wait row without an accepted receipt"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_while_input_required_never_revives_or_dispatches() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "cancel-wait").await;
|
||||
let app = application(®istry).await;
|
||||
|
||||
let (task_id, wait_id, expected_version) =
|
||||
start_and_suspend(®istry, &app, &caller, "input-key-cancel").await;
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)]
|
||||
);
|
||||
|
||||
let outcome = app
|
||||
.cancel(&caller, TOOL_NAME, task_id, None)
|
||||
.await
|
||||
.expect("cancel");
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
crank_core::TaskCancelOutcome::Cancelled(_)
|
||||
));
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
assert_eq!(admission_entries(®istry, task_id).await, Vec::new());
|
||||
|
||||
let late = app
|
||||
.update_input(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
task_id,
|
||||
&wait_id,
|
||||
expected_version,
|
||||
"update-late",
|
||||
&json!({"email": "user@example.com"}),
|
||||
)
|
||||
.await
|
||||
.expect_err("late answer must not revive");
|
||||
assert!(
|
||||
matches!(
|
||||
late,
|
||||
crank_community_mcp::TaskApplicationError::Task(
|
||||
crank_core::TaskError::StateConflict | crank_core::TaskError::InputExpired
|
||||
)
|
||||
),
|
||||
"unexpected: {late:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("cancelled")
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let events = audit_events(®istry, task_id).await;
|
||||
assert!(events.iter().any(|(e, _)| e == "input_required"));
|
||||
assert!(!events.iter().any(|(e, _)| e == "input_accepted"));
|
||||
let attempts = app
|
||||
.attempts(&caller, TOOL_NAME, task_id)
|
||||
.await
|
||||
.expect("attempts");
|
||||
assert!(attempts.is_empty());
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_lease_fence_cannot_suspend_or_dispatch() {
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
move || {
|
||||
let calls = Arc::clone(&calls);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let policy = ToolExecutionPolicy {
|
||||
mode: ToolExecutionMode::Background,
|
||||
deadline_ms: 60_000,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
};
|
||||
seed_tool(®istry, &format!("http://{address}"), policy).await;
|
||||
let caller = principal(®istry, "stale-fence").await;
|
||||
let app = application(®istry).await;
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"stale-key",
|
||||
&json!({"report": "q3"}),
|
||||
&unsampled_correlation(),
|
||||
)
|
||||
.await
|
||||
.expect("accept");
|
||||
let task_id = acceptance.task_id;
|
||||
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("store"),
|
||||
);
|
||||
let mut old_claims = store
|
||||
.claim_batch("worker-old", 1, 50)
|
||||
.await
|
||||
.expect("old claim");
|
||||
assert!(!old_claims.is_empty());
|
||||
let old = old_claims.remove(0);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(120)).await;
|
||||
let mut new_claims = store
|
||||
.claim_batch("worker-new", 1, 10_000)
|
||||
.await
|
||||
.expect("new claim");
|
||||
assert!(!new_claims.is_empty());
|
||||
let new = new_claims.remove(0);
|
||||
assert!(new.fence > old.fence);
|
||||
assert_eq!(new.record.task_id, task_id);
|
||||
|
||||
let request = TaskInputRequest {
|
||||
wait_id: "stale-wait".to_string(),
|
||||
expected_version: old.record.version + 1,
|
||||
schema_revision: 1,
|
||||
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot: json!({}),
|
||||
projected_schema: json!({}),
|
||||
prompt_code: "required_input_missing".to_string(),
|
||||
missing_fields: vec!["$.email".to_string()],
|
||||
expires_at_unix_ms: OffsetDateTime::now_utc().unix_timestamp() * 1000 + 30_000,
|
||||
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
};
|
||||
|
||||
let suspended =
|
||||
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &old, request).await;
|
||||
assert!(
|
||||
suspended.is_err(),
|
||||
"stale fence must not suspend: {suspended:?}"
|
||||
);
|
||||
|
||||
let attempt_id = "attempt-stale".to_string();
|
||||
let checkpoint = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id,
|
||||
owner: old.owner.clone(),
|
||||
fence: old.fence,
|
||||
expected_version: old.record.version,
|
||||
attempt_id,
|
||||
input_fingerprint: old.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
checkpoint.is_err(),
|
||||
"stale fence must not checkpoint: {checkpoint:?}"
|
||||
);
|
||||
|
||||
assert_ne!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
let attempt_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempt_count, 0);
|
||||
let req_count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from task_input_requests where task_id = $1")
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(req_count, 0);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
// Positive control: the same request under the current lease suspends the Task exactly once, so
|
||||
// the refusal above is the stale fence and not an invalid request shape.
|
||||
let request = TaskInputRequest {
|
||||
wait_id: "current-wait".to_string(),
|
||||
expected_version: new.record.version + 1,
|
||||
schema_revision: 1,
|
||||
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot: json!({}),
|
||||
projected_schema: json!({}),
|
||||
prompt_code: "required_input_missing".to_string(),
|
||||
missing_fields: vec!["$.email".to_string()],
|
||||
expires_at_unix_ms: new.record.pins.execution_deadline_unix_ms,
|
||||
};
|
||||
let suspended =
|
||||
<PostgresTaskStore as TaskInputStore>::suspend_for_input(&*store, &new, request)
|
||||
.await
|
||||
.expect("the current lease suspends");
|
||||
assert_eq!(suspended.state, TaskState::InputRequired);
|
||||
assert_eq!(
|
||||
state_of(®istry, task_id).await.as_deref(),
|
||||
Some("input_required")
|
||||
);
|
||||
assert_eq!(
|
||||
admission_entries(®istry, task_id).await,
|
||||
vec![("executing".to_owned(), 1)]
|
||||
);
|
||||
// The stale fence still cannot checkpoint a dispatch of this generation.
|
||||
let checkpoint = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id,
|
||||
owner: old.owner.clone(),
|
||||
fence: old.fence,
|
||||
expected_version: old.record.version,
|
||||
attempt_id: "attempt-stale-after-suspend".to_string(),
|
||||
input_fingerprint: old.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
checkpoint.is_err(),
|
||||
"stale fence must not checkpoint: {checkpoint:?}"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
+82
-14
@@ -14,13 +14,14 @@ use std::time::Duration;
|
||||
|
||||
use crank_community_mcp::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
use crank_core::{
|
||||
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, TaskState,
|
||||
ToolExecutionMode, ToolExecutionPolicy,
|
||||
AgentOperationBinding, CorrelationContext, NoTasksFallback, PlatformApiKeyScope, RequestId,
|
||||
TaskAcceptRequest, TaskEnvelope, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode, ToolExecutionPolicy,
|
||||
};
|
||||
use crank_registry::{
|
||||
AdmissionLimits, CreateVersionRequest, PostgresRegistry, PostgresTaskStore, PublishRequest,
|
||||
};
|
||||
use crank_runtime::{InMemoryCoordinationStateStore, TaskService};
|
||||
use crank_runtime::{InMemoryCoordinationStateStore, SecretCrypto, TaskService};
|
||||
use serde_json::json;
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
@@ -149,7 +150,9 @@ async fn publish_tool_version(registry: &PostgresRegistry, version: u32) {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn task_service(pool: PgPool) -> TaskService {
|
||||
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
|
||||
|
||||
fn task_service(pool: PgPool, crypto: SecretCrypto) -> TaskService {
|
||||
let store = PostgresTaskStore::new(
|
||||
pool,
|
||||
AdmissionLimits {
|
||||
@@ -161,15 +164,27 @@ fn task_service(pool: PgPool) -> TaskService {
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration");
|
||||
TaskService::new(std::sync::Arc::new(store))
|
||||
TaskService::with_secret_crypto(std::sync::Arc::new(store), std::sync::Arc::new(crypto))
|
||||
}
|
||||
|
||||
fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
/// The application entry under the same encrypted-input authority the production process uses.
|
||||
async fn application(registry: &PostgresRegistry) -> TaskApplication {
|
||||
let crypto = SecretCrypto::new(TEST_MASTER_KEY).expect("test master key");
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_millis(0),
|
||||
std::sync::Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
task_service(registry.pool().clone()),
|
||||
task_service(registry.pool().clone(), crypto),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -243,7 +258,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
|
||||
let acceptance = app
|
||||
.accept_background(
|
||||
@@ -273,7 +288,7 @@ async fn a_verified_caller_receives_a_durable_task_that_a_new_instance_reads() {
|
||||
assert_eq!(counts(registry.pool()).await, (1, 1));
|
||||
|
||||
// A freshly constructed application + service reads the same durable identity and pins.
|
||||
let reader = application(®istry);
|
||||
let reader = application(®istry).await;
|
||||
let record = reader
|
||||
.status(&caller, TOOL_NAME, acceptance.task_id)
|
||||
.await
|
||||
@@ -317,7 +332,7 @@ async fn a_new_publication_never_rebinds_an_accepted_task() {
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&caller,
|
||||
@@ -397,7 +412,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
let accepted = app
|
||||
.accept_background(
|
||||
&owner,
|
||||
@@ -433,7 +448,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
));
|
||||
|
||||
// A synchronous publication is not accepted as background work at all.
|
||||
let synchronous_app = application(®istry);
|
||||
let synchronous_app = application(®istry).await;
|
||||
let refused = synchronous_app
|
||||
.accept_background(
|
||||
&owner,
|
||||
@@ -453,7 +468,7 @@ async fn foreign_and_insufficient_callers_receive_no_task_metadata() {
|
||||
async fn admin_owned_tasks_are_scoped_to_that_admin_and_workspace() {
|
||||
let registry = common::test_registry().await;
|
||||
seed_tool(®istry, background_policy()).await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
let admin = TaskPrincipal::Admin {
|
||||
workspace_slug: common::test_workspace_slug().to_owned(),
|
||||
agent_slug: AGENT_SLUG.to_owned(),
|
||||
@@ -549,7 +564,7 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = application(®istry);
|
||||
let app = application(®istry).await;
|
||||
assert!(app.health().await.is_ok());
|
||||
|
||||
// Fail closed: without the Task schema the entry accepts nothing and reports dependency failure.
|
||||
@@ -589,3 +604,56 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
|
||||
assert_eq!(body["checks"]["durable_tasks"], json!("not_ready"));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
/// Commits and resolves exactly one durable attempt of the current lease through the real store.
|
||||
///
|
||||
/// Story 4.4 moved the dispatch checkpoint and the attempt resolution into the runtime seam, so a
|
||||
/// worker-level test double persists exactly one attempt instead of inventing a second retry loop.
|
||||
pub(crate) async fn commit_attempt(
|
||||
context: &crank_runtime::TaskAttemptContext,
|
||||
outcome: crank_core::AttemptOutcome,
|
||||
) -> crank_runtime::TaskAttemptOutcome {
|
||||
use crank_core::{AttemptResolution, BeginAttempt, TaskState};
|
||||
|
||||
let lease = context.lease();
|
||||
let begin = BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: lease.owner.clone(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: lease.record.pins.retry.max_attempts > 1,
|
||||
};
|
||||
match context.store().checkpoint_dispatch(begin).await {
|
||||
Ok(attempt) => match context.store().resolve_attempt(&attempt, outcome).await {
|
||||
Ok(AttemptResolution::Finished(task)) if task.state == TaskState::Completed => {
|
||||
crank_runtime::TaskAttemptOutcome::Completed
|
||||
}
|
||||
Ok(AttemptResolution::Finished(_)) => crank_runtime::TaskAttemptOutcome::Failed {
|
||||
code: "test_failure".to_owned(),
|
||||
},
|
||||
Ok(AttemptResolution::Unknown(_)) => crank_runtime::TaskAttemptOutcome::OutcomeUnknown,
|
||||
Ok(AttemptResolution::RetryAt { .. }) | Err(_) => {
|
||||
crank_runtime::TaskAttemptOutcome::Stopped
|
||||
}
|
||||
},
|
||||
Err(_) => crank_runtime::TaskAttemptOutcome::Stopped,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded success outcome used by the worker-level test doubles.
|
||||
pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::AttemptOutcome {
|
||||
crank_core::AttemptOutcome::Completed(
|
||||
crank_core::TaskEnvelope::try_redacted(&value).expect("test result is bounded"),
|
||||
)
|
||||
}
|
||||
|
||||
mod attempt_guards;
|
||||
mod attempt_trace;
|
||||
mod cancel_chain;
|
||||
mod input_continuation;
|
||||
mod input_continuation_guards;
|
||||
mod shutdown_window;
|
||||
mod task_worker;
|
||||
@@ -0,0 +1,220 @@
|
||||
//! Story 4.3 QA caveat proof: the narrow post-checkpoint / pre-permit stop window.
|
||||
//!
|
||||
//! Split from `task_worker.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A stop that lands after the durable dispatch checkpoint but before the sealed dispatch permit
|
||||
/// exists finalizes the Task as failed(`shutdown_before_dispatch`) without any outbound call, and
|
||||
/// does not consume work that is still queued.
|
||||
///
|
||||
/// The window is made deterministic instead of raced: a `BEFORE UPDATE` trigger delays only the
|
||||
/// durable dispatch checkpoint, so the stop flag is set while that checkpoint statement is still in
|
||||
/// flight (before `TaskDispatchPermit`/`TaskHandler::execute` can exist).
|
||||
#[tokio::test]
|
||||
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
// The test registry exposes a single pooled connection: delaying the checkpoint inside its own
|
||||
// statement keeps the window deterministic without competing for a second connection.
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"create function delay_task_dispatch_checkpoint() returns trigger language plpgsql as $$
|
||||
begin
|
||||
if new.current_attempt_id is not null and old.current_attempt_id is null then
|
||||
perform pg_sleep(1.5);
|
||||
end if;
|
||||
return new;
|
||||
end $$;
|
||||
create trigger delay_task_dispatch_checkpoint before update on tasks
|
||||
for each row execute function delay_task_dispatch_checkpoint();",
|
||||
))
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.expect("checkpoint delay trigger");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let mut accepted = Vec::new();
|
||||
for (seed, scoped_key) in [("q", "a"), ("r", "c")] {
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_stop_window".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: scoped_key.repeat(64),
|
||||
input_fingerprint: scoped_key.repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({ "report": seed })).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
accepted.push(store.accept(request).await.expect("PG accepts"));
|
||||
}
|
||||
|
||||
// The first revalidation waits for the test; the second Task is never claimed (batch_limit = 1).
|
||||
let revalidate_entered = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let revalidate_release = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let executed = Arc::new(AtomicUsize::new(0));
|
||||
struct GatedHandler {
|
||||
entered: Arc<tokio::sync::Semaphore>,
|
||||
release: Arc<tokio::sync::Semaphore>,
|
||||
executed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for GatedHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
self.entered.add_permits(1);
|
||||
let _ = self.release.acquire().await;
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
// The trigger delays the durable checkpoint; a stop that lands while it is in flight must
|
||||
// finalize failed(shutdown_before_dispatch) without an outbound send.
|
||||
let begin = crank_core::BeginAttempt {
|
||||
task_id: context.lease().record.task_id,
|
||||
owner: context.lease().owner.clone(),
|
||||
fence: context.lease().fence,
|
||||
expected_version: context.lease().record.version,
|
||||
attempt_id: "attempt-stop-window".to_owned(),
|
||||
input_fingerprint: context.lease().record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
};
|
||||
let Ok(attempt) = context.store().checkpoint_dispatch(begin).await else {
|
||||
return TaskAttemptOutcome::Stopped;
|
||||
};
|
||||
if context.is_stopping() {
|
||||
let _ = context
|
||||
.store()
|
||||
.resolve_attempt(
|
||||
&attempt,
|
||||
crank_core::AttemptOutcome::Failed {
|
||||
error: crank_core::TaskEnvelope::try_redacted(&json!({
|
||||
"code": "shutdown_before_dispatch",
|
||||
"phase": "before_dispatch",
|
||||
}))
|
||||
.expect("bounded error envelope"),
|
||||
retryable: false,
|
||||
phase: crank_core::AttemptPhase::BeforeDispatch,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return TaskAttemptOutcome::Failed {
|
||||
code: "shutdown_before_dispatch".to_owned(),
|
||||
};
|
||||
}
|
||||
self.executed.fetch_add(1, Ordering::SeqCst);
|
||||
commit_attempt(&context, completed_outcome(json!({ "ok": true }))).await
|
||||
}
|
||||
}
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
|
||||
entered: revalidate_entered.clone(),
|
||||
release: revalidate_release.clone(),
|
||||
executed: executed.clone(),
|
||||
});
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-stop-window".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 30_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
renewal_interval: Duration::from_secs(20),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(20),
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), revalidate_entered.acquire())
|
||||
.await
|
||||
.expect("worker must claim the first Task and enter revalidation")
|
||||
.expect("revalidation gate is open")
|
||||
.forget();
|
||||
|
||||
// Let revalidation continue; the checkpoint now runs while the trigger holds it for 1.5s.
|
||||
revalidate_release.add_permits(1);
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
// The stop now lands while the durable checkpoint statement is still in flight: the worker may
|
||||
// finalize the Task, but no permit exists yet, so no outbound call can start.
|
||||
let stop_requested = std::time::Instant::now();
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the 30s budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
assert!(
|
||||
stop_requested.elapsed() >= Duration::from_millis(500),
|
||||
"the checkpoint must have still been in flight when the stop arrived"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
executed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"a stop between checkpoint and permit must never reach the upstream adapter"
|
||||
);
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_stop_window".to_owned(),
|
||||
};
|
||||
let record = store.read(&access, accepted[0].task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Failed);
|
||||
assert_eq!(record.attempts, 1, "the durable checkpoint did commit");
|
||||
assert!(record.current_attempt_id.is_some());
|
||||
assert_eq!(
|
||||
record
|
||||
.error
|
||||
.as_ref()
|
||||
.and_then(|error| error.value().get("code"))
|
||||
.and_then(|code| code.as_str()),
|
||||
Some("shutdown_before_dispatch")
|
||||
);
|
||||
|
||||
// Work that was still queued is not consumed by the stopped worker.
|
||||
let queued = store.read(&access, accepted[1].task_id).await.unwrap();
|
||||
assert_eq!(queued.state, TaskState::Queued);
|
||||
let recovered = store
|
||||
.claim_batch("worker_after_stop", 4, 10_000)
|
||||
.await
|
||||
.expect("queued work stays claimable");
|
||||
assert_eq!(recovered.len(), 1);
|
||||
assert_eq!(recovered[0].record.task_id, accepted[1].task_id);
|
||||
}
|
||||
@@ -0,0 +1,821 @@
|
||||
//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch.
|
||||
//!
|
||||
//! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a
|
||||
/// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan,
|
||||
/// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented.
|
||||
#[tokio::test]
|
||||
async fn task_worker_store_lease_recovery_contract() {
|
||||
let registry = common::test_registry().await;
|
||||
let store = PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration");
|
||||
|
||||
// Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and
|
||||
// leases durable work, preserving the same admission-backed intent.
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "a".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
// This store-level control exercises lease/fence behavior, not the encrypted acceptance path.
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
let leases = store
|
||||
.claim_batch("worker_1", 4, 10_000)
|
||||
.await
|
||||
.expect("claim");
|
||||
assert_eq!(leases.len(), 1);
|
||||
assert_eq!(leases[0].record.task_id, accepted.task_id);
|
||||
let lease = leases[0].clone();
|
||||
|
||||
// A stale or competing owner cannot win CAS with the issued fencing token.
|
||||
assert!(
|
||||
store
|
||||
.renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.write_progress(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence + 1,
|
||||
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
|
||||
assert!(
|
||||
store
|
||||
.checkpoint_dispatch(crank_core::BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker_1".to_owned(),
|
||||
fence: lease.fence + 1,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-stale".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
// The valid owner still commits the dispatch checkpoint before any outbound work.
|
||||
let committed = store
|
||||
.checkpoint_dispatch(crank_core::BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker_1".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-1".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(committed.attempt.attempt_id, "attempt-1");
|
||||
assert_eq!(
|
||||
committed.task.current_attempt_id.as_deref(),
|
||||
Some("attempt-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_worker_runtime_dispatch_and_recovery_contract() {
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let pool = registry.pool().clone();
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
pool.clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// Positive control: PostgreSQL alone accepts the durable Task.
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_runtime".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "r".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
let task_id = accepted.task_id;
|
||||
|
||||
struct RuntimeHandler {
|
||||
observed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait]
|
||||
impl TaskHandler for RuntimeHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.observed.fetch_add(1, Ordering::SeqCst);
|
||||
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
|
||||
observed: observed.clone(),
|
||||
});
|
||||
|
||||
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-1".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(100),
|
||||
renewal_interval: Duration::from_millis(1_000),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_runtime".to_owned(),
|
||||
};
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
if let Ok(record) = store.read(&access, task_id).await {
|
||||
state = Some(record.state);
|
||||
if record.state == TaskState::Completed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state, Some(TaskState::Completed));
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
1,
|
||||
"exactly one outbound attempt"
|
||||
);
|
||||
// Terminal state is not duplicated.
|
||||
let record = store.read(&access, task_id).await.expect("terminal read");
|
||||
assert_eq!(record.state, TaskState::Completed);
|
||||
|
||||
// Bounded shutdown with no fresh work returns promptly.
|
||||
let idle_config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-idle".to_owned(),
|
||||
poll_interval: Duration::from_millis(50),
|
||||
shutdown_timeout: Duration::from_secs(2),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let idle_worker = TaskWorker::new(
|
||||
store.clone(),
|
||||
handler_for_no_work(store.clone()),
|
||||
idle_config,
|
||||
)
|
||||
.expect("valid idle worker");
|
||||
idle_worker.stop();
|
||||
let started = std::time::Instant::now();
|
||||
idle_worker.run().await.expect("idle worker stops");
|
||||
assert!(started.elapsed() <= Duration::from_secs(30));
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call.
|
||||
#[tokio::test]
|
||||
async fn task_worker_revalidation_denial_never_dispatches() {
|
||||
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_denied".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "d".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
|
||||
struct DeniedHandler {
|
||||
observed: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for DeniedHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Err(TaskWorkerError::Store(TaskError::invalid(
|
||||
"publication or credential revoked",
|
||||
)))
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.observed.fetch_add(1, Ordering::SeqCst);
|
||||
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
|
||||
observed: observed.clone(),
|
||||
});
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-denied".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_denied".to_owned(),
|
||||
};
|
||||
let mut state = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
if let Ok(record) = store.read(&access, accepted.task_id).await {
|
||||
state = Some(record.state);
|
||||
if record.state == TaskState::Failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
|
||||
assert_eq!(state, Some(TaskState::Failed));
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
0,
|
||||
"invalidated work never reaches the upstream"
|
||||
);
|
||||
}
|
||||
|
||||
fn handler_for_no_work(
|
||||
_store: std::sync::Arc<PostgresTaskStore>,
|
||||
) -> std::sync::Arc<dyn crank_runtime::TaskHandler> {
|
||||
struct NoWorkHandler;
|
||||
#[async_trait::async_trait]
|
||||
impl crank_runtime::TaskHandler for NoWorkHandler {
|
||||
async fn revalidate(
|
||||
&self,
|
||||
_lease: &crank_core::TaskLease,
|
||||
) -> Result<(), crank_runtime::TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
context: crank_runtime::TaskAttemptContext,
|
||||
) -> crank_runtime::TaskAttemptOutcome {
|
||||
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
std::sync::Arc::new(NoWorkHandler)
|
||||
}
|
||||
|
||||
/// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a
|
||||
/// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the
|
||||
/// accepted parent trace is continued on the wire.
|
||||
#[tokio::test]
|
||||
async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
|
||||
use axum::Router;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let seen_traceparent = Arc::new(Mutex::new(None::<String>));
|
||||
|
||||
let app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post({
|
||||
let calls = Arc::clone(&calls);
|
||||
let seen = Arc::clone(&seen_traceparent);
|
||||
move |headers: HeaderMap| {
|
||||
let calls = Arc::clone(&calls);
|
||||
let seen = Arc::clone(&seen);
|
||||
async move {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
if let Some(value) = headers
|
||||
.get("traceparent")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
{
|
||||
*seen.lock().unwrap() = Some(value.to_owned());
|
||||
}
|
||||
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-worker-e2e",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"worker-e2e-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance with executable input");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-e2e".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("completed") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("completed"));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call");
|
||||
assert_eq!(
|
||||
seen_traceparent.lock().unwrap().as_deref(),
|
||||
Some(correlation.trace_context().traceparent()),
|
||||
"the durable attempt continues the accepted parent trace"
|
||||
);
|
||||
}
|
||||
|
||||
/// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as
|
||||
/// `outcome_unknown` (not a retryable failure) and is not redispatched automatically.
|
||||
#[tokio::test]
|
||||
async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
|
||||
use axum::Router;
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::post;
|
||||
use crank_community_mcp::TaskWorkerHandler;
|
||||
use crank_core::{AgentOperationBinding, PlatformApiKeyScope};
|
||||
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let upstream_app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
|
||||
let upstream = format!("http://{address}");
|
||||
|
||||
let mut operation = common::test_operation(&upstream, TOOL_NAME);
|
||||
operation.execution_config.execution_policy = Some(background_policy());
|
||||
registry
|
||||
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish(®istry, &operation.id, 1).await;
|
||||
common::publish_agent_with_bindings(
|
||||
®istry,
|
||||
AGENT_SLUG,
|
||||
vec![AgentOperationBinding {
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: 1,
|
||||
tool_name: operation.name.clone(),
|
||||
tool_title: operation.tool_description.title.clone(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
let caller = principal(
|
||||
®istry,
|
||||
"task-worker-uncertain",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let correlation = unsampled_correlation();
|
||||
let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap();
|
||||
let observed_at = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
|
||||
epoch: crypto.master_key_epoch(),
|
||||
fingerprint: crypto.master_key_fingerprint(),
|
||||
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
|
||||
observed_at: &observed_at,
|
||||
})
|
||||
.await
|
||||
.expect("test master-key identity registered");
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let tasks = TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
|
||||
);
|
||||
let acceptance = tasks
|
||||
.accept_background(
|
||||
&caller,
|
||||
TOOL_NAME,
|
||||
None,
|
||||
"worker-uncertain-key",
|
||||
&json!({"email": "user@example.com"}),
|
||||
&correlation,
|
||||
)
|
||||
.await
|
||||
.expect("durable acceptance with executable input");
|
||||
|
||||
let runtime = crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build();
|
||||
let handler = Arc::new(TaskWorkerHandler::from_application(
|
||||
&tasks,
|
||||
registry.clone(),
|
||||
runtime,
|
||||
crypto,
|
||||
));
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-uncertain".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(5),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
let mut state = None;
|
||||
for _ in 0..200 {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let current: Option<String> =
|
||||
sqlx::query_scalar("select state from tasks where task_id = $1")
|
||||
.bind(acceptance.task_id.as_uuid())
|
||||
.fetch_optional(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
state = current;
|
||||
if state.as_deref() == Some("outcome_unknown") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
|
||||
server.abort();
|
||||
|
||||
assert_eq!(state.as_deref(), Some("outcome_unknown"));
|
||||
// No automatic retry: another authoritative scan never redispaches the uncertain attempt.
|
||||
assert!(
|
||||
store
|
||||
.claim_batch("worker_after", 4, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced
|
||||
/// terminal commit within the 30s budget.
|
||||
#[tokio::test]
|
||||
async fn task_worker_graceful_stop_drains_active_attempt() {
|
||||
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
|
||||
use crank_runtime::{
|
||||
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let registry = common::test_registry().await;
|
||||
let store = Arc::new(
|
||||
PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits {
|
||||
global_limit: 4,
|
||||
agent_limit: 4,
|
||||
tool_limit: 4,
|
||||
queue_limit: 8,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.expect("valid task store configuration"),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_drain".to_owned(),
|
||||
platform_api_key_id: None,
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 1,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
execution_deadline_unix_ms: 1_900_000_000_000,
|
||||
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: "s".repeat(64),
|
||||
input_fingerprint: "d".repeat(64),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
||||
support: TaskSupportIds::new(
|
||||
RequestId::generate(),
|
||||
&crank_core::TraceContext::parse(
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00",
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
};
|
||||
let accepted = store.accept(request).await.expect("PG accepts");
|
||||
|
||||
struct SlowHandler {
|
||||
started: Arc<AtomicUsize>,
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl TaskHandler for SlowHandler {
|
||||
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.started.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
|
||||
}
|
||||
}
|
||||
let started = Arc::new(AtomicUsize::new(0));
|
||||
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
|
||||
started: started.clone(),
|
||||
});
|
||||
let config = TaskWorkerConfig {
|
||||
owner: "runtime-worker-drain".to_owned(),
|
||||
batch_limit: 1,
|
||||
lease_ms: 10_000,
|
||||
poll_interval: Duration::from_millis(50),
|
||||
max_inflight: 1,
|
||||
shutdown_timeout: Duration::from_secs(20),
|
||||
..TaskWorkerConfig::default()
|
||||
};
|
||||
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
|
||||
let stop_flag = worker.stop_flag();
|
||||
let worker_task = tokio::spawn(async move { worker.run().await });
|
||||
|
||||
for _ in 0..100 {
|
||||
if started.load(Ordering::SeqCst) > 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started");
|
||||
let stop_requested = std::time::Instant::now();
|
||||
stop_flag.store(true, Ordering::SeqCst);
|
||||
tokio::time::timeout(Duration::from_secs(30), worker_task)
|
||||
.await
|
||||
.expect("worker stops inside the 30s budget")
|
||||
.expect("worker exits cleanly")
|
||||
.expect("worker reports no error");
|
||||
assert!(stop_requested.elapsed() <= Duration::from_secs(30));
|
||||
|
||||
let access = TaskAccess {
|
||||
workspace_id: common::test_workspace_id().to_string(),
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_drain".to_owned(),
|
||||
};
|
||||
let record = store.read(&access, accepted.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Completed);
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
FROM node:22-alpine AS build
|
||||
|
||||
# The CI host uplink drops TLS 1.3 ClientHello carrying the hybrid
|
||||
# X25519MLKEM768 key share; pin the npm stage to TLS 1.2 until that network
|
||||
# defect is fixed. Certificates, SRI and the lockfile are unaffected.
|
||||
ENV NODE_OPTIONS=--tls-max-v1.2
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY apps/ui/package.json apps/ui/package-lock.json ./
|
||||
|
||||
@@ -21,6 +21,7 @@ crank-schema = { path = "../crank-schema" }
|
||||
crank-trace = { path = "../crank-trace" }
|
||||
futures-util = "0.3"
|
||||
reqwest.workspace = true
|
||||
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
@@ -31,7 +32,11 @@ tokio = { workspace = true, features = ["sync"] }
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
# Typed Tools dispatch uses the SDK; its HTTP transport remains test-only.
|
||||
rmcp = { version = "=3.9.0" }
|
||||
|
||||
[dev-dependencies]
|
||||
rmcp = { version = "=3.9.0", features = ["transport-streamable-http-server"] }
|
||||
crank-mapping = { path = "../crank-mapping" }
|
||||
crank-test-support = { path = "../crank-test-support" }
|
||||
metrics-util = "0.20.4"
|
||||
|
||||
@@ -272,7 +272,10 @@ fn record_unknown_approval_access_denial(reason: &'static str) {
|
||||
);
|
||||
}
|
||||
|
||||
fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool {
|
||||
pub(super) fn allows_scope(
|
||||
scopes: &[PlatformApiKeyScope],
|
||||
required_scope: PlatformApiKeyScope,
|
||||
) -> bool {
|
||||
match required_scope {
|
||||
PlatformApiKeyScope::Read => scopes.iter().any(|scope| {
|
||||
matches!(
|
||||
|
||||
@@ -34,6 +34,19 @@ pub(crate) mod lifecycle;
|
||||
mod metrics;
|
||||
mod request;
|
||||
mod response;
|
||||
mod rmcp_dispatch;
|
||||
#[cfg(test)]
|
||||
mod rmcp_http;
|
||||
#[cfg(test)]
|
||||
#[path = "rmcp_http_tests.rs"]
|
||||
mod rmcp_http_tests;
|
||||
#[cfg(test)]
|
||||
mod rmcp_session_tests;
|
||||
#[cfg(test)]
|
||||
mod rmcp_tools;
|
||||
#[cfg(test)]
|
||||
#[path = "rmcp_tools_tests.rs"]
|
||||
mod rmcp_tools_tests;
|
||||
mod stages;
|
||||
mod tool_resolution;
|
||||
pub(crate) use self::approval_policy::{
|
||||
@@ -51,15 +64,15 @@ mod auth_resolution;
|
||||
mod dispatch;
|
||||
mod endpoint;
|
||||
mod legacy_dispatch;
|
||||
mod task_worker;
|
||||
mod tasks;
|
||||
|
||||
use self::approvals::{
|
||||
approve_request, deny_request, get_approval_request, list_pending_approvals,
|
||||
};
|
||||
pub(super) use self::auth_resolution::resolve_operation_auth;
|
||||
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
|
||||
pub use self::task_worker::{TaskWorkerHandler, VerifiedIdempotentWrite};
|
||||
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
|
||||
|
||||
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
|
||||
// through `super`, and the plain lib build does not use them (same pattern as
|
||||
// `#[cfg(test)] use invocation_history::observe_invocation_history_outcome;` above).
|
||||
|
||||
@@ -19,18 +19,31 @@ pub(crate) async fn resolve_operation_auth(
|
||||
workspace_id: &crank_core::WorkspaceId,
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<Option<ResolvedAuth>, RuntimeError> {
|
||||
if execution_config.auth_profile_ref.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let span = Stage::AuthResolve.span();
|
||||
let result = resolve_runtime_auth_for_task(
|
||||
resolve_operation_auth_with(
|
||||
&state.registry,
|
||||
&state.secret_crypto,
|
||||
workspace_id,
|
||||
execution_config,
|
||||
)
|
||||
.instrument(span.clone())
|
||||
.await;
|
||||
.await
|
||||
}
|
||||
|
||||
/// Same resolution for callers that hold the authoritative registry/secret-crypto pair without the
|
||||
/// HTTP application state (the durable Task worker).
|
||||
pub(crate) async fn resolve_operation_auth_with(
|
||||
registry: &PostgresRegistry,
|
||||
secret_crypto: &SecretCrypto,
|
||||
workspace_id: &crank_core::WorkspaceId,
|
||||
execution_config: &crank_core::ExecutionConfig,
|
||||
) -> Result<Option<ResolvedAuth>, RuntimeError> {
|
||||
if execution_config.auth_profile_ref.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let span = Stage::AuthResolve.span();
|
||||
let result =
|
||||
resolve_runtime_auth_for_task(registry, secret_crypto, workspace_id, execution_config)
|
||||
.instrument(span.clone())
|
||||
.await;
|
||||
match &result {
|
||||
Ok(_) => StageOutcome::Success.record(&span),
|
||||
Err(_) => {
|
||||
|
||||
@@ -7,6 +7,9 @@
|
||||
|
||||
use super::request::{InitializeParams, ToolCallParams};
|
||||
use super::response::take_confirmation_token;
|
||||
use super::rmcp_dispatch::{
|
||||
handle_resources_command, handle_tools_command, list_resource_templates,
|
||||
};
|
||||
use crate::{
|
||||
auth::VerifiedMachineCredential,
|
||||
jsonrpc::{
|
||||
@@ -18,8 +21,7 @@ use crate::{
|
||||
PromptGetCommand, PromptsCommand, PromptsListCommand, PromptsOutcome, RequestIdentity,
|
||||
ResourceReadCommand, ResourcesCommand, ResourcesListCommand, ResourcesOutcome,
|
||||
ServiceContext, ToolCallCommand, ToolsCommand, ToolsListCommand, ToolsOutcome,
|
||||
declared_mcp_method, discover_result, handle_completion_command, handle_prompts_command,
|
||||
handle_resources_command, handle_tools_command, validate_modern_request_name,
|
||||
declared_mcp_method, discover_result, validate_modern_request_name,
|
||||
},
|
||||
session::SessionState,
|
||||
tool_error::execution_locale,
|
||||
@@ -42,7 +44,7 @@ use super::response::{
|
||||
resource_templates_list_result, resources_list_result, tools_list_result,
|
||||
};
|
||||
use super::{AgentRoutePath, AppState, TRANSPORT_SESSION_TTL_MS};
|
||||
use crate::protocol::{ResourceTemplatesListError, list_resource_templates};
|
||||
use crate::protocol::ResourceTemplatesListError;
|
||||
const SERVER_NAME: &str = "crank-mcp-server";
|
||||
const SERVER_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
@@ -160,19 +162,18 @@ pub(super) async fn handle_modern_request(
|
||||
)
|
||||
.await;
|
||||
match outcome {
|
||||
ToolsOutcome::List { tools, next_cursor } => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
crate::protocol::project_modern_cacheable_result(
|
||||
version,
|
||||
tools_list_result(tools, next_cursor),
|
||||
ToolsOutcome::List { tools, next_cursor } => {
|
||||
match tools_list_result(tools, next_cursor, version) {
|
||||
Ok(result) => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(request_id(message), result),
|
||||
response_mode,
|
||||
None,
|
||||
Some(version),
|
||||
),
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(version),
|
||||
),
|
||||
Err(error) => internal_jsonrpc_error(message, error),
|
||||
}
|
||||
}
|
||||
ToolsOutcome::InvalidParams(message_text) => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_error(request_id(message), -32602, message_text),
|
||||
@@ -286,10 +287,10 @@ pub(super) async fn handle_modern_request(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
crate::protocol::project_modern_cacheable_result(
|
||||
version,
|
||||
resources_list_result(resources, next_cursor),
|
||||
),
|
||||
match resources_list_result(resources, next_cursor, version) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
@@ -345,10 +346,10 @@ pub(super) async fn handle_modern_request(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
crate::protocol::project_modern_cacheable_result(
|
||||
version,
|
||||
resource_read_result(contents),
|
||||
),
|
||||
match resource_read_result(contents, version) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
@@ -434,10 +435,10 @@ pub(super) async fn handle_modern_request(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
crate::protocol::project_modern_cacheable_result(
|
||||
version,
|
||||
resource_templates_list_result(templates, next_cursor),
|
||||
),
|
||||
match resource_templates_list_result(templates, next_cursor, version) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
@@ -466,7 +467,7 @@ pub(super) async fn handle_modern_request(
|
||||
);
|
||||
}
|
||||
};
|
||||
let outcome = handle_prompts_command(
|
||||
let outcome = super::rmcp_dispatch::handle_prompts_command(
|
||||
state,
|
||||
resource_service_context(path, credential, transport_correlation, message),
|
||||
version,
|
||||
@@ -483,7 +484,10 @@ pub(super) async fn handle_modern_request(
|
||||
request_id(message),
|
||||
crate::protocol::project_modern_cacheable_result(
|
||||
version,
|
||||
prompts_list_result(prompts, next_cursor),
|
||||
match prompts_list_result(prompts, next_cursor) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
),
|
||||
response_mode,
|
||||
@@ -525,7 +529,7 @@ pub(super) async fn handle_modern_request(
|
||||
if let Err(response) = validate_modern_request_name(message, headers, &command.name) {
|
||||
return *response;
|
||||
}
|
||||
let outcome = handle_prompts_command(
|
||||
let outcome = super::rmcp_dispatch::handle_prompts_command(
|
||||
state,
|
||||
resource_service_context(path, credential, transport_correlation, message),
|
||||
version,
|
||||
@@ -542,7 +546,10 @@ pub(super) async fn handle_modern_request(
|
||||
request_id(message),
|
||||
crate::protocol::project_modern_result(
|
||||
version,
|
||||
prompt_get_result(description, messages),
|
||||
match prompt_get_result(description, messages) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
),
|
||||
response_mode,
|
||||
@@ -616,7 +623,7 @@ pub(super) async fn handle_modern_request(
|
||||
);
|
||||
}
|
||||
};
|
||||
let outcome = handle_completion_command(
|
||||
let outcome = super::rmcp_dispatch::handle_completion_command(
|
||||
state,
|
||||
resource_service_context(path, credential, transport_correlation, message),
|
||||
version,
|
||||
|
||||
@@ -51,9 +51,87 @@ pub(crate) struct CounterSet {
|
||||
era_modern: AtomicUsize,
|
||||
era_legacy: AtomicUsize,
|
||||
tools_service: AtomicUsize,
|
||||
sdk_tools_dispatch: AtomicUsize,
|
||||
sdk_tools_list_result: AtomicUsize,
|
||||
sdk_tools_call_result: AtomicUsize,
|
||||
sdk_resources_dispatch: AtomicUsize,
|
||||
sdk_resources_list_result: AtomicUsize,
|
||||
sdk_resources_read_result: AtomicUsize,
|
||||
sdk_resource_templates_list_result: AtomicUsize,
|
||||
sdk_prompts_dispatch: AtomicUsize,
|
||||
sdk_prompts_list_result: AtomicUsize,
|
||||
sdk_prompts_get_result: AtomicUsize,
|
||||
sdk_completion_dispatch: AtomicUsize,
|
||||
sdk_completion_result: AtomicUsize,
|
||||
sdk_logging_dispatch: AtomicUsize,
|
||||
sdk_logging_level: AtomicUsize,
|
||||
sdk_logging_notification: AtomicUsize,
|
||||
}
|
||||
|
||||
impl CounterSet {
|
||||
pub(crate) fn sdk_logging_dispatch(&self) -> usize {
|
||||
self.sdk_logging_dispatch.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_logging_level(&self) -> usize {
|
||||
self.sdk_logging_level.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_logging_notification(&self) -> usize {
|
||||
self.sdk_logging_notification.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_completion_dispatch(&self) -> usize {
|
||||
self.sdk_completion_dispatch.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_completion_result(&self) -> usize {
|
||||
self.sdk_completion_result.load(Ordering::SeqCst)
|
||||
}
|
||||
pub(crate) fn sdk_prompts_dispatch(&self) -> usize {
|
||||
self.sdk_prompts_dispatch.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_prompts_list_result(&self) -> usize {
|
||||
self.sdk_prompts_list_result.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_prompts_get_result(&self) -> usize {
|
||||
self.sdk_prompts_get_result.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_resources_dispatch(&self) -> usize {
|
||||
self.sdk_resources_dispatch.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_resources_list_result(&self) -> usize {
|
||||
self.sdk_resources_list_result.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_resources_read_result(&self) -> usize {
|
||||
self.sdk_resources_read_result.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn sdk_resource_templates_list_result(&self) -> usize {
|
||||
self.sdk_resource_templates_list_result
|
||||
.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// SDK typed Tools dispatch entries, separate from the existing era/service snapshot.
|
||||
pub(crate) fn sdk_tools_dispatch(&self) -> usize {
|
||||
self.sdk_tools_dispatch.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Successfully serialized SDK ListToolsResult values, not dispatch entries.
|
||||
pub(crate) fn sdk_tools_list_result(&self) -> usize {
|
||||
self.sdk_tools_list_result.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Successfully serialized SDK CallToolResult values, not dispatch entries.
|
||||
pub(crate) fn sdk_tools_call_result(&self) -> usize {
|
||||
self.sdk_tools_call_result.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Read the counters of one observed request as one comparable snapshot.
|
||||
pub(crate) fn snapshot(&self) -> Snapshot {
|
||||
Snapshot {
|
||||
@@ -134,6 +212,96 @@ pub(crate) fn tools_service_enter() {
|
||||
increment(|counters| &counters.tools_service);
|
||||
}
|
||||
|
||||
/// `SDK_TOOLS_DISPATCH`: entered the real SDK typed Tools dispatch, not just admission.
|
||||
pub(crate) fn sdk_tools_dispatch_enter() {
|
||||
increment(|counters| &counters.sdk_tools_dispatch);
|
||||
}
|
||||
|
||||
/// Mark only after serializing a real SDK ListToolsResult successfully.
|
||||
pub(crate) fn sdk_tools_list_result_serialized() {
|
||||
increment(|counters| &counters.sdk_tools_list_result);
|
||||
}
|
||||
|
||||
/// Mark only after serializing a real SDK CallToolResult successfully.
|
||||
pub(crate) fn sdk_tools_call_result_serialized() {
|
||||
increment(|counters| &counters.sdk_tools_call_result);
|
||||
}
|
||||
|
||||
/// Mark only after converting a Resources request through the real SDK typed path.
|
||||
pub(crate) fn sdk_resources_dispatch_enter() {
|
||||
increment(|counters| &counters.sdk_resources_dispatch);
|
||||
}
|
||||
|
||||
/// Mark only after successfully serializing a real SDK ListResourcesResult.
|
||||
pub(crate) fn sdk_resources_list_result_serialized() {
|
||||
increment(|counters| &counters.sdk_resources_list_result);
|
||||
}
|
||||
|
||||
/// Mark only after successfully serializing a real SDK ReadResourceResult.
|
||||
pub(crate) fn sdk_resources_read_result_serialized() {
|
||||
increment(|counters| &counters.sdk_resources_read_result);
|
||||
}
|
||||
|
||||
/// Mark only after successfully serializing a real SDK ListResourceTemplatesResult.
|
||||
pub(crate) fn sdk_resource_templates_list_result_serialized() {
|
||||
increment(|counters| &counters.sdk_resource_templates_list_result);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../rmcp_resources_runtime_tests.rs"]
|
||||
mod rmcp_resources_runtime_tests;
|
||||
|
||||
/// Mark only after converting a Prompts request through the real SDK typed path.
|
||||
pub(crate) fn sdk_prompts_dispatch_enter() {
|
||||
increment(|counters| &counters.sdk_prompts_dispatch);
|
||||
}
|
||||
|
||||
/// Mark only after successfully serializing a real SDK ListPromptsResult.
|
||||
pub(crate) fn sdk_prompts_list_result_serialized() {
|
||||
increment(|counters| &counters.sdk_prompts_list_result);
|
||||
}
|
||||
|
||||
/// Mark only after successfully serializing a real SDK GetPromptResult.
|
||||
pub(crate) fn sdk_prompts_get_result_serialized() {
|
||||
increment(|counters| &counters.sdk_prompts_get_result);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../rmcp_prompts_runtime_tests.rs"]
|
||||
mod rmcp_prompts_runtime_tests;
|
||||
|
||||
/// Mark only after constructing real SDK CompleteRequestParams.
|
||||
pub(crate) fn sdk_completion_dispatch_enter() {
|
||||
increment(|counters| &counters.sdk_completion_dispatch);
|
||||
}
|
||||
|
||||
/// Mark only after successfully serializing real SDK CompleteResult/CompletionInfo.
|
||||
pub(crate) fn sdk_completion_result_serialized() {
|
||||
increment(|counters| &counters.sdk_completion_result);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../rmcp_completion_runtime_tests.rs"]
|
||||
mod rmcp_completion_runtime_tests;
|
||||
|
||||
/// Mark only after constructing the real SDK typed legacy logging request.
|
||||
pub(crate) fn sdk_logging_dispatch_enter() {
|
||||
increment(|counters| &counters.sdk_logging_dispatch);
|
||||
}
|
||||
|
||||
/// Mark only after successfully decoding a real SDK LoggingLevel at either input.
|
||||
pub(crate) fn sdk_logging_level_parsed() {
|
||||
increment(|counters| &counters.sdk_logging_level);
|
||||
}
|
||||
|
||||
/// Mark only after serializing real SDK LoggingMessageNotificationParam params, not an envelope.
|
||||
pub(crate) fn sdk_logging_notification_serialized() {
|
||||
increment(|counters| &counters.sdk_logging_notification);
|
||||
}
|
||||
|
||||
#[path = "../rmcp_logging_runtime_tests.rs"]
|
||||
mod rmcp_logging_runtime_tests;
|
||||
|
||||
fn increment(counter: impl Fn(&CounterSet) -> &AtomicUsize) {
|
||||
let _ = OBSERVER.try_with(|counters| {
|
||||
counter(counters).fetch_add(1, Ordering::SeqCst);
|
||||
@@ -144,7 +312,7 @@ fn increment(counter: impl Fn(&CounterSet) -> &AtomicUsize) {
|
||||
/// execution-boundary reason as the tests below).
|
||||
#[cfg(test)]
|
||||
#[path = "../dispatch_observer_support.rs"]
|
||||
mod support;
|
||||
pub(crate) mod support;
|
||||
|
||||
/// The behavioural section 7.2 harness: real app, real PostgreSQL, one counted loopback upstream and
|
||||
/// one request-scoped observer per request. Its file lives outside `src/app/` only because the
|
||||
@@ -153,3 +321,7 @@ mod support;
|
||||
#[cfg(test)]
|
||||
#[path = "../dispatch_observer_harness.rs"]
|
||||
mod tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../rmcp_runtime_tests.rs"]
|
||||
mod rmcp_runtime_tests;
|
||||
@@ -24,6 +24,9 @@ use super::response::{
|
||||
resource_templates_list_result, resources_list_result, take_confirmation_token,
|
||||
tools_list_result,
|
||||
};
|
||||
use super::rmcp_dispatch::{
|
||||
handle_resources_command, handle_tools_command, list_resource_templates,
|
||||
};
|
||||
use super::{AgentRoutePath, AppState};
|
||||
use crate::{
|
||||
auth::VerifiedMachineCredential,
|
||||
@@ -36,9 +39,7 @@ use crate::{
|
||||
EVENT_LEVEL_APPLIED, INVALID_LEVEL, Kind, LoggingOutcome, MethodHandler, PromptGetCommand,
|
||||
PromptsCommand, PromptsListCommand, PromptsOutcome, RequestIdentity, ResourceReadCommand,
|
||||
ResourcesCommand, ResourcesListCommand, ResourcesOutcome, ServiceContext, SetLevelCommand,
|
||||
ToolCallCommand, ToolsCommand, ToolsListCommand, ToolsOutcome, handle_completion_command,
|
||||
handle_logging_command, handle_prompts_command, handle_resources_command,
|
||||
handle_tools_command,
|
||||
ToolCallCommand, ToolsCommand, ToolsListCommand, ToolsOutcome, handle_logging_command,
|
||||
},
|
||||
tool_error::execution_locale,
|
||||
transport::{ResponseMode, session_id_from_headers, transport_response},
|
||||
@@ -143,13 +144,18 @@ pub(super) async fn handle_legacy_request(
|
||||
)
|
||||
.await;
|
||||
match outcome {
|
||||
ToolsOutcome::List { tools, next_cursor } => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(request_id(message), tools_list_result(tools, next_cursor)),
|
||||
response_mode,
|
||||
None,
|
||||
Some(&session.protocol_version),
|
||||
),
|
||||
ToolsOutcome::List { tools, next_cursor } => {
|
||||
match tools_list_result(tools, next_cursor, &session.protocol_version) {
|
||||
Ok(result) => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(request_id(message), result),
|
||||
response_mode,
|
||||
None,
|
||||
Some(&session.protocol_version),
|
||||
),
|
||||
Err(error) => internal_jsonrpc_error(message, error),
|
||||
}
|
||||
}
|
||||
ToolsOutcome::InvalidParams(message_text) => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_error(request_id(message), -32602, message_text),
|
||||
@@ -278,7 +284,14 @@ pub(super) async fn handle_legacy_request(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
resources_list_result(resources, next_cursor),
|
||||
match resources_list_result(
|
||||
resources,
|
||||
next_cursor,
|
||||
&session.protocol_version,
|
||||
) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
@@ -335,7 +348,13 @@ pub(super) async fn handle_legacy_request(
|
||||
match outcome {
|
||||
ResourcesOutcome::Read { contents } => transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(request_id(message), resource_read_result(contents)),
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
match resource_read_result(contents, &session.protocol_version) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(&session.protocol_version),
|
||||
@@ -394,7 +413,7 @@ pub(super) async fn handle_legacy_request(
|
||||
);
|
||||
}
|
||||
};
|
||||
let (templates, next_cursor) = match crate::protocol::list_resource_templates(
|
||||
let (templates, next_cursor) = match list_resource_templates(
|
||||
&state,
|
||||
&path.workspace_slug,
|
||||
&path.agent_slug,
|
||||
@@ -425,7 +444,14 @@ pub(super) async fn handle_legacy_request(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
resource_templates_list_result(templates, next_cursor),
|
||||
match resource_templates_list_result(
|
||||
templates,
|
||||
next_cursor,
|
||||
&session.protocol_version,
|
||||
) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
@@ -458,7 +484,7 @@ pub(super) async fn handle_legacy_request(
|
||||
);
|
||||
}
|
||||
};
|
||||
let outcome = handle_prompts_command(
|
||||
let outcome = super::rmcp_dispatch::handle_prompts_command(
|
||||
&state,
|
||||
ServiceContext {
|
||||
workspace_slug: &session.workspace_slug,
|
||||
@@ -480,7 +506,10 @@ pub(super) async fn handle_legacy_request(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
prompts_list_result(prompts, next_cursor),
|
||||
match prompts_list_result(prompts, next_cursor) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
@@ -520,7 +549,7 @@ pub(super) async fn handle_legacy_request(
|
||||
);
|
||||
}
|
||||
};
|
||||
let outcome = handle_prompts_command(
|
||||
let outcome = super::rmcp_dispatch::handle_prompts_command(
|
||||
&state,
|
||||
ServiceContext {
|
||||
workspace_slug: &session.workspace_slug,
|
||||
@@ -542,7 +571,10 @@ pub(super) async fn handle_legacy_request(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
prompt_get_result(description, messages),
|
||||
match prompt_get_result(description, messages) {
|
||||
Ok(result) => result,
|
||||
Err(error) => return internal_jsonrpc_error(message, error),
|
||||
},
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
@@ -615,7 +647,7 @@ pub(super) async fn handle_legacy_request(
|
||||
);
|
||||
}
|
||||
};
|
||||
let outcome = handle_completion_command(
|
||||
let outcome = super::rmcp_dispatch::handle_completion_command(
|
||||
&state,
|
||||
ServiceContext {
|
||||
workspace_slug: &session.workspace_slug,
|
||||
|
||||
@@ -37,16 +37,14 @@ use futures_util::stream;
|
||||
use serde_json::{Number, Value, json};
|
||||
use tokio::sync::{mpsc, watch};
|
||||
|
||||
use super::rmcp_dispatch::handle_tools_command;
|
||||
use super::{
|
||||
AppState,
|
||||
response::{ToolCommandPayload, tool_command_payload, tool_command_response},
|
||||
};
|
||||
use crate::{
|
||||
jsonrpc::request_id,
|
||||
protocol::{
|
||||
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsOutcome,
|
||||
handle_tools_command,
|
||||
},
|
||||
protocol::{ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsOutcome},
|
||||
transport::{ResponseMode, sse_response},
|
||||
};
|
||||
|
||||
@@ -133,7 +131,7 @@ pub(crate) struct ProgressEvent {
|
||||
|
||||
/// The pinned `notifications/progress` frame. Both supported revisions declare the same parameters,
|
||||
/// so the projection is shared; the token is echoed verbatim and never validated as a credential.
|
||||
fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
|
||||
pub(super) fn progress_frame(token: &ProgressToken, event: &ProgressEvent) -> Value {
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/progress",
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use axum::{http::StatusCode, response::Response};
|
||||
use rmcp::model::{CacheScope, CallToolResult, ContentBlock, ListToolsResult, ResultType, Tool};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{
|
||||
jsonrpc::{jsonrpc_result, request_id},
|
||||
protocol::project_modern_result,
|
||||
tool_error::{ToolErrorContract, tool_error_text, tool_error_value},
|
||||
transport::{ResponseMode, transport_response},
|
||||
};
|
||||
@@ -30,22 +30,7 @@ pub(super) fn success_tool_payload(
|
||||
protocol_version: &str,
|
||||
output: Value,
|
||||
) -> Value {
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
project_modern_result(
|
||||
protocol_version,
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
|
||||
}
|
||||
],
|
||||
"structuredContent": output,
|
||||
"isError": false
|
||||
}),
|
||||
),
|
||||
)
|
||||
call_result_payload(message, protocol_version, sdk_success_tool_result(output))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -69,85 +54,273 @@ pub(super) fn tool_error_payload(
|
||||
protocol_version: &str,
|
||||
error: ToolErrorContract,
|
||||
) -> Value {
|
||||
let error_message = tool_error_text(&error);
|
||||
let error_value = tool_error_value(&error);
|
||||
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
project_modern_result(
|
||||
protocol_version,
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": error_message
|
||||
}
|
||||
],
|
||||
"structuredContent": {
|
||||
"error": error_value
|
||||
},
|
||||
"isError": true
|
||||
}),
|
||||
),
|
||||
)
|
||||
call_result_payload(message, protocol_version, sdk_error_tool_result(error))
|
||||
}
|
||||
|
||||
/// The era-neutral `tools/list` result shape built from the shared command outcome. Object keys end
|
||||
/// up byte-lexicographically ordered, exactly as before the extraction.
|
||||
pub(super) fn tools_list_result(tools: Vec<Value>, next_cursor: Option<String>) -> Value {
|
||||
let mut result = json!({ "tools": tools });
|
||||
if let Some(next_cursor) = next_cursor {
|
||||
result["nextCursor"] = Value::String(next_cursor);
|
||||
}
|
||||
pub(super) fn sdk_success_tool_result(output: Value) -> CallToolResult {
|
||||
let text = serde_json::to_string_pretty(&output).expect("JSON value serializes");
|
||||
let mut result = CallToolResult::success(vec![ContentBlock::text(text)]);
|
||||
result.structured_content = Some(output);
|
||||
result
|
||||
}
|
||||
|
||||
/// The era-neutral `resources/list` result shape (Story 3.8). Same pagination wire keys as
|
||||
/// `tools/list`; `nextCursor` is absent on the last page.
|
||||
pub(super) fn resources_list_result(resources: Vec<Value>, next_cursor: Option<String>) -> Value {
|
||||
let mut result = json!({ "resources": resources });
|
||||
if let Some(next_cursor) = next_cursor {
|
||||
result["nextCursor"] = Value::String(next_cursor);
|
||||
}
|
||||
pub(super) fn sdk_error_tool_result(error: ToolErrorContract) -> CallToolResult {
|
||||
let mut result = CallToolResult::error(vec![ContentBlock::text(tool_error_text(&error))]);
|
||||
result.structured_content = Some(json!({ "error": tool_error_value(&error) }));
|
||||
result
|
||||
}
|
||||
|
||||
/// The era-neutral `resources/read` result shape: exactly the typed `contents[]` the revision
|
||||
/// converter produced.
|
||||
pub(super) fn resource_read_result(contents: Value) -> Value {
|
||||
json!({ "contents": contents })
|
||||
fn call_result_payload(message: &Value, version: &str, mut result: CallToolResult) -> Value {
|
||||
result.result_type =
|
||||
(version == crate::jsonrpc::MODERN_PROTOCOL_VERSION).then_some(ResultType::COMPLETE);
|
||||
let result = serde_json::to_value(result).expect("SDK call result contains JSON values");
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_tools_call_result_serialized();
|
||||
jsonrpc_result(request_id(message), result)
|
||||
}
|
||||
|
||||
pub(super) fn sdk_tools_list_result(
|
||||
tools: &[Value],
|
||||
next_cursor: Option<String>,
|
||||
) -> Result<ListToolsResult, serde_json::Error> {
|
||||
Ok(ListToolsResult {
|
||||
tools: tools
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(serde_json::from_value::<Tool>)
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
next_cursor,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Serialize the SDK model, preserving catalog extensions that pinned Tool serde cannot represent.
|
||||
pub(super) fn tools_list_result(
|
||||
tools: Vec<Value>,
|
||||
next_cursor: Option<String>,
|
||||
version: &str,
|
||||
) -> Result<Value, serde_json::Error> {
|
||||
let mut result = sdk_tools_list_result(&tools, next_cursor)?;
|
||||
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
|
||||
result.result_type = modern.then_some(ResultType::COMPLETE);
|
||||
result.ttl_ms = modern.then_some(0);
|
||||
result.cache_scope = modern.then_some(CacheScope::Private);
|
||||
let mut result = serde_json::to_value(result)?;
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_tools_list_result_serialized();
|
||||
for (raw, serialized) in tools
|
||||
.iter()
|
||||
.zip(result["tools"].as_array_mut().expect("SDK tools array"))
|
||||
{
|
||||
let serialized = serialized.as_object_mut().expect("SDK Tool object");
|
||||
for (key, value) in raw.as_object().expect("validated Tool object") {
|
||||
if matches!(key.as_str(), "annotations" | "icons")
|
||||
|| !matches!(
|
||||
key.as_str(),
|
||||
"name" | "title" | "description" | "inputSchema" | "outputSchema" | "_meta"
|
||||
)
|
||||
|| value.is_null()
|
||||
{
|
||||
serialized.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// SDK results are serialized before the minimal lossless projection of descriptor extensions.
|
||||
pub(super) fn resources_list_result(
|
||||
resources: Vec<Value>,
|
||||
next_cursor: Option<String>,
|
||||
version: &str,
|
||||
) -> Result<Value, serde_json::Error> {
|
||||
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
|
||||
let result = rmcp::model::ListResourcesResult {
|
||||
resources: resources
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(serde_json::from_value)
|
||||
.collect::<Result<_, _>>()?,
|
||||
next_cursor,
|
||||
result_type: modern.then_some(ResultType::COMPLETE),
|
||||
ttl_ms: modern.then_some(0),
|
||||
cache_scope: modern.then_some(CacheScope::Private),
|
||||
meta: None,
|
||||
};
|
||||
let mut result = serde_json::to_value(result)?;
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_resources_list_result_serialized();
|
||||
preserve_resource_fields(
|
||||
&resources,
|
||||
&mut result["resources"],
|
||||
&[
|
||||
"uri",
|
||||
"name",
|
||||
"title",
|
||||
"description",
|
||||
"mimeType",
|
||||
"size",
|
||||
"_meta",
|
||||
],
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub(super) fn resource_read_result(
|
||||
contents: Value,
|
||||
version: &str,
|
||||
) -> Result<Value, serde_json::Error> {
|
||||
let mut result =
|
||||
rmcp::model::ReadResourceResult::new(serde_json::from_value(contents.clone())?);
|
||||
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
|
||||
result.result_type = modern.then_some(ResultType::COMPLETE);
|
||||
result.ttl_ms = modern.then_some(0);
|
||||
result.cache_scope = modern.then_some(CacheScope::Private);
|
||||
let mut result = serde_json::to_value(result)?;
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_resources_read_result_serialized();
|
||||
preserve_resource_fields(
|
||||
contents.as_array().expect("validated contents array"),
|
||||
&mut result["contents"],
|
||||
&["uri", "mimeType", "text", "blob", "_meta"],
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// The era-neutral `resources/templates/list` result shape.
|
||||
pub(super) fn resource_templates_list_result(
|
||||
templates: Vec<Value>,
|
||||
next_cursor: Option<String>,
|
||||
) -> Value {
|
||||
let mut result = json!({ "resourceTemplates": templates });
|
||||
if let Some(cursor) = next_cursor {
|
||||
result["nextCursor"] = Value::String(cursor);
|
||||
}
|
||||
result
|
||||
version: &str,
|
||||
) -> Result<Value, serde_json::Error> {
|
||||
let modern = version == crate::jsonrpc::MODERN_PROTOCOL_VERSION;
|
||||
let result = rmcp::model::ListResourceTemplatesResult {
|
||||
resource_templates: templates
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(serde_json::from_value)
|
||||
.collect::<Result<_, _>>()?,
|
||||
next_cursor,
|
||||
result_type: modern.then_some(ResultType::COMPLETE),
|
||||
ttl_ms: modern.then_some(0),
|
||||
cache_scope: modern.then_some(CacheScope::Private),
|
||||
meta: None,
|
||||
};
|
||||
let mut result = serde_json::to_value(result)?;
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_resource_templates_list_result_serialized();
|
||||
preserve_resource_fields(
|
||||
&templates,
|
||||
&mut result["resourceTemplates"],
|
||||
&[
|
||||
"uriTemplate",
|
||||
"name",
|
||||
"title",
|
||||
"description",
|
||||
"mimeType",
|
||||
"_meta",
|
||||
],
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// The era-neutral `prompts/list` result shape (Story 3.14). Same pagination wire keys as
|
||||
/// `resources/list`; `nextCursor` is absent on the last page.
|
||||
pub(super) fn prompts_list_result(prompts: Vec<Value>, next_cursor: Option<String>) -> Value {
|
||||
let mut result = json!({ "prompts": prompts });
|
||||
if let Some(cursor) = next_cursor {
|
||||
result["nextCursor"] = Value::String(cursor);
|
||||
// Pinned SDK serde drops unknown fields and optional nulls, including Crank template parameters.
|
||||
// Treat annotations/icons as opaque too: their nested extensions are not represented by the SDK.
|
||||
fn preserve_resource_fields(raw: &[Value], serialized: &mut Value, sdk_fields: &[&str]) {
|
||||
for (raw, serialized) in raw
|
||||
.iter()
|
||||
.zip(serialized.as_array_mut().expect("SDK array"))
|
||||
{
|
||||
let serialized = serialized.as_object_mut().expect("SDK object");
|
||||
for (key, value) in raw.as_object().expect("validated resource object") {
|
||||
if !sdk_fields.contains(&key.as_str()) || value.is_null() {
|
||||
serialized.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// The era-neutral `prompts/get` result shape: the rendered `messages[]` and the description of the
|
||||
/// exact bound Published Prompt Version it came from.
|
||||
pub(super) fn prompt_get_result(description: Option<String>, messages: Vec<Value>) -> Value {
|
||||
let mut result = json!({ "messages": messages });
|
||||
if let Some(description) = description {
|
||||
result["description"] = Value::String(description);
|
||||
/// SDK pagination with lossless Crank argument schemas/defaults and optional nulls.
|
||||
pub(super) fn prompts_list_result(
|
||||
prompts: Vec<Value>,
|
||||
next_cursor: Option<String>,
|
||||
) -> Result<Value, serde_json::Error> {
|
||||
let result = rmcp::model::ListPromptsResult {
|
||||
prompts: prompts
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(serde_json::from_value)
|
||||
.collect::<Result<Vec<rmcp::model::Prompt>, _>>()?,
|
||||
next_cursor,
|
||||
result_type: None,
|
||||
ttl_ms: None,
|
||||
cache_scope: None,
|
||||
meta: None,
|
||||
};
|
||||
let mut result = serde_json::to_value(result)?;
|
||||
preserve_resource_fields(
|
||||
&prompts,
|
||||
&mut result["prompts"],
|
||||
&["name", "title", "description", "arguments", "_meta"],
|
||||
);
|
||||
for (raw, serialized) in prompts
|
||||
.iter()
|
||||
.zip(result["prompts"].as_array_mut().expect("SDK prompts"))
|
||||
{
|
||||
if let Some(arguments) = raw["arguments"].as_array() {
|
||||
preserve_resource_fields(
|
||||
arguments,
|
||||
&mut serialized["arguments"],
|
||||
&["name", "title", "description", "required"],
|
||||
);
|
||||
}
|
||||
}
|
||||
result
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_prompts_list_result_serialized();
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Preserve Crank's system role, which the pinned SDK's user/assistant Role cannot express.
|
||||
pub(super) fn prompt_get_result(
|
||||
description: Option<String>,
|
||||
messages: Vec<Value>,
|
||||
) -> Result<Value, serde_json::Error> {
|
||||
let sdk_messages = messages
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut message| {
|
||||
if message["role"] == "system" {
|
||||
message["role"] = json!("user");
|
||||
}
|
||||
serde_json::from_value::<rmcp::model::PromptMessage>(message)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let mut result = rmcp::model::GetPromptResult::new(sdk_messages);
|
||||
result.description = description;
|
||||
// Era-specific resultType/cache projection remains at the existing dispatch boundary.
|
||||
result.result_type = None;
|
||||
let mut result = serde_json::to_value(result)?;
|
||||
preserve_resource_fields(&messages, &mut result["messages"], &["role", "content"]);
|
||||
for (raw, serialized) in messages
|
||||
.iter()
|
||||
.zip(result["messages"].as_array_mut().expect("SDK messages"))
|
||||
{
|
||||
if raw["role"] == "system" {
|
||||
serialized["role"] = raw["role"].clone();
|
||||
}
|
||||
// Content extensions/optional nulls are opaque to the pinned SDK.
|
||||
let content = serialized["content"].as_object_mut().expect("SDK content");
|
||||
for (key, value) in raw["content"].as_object().expect("validated content") {
|
||||
if !content.contains_key(key)
|
||||
|| value.is_null()
|
||||
|| matches!(key.as_str(), "annotations" | "resource")
|
||||
{
|
||||
content.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_prompts_get_result_serialized();
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// The era-neutral `completion/complete` result shape (Story 3.17): the bounded `values[]`, the
|
||||
@@ -158,14 +331,18 @@ pub(super) fn completion_result(
|
||||
total: Option<usize>,
|
||||
has_more: bool,
|
||||
) -> Value {
|
||||
let mut completion = json!({ "values": values });
|
||||
if let Some(total) = total {
|
||||
completion["total"] = json!(total);
|
||||
}
|
||||
if has_more {
|
||||
completion["hasMore"] = json!(true);
|
||||
}
|
||||
json!({ "completion": completion })
|
||||
let mut completion = rmcp::model::CompletionInfo::default();
|
||||
completion.values = values;
|
||||
// The domain scans at most 1024 candidates, so its exact count fits the SDK's u32.
|
||||
completion.total = total.map(|total| u32::try_from(total).expect("bounded completion count"));
|
||||
completion.has_more = has_more.then_some(true);
|
||||
let mut result = rmcp::model::CompleteResult::new(completion);
|
||||
// Keep the shared result era-neutral; the existing caller projects modern resultType.
|
||||
result.result_type = None;
|
||||
let result = serde_json::to_value(result).expect("SDK completion contains strings and counts");
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_completion_result_serialized();
|
||||
result
|
||||
}
|
||||
|
||||
/// The structured `prompts/get` argument rejection (AT-3.14-02): a `-32602` error whose `data`
|
||||
@@ -310,3 +487,195 @@ pub(super) fn internal_jsonrpc_error(message: &Value, _error: impl std::fmt::Dis
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod sdk_result_tests {
|
||||
use super::*;
|
||||
use crate::jsonrpc::{LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION};
|
||||
|
||||
#[test]
|
||||
fn prompts_sdk_projection_preserves_typed_extensions_and_system_content() {
|
||||
let prompt = json!({"name":"p", "description":null, "icons":null,
|
||||
"x-prompt":{}, "arguments":[{"name":"count", "description":null,
|
||||
"required":false, "schema":{"type":"integer"}, "default":7}]});
|
||||
for cursor in [None, Some(String::new()), Some("opaque".to_owned())] {
|
||||
let result = prompts_list_result(vec![prompt.clone()], cursor.clone()).unwrap();
|
||||
assert_eq!(result["prompts"], json!([prompt]));
|
||||
assert_eq!(result.get("nextCursor"), cursor.map(Value::String).as_ref());
|
||||
assert!(result.get("resultType").is_none());
|
||||
}
|
||||
let messages = vec![
|
||||
json!({"role":"system", "x-message":null,
|
||||
"content":{"type":"text", "text":"\0é\n", "annotations":null, "x-content":[]}}),
|
||||
json!({"role":"assistant", "content":{"type":"text", "text":""}}),
|
||||
];
|
||||
assert_eq!(
|
||||
prompt_get_result(None, messages.clone()).unwrap(),
|
||||
json!({"messages":messages})
|
||||
);
|
||||
assert_eq!(
|
||||
prompts_list_result(vec![], None).unwrap(),
|
||||
json!({"prompts":[]})
|
||||
);
|
||||
assert_eq!(
|
||||
prompt_get_result(Some(String::new()), vec![]).unwrap(),
|
||||
json!({"description":"", "messages":[]})
|
||||
);
|
||||
assert!(prompts_list_result(vec![json!({"name":42})], None).is_err());
|
||||
assert!(
|
||||
prompt_get_result(
|
||||
None,
|
||||
vec![json!({"role":"invalid", "content":{"type":"text", "text":"secret"}})]
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resources_sdk_projection_is_lossless_for_templates_contents_and_catalog() {
|
||||
for version in [LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION] {
|
||||
for optional in [Value::Null, json!({}), json!({"x-extension":[null,{}]})] {
|
||||
let raw = json!({"uri":"crank://r","name":"r","description":null,
|
||||
"mimeType":"text/plain","_meta":optional,"x-resource":[],
|
||||
"annotations":{"x-nested":null},"icons":[{"src":"icon","x-icon":{}}]});
|
||||
for cursor in [None, Some(String::new()), Some("opaque".to_owned())] {
|
||||
let list =
|
||||
resources_list_result(vec![raw.clone()], cursor.clone(), version).unwrap();
|
||||
assert_eq!(list["resources"], json!([raw]));
|
||||
assert_eq!(
|
||||
list.get("nextCursor"),
|
||||
cursor.as_ref().map(|s| json!(s)).as_ref()
|
||||
);
|
||||
assert_eq!(
|
||||
list.get("resultType").is_some(),
|
||||
version == MODERN_PROTOCOL_VERSION
|
||||
);
|
||||
assert_eq!(
|
||||
list.get("ttlMs").is_some(),
|
||||
version == MODERN_PROTOCOL_VERSION
|
||||
);
|
||||
assert_eq!(
|
||||
list.get("cacheScope").is_some(),
|
||||
version == MODERN_PROTOCOL_VERSION
|
||||
);
|
||||
assert!(list.get("_meta").is_none());
|
||||
let template = json!({"uriTemplate":"crank://r/{id}","name":"r",
|
||||
"description":null,"mimeType":null,"_meta":optional,
|
||||
"parameters":[{"name":"id","required":true,"opaque":null}],
|
||||
"annotations":{"x-nested":[]},"icons":null});
|
||||
let list = resource_templates_list_result(
|
||||
vec![template.clone()],
|
||||
cursor.clone(),
|
||||
version,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(list["resourceTemplates"], json!([template]));
|
||||
assert_eq!(
|
||||
list.get("nextCursor"),
|
||||
cursor.as_ref().map(|s| json!(s)).as_ref()
|
||||
);
|
||||
}
|
||||
let contents = json!([
|
||||
{"uri":"crank://r","mimeType":null,"text":"\0é\n","_meta":optional,"x-content":[]},
|
||||
{"uri":"crank://b","mimeType":"application/octet-stream","blob":"AP8=","_meta":null}
|
||||
]);
|
||||
assert_eq!(
|
||||
resource_read_result(contents.clone(), version).unwrap()["contents"],
|
||||
contents
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
resources_list_result(vec![], None, version).unwrap()["resources"],
|
||||
json!([])
|
||||
);
|
||||
assert_eq!(
|
||||
resource_templates_list_result(vec![], None, version).unwrap()["resourceTemplates"],
|
||||
json!([])
|
||||
);
|
||||
assert_eq!(
|
||||
resource_read_result(json!([]), version).unwrap()["contents"],
|
||||
json!([])
|
||||
);
|
||||
assert!(resources_list_result(vec![json!({"uri":42})], None, version).is_err());
|
||||
assert!(
|
||||
resource_templates_list_result(vec![json!({"name":"r"})], None, version).is_err()
|
||||
);
|
||||
assert!(resource_read_result(json!([{"uri":"r"}]), version).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalog_projection_preserves_extensions_null_and_empty() {
|
||||
let descriptor = json!({
|
||||
"name":"tool", "title":"", "description":null,
|
||||
"inputSchema":{"type":"object", "x-schema":{"nested":null}},
|
||||
"outputSchema":{"type":"object", "x-output":[]},
|
||||
"_meta":{"custom":{"empty":{}}}, "x-tool":{"nested":[]},
|
||||
"annotations":{"readOnlyHint":true,"x-annotation":{"nested":null}},
|
||||
"icons":[{"src":"https://example.test/icon", "x-icon":{"nested":[]}}]
|
||||
});
|
||||
for version in [LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION] {
|
||||
for raw in [
|
||||
descriptor.clone(),
|
||||
json!({"name":"tool","inputSchema":{},
|
||||
"title":null,"description":"","outputSchema":null,"_meta":null,
|
||||
"annotations":null,"icons":null}),
|
||||
json!({"name":"tool","inputSchema":{},
|
||||
"annotations":{},"icons":[]}),
|
||||
] {
|
||||
for cursor in [None, Some(String::new()), Some("next".to_owned())] {
|
||||
let value =
|
||||
tools_list_result(vec![raw.clone()], cursor.clone(), version).unwrap();
|
||||
assert_eq!(value["tools"], json!([raw]));
|
||||
assert_eq!(
|
||||
value.get("nextCursor"),
|
||||
cursor.as_ref().map(|s| Value::String(s.clone())).as_ref()
|
||||
);
|
||||
assert_eq!(
|
||||
value.get("resultType").is_some(),
|
||||
version == MODERN_PROTOCOL_VERSION
|
||||
);
|
||||
assert_eq!(
|
||||
value.get("ttlMs").is_some(),
|
||||
version == MODERN_PROTOCOL_VERSION
|
||||
);
|
||||
assert_eq!(
|
||||
value.get("cacheScope").is_some(),
|
||||
version == MODERN_PROTOCOL_VERSION
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
tools_list_result(vec![], None, version).unwrap()["tools"],
|
||||
json!([])
|
||||
);
|
||||
assert!(tools_list_result(vec![json!({"name":42})], None, version).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_success_and_rpc_failures_keep_their_distinct_shapes() {
|
||||
for version in [LEGACY_PROTOCOL_VERSION, MODERN_PROTOCOL_VERSION] {
|
||||
let message = json!({"id":9007199254740993_u64});
|
||||
let payload = success_tool_payload(&message, version, Value::Null);
|
||||
assert_eq!(payload["id"], message["id"]);
|
||||
assert_eq!(
|
||||
payload["result"]["content"],
|
||||
json!([{"type":"text","text":"null"}])
|
||||
);
|
||||
assert_eq!(
|
||||
payload["result"].get("structuredContent"),
|
||||
Some(&Value::Null)
|
||||
);
|
||||
assert_eq!(payload["result"]["isError"], false);
|
||||
let correlation = CorrelationContext::generate();
|
||||
let internal = tool_command_payload(
|
||||
&message,
|
||||
version,
|
||||
&correlation,
|
||||
ToolsOutcome::Call(ToolCallOutcome::InternalFailure),
|
||||
);
|
||||
assert!(matches!(internal, ToolCommandPayload::Internal(_)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
//! Runtime SDK dispatch; admission, lifecycle and response projection stay with the caller.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rmcp::model::{CallToolRequestParams, PaginatedRequestParams, ReadResourceRequestParams};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{AppState, lifecycle::ToolCallLifecycle};
|
||||
use crate::protocol::{
|
||||
ServiceContext, ToolCallCommand, ToolCallOutcome, ToolsCommand, ToolsListCommand, ToolsOutcome,
|
||||
};
|
||||
|
||||
// Crank extensions are not SDK fields and must survive the typed request boundary unchanged.
|
||||
enum SdkToolsCommand {
|
||||
List {
|
||||
params: PaginatedRequestParams,
|
||||
limit: Option<usize>,
|
||||
},
|
||||
Call {
|
||||
params: CallToolRequestParams,
|
||||
confirmation_token: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
pub(super) async fn handle_tools_command(
|
||||
state: &Arc<AppState>,
|
||||
context: ServiceContext<'_>,
|
||||
command: ToolsCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
let request = match command {
|
||||
ToolsCommand::List(list) => SdkToolsCommand::List {
|
||||
params: PaginatedRequestParams::default().with_cursor(list.cursor),
|
||||
limit: list.limit,
|
||||
},
|
||||
ToolsCommand::Call(call) => {
|
||||
let Value::Object(arguments) = call.arguments else {
|
||||
return ToolsOutcome::Call(ToolCallOutcome::InvalidParams(
|
||||
"tool arguments must be an object".to_owned(),
|
||||
));
|
||||
};
|
||||
SdkToolsCommand::Call {
|
||||
params: CallToolRequestParams::new(call.tool_name).with_arguments(arguments),
|
||||
confirmation_token: call.confirmation_token,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_tools_dispatch_enter();
|
||||
|
||||
let command = match request {
|
||||
SdkToolsCommand::List { params, limit } => ToolsCommand::List(ToolsListCommand {
|
||||
cursor: params.cursor,
|
||||
limit,
|
||||
}),
|
||||
SdkToolsCommand::Call {
|
||||
params,
|
||||
confirmation_token,
|
||||
} => ToolsCommand::Call(ToolCallCommand {
|
||||
tool_name: params.name.into_owned(),
|
||||
arguments: Value::Object(params.arguments.unwrap_or_default()),
|
||||
confirmation_token,
|
||||
}),
|
||||
};
|
||||
crate::protocol::handle_tools_command(state, context, command, lifecycle).await
|
||||
}
|
||||
|
||||
pub(super) async fn handle_prompts_command(
|
||||
state: &Arc<AppState>,
|
||||
context: ServiceContext<'_>,
|
||||
revision: &str,
|
||||
command: crate::protocol::PromptsCommand,
|
||||
) -> crate::protocol::PromptsOutcome {
|
||||
use crate::protocol::{PromptGetCommand, PromptsCommand, PromptsListCommand};
|
||||
let command = match command {
|
||||
PromptsCommand::List(list) => {
|
||||
let params = PaginatedRequestParams::default().with_cursor(list.cursor);
|
||||
PromptsCommand::List(PromptsListCommand {
|
||||
cursor: params.cursor,
|
||||
limit: list.limit,
|
||||
})
|
||||
}
|
||||
PromptsCommand::Get(get) => {
|
||||
let mut params = rmcp::model::GetPromptRequestParams::new(get.name);
|
||||
params.arguments = get.arguments;
|
||||
PromptsCommand::Get(PromptGetCommand {
|
||||
name: params.name,
|
||||
arguments: params.arguments,
|
||||
})
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_prompts_dispatch_enter();
|
||||
crate::protocol::handle_prompts_command(state, context, revision, command).await
|
||||
}
|
||||
|
||||
pub(super) async fn handle_resources_command(
|
||||
state: &Arc<AppState>,
|
||||
context: ServiceContext<'_>,
|
||||
revision: &str,
|
||||
command: crate::protocol::ResourcesCommand,
|
||||
) -> crate::protocol::ResourcesOutcome {
|
||||
use crate::protocol::{ResourceReadCommand, ResourcesCommand};
|
||||
let command = match command {
|
||||
ResourcesCommand::List(list) => ResourcesCommand::List(sdk_resource_page(list)),
|
||||
ResourcesCommand::Read(read) => {
|
||||
let params = ReadResourceRequestParams::new(read.uri);
|
||||
ResourcesCommand::Read(ResourceReadCommand { uri: params.uri })
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_resources_dispatch_enter();
|
||||
crate::protocol::handle_resources_command(state, context, revision, command).await
|
||||
}
|
||||
|
||||
pub(super) async fn list_resource_templates(
|
||||
state: &Arc<AppState>,
|
||||
workspace_slug: &str,
|
||||
agent_slug: &str,
|
||||
revision: &str,
|
||||
command: crate::protocol::ResourcesListCommand,
|
||||
) -> Result<(Vec<Value>, Option<String>), crate::protocol::ResourceTemplatesListError> {
|
||||
let command = sdk_resource_page(command);
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_resources_dispatch_enter();
|
||||
crate::protocol::list_resource_templates(state, workspace_slug, agent_slug, revision, command)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn handle_completion_command(
|
||||
state: &Arc<AppState>,
|
||||
context: ServiceContext<'_>,
|
||||
revision: &str,
|
||||
command: crate::protocol::CompletionCommand,
|
||||
) -> crate::protocol::CompletionOutcome {
|
||||
use crate::protocol::{
|
||||
CompletionArgument, CompletionCommand, CompletionOutcome, CompletionReference,
|
||||
};
|
||||
use rmcp::model::{ArgumentInfo, CompleteRequestParams, Reference};
|
||||
|
||||
let reference = match command.reference {
|
||||
CompletionReference::Prompt { name } => Reference::for_prompt(name),
|
||||
CompletionReference::Resource { uri } => Reference::for_resource(uri),
|
||||
};
|
||||
let params = CompleteRequestParams::new(
|
||||
reference,
|
||||
ArgumentInfo::new(command.argument.name, command.argument.value),
|
||||
);
|
||||
// SDK context only supports strings; retain arbitrary JSON unchanged for domain validation.
|
||||
let context_arguments = command.context;
|
||||
#[cfg(test)]
|
||||
super::dispatch_observer::sdk_completion_dispatch_enter();
|
||||
let reference = match params.r#ref {
|
||||
Reference::Prompt(reference) => CompletionReference::Prompt {
|
||||
name: reference.name,
|
||||
},
|
||||
Reference::Resource(reference) => CompletionReference::Resource { uri: reference.uri },
|
||||
_ => return CompletionOutcome::InvalidParams("invalid completion/complete parameters"),
|
||||
};
|
||||
let command = CompletionCommand {
|
||||
reference,
|
||||
argument: CompletionArgument {
|
||||
name: params.argument.name,
|
||||
value: params.argument.value,
|
||||
},
|
||||
context: context_arguments,
|
||||
};
|
||||
crate::protocol::handle_completion_command(state, context, revision, command).await
|
||||
}
|
||||
|
||||
fn sdk_resource_page(
|
||||
list: crate::protocol::ResourcesListCommand,
|
||||
) -> crate::protocol::ResourcesListCommand {
|
||||
let params = PaginatedRequestParams::default().with_cursor(list.cursor);
|
||||
// Crank's page limit is not an SDK field; retain its existing validation and clamping.
|
||||
crate::protocol::ResourcesListCommand {
|
||||
cursor: params.cursor,
|
||||
limit: list.limit,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
//! Isolated single-Agent loopback experiment. Not installed in the production router.
|
||||
//! Each instance owns one route and one SDK manager; no durable store is replaced.
|
||||
|
||||
use super::{
|
||||
AgentRoutePath, AppState,
|
||||
lifecycle::InFlightScope,
|
||||
metrics::ActiveStreamGuard,
|
||||
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
|
||||
stages::{enforce_traced_rate_limit, require_traced_machine_access},
|
||||
};
|
||||
use crate::{
|
||||
protocol::{self, ContinuationSupport, RequestEra},
|
||||
transport,
|
||||
};
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
extract::Request,
|
||||
http::{Method, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use crank_core::PlatformApiKeyScope;
|
||||
use futures_util::StreamExt;
|
||||
use rmcp::transport::streamable_http_server::{
|
||||
StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use std::sync::Arc;
|
||||
|
||||
const BODY_LIMIT: usize = 2 * 1024 * 1024;
|
||||
|
||||
pub(super) fn router(
|
||||
state: Arc<AppState>,
|
||||
path: AgentRoutePath,
|
||||
active: Arc<std::sync::atomic::AtomicUsize>,
|
||||
) -> Router {
|
||||
let handler_state = state.clone();
|
||||
let mut config = StreamableHttpServerConfig::default().with_json_response(true);
|
||||
config.max_request_body_bytes = BODY_LIMIT;
|
||||
let shutdown = Arc::new(config.cancellation_token.clone().drop_guard());
|
||||
let mut manager = LocalSessionManager::default();
|
||||
manager.session_config.channel_capacity = 16;
|
||||
let service = StreamableHttpService::new(
|
||||
move || {
|
||||
Ok(RmcpTools {
|
||||
state: handler_state.clone(),
|
||||
active: active.clone(),
|
||||
})
|
||||
},
|
||||
Arc::new(manager),
|
||||
config,
|
||||
);
|
||||
let route = format!("/v1/{}/{}", path.workspace_slug, path.agent_slug);
|
||||
Router::new()
|
||||
.route(
|
||||
&route,
|
||||
axum::routing::any(move |request: Request| {
|
||||
let state = state.clone();
|
||||
let path = path.clone();
|
||||
let service = service.clone();
|
||||
let shutdown = shutdown.clone();
|
||||
async move {
|
||||
let _shutdown = shutdown;
|
||||
let (mut parts, body) = request.into_parts();
|
||||
if let Err(status) =
|
||||
transport::validate_origin(&state.allowed_origins, &parts.headers)
|
||||
{
|
||||
return status.into_response();
|
||||
}
|
||||
let bytes = match to_bytes(body, BODY_LIMIT).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(),
|
||||
};
|
||||
let message: Value = if parts.method == Method::POST {
|
||||
match serde_json::from_slice(&bytes) {
|
||||
Ok(message) => message,
|
||||
Err(_) => return StatusCode::BAD_REQUEST.into_response(),
|
||||
}
|
||||
} else {
|
||||
Value::Null
|
||||
};
|
||||
let method = message["method"].as_str().unwrap_or("");
|
||||
let required = if method == "tools/call" {
|
||||
PlatformApiKeyScope::Write
|
||||
} else {
|
||||
PlatformApiKeyScope::Read
|
||||
};
|
||||
let credential = match require_traced_machine_access(
|
||||
&state,
|
||||
&path,
|
||||
&parts.headers,
|
||||
required,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(credential) => credential,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let session = match transport::session_id_from_headers(&parts.headers) {
|
||||
Ok(session) => session,
|
||||
Err(_) => {
|
||||
return protocol::ambiguous_session_carrier_response(
|
||||
&parts.headers,
|
||||
&message,
|
||||
);
|
||||
}
|
||||
};
|
||||
let era = if parts.method == Method::POST {
|
||||
if let Err(response) = protocol::validate_jsonrpc_envelope(&message) {
|
||||
return *response;
|
||||
}
|
||||
if let Err(status) = transport::negotiate_post_response_mode(&parts.headers)
|
||||
{
|
||||
return status.into_response();
|
||||
}
|
||||
match protocol::resolve_request_identity(&parts.headers, &message) {
|
||||
Ok(identity) => identity.era,
|
||||
Err(response) => return *response,
|
||||
}
|
||||
} else {
|
||||
if let Err(response) = protocol::session_lifecycle_admission(&parts.headers)
|
||||
{
|
||||
return *response;
|
||||
}
|
||||
RequestEra::Legacy
|
||||
};
|
||||
if matches!(era, RequestEra::Modern) && session.is_some() {
|
||||
return protocol::modern_carrier_conflict_response(&message);
|
||||
}
|
||||
if let Err(error) =
|
||||
enforce_traced_rate_limit(&state, &path, &parts.headers).await
|
||||
{
|
||||
return crate::rate_limit::rate_limited_status_response(error);
|
||||
}
|
||||
// SDK cancellation is keyed only by request id. Never forward an unchecked cancel.
|
||||
if method == "notifications/cancelled" {
|
||||
if let Some(session_id) = session {
|
||||
state.in_flight.cancel(
|
||||
&InFlightScope {
|
||||
session_id,
|
||||
owner: credential
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str())
|
||||
.unwrap_or("")
|
||||
.to_owned(),
|
||||
},
|
||||
&message["params"]["requestId"],
|
||||
);
|
||||
}
|
||||
return StatusCode::ACCEPTED.into_response();
|
||||
}
|
||||
let Some(context) = parts
|
||||
.extensions
|
||||
.get::<crate::request_context::RequestContext>()
|
||||
else {
|
||||
return StatusCode::INTERNAL_SERVER_ERROR.into_response();
|
||||
};
|
||||
let correlation = context.correlation.clone();
|
||||
|
||||
parts.extensions.insert(RmcpRequestIdentity {
|
||||
workspace_slug: path.workspace_slug,
|
||||
agent_slug: path.agent_slug,
|
||||
credential,
|
||||
correlation,
|
||||
continuation: ContinuationSupport::from_supported(false),
|
||||
locale: crate::tool_error::execution_locale(&message),
|
||||
session_id: session,
|
||||
progress_meta: message.pointer("/params/_meta").cloned(),
|
||||
sse_acceptable: transport::accepts_event_stream(&parts.headers),
|
||||
});
|
||||
// Legacy SDK calls outlive disconnected bodies: both lifetimes share one slot.
|
||||
let permit = match ActiveStreamGuard::try_acquire(&state.session_slots) {
|
||||
Ok(permit) => Arc::new(permit),
|
||||
Err(_) => return StatusCode::TOO_MANY_REQUESTS.into_response(),
|
||||
};
|
||||
if method == "tools/call" {
|
||||
parts.extensions.insert(permit.clone());
|
||||
}
|
||||
let response = service
|
||||
.handle(Request::from_parts(parts, Body::from(bytes)))
|
||||
.await;
|
||||
let (parts, body) = response.into_parts();
|
||||
let stream = futures_util::stream::unfold(
|
||||
(Box::pin(Body::new(body).into_data_stream()), permit),
|
||||
|(mut stream, permit)| async move {
|
||||
stream.next().await.map(|chunk| (chunk, (stream, permit)))
|
||||
},
|
||||
);
|
||||
Response::from_parts(parts, Body::from_stream(stream))
|
||||
}
|
||||
}),
|
||||
)
|
||||
.layer(axum::middleware::from_fn(
|
||||
crate::request_context::apply_request_context,
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
//! Behavioural ATDD for rmcp 3.7.0 LocalSessionManager reconnect/restart semantics.
|
||||
|
||||
use rmcp::transport::streamable_http_server::session::{
|
||||
SessionManager, local::LocalSessionManager,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_local_session_manager_preserves_access_to_created_session() {
|
||||
let manager = LocalSessionManager::default();
|
||||
|
||||
let (session_id, _transport) = manager
|
||||
.create_session()
|
||||
.await
|
||||
.expect("local manager must create a session");
|
||||
|
||||
assert!(
|
||||
manager
|
||||
.has_session(&session_id)
|
||||
.await
|
||||
.expect("live manager must answer has_session"),
|
||||
"a live manager must keep access to a session it created"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_fresh_session_manager_without_external_store_rejects_known_session_id() {
|
||||
let first_manager = LocalSessionManager::default();
|
||||
|
||||
let (session_id, _transport) = first_manager
|
||||
.create_session()
|
||||
.await
|
||||
.expect("first local manager must create a session");
|
||||
|
||||
let second_manager = LocalSessionManager::default();
|
||||
|
||||
assert!(
|
||||
!second_manager
|
||||
.has_session(&session_id)
|
||||
.await
|
||||
.expect("fresh manager must answer has_session"),
|
||||
"a fresh in-memory manager must not claim ownership of another manager's session"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
//! Isolated rmcp 3.7.0 Tools adapter over Crank's existing typed Tools service.
|
||||
//!
|
||||
//! This module deliberately does not provide a second command engine. It owns only
|
||||
//! the verified request identity (`ServiceContext` data) and delegates both
|
||||
//! `tools/list` and `tools/call` to `handle_tools_command` via rmcp's
|
||||
//! `ServerHandler` trait. HTTP admission must supply identity for each request.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crank_core::{CorrelationContext, ExecutionLocale};
|
||||
|
||||
use crate::app::{AppState, lifecycle::ToolCallLifecycle};
|
||||
use crate::auth::VerifiedMachineCredential;
|
||||
use crate::protocol::{
|
||||
ContinuationSupport, ServiceContext, ToolCallCommand, ToolsCommand, ToolsListCommand,
|
||||
ToolsOutcome, handle_tools_command,
|
||||
};
|
||||
|
||||
/// Long-lived handler: never stores credentials obtained on initialization.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct RmcpTools {
|
||||
pub(super) state: Arc<AppState>,
|
||||
pub(super) active: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ActiveCall(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ActiveCall {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// Server-created identity attached to the individual admitted HTTP request.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct RmcpRequestIdentity {
|
||||
pub(super) workspace_slug: String,
|
||||
pub(super) agent_slug: String,
|
||||
pub(super) credential: VerifiedMachineCredential,
|
||||
pub(super) correlation: CorrelationContext,
|
||||
pub(super) continuation: ContinuationSupport,
|
||||
pub(super) locale: ExecutionLocale,
|
||||
pub(super) session_id: Option<String>,
|
||||
pub(super) progress_meta: Option<serde_json::Value>,
|
||||
pub(super) sse_acceptable: bool,
|
||||
}
|
||||
|
||||
impl RmcpRequestIdentity {
|
||||
/// Build the neutral `ServiceContext` borrowed from this adapter's owned data.
|
||||
pub(super) fn service_context(&self) -> ServiceContext<'_> {
|
||||
ServiceContext {
|
||||
workspace_slug: &self.workspace_slug,
|
||||
agent_slug: &self.agent_slug,
|
||||
credential: &self.credential,
|
||||
correlation: &self.correlation,
|
||||
continuation: self.continuation,
|
||||
locale: self.locale,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RmcpTools {
|
||||
/// Delegate one neutral Tools command to the existing shared service.
|
||||
pub(super) async fn handle(
|
||||
&self,
|
||||
identity: &RmcpRequestIdentity,
|
||||
command: ToolsCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
handle_tools_command(&self.state, identity.service_context(), command, lifecycle).await
|
||||
}
|
||||
|
||||
pub(super) async fn list_tools(
|
||||
&self,
|
||||
identity: &RmcpRequestIdentity,
|
||||
command: ToolsListCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
self.handle(identity, ToolsCommand::List(command), lifecycle)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn call_tool(
|
||||
&self,
|
||||
identity: &RmcpRequestIdentity,
|
||||
command: ToolCallCommand,
|
||||
lifecycle: Option<ToolCallLifecycle>,
|
||||
) -> ToolsOutcome {
|
||||
self.handle(identity, ToolsCommand::Call(command), lifecycle)
|
||||
.await
|
||||
}
|
||||
}
|
||||
use rmcp::ErrorData as McpError;
|
||||
use rmcp::model::{CallToolRequestParams, ListToolsResult, PaginatedRequestParams};
|
||||
use rmcp::service::RequestContext;
|
||||
use rmcp::{RoleServer, ServerHandler};
|
||||
|
||||
fn request_identity(context: &RequestContext<RoleServer>) -> Result<RmcpRequestIdentity, McpError> {
|
||||
context
|
||||
.extensions
|
||||
.get::<axum::http::request::Parts>()
|
||||
.and_then(|parts| parts.extensions.get::<RmcpRequestIdentity>())
|
||||
.cloned()
|
||||
.ok_or_else(|| McpError::internal_error("missing authenticated request context", None))
|
||||
}
|
||||
|
||||
impl ServerHandler for RmcpTools {
|
||||
fn supported_protocol_versions(
|
||||
&self,
|
||||
) -> std::borrow::Cow<'static, [rmcp::model::ProtocolVersion]> {
|
||||
std::borrow::Cow::Borrowed(&[
|
||||
rmcp::model::ProtocolVersion::V_2025_11_25,
|
||||
rmcp::model::ProtocolVersion::V_2026_07_28,
|
||||
])
|
||||
}
|
||||
|
||||
fn get_info(&self) -> rmcp::model::ServerConfig {
|
||||
let mut info = rmcp::model::ServerConfig::default();
|
||||
info.capabilities = rmcp::model::ServerCapabilities::builder()
|
||||
.enable_tools()
|
||||
.build();
|
||||
info
|
||||
}
|
||||
fn list_tools(
|
||||
&self,
|
||||
request: Option<PaginatedRequestParams>,
|
||||
context: RequestContext<RoleServer>,
|
||||
) -> impl std::future::Future<Output = Result<ListToolsResult, McpError>> + Send + '_ {
|
||||
let request = request.unwrap_or_default();
|
||||
async move {
|
||||
let identity = request_identity(&context)?;
|
||||
let cursor = request.cursor;
|
||||
let command = crate::protocol::ToolsListCommand {
|
||||
cursor,
|
||||
limit: None,
|
||||
};
|
||||
match self.list_tools(&identity, command, None).await {
|
||||
crate::protocol::ToolsOutcome::List { tools, next_cursor } => {
|
||||
super::response::sdk_tools_list_result(&tools, next_cursor)
|
||||
.map_err(|_| McpError::internal_error("invalid tool catalog entry", None))
|
||||
}
|
||||
crate::protocol::ToolsOutcome::InvalidParams(message) => {
|
||||
Err(McpError::invalid_params(message, None))
|
||||
}
|
||||
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn call_tool(
|
||||
&self,
|
||||
request: CallToolRequestParams,
|
||||
context: RequestContext<RoleServer>,
|
||||
) -> Result<rmcp::model::CallToolResponse, McpError> {
|
||||
let identity = request_identity(&context)?;
|
||||
let arguments = request
|
||||
.arguments
|
||||
.map(serde_json::Value::Object)
|
||||
.unwrap_or_else(|| serde_json::json!({}));
|
||||
let command = crate::protocol::ToolCallCommand {
|
||||
tool_name: request.name.to_string(),
|
||||
arguments,
|
||||
confirmation_token: None,
|
||||
};
|
||||
let token = identity
|
||||
.sse_acceptable
|
||||
.then(|| super::lifecycle::token_from_meta(identity.progress_meta.as_ref()))
|
||||
.flatten();
|
||||
// Observe future cleanup directly in the isolated loopback oracle.
|
||||
self.active
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let _active = ActiveCall(self.active.clone());
|
||||
let (mut lifecycle, mut events) = ToolCallLifecycle::new(token.clone());
|
||||
let entry = identity.session_id.as_ref().and_then(|session_id| {
|
||||
self.state.in_flight.register(
|
||||
&super::lifecycle::InFlightScope {
|
||||
session_id: session_id.clone(),
|
||||
owner: identity
|
||||
.credential
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str())
|
||||
.unwrap_or("")
|
||||
.to_owned(),
|
||||
},
|
||||
&serde_json::to_value(&context.id).expect("request id serializes"),
|
||||
)
|
||||
});
|
||||
if let Some(entry) = &entry {
|
||||
lifecycle = lifecycle.with_cancel(entry.cancel_signal());
|
||||
}
|
||||
let call = self.call_tool(&identity, command, Some(lifecycle.clone()));
|
||||
tokio::pin!(call);
|
||||
let outcome = loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
event = async { match events.as_mut() { Some(events) => events.recv().await, None => std::future::pending().await } } => {
|
||||
if let Some(event) = event {
|
||||
if let Some(token) = &token {
|
||||
let params = serde_json::from_value(super::lifecycle::progress_frame(token, &event)["params"].clone()).map_err(|_| McpError::internal_error("invalid progress event", None))?;
|
||||
let _ = context.peer.notify_progress(params).await;
|
||||
}
|
||||
} else { events = None; }
|
||||
}
|
||||
outcome = &mut call => break outcome,
|
||||
_ = lifecycle.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
|
||||
_ = context.ct.cancelled() => { return Err(McpError::internal_error("request cancelled", None)); },
|
||||
}
|
||||
};
|
||||
match outcome {
|
||||
crate::protocol::ToolsOutcome::Call(outcome) => match outcome {
|
||||
crate::protocol::ToolCallOutcome::Completed(value)
|
||||
| crate::protocol::ToolCallOutcome::ApprovalPending(value) => {
|
||||
Ok(super::response::sdk_success_tool_result(value).into())
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::ApprovalUnavailable(error) => {
|
||||
Ok(super::response::sdk_error_tool_result(
|
||||
crate::tool_error::control_tool_error_contract(
|
||||
error,
|
||||
identity.correlation.request_id().as_str(),
|
||||
identity.correlation.trace_id().as_str(),
|
||||
identity.locale,
|
||||
),
|
||||
)
|
||||
.into())
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::InvalidParams(message) => {
|
||||
Err(McpError::invalid_params(message, None))
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::Error(contract) => {
|
||||
Ok(super::response::sdk_error_tool_result(contract).into())
|
||||
}
|
||||
crate::protocol::ToolCallOutcome::InternalFailure => {
|
||||
Err(McpError::internal_error("tool execution failed", None))
|
||||
}
|
||||
},
|
||||
crate::protocol::ToolsOutcome::InvalidParams(message) => {
|
||||
Err(McpError::invalid_params(message, None))
|
||||
}
|
||||
_ => Err(McpError::internal_error("crate catalog unavailable", None)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
//! Application-level durable Task handler and attempt authority (Story 4.3/4.4).
|
||||
//!
|
||||
//! The handler owns everything that touches product authority: the exact published pin, the typed
|
||||
//! Platform API key, the approval policy, the encrypted executable input and the resolved upstream
|
||||
//! auth. The runtime owns the single bounded retry loop and the durable attempt checkpoint/resolution;
|
||||
//! this module only revalidates and prepares one attempt at a time.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AgentId, ExecutionOrigin, MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
|
||||
TaskError, TaskLease, TaskOwnerKind, TraceContext, VerifiedMachineCredential, WorkspaceId,
|
||||
current_unix_ms,
|
||||
};
|
||||
use crank_registry::PostgresRegistry;
|
||||
use crank_registry::PublishedAgentTool;
|
||||
use crank_runtime::{
|
||||
ExecutionAuthorization, PrepareAttempt, RuntimeExecutor, RuntimeOperation, SecretCrypto,
|
||||
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
|
||||
TaskAttemptRefusal, TaskHandler, TaskService, TaskWorkerError,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::TaskApplication;
|
||||
use super::auth_resolution::resolve_operation_auth_with;
|
||||
use super::tool_resolution::runtime_operation;
|
||||
use crate::access::{allows_scope, credential_allows_security_level};
|
||||
use crate::catalog::PublishedToolCatalog;
|
||||
|
||||
/// Server-owned evidence that one published operation deduplicates repeated sends of the same stable
|
||||
/// idempotency key.
|
||||
///
|
||||
/// The descriptor is constructed by the application for an integration whose upstream contract was
|
||||
/// actually qualified (key delivery, dedup scope, canonical input binding and a dedup window that
|
||||
/// covers the pinned retry budget). It is never deserialized from Task input, so a caller key or
|
||||
/// header alone can never make a write retryable. Production wiring passes an empty set until such a
|
||||
/// contract exists.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct VerifiedIdempotentWrite {
|
||||
pub operation_id: String,
|
||||
pub operation_version: u32,
|
||||
/// Header through which the stable key is delivered to this exact upstream.
|
||||
pub key_header: String,
|
||||
/// Dedup window of the upstream contract, which must cover the pinned retry budget.
|
||||
pub validity_ms: i64,
|
||||
}
|
||||
|
||||
pub struct TaskWorkerHandler {
|
||||
registry: PostgresRegistry,
|
||||
catalog: PublishedToolCatalog,
|
||||
runtime: RuntimeExecutor,
|
||||
secret_crypto: SecretCrypto,
|
||||
service: TaskService,
|
||||
verified_writes: Vec<VerifiedIdempotentWrite>,
|
||||
}
|
||||
|
||||
impl TaskWorkerHandler {
|
||||
/// Production construction from the same application entry, registry and runtime the MCP
|
||||
/// process serves requests with. The worker needs no HTTP state, only these authoritative
|
||||
/// dependencies.
|
||||
pub fn from_application(
|
||||
tasks: &TaskApplication,
|
||||
registry: PostgresRegistry,
|
||||
runtime: RuntimeExecutor,
|
||||
secret_crypto: SecretCrypto,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
catalog: tasks.catalog(),
|
||||
runtime,
|
||||
secret_crypto,
|
||||
service: tasks.task_service(),
|
||||
verified_writes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers server-owned verified-idempotency evidence of qualified integrations.
|
||||
pub fn with_verified_writes(mut self, verified_writes: Vec<VerifiedIdempotentWrite>) -> Self {
|
||||
self.verified_writes = verified_writes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
|
||||
///
|
||||
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
|
||||
/// policy and revalidates the typed persisted Platform API key identity against the registry:
|
||||
/// the key must still be active, unexpired and bound to this workspace/agent, and its exact
|
||||
/// scopes/security level must still authorize the operation. Any denial is a typed error, so the
|
||||
/// worker finishes the attempt without dispatch (fail closed before any network use).
|
||||
///
|
||||
/// An administrator Task has no authoritative Task revalidation API in the registry, so it stays
|
||||
/// fail-closed here instead of inventing authority.
|
||||
async fn authorize(&self, lease: &TaskLease) -> Result<PublishedAgentTool, TaskError> {
|
||||
if lease.record.owner_kind != TaskOwnerKind::Agent {
|
||||
return Err(TaskError::invalid(
|
||||
"admin task authority revalidation is not implemented",
|
||||
));
|
||||
}
|
||||
let Some(key_id) = lease.record.platform_api_key_id.as_ref() else {
|
||||
return Err(TaskError::invalid("task authority reference is missing"));
|
||||
};
|
||||
|
||||
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
||||
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
||||
let workspace = self
|
||||
.registry
|
||||
.get_workspace(&workspace_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?
|
||||
.ok_or_else(|| TaskError::invalid("task workspace no longer exists"))?;
|
||||
let agents = self
|
||||
.registry
|
||||
.list_agents(&workspace_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
||||
let agent = agents
|
||||
.into_iter()
|
||||
.find(|agent| agent.id.as_str() == lease.record.owner_id.as_str())
|
||||
.ok_or_else(|| TaskError::invalid("task agent no longer exists"))?;
|
||||
|
||||
let catalog = self
|
||||
.catalog
|
||||
.get_catalog(&workspace.workspace.slug, &agent.slug)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("published catalog unavailable"))?;
|
||||
let tool = catalog
|
||||
.tools
|
||||
.into_iter()
|
||||
.find(|tool| {
|
||||
tool.operation.id.as_str() == lease.record.operation_id.as_str()
|
||||
&& tool.operation.version == lease.record.pins.operation_version
|
||||
})
|
||||
.ok_or_else(|| TaskError::invalid("task published operation pin no longer matches"))?;
|
||||
|
||||
if tool.workspace_id.as_str() != lease.record.workspace_id.as_str() {
|
||||
return Err(TaskError::invalid("task workspace binding mismatch"));
|
||||
}
|
||||
if tool.agent_id.as_str() != lease.record.owner_id.as_str() {
|
||||
return Err(TaskError::invalid("task agent binding mismatch"));
|
||||
}
|
||||
if tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()
|
||||
.is_some_and(|policy| policy.required)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"missing approval proof for task operation",
|
||||
));
|
||||
}
|
||||
|
||||
// Authoritative revalidation of the exact immutable key identity. A revoked, expired,
|
||||
// foreign-workspace or foreign-agent key is absent here and never reaches the network.
|
||||
let key = self
|
||||
.registry
|
||||
.revalidate_platform_api_key(&workspace_id, &agent_id, key_id)
|
||||
.await
|
||||
.map_err(|_| TaskError::unavailable("task authority store unavailable"))?
|
||||
.ok_or_else(|| TaskError::invalid("task authority is no longer active"))?;
|
||||
let credential = VerifiedMachineCredential {
|
||||
machine_access_mode: MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: OperationSecurityLevel::Standard,
|
||||
scopes: key.api_key.scopes.clone(),
|
||||
platform_api_key_id: Some(key.api_key.id.clone()),
|
||||
};
|
||||
if !allows_scope(&credential.scopes, PlatformApiKeyScope::Write) {
|
||||
return Err(TaskError::invalid("task authority scope is insufficient"));
|
||||
}
|
||||
if !credential_allows_security_level(&credential, tool.operation.security_level) {
|
||||
return Err(TaskError::invalid(
|
||||
"task authority security level is insufficient",
|
||||
));
|
||||
}
|
||||
Ok(tool)
|
||||
}
|
||||
|
||||
/// Safe-retry classification of one attempt and the stable upstream key digest.
|
||||
///
|
||||
/// A read is always safe to repeat. A write is only safe to repeat when this process holds
|
||||
/// server-owned evidence for the exact published operation and that evidence's dedup window still
|
||||
/// covers the accepted retry budget; the presence of a client key or an idempotency header alone
|
||||
/// never qualifies.
|
||||
fn retry_classification(
|
||||
&self,
|
||||
operation: &RuntimeOperation,
|
||||
input: &serde_json::Value,
|
||||
retry_window_ms: i64,
|
||||
) -> (bool, Option<String>) {
|
||||
if matches!(
|
||||
&operation.target,
|
||||
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
|
||||
) {
|
||||
return (true, None);
|
||||
}
|
||||
let Some(policy) = operation.execution_config.idempotency.as_ref() else {
|
||||
return (false, None);
|
||||
};
|
||||
let Some(field) = policy
|
||||
.input_field
|
||||
.as_deref()
|
||||
.filter(|field| !field.trim().is_empty())
|
||||
else {
|
||||
return (false, None);
|
||||
};
|
||||
let Some(header) = policy
|
||||
.header_name
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return (false, None);
|
||||
};
|
||||
let key = match input.get(field) {
|
||||
Some(serde_json::Value::String(value)) if !value.trim().is_empty() => value.clone(),
|
||||
Some(serde_json::Value::Number(value)) => value.to_string(),
|
||||
_ => return (false, None),
|
||||
};
|
||||
let evidence = self.verified_writes.iter().find(|evidence| {
|
||||
evidence.operation_id == operation.operation_id.as_str()
|
||||
&& evidence.operation_version == operation.operation_version
|
||||
&& evidence.key_header.eq_ignore_ascii_case(header)
|
||||
&& evidence.validity_ms >= retry_window_ms
|
||||
});
|
||||
let Some(evidence) = evidence else {
|
||||
return (false, None);
|
||||
};
|
||||
// Only a digest of the stable key is persisted; the key itself stays in the encrypted input.
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(evidence.key_header.as_bytes());
|
||||
digest.update([0u8]);
|
||||
digest.update(key.as_bytes());
|
||||
(true, Some(format!("{:x}", digest.finalize())))
|
||||
}
|
||||
|
||||
/// Story 4.5 producer: the pinned operation still lacks a required input item, so the Task must be
|
||||
/// suspended before any dispatch. The schema snapshot and its presentation projection are frozen
|
||||
/// here; the TTL never exceeds the accepted absolute execution deadline.
|
||||
fn input_request(
|
||||
&self,
|
||||
tool: &PublishedAgentTool,
|
||||
lease: &TaskLease,
|
||||
missing_path: &str,
|
||||
expires_at_unix_ms: i64,
|
||||
) -> Result<crank_core::TaskInputRequest, TaskAttemptRefusal> {
|
||||
let schema_snapshot = serde_json::to_value(&tool.operation.input_schema).map_err(|_| {
|
||||
TaskAttemptRefusal::Failed {
|
||||
code: "task_input_schema_unavailable".to_owned(),
|
||||
}
|
||||
})?;
|
||||
let request = crank_core::TaskInputRequest {
|
||||
wait_id: format!("wait-{}", uuid::Uuid::now_v7()),
|
||||
expected_version: lease.record.version.saturating_add(1),
|
||||
schema_revision: tool.operation.version,
|
||||
schema_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot,
|
||||
projected_schema: crate::manifest::schema_to_json_schema(&tool.operation.input_schema),
|
||||
prompt_code: "required_input_missing".to_owned(),
|
||||
missing_fields: vec![missing_path.to_owned()],
|
||||
expires_at_unix_ms,
|
||||
fingerprint_format_version: crank_core::TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
};
|
||||
request.validate().map_err(|_| TaskAttemptRefusal::Failed {
|
||||
code: "task_input_request_invalid".to_owned(),
|
||||
})?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
/// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store
|
||||
/// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal.
|
||||
fn refusal(error: TaskError) -> TaskAttemptRefusal {
|
||||
match error {
|
||||
TaskError::Unavailable { .. } => TaskAttemptRefusal::Stopped,
|
||||
_ => TaskAttemptRefusal::Failed {
|
||||
code: "task_authority_rejected".to_owned(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskHandler for TaskWorkerHandler {
|
||||
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError> {
|
||||
self.authorize(lease)
|
||||
.await
|
||||
.map_err(TaskWorkerError::Store)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome {
|
||||
self.runtime.execute_task_outcome(&context, self).await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskAttemptAuthority for TaskWorkerHandler {
|
||||
async fn prepare_attempt(
|
||||
&self,
|
||||
lease: &TaskLease,
|
||||
) -> Result<PrepareAttempt, TaskAttemptRefusal> {
|
||||
let tool = self.authorize(lease).await.map_err(Self::refusal)?;
|
||||
let input = self
|
||||
.service
|
||||
.executable_input(lease)
|
||||
.await
|
||||
.map_err(Self::refusal)?;
|
||||
let operation = runtime_operation(&tool);
|
||||
let now_ms = current_unix_ms();
|
||||
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
|
||||
if deadline_ms <= now_ms {
|
||||
// The accepted absolute deadline already elapsed: no further attempt may be dispatched,
|
||||
// and any previous uncertainty was already recorded as such.
|
||||
return Err(TaskAttemptRefusal::Failed {
|
||||
code: "task_deadline_elapsed".to_owned(),
|
||||
});
|
||||
}
|
||||
// Only a missing required item of the pinned schema becomes a typed input request. Any other
|
||||
// schema violation keeps the existing pre-dispatch attempt validation, and a generation that
|
||||
// already accepted a continuation can never open a second wait for the same Task.
|
||||
if let Some(missing_path) =
|
||||
crank_runtime::missing_required_input(&tool.operation.input_schema, &input)
|
||||
{
|
||||
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
|
||||
return Err(TaskAttemptRefusal::Failed {
|
||||
code: "task_input_still_incomplete".to_owned(),
|
||||
});
|
||||
}
|
||||
return Ok(PrepareAttempt::InputRequired(self.input_request(
|
||||
&tool,
|
||||
lease,
|
||||
&missing_path,
|
||||
deadline_ms,
|
||||
)?));
|
||||
}
|
||||
let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64);
|
||||
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
|
||||
let agent_id = AgentId::new(lease.record.owner_id.as_str());
|
||||
let resolved_auth = resolve_operation_auth_with(
|
||||
&self.registry,
|
||||
&self.secret_crypto,
|
||||
&workspace_id,
|
||||
&operation.execution_config,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| TaskAttemptRefusal::Failed {
|
||||
code: "task_auth_resolution_failed".to_owned(),
|
||||
})?;
|
||||
// Continue the parent trace persisted with the accepted Task; every attempt adds its own
|
||||
// child `adapter_attempt` span carrying the committed durable attempt ID.
|
||||
let trace_context =
|
||||
TraceContext::parse(&lease.record.support.traceparent).map_err(|_| {
|
||||
TaskAttemptRefusal::Failed {
|
||||
code: "task_trace_parent_invalid".to_owned(),
|
||||
}
|
||||
})?;
|
||||
let request_context = crank_runtime::RuntimeRequestContext::new(
|
||||
lease.record.support.request_id.clone(),
|
||||
trace_context,
|
||||
)
|
||||
.with_metering_context(
|
||||
workspace_id.clone(),
|
||||
Some(agent_id.clone()),
|
||||
crank_core::InvocationSource::AgentToolCall,
|
||||
);
|
||||
let (retry_allowed, upstream_key_digest) =
|
||||
self.retry_classification(&operation, &input, deadline_ms - now_ms);
|
||||
Ok(PrepareAttempt::Ready(Box::new(TaskAttemptPlan {
|
||||
workspace_id,
|
||||
origin: ExecutionOrigin::AgentSnapshot,
|
||||
agent_id: Some(agent_id),
|
||||
operation,
|
||||
input,
|
||||
authorization: ExecutionAuthorization::Authorized,
|
||||
resolved_auth,
|
||||
request_context,
|
||||
deadline,
|
||||
retry_allowed,
|
||||
upstream_key_digest,
|
||||
})))
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,8 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crank_core::{
|
||||
CorrelationContext, TaskAcceptance, TaskAccess, TaskError, TaskId, TaskOwnerKind, TaskRecord,
|
||||
ToolExecutionMode,
|
||||
CorrelationContext, PlatformApiKeyId, TaskAcceptance, TaskAccess, TaskError, TaskId,
|
||||
TaskOwnerKind, TaskRecord, ToolExecutionMode,
|
||||
};
|
||||
use crank_registry::{PostgresRegistry, PublishedAgentTool};
|
||||
use crank_runtime::{TaskAcceptDraft, TaskOutcome, TaskService};
|
||||
@@ -64,6 +64,20 @@ impl TaskPrincipal {
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed immutable Platform API key identity of this principal. A verified agent credential
|
||||
/// without a key ID cannot be revalidated by a worker, so it is refused instead of accepted;
|
||||
/// an administrator has no machine key and is stored as `None` (the worker fails closed).
|
||||
fn platform_api_key_id(&self) -> Result<Option<PlatformApiKeyId>, TaskApplicationError> {
|
||||
match self {
|
||||
Self::Agent { credential, .. } => credential
|
||||
.platform_api_key_id
|
||||
.clone()
|
||||
.map(Some)
|
||||
.ok_or(TaskApplicationError::Denied),
|
||||
Self::Admin { .. } => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded server-derived authorization scope identity of this principal.
|
||||
fn authorization_scope(&self) -> String {
|
||||
let identity = match self {
|
||||
@@ -131,6 +145,17 @@ impl TaskApplication {
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime task service carrying encrypted executable input.
|
||||
pub fn task_service(&self) -> TaskService {
|
||||
self.service.clone()
|
||||
}
|
||||
|
||||
/// Published catalog of the same application entry, for the durable Task worker's pre-dispatch
|
||||
/// revalidation.
|
||||
pub(super) fn catalog(&self) -> PublishedToolCatalog {
|
||||
self.catalog.clone()
|
||||
}
|
||||
|
||||
/// Readiness of the durable Task store this entry depends on.
|
||||
pub async fn health(&self) -> Result<(), TaskError> {
|
||||
self.service.health().await
|
||||
@@ -170,6 +195,7 @@ impl TaskApplication {
|
||||
// of accepting work a worker could never safely execute.
|
||||
return Err(TaskApplicationError::ApprovalRequired);
|
||||
}
|
||||
let platform_api_key_id = principal.platform_api_key_id()?;
|
||||
let draft = |operation_version: u32| {
|
||||
let (owner_kind, owner_id) = principal.owner(&tool);
|
||||
TaskAcceptDraft {
|
||||
@@ -177,6 +203,7 @@ impl TaskApplication {
|
||||
owner_kind,
|
||||
owner_id,
|
||||
authorization_scope: principal.authorization_scope(),
|
||||
platform_api_key_id: platform_api_key_id.clone(),
|
||||
operation_id: tool.operation.id.as_str().to_owned(),
|
||||
operation_version,
|
||||
policy_mode: policy.mode,
|
||||
@@ -186,6 +213,7 @@ impl TaskApplication {
|
||||
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
|
||||
retention_ms: TaskService::default_retention_ms(),
|
||||
approval_reference: None,
|
||||
retry: task_retry_policy(&tool),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -247,6 +275,82 @@ impl TaskApplication {
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped cancellation of one Task of the addressed published Tool (Story 4.4).
|
||||
///
|
||||
/// Cancellation is decided by the durable store under a row lock: before the dispatch checkpoint
|
||||
/// it is an honest cancellation with zero outbound calls, after it the Task becomes the
|
||||
/// non-terminal uncertain outcome instead of a plain cancel or a retry.
|
||||
pub async fn cancel(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
expected_version: Option<u32>,
|
||||
) -> Result<crank_core::TaskCancelOutcome, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self
|
||||
.service
|
||||
.cancel(&access, task_id, expected_version, "user_cancelled")
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies one additional validated input to a waiting Task (Story 4.5).
|
||||
///
|
||||
/// This is the server-internal application entry: the public Task MCP API stays Story 4.7. The
|
||||
/// principal is already authenticated here, the owner scope is derived from the verified
|
||||
/// credential and the durable transaction revalidates the same key before any receipt lookup.
|
||||
/// An identical authorized replay returns the committed outcome; a conflicting or stale answer is
|
||||
/// typed, and an elapsed TTL never revives the Task.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn update_input(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
wait_id: &str,
|
||||
expected_version: u32,
|
||||
update_key: &str,
|
||||
input: &Value,
|
||||
) -> Result<crank_core::TaskInputUpdateOutcome, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self
|
||||
.service
|
||||
.update_input(
|
||||
&access,
|
||||
task_id,
|
||||
wait_id,
|
||||
expected_version,
|
||||
update_key,
|
||||
input,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-scoped attempt history of one Task of the addressed published Tool.
|
||||
pub async fn attempts(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
tool_name: &str,
|
||||
task_id: TaskId,
|
||||
) -> Result<Vec<crank_core::TaskAttempt>, TaskApplicationError> {
|
||||
let access = self.task_access(principal, tool_name).await?;
|
||||
match self.service.attempts(&access, task_id).await {
|
||||
Ok(attempts) => Ok(attempts),
|
||||
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
|
||||
Err(error) => Err(TaskApplicationError::Task(error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn task_access(
|
||||
&self,
|
||||
principal: &TaskPrincipal,
|
||||
@@ -293,3 +397,18 @@ impl TaskApplication {
|
||||
Ok(tool)
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded retry snapshot pinned from the exact published operation at acceptance. A later
|
||||
/// publication, a restart or a new binary never changes the numbers an accepted Task was admitted
|
||||
/// with; a missing or oversized policy is clamped to the closed story bound.
|
||||
fn task_retry_policy(tool: &PublishedAgentTool) -> crank_core::TaskRetryPolicy {
|
||||
let max_attempts = tool
|
||||
.operation
|
||||
.execution_config
|
||||
.retry_policy
|
||||
.as_ref()
|
||||
.map_or(1, |policy| {
|
||||
policy.max_attempts.clamp(1, crank_core::TASK_MAX_ATTEMPTS)
|
||||
});
|
||||
crank_core::TaskRetryPolicy::new(max_attempts, 1)
|
||||
}
|
||||
@@ -415,7 +415,7 @@ pub(super) fn agent_id() -> AgentId {
|
||||
AgentId::new(format!("agent_{AGENT_SLUG}"))
|
||||
}
|
||||
|
||||
pub(super) async fn publish_agent(
|
||||
pub(crate) async fn publish_agent(
|
||||
registry: &PostgresRegistry,
|
||||
operation: &Operation<Schema, MappingSet>,
|
||||
) {
|
||||
@@ -470,7 +470,7 @@ pub(super) async fn publish_agent(
|
||||
.expect("the agent must be published");
|
||||
}
|
||||
|
||||
pub(super) async fn create_api_key(registry: &PostgresRegistry) -> String {
|
||||
pub(crate) async fn create_api_key(registry: &PostgresRegistry) -> String {
|
||||
let secret = format!("crk_dispatch_observer_{}", uuid::Uuid::now_v7().simple());
|
||||
let api_key = PlatformApiKey {
|
||||
id: PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
@@ -500,7 +500,7 @@ pub(super) fn hash_access_secret(secret: &str) -> String {
|
||||
URL_SAFE_NO_PAD.encode(Sha256::digest(secret.as_bytes()))
|
||||
}
|
||||
|
||||
pub(super) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
|
||||
pub(crate) fn test_operation(base_url: &str) -> Operation<Schema, MappingSet> {
|
||||
Operation {
|
||||
id: OperationId::new(format!("op_{TOOL_NAME}")),
|
||||
name: TOOL_NAME.to_owned(),
|
||||
|
||||
@@ -16,7 +16,8 @@ mod tool_search;
|
||||
mod transport;
|
||||
|
||||
pub use app::{
|
||||
TaskApplication, TaskApplicationError, TaskPrincipal, build_app, build_app_with_artifacts,
|
||||
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler,
|
||||
VerifiedIdempotentWrite, build_app, build_app_with_artifacts,
|
||||
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
|
||||
build_app_with_background_workers_and_limits,
|
||||
};
|
||||
@@ -41,6 +41,9 @@ use std::{
|
||||
sync::{Mutex, MutexGuard},
|
||||
};
|
||||
|
||||
// Both pinned revisions still require logging despite the SDK's SEP-2577 deprecation.
|
||||
#[allow(deprecated)]
|
||||
use rmcp::model::{LoggingLevel, LoggingMessageNotificationParam, SetLevelRequestParams};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::task_local;
|
||||
@@ -61,7 +64,8 @@ pub(crate) enum LogLevel {
|
||||
}
|
||||
|
||||
impl LogLevel {
|
||||
/// The exact wire literal of this level.
|
||||
/// The exact wire literal of this level (contract assertions only; runtime uses SDK serde).
|
||||
#[cfg(test)]
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Debug => "debug",
|
||||
@@ -80,17 +84,47 @@ impl LogLevel {
|
||||
/// Nothing is coerced: no case folding, no aliases (`warn`), no numeric severities. A value that
|
||||
/// is not one of the pinned strings is malformed, which is what the pinned `LoggingLevel` type
|
||||
/// says and what `InvalidParamsError` is for.
|
||||
#[allow(deprecated)]
|
||||
pub(crate) fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"debug" => Some(Self::Debug),
|
||||
"info" => Some(Self::Info),
|
||||
"notice" => Some(Self::Notice),
|
||||
"warning" => Some(Self::Warning),
|
||||
"error" => Some(Self::Error),
|
||||
"critical" => Some(Self::Critical),
|
||||
"alert" => Some(Self::Alert),
|
||||
"emergency" => Some(Self::Emergency),
|
||||
_ => None,
|
||||
if value.len() > MAX_LEVEL_BYTES {
|
||||
return None;
|
||||
}
|
||||
let level: LoggingLevel = serde_json::from_value(Value::String(value.to_owned())).ok()?;
|
||||
#[cfg(test)]
|
||||
crate::app::dispatch_observer::sdk_logging_level_parsed();
|
||||
Some(level.into())
|
||||
}
|
||||
}
|
||||
|
||||
// Keep domain severity ordering independent of SDK enum discriminants (which have no Ord).
|
||||
#[allow(deprecated)]
|
||||
impl From<LoggingLevel> for LogLevel {
|
||||
fn from(level: LoggingLevel) -> Self {
|
||||
match level {
|
||||
LoggingLevel::Debug => Self::Debug,
|
||||
LoggingLevel::Info => Self::Info,
|
||||
LoggingLevel::Notice => Self::Notice,
|
||||
LoggingLevel::Warning => Self::Warning,
|
||||
LoggingLevel::Error => Self::Error,
|
||||
LoggingLevel::Critical => Self::Critical,
|
||||
LoggingLevel::Alert => Self::Alert,
|
||||
LoggingLevel::Emergency => Self::Emergency,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
impl From<LogLevel> for LoggingLevel {
|
||||
fn from(level: LogLevel) -> Self {
|
||||
match level {
|
||||
LogLevel::Debug => Self::Debug,
|
||||
LogLevel::Info => Self::Info,
|
||||
LogLevel::Notice => Self::Notice,
|
||||
LogLevel::Warning => Self::Warning,
|
||||
LogLevel::Error => Self::Error,
|
||||
LogLevel::Critical => Self::Critical,
|
||||
LogLevel::Alert => Self::Alert,
|
||||
LogLevel::Emergency => Self::Emergency,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -135,11 +169,20 @@ pub(crate) struct SetLevelCommand {
|
||||
impl SetLevelCommand {
|
||||
/// The bounded input contract of AC2: the level is length-bounded and compared exactly, so a
|
||||
/// malformed or oversized value is refused with bounded work and one safe detail.
|
||||
pub(crate) fn validate(&self) -> Result<LogLevel, &'static str> {
|
||||
#[allow(deprecated)]
|
||||
pub(crate) fn validate(&self) -> Result<SetLevelRequestParams, &'static str> {
|
||||
if self.level.is_empty() || self.level.len() > MAX_LEVEL_BYTES {
|
||||
return Err(INVALID_LEVEL);
|
||||
}
|
||||
LogLevel::parse(&self.level).ok_or(INVALID_LEVEL)
|
||||
// Only level was part of the old command: unknown members (including _meta) stay ignored.
|
||||
let request =
|
||||
serde_json::from_value(json!({"level": self.level})).map_err(|_| INVALID_LEVEL)?;
|
||||
#[cfg(test)]
|
||||
{
|
||||
crate::app::dispatch_observer::sdk_logging_level_parsed();
|
||||
crate::app::dispatch_observer::sdk_logging_dispatch_enter();
|
||||
}
|
||||
Ok(request)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,19 +325,22 @@ pub(crate) fn take_notifications() -> Vec<Value> {
|
||||
}
|
||||
std::mem::take(&mut log.pending)
|
||||
.into_iter()
|
||||
.map(|event| {
|
||||
json!({
|
||||
.filter_map(|event| {
|
||||
#[allow(deprecated)]
|
||||
let params = LoggingMessageNotificationParam::new(
|
||||
event.level.into(),
|
||||
json!({"event": event.event, "outcome": event.outcome}),
|
||||
)
|
||||
.with_logger(LOGGER_NAME);
|
||||
// Logging remains best-effort: a serialization failure drops only this event.
|
||||
let params = serde_json::to_value(params).ok()?;
|
||||
#[cfg(test)]
|
||||
crate::app::dispatch_observer::sdk_logging_notification_serialized();
|
||||
Some(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/message",
|
||||
"params": {
|
||||
"level": event.level.as_str(),
|
||||
"logger": LOGGER_NAME,
|
||||
"data": {
|
||||
"event": event.event,
|
||||
"outcome": event.outcome,
|
||||
}
|
||||
}
|
||||
})
|
||||
"params": params,
|
||||
}))
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
@@ -354,7 +400,9 @@ pub(crate) fn handle_logging_command(
|
||||
command: SetLevelCommand,
|
||||
) -> LoggingOutcome {
|
||||
match command.validate() {
|
||||
Ok(level) => {
|
||||
Ok(request) => {
|
||||
#[allow(deprecated)]
|
||||
let level = request.level.into();
|
||||
store.set(session_id, level);
|
||||
LoggingOutcome::Applied(level)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,8 @@ use super::*;
|
||||
/// is malformed (which the adapter answers with the same safe detail).
|
||||
fn validate(level: &str) -> Option<Result<LogLevel, &'static str>> {
|
||||
let command: SetLevelCommand = serde_json::from_value(json!({ "level": level })).ok()?;
|
||||
Some(command.validate())
|
||||
#[allow(deprecated)]
|
||||
Some(command.validate().map(|request| request.level.into()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -291,8 +292,11 @@ fn the_modern_level_is_read_from_the_pinned_meta_key_only() {
|
||||
fn the_command_records_the_level_of_its_own_session_only() {
|
||||
let store = LogLevelStore::new();
|
||||
store.set("other-session", LogLevel::Debug);
|
||||
let command: SetLevelCommand =
|
||||
serde_json::from_value(json!({ "level": "critical" })).expect("a pinned level parses");
|
||||
// Unknown members were ignored by the legacy command; SDK metadata must not narrow that wire.
|
||||
let command: SetLevelCommand = serde_json::from_value(json!({
|
||||
"level": "critical", "_meta": 7, "unknown": {"private": "ignored"}
|
||||
}))
|
||||
.expect("a pinned level parses regardless of unknown members");
|
||||
let outcome = handle_logging_command(&store, "this-session", command);
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
|
||||
@@ -36,7 +36,10 @@ pub(crate) use command::{
|
||||
ContinuationSupport, ServiceContext, ToolCallCommand, ToolCallOutcome, ToolsCommand,
|
||||
ToolsListCommand, ToolsOutcome, handle_tools_command,
|
||||
};
|
||||
pub(crate) use completion::{CompletionCommand, CompletionOutcome, handle_completion_command};
|
||||
pub(crate) use completion::{
|
||||
CompletionArgument, CompletionCommand, CompletionOutcome, CompletionReference,
|
||||
handle_completion_command,
|
||||
};
|
||||
pub(crate) use logging::{
|
||||
CarriedScope, EVENT_FAILED, EVENT_LEVEL_APPLIED, EVENT_REJECTED, EVENT_SERVED, INVALID_LEVEL,
|
||||
LogLevel, LogLevelStore, LoggingOutcome, SetLevelCommand, carry_scope, handle_logging_command,
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
//! #107 Completion: real build_app HTTP wire controls and a separate SDK-path oracle.
|
||||
use super::{CounterSet, observe, support::*};
|
||||
use axum::{body::Body, response::Response};
|
||||
use futures_util::{StreamExt, stream};
|
||||
use serde_json::{Value, json};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
#[path = "rmcp_completion_test_support.rs"]
|
||||
mod fixtures;
|
||||
use fixtures::{BOUNDED_URI, Endpoint, LARGE_URI, URI};
|
||||
|
||||
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = stream::unfold(
|
||||
(body.into_data_stream(), counters),
|
||||
|(mut body, c)| async move {
|
||||
observe(c.clone(), body.next())
|
||||
.await
|
||||
.map(|chunk| (chunk, (body, c)))
|
||||
},
|
||||
);
|
||||
Response::from_parts(parts, Body::from_stream(body))
|
||||
}
|
||||
fn counts(c: &CounterSet) -> [usize; 2] {
|
||||
[c.sdk_completion_dispatch(), c.sdk_completion_result()]
|
||||
}
|
||||
fn params(reference: Value, name: &str, value: &str) -> Value {
|
||||
json!({"ref":reference,"argument":{"name":name,"value":value}})
|
||||
}
|
||||
fn prompt(name: &str) -> Value {
|
||||
json!({"type":"ref/prompt","name":name})
|
||||
}
|
||||
fn resource(uri: &str) -> Value {
|
||||
json!({"type":"ref/resource","uri":uri})
|
||||
}
|
||||
fn request(revision: &str, mut params: Value) -> Value {
|
||||
if revision == MODERN_REVISION {
|
||||
params["_meta"] = modern_meta();
|
||||
}
|
||||
json!({"jsonrpc":"2.0","id":9007199254740993_u64,"method":"completion/complete","params":params})
|
||||
}
|
||||
async fn wire(response: reqwest::Response, sse: bool) -> Value {
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(response.headers()["x-request-id"], "sdk-completion-wire");
|
||||
assert_eq!(
|
||||
response.headers()["x-trace-id"],
|
||||
"11111111111111111111111111111111"
|
||||
);
|
||||
let content_type = response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let body: Value = if sse {
|
||||
assert!(
|
||||
content_type.starts_with("text/event-stream"),
|
||||
"{content_type}: {text}"
|
||||
);
|
||||
let frames: Vec<Value> = text
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
line.strip_prefix("data:")
|
||||
.map(|s| serde_json::from_str(s.trim()).unwrap())
|
||||
})
|
||||
.collect();
|
||||
let terminal: Vec<_> = frames
|
||||
.iter()
|
||||
.filter(|f| f.get("id") == Some(&json!(9007199254740993_u64)))
|
||||
.collect();
|
||||
assert_eq!(terminal.len(), 1, "{frames:?}");
|
||||
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
|
||||
terminal[0].clone()
|
||||
} else {
|
||||
assert!(
|
||||
content_type.starts_with("application/json"),
|
||||
"{content_type}: {text}"
|
||||
);
|
||||
serde_json::from_str(&text).unwrap()
|
||||
};
|
||||
assert_eq!(body["jsonrpc"], "2.0");
|
||||
assert_eq!(body["id"], json!(9007199254740993_u64));
|
||||
body
|
||||
}
|
||||
async fn success(
|
||||
h: &Endpoint,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
p: Value,
|
||||
sse: bool,
|
||||
) -> (Value, [usize; 2]) {
|
||||
let before = counts(&h.counters);
|
||||
let body = wire(
|
||||
h.post(revision, session, &request(revision, p), sse).await,
|
||||
sse,
|
||||
)
|
||||
.await;
|
||||
assert!(body.get("error").is_none(), "{body}");
|
||||
let result = &body["result"];
|
||||
if revision == MODERN_REVISION {
|
||||
assert_eq!(result["resultType"], "complete", "{body}");
|
||||
} else {
|
||||
assert!(result.get("resultType").is_none(), "{body}");
|
||||
}
|
||||
for key in ["ttlMs", "cacheScope", "_meta"] {
|
||||
assert!(result.get(key).is_none(), "{body}");
|
||||
}
|
||||
let after = counts(&h.counters);
|
||||
(
|
||||
result["completion"].clone(),
|
||||
std::array::from_fn(|i| after[i] - before[i]),
|
||||
)
|
||||
}
|
||||
async fn control(revision: &str) -> Vec<[usize; 2]> {
|
||||
let h = Endpoint::started().await;
|
||||
let session = h.session(revision).await;
|
||||
let session = session.as_deref();
|
||||
let modes: &[bool] = if revision == MODERN_REVISION {
|
||||
&[false, true]
|
||||
} else {
|
||||
&[false]
|
||||
};
|
||||
let mut deltas = Vec::new();
|
||||
for &sse in modes {
|
||||
for (p, expected) in [
|
||||
(
|
||||
params(prompt("greeting"), "tone", ""),
|
||||
json!({"values":["warm","dry"],"total":2}),
|
||||
),
|
||||
(
|
||||
params(prompt("archived-prompt"), "tone", "W"),
|
||||
json!({"values":["warm"],"total":1}),
|
||||
),
|
||||
(
|
||||
params(prompt("greeting"), "name", "A"),
|
||||
json!({"values":[],"total":0}),
|
||||
),
|
||||
(
|
||||
params(resource(URI), "region", ""),
|
||||
json!({"values":["eu","us","apac"],"total":3}),
|
||||
),
|
||||
(
|
||||
params(resource(URI), "region", "A"),
|
||||
json!({"values":["apac"],"total":1}),
|
||||
),
|
||||
(
|
||||
params(resource(URI), "region", "zzz"),
|
||||
json!({"values":[],"total":0}),
|
||||
),
|
||||
] {
|
||||
let (completion, delta) = success(&h, revision, session, p, sse).await;
|
||||
assert_eq!(completion, expected);
|
||||
deltas.push(delta);
|
||||
}
|
||||
for (uri, total) in [(BOUNDED_URI, Some(150)), (LARGE_URI, None)] {
|
||||
let (completion, delta) = success(
|
||||
&h,
|
||||
revision,
|
||||
session,
|
||||
params(resource(uri), "region", "item"),
|
||||
sse,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
completion["values"],
|
||||
json!((0..100).map(|i| format!("item{i:04}")).collect::<Vec<_>>())
|
||||
);
|
||||
assert_eq!(completion["hasMore"], true);
|
||||
if let Some(total) = total {
|
||||
assert_eq!(completion["total"], total);
|
||||
} else {
|
||||
assert!(
|
||||
completion.get("total").is_none(),
|
||||
"partial scan: {completion}"
|
||||
);
|
||||
}
|
||||
deltas.push(delta);
|
||||
}
|
||||
let mut with_context = params(prompt("greeting"), "tone", "w");
|
||||
with_context["context"] = json!({"arguments":{"name":"Ada","tone":"dry"}});
|
||||
let (completion, delta) = success(&h, revision, session, with_context, sse).await;
|
||||
assert_eq!(completion, json!({"values":["warm"],"total":1}));
|
||||
deltas.push(delta);
|
||||
let mut unavailable = None;
|
||||
for p in [
|
||||
params(prompt("draft-prompt"), "tone", ""),
|
||||
params(prompt("unbound-prompt"), "tone", ""),
|
||||
params(prompt("other-agent-prompt"), "tone", ""),
|
||||
params(prompt("never-existed"), "tone", ""),
|
||||
params(prompt("changed-draft"), "tone", ""),
|
||||
params(prompt("greeting"), "unknown-argument", ""),
|
||||
params(resource("crank://foreign/{region}"), "region", ""),
|
||||
params(resource(URI), "unknown-parameter", ""),
|
||||
] {
|
||||
let body = wire(
|
||||
h.post(revision, session, &request(revision, p), sse).await,
|
||||
sse,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body["error"]["code"], -32602, "{body}");
|
||||
assert_eq!(
|
||||
body["error"]["message"],
|
||||
"the completion reference is not available"
|
||||
);
|
||||
assert!(body.get("result").is_none());
|
||||
if let Some(ref previous) = unavailable {
|
||||
assert_eq!(&body, previous);
|
||||
}
|
||||
unavailable = Some(body);
|
||||
}
|
||||
let mut invalid = vec![
|
||||
json!({"argument":{"name":"tone","value":""}}),
|
||||
params(json!({"type":"ref/tool","name":"greeting"}), "tone", ""),
|
||||
params(prompt("greeting"), "tone", &"s".repeat(1025)),
|
||||
params(prompt("greeting"), &"n".repeat(129), ""),
|
||||
];
|
||||
// Preserve domain validation on arbitrary JSON, never drop or stringify context values for SDK.
|
||||
for value in [
|
||||
json!(7),
|
||||
json!(true),
|
||||
json!(null),
|
||||
json!(["eu"]),
|
||||
json!({"region":"eu"}),
|
||||
json!("s".repeat(1025)),
|
||||
] {
|
||||
let mut p = params(prompt("greeting"), "tone", "");
|
||||
p["context"] = json!({"arguments":{"secret-context":value}});
|
||||
invalid.push(p);
|
||||
}
|
||||
let mut p = params(prompt("greeting"), "tone", "");
|
||||
p["context"] = json!({"arguments":(0..33).map(|i| (format!("arg{i}"), json!("eu"))).collect::<serde_json::Map<_,_>>()});
|
||||
invalid.push(p);
|
||||
for p in invalid {
|
||||
let body = wire(
|
||||
h.post(revision, session, &request(revision, p), sse).await,
|
||||
sse,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body["error"]["code"], -32602, "{body}");
|
||||
assert_eq!(
|
||||
body["error"]["message"],
|
||||
"invalid completion/complete parameters"
|
||||
);
|
||||
assert!(body.get("result").is_none());
|
||||
for leaked in ["secret-context", &"s".repeat(1025), &"n".repeat(129)] {
|
||||
assert!(!body.to_string().contains(leaked), "{body}");
|
||||
}
|
||||
}
|
||||
}
|
||||
h.registry
|
||||
.revoke_platform_api_key(
|
||||
&workspace_id(),
|
||||
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
&time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let before = h.counters.snapshot();
|
||||
let sdk_before = counts(&h.counters);
|
||||
let response = h
|
||||
.post(
|
||||
revision,
|
||||
session,
|
||||
&request(revision, params(prompt("greeting"), "tone", "")),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 401);
|
||||
let _ = response.bytes().await.unwrap();
|
||||
assert_eq!(
|
||||
counts(&h.counters),
|
||||
sdk_before,
|
||||
"revoked key must not dispatch to SDK"
|
||||
);
|
||||
let after = h.counters.snapshot();
|
||||
assert_eq!(after.era_modern, before.era_modern);
|
||||
assert_eq!(after.era_legacy, before.era_legacy);
|
||||
assert_eq!(after.tools_service, before.tools_service);
|
||||
eprintln!(
|
||||
"existing GREEN Completion HTTP {revision}: Prompt+Resource refs, exact bound v1/archive, prefix/empty/bounded/truncated, JSON/context/errors/correlation/auth; SDK actual={deltas:?}, expected=[1,1] per success"
|
||||
);
|
||||
deltas
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_completion_wire_legacy() {
|
||||
control(LEGACY_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_completion_wire_modern() {
|
||||
control(MODERN_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_completion_sdk_legacy() {
|
||||
let deltas = control(LEGACY_REVISION).await;
|
||||
assert!(
|
||||
deltas.iter().all(|d| *d == [1, 1]),
|
||||
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_completion_sdk_modern() {
|
||||
let deltas = control(MODERN_REVISION).await;
|
||||
assert!(
|
||||
deltas.iter().all(|d| *d == [1, 1]),
|
||||
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_completion_observer_scope_control() {
|
||||
use super::{sdk_completion_dispatch_enter, sdk_completion_result_serialized};
|
||||
let a = Arc::new(CounterSet::default());
|
||||
let b = Arc::new(CounterSet::default());
|
||||
sdk_completion_dispatch_enter();
|
||||
sdk_completion_result_serialized();
|
||||
assert_eq!(counts(&a), [0; 2]);
|
||||
let snapshot = a.snapshot();
|
||||
tokio::join!(
|
||||
observe(a.clone(), async {
|
||||
sdk_completion_dispatch_enter();
|
||||
tokio::task::yield_now().await;
|
||||
sdk_completion_result_serialized();
|
||||
}),
|
||||
observe(b.clone(), async {
|
||||
sdk_completion_result_serialized();
|
||||
})
|
||||
);
|
||||
assert_eq!(counts(&a), [1, 1]);
|
||||
assert_eq!(counts(&b), [0, 1]);
|
||||
assert_eq!(a.snapshot(), snapshot);
|
||||
let body = Body::from_stream(stream::once(async {
|
||||
sdk_completion_result_serialized();
|
||||
Ok::<_, std::convert::Infallible>("data: terminal\n\n")
|
||||
}));
|
||||
let response = observe_body(Response::new(body), b.clone());
|
||||
assert_eq!(counts(&b), [0, 1], "body must be deferred");
|
||||
assert_eq!(
|
||||
&axum::body::to_bytes(response.into_body(), 100)
|
||||
.await
|
||||
.unwrap()[..],
|
||||
b"data: terminal\n\n"
|
||||
);
|
||||
assert_eq!(counts(&b), [0, 2]);
|
||||
assert_eq!(counts(&a), [1, 1]);
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
//! Completion HTTP fixture using the existing observed HTTP app and disposable PostgreSQL.
|
||||
use super::*;
|
||||
use crank_core::prompt::{PromptDefinition, PromptSensitivity};
|
||||
use crank_core::{
|
||||
Agent, AgentStatus, AgentVersion, HttpMethod, RestTarget, Target, ToolSelectionPolicy,
|
||||
ids::PromptId,
|
||||
};
|
||||
use crank_registry::PostgresRegistry;
|
||||
use crank_registry::model::{
|
||||
AgentPromptBinding, ArchivePromptRequest, CreatePromptDraftRequest, PublishPromptRequest,
|
||||
SaveAgentCatalogConfigRequest, UpdatePromptDraftRequest,
|
||||
};
|
||||
use crank_registry::{CreateAgentRequest, PublishAgentRequest};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
|
||||
};
|
||||
|
||||
pub(super) fn definition(name: &str, content: &str) -> PromptDefinition {
|
||||
PromptDefinition::parse(&json!({
|
||||
"name":name, "description":format!("Published {name}"),
|
||||
"arguments":[
|
||||
{"name":"name","schema":{"type":"string"},"required":true},
|
||||
{"name":"count","schema":{"type":"integer"},"required":false,"default":2},
|
||||
{"name":"enabled","schema":{"type":"boolean"},"required":false,"default":true},
|
||||
{"name":"tone","schema":{"type":"string","enum":["warm","dry"]},"required":false,"default":"warm"}
|
||||
],
|
||||
"messages":[{"role":"system","content":"Be brief."},{"role":"user","content":content}]
|
||||
})).unwrap()
|
||||
}
|
||||
|
||||
pub(super) struct Endpoint {
|
||||
pub registry: crank_registry::PostgresRegistry,
|
||||
pub counters: Arc<CounterSet>,
|
||||
pub key: String,
|
||||
pub url: String,
|
||||
server: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
impl Drop for Endpoint {
|
||||
fn drop(&mut self) {
|
||||
self.server.abort();
|
||||
}
|
||||
}
|
||||
impl Endpoint {
|
||||
pub async fn started() -> Self {
|
||||
crank_test_support::postgres_schema_url_for_task("completion_owner", "t_3fa1ad63").await;
|
||||
let registry = test_registry().await;
|
||||
publish_region_operation(®istry, "completion_lookup").await;
|
||||
for (id, uri, size) in [
|
||||
("res_completion_small", URI, 3),
|
||||
("res_completion_large", LARGE_URI, 1025),
|
||||
("res_completion_bounded", BOUNDED_URI, 150),
|
||||
] {
|
||||
publish_template(®istry, &ResourceId::new(id), uri, size).await;
|
||||
}
|
||||
let mut archived_at = test_time();
|
||||
for (id, name, publish) in [
|
||||
("prm_a", "greeting", true),
|
||||
("prm_b", "archived-prompt", true),
|
||||
("prm_c", "unbound-prompt", true),
|
||||
("prm_d", "draft-prompt", false),
|
||||
("prm_e", "other-agent-prompt", true),
|
||||
] {
|
||||
let row = registry
|
||||
.create_prompt_draft(CreatePromptDraftRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new(id),
|
||||
sensitivity: PromptSensitivity::Internal,
|
||||
definition: definition(
|
||||
name,
|
||||
"Hello {{name}} count={{count}} enabled={{enabled}} tone={{tone}}",
|
||||
),
|
||||
created_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if publish {
|
||||
let row = registry
|
||||
.publish_prompt_draft(PublishPromptRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new(id),
|
||||
expected_updated_at: row.updated_at,
|
||||
published_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if id == "prm_b" {
|
||||
archived_at = row.updated_at;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (id, slug) in [
|
||||
(agent_id(), AGENT_SLUG),
|
||||
(
|
||||
crank_core::AgentId::new("agent_other_prompts"),
|
||||
"other-prompts",
|
||||
),
|
||||
] {
|
||||
registry
|
||||
.create_agent(CreateAgentRequest {
|
||||
agent: &Agent {
|
||||
id: id.clone(),
|
||||
workspace_id: workspace_id(),
|
||||
slug: slug.into(),
|
||||
display_name: "SDK Prompts".into(),
|
||||
description: String::new(),
|
||||
status: AgentStatus::Draft,
|
||||
current_draft_version: 1,
|
||||
latest_published_version: None,
|
||||
created_at: test_time(),
|
||||
updated_at: test_time(),
|
||||
published_at: None,
|
||||
},
|
||||
version: &AgentVersion {
|
||||
agent_id: id,
|
||||
version: 1,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: test_time(),
|
||||
},
|
||||
bindings: &[],
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let other_agent = crank_core::AgentId::new("agent_other_prompts");
|
||||
registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &other_agent,
|
||||
agent_version: 1,
|
||||
bindings: &[],
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[],
|
||||
expected_state: None,
|
||||
},
|
||||
&[AgentPromptBinding {
|
||||
prompt_id: PromptId::new("prm_e"),
|
||||
prompt_version: 1,
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &other_agent,
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
agent_version: 1,
|
||||
bindings: &[],
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[
|
||||
AgentResourceBinding {
|
||||
resource_id: ResourceId::new("res_completion_small"),
|
||||
resource_version: 1,
|
||||
},
|
||||
AgentResourceBinding {
|
||||
resource_id: ResourceId::new("res_completion_large"),
|
||||
resource_version: 1,
|
||||
},
|
||||
AgentResourceBinding {
|
||||
resource_id: ResourceId::new("res_completion_bounded"),
|
||||
resource_version: 1,
|
||||
},
|
||||
],
|
||||
expected_state: None,
|
||||
},
|
||||
&[
|
||||
AgentPromptBinding {
|
||||
prompt_id: PromptId::new("prm_a"),
|
||||
prompt_version: 1,
|
||||
},
|
||||
AgentPromptBinding {
|
||||
prompt_id: PromptId::new("prm_b"),
|
||||
prompt_version: 1,
|
||||
},
|
||||
],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.archive_prompt_draft(ArchivePromptRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new("prm_b"),
|
||||
expected_updated_at: archived_at,
|
||||
archived_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
// A mutable v2 draft must not change the bound v1 rendering or discovery metadata.
|
||||
let draft = registry
|
||||
.get_prompt_draft(&workspace_id(), &PromptId::new("prm_a"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.update_prompt_draft(UpdatePromptDraftRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new("prm_a"),
|
||||
sensitivity: PromptSensitivity::Internal,
|
||||
definition: definition("changed-draft", "Changed {{name}}"),
|
||||
expected_updated_at: draft.updated_at,
|
||||
updated_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let key = create_api_key(®istry).await;
|
||||
let counters = Arc::new(CounterSet::default());
|
||||
let app = crate::build_app(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
None,
|
||||
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
|
||||
crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
Arc::new(crate::session::InMemorySessionStore::default()),
|
||||
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
|
||||
);
|
||||
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
|
||||
let counters = counters.clone();
|
||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||
let counters = counters.clone();
|
||||
async move { observe_body(next.run(request).await, counters) }
|
||||
}
|
||||
}));
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!(
|
||||
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
|
||||
listener.local_addr().unwrap()
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Self {
|
||||
registry,
|
||||
counters,
|
||||
key,
|
||||
url,
|
||||
server,
|
||||
}
|
||||
}
|
||||
pub async fn post(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
body: &Value,
|
||||
sse: bool,
|
||||
) -> reqwest::Response {
|
||||
let mut request = reqwest::Client::new()
|
||||
.post(&self.url)
|
||||
.bearer_auth(&self.key)
|
||||
.header(
|
||||
"Accept",
|
||||
if sse {
|
||||
"text/event-stream, application/json"
|
||||
} else {
|
||||
"application/json"
|
||||
},
|
||||
)
|
||||
.header("x-request-id", "sdk-completion-wire")
|
||||
.header(
|
||||
"traceparent",
|
||||
"00-11111111111111111111111111111111-2222222222222222-01",
|
||||
)
|
||||
.header("MCP-Protocol-Version", revision);
|
||||
if let Some(session) = session {
|
||||
request = request.header("MCP-Session-Id", session);
|
||||
}
|
||||
if revision == MODERN_REVISION {
|
||||
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
|
||||
}
|
||||
request.json(body).send().await.unwrap()
|
||||
}
|
||||
pub async fn session(&self, revision: &str) -> Option<String> {
|
||||
if revision == MODERN_REVISION {
|
||||
return None;
|
||||
}
|
||||
let response = self
|
||||
.post(revision, None, &legacy_initialize(1), false)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let session = response.headers()["MCP-Session-Id"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
assert_eq!(
|
||||
json_body(response).await["result"]["protocolVersion"],
|
||||
revision
|
||||
);
|
||||
assert_eq!(
|
||||
self.post(revision, Some(&session), &initialized_notification(), false)
|
||||
.await
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
Some(session)
|
||||
}
|
||||
}
|
||||
|
||||
use crank_core::{ids::ResourceId, resource::TemplateSource};
|
||||
use crank_mapping::{MappingRule, MappingSet};
|
||||
use crank_registry::model::{
|
||||
CreateResourceTemplateDraftRequest, PublishResourceTemplateVersionRequest,
|
||||
};
|
||||
use crank_registry::{AgentResourceBinding, PublishRequest};
|
||||
use sha2::{Digest, Sha256};
|
||||
pub(super) const URI: &str = "crank://completion-orders/{region}";
|
||||
pub(super) const LARGE_URI: &str = "crank://completion-large/{region}";
|
||||
pub(super) const BOUNDED_URI: &str = "crank://completion-bounded/{region}";
|
||||
async fn publish_region_operation(registry: &PostgresRegistry, name: &str) {
|
||||
let mut operation = test_operation("http://127.0.0.1:9");
|
||||
operation.id = crank_core::OperationId::new(format!("op_{name}"));
|
||||
operation.name = name.to_owned();
|
||||
operation.display_name = "Region lookup".to_owned();
|
||||
operation.target = Target::Rest(RestTarget {
|
||||
base_url: "http://127.0.0.1:9".to_owned(),
|
||||
method: HttpMethod::Get,
|
||||
path_template: "/regions/{region}".to_owned(),
|
||||
static_headers: Default::default(),
|
||||
request_contract: None,
|
||||
response_contract: Vec::new(),
|
||||
multipart_field_mime_types: Default::default(),
|
||||
});
|
||||
operation.input_schema = object_schema("region");
|
||||
operation
|
||||
.input_schema
|
||||
.fields
|
||||
.get_mut("region")
|
||||
.unwrap()
|
||||
.enum_values = vec!["eu".to_owned(), "us".to_owned(), "apac".to_owned()];
|
||||
operation.input_mapping = MappingSet {
|
||||
rules: vec![MappingRule {
|
||||
source: "$.mcp.region".to_owned(),
|
||||
target: "$.request.path.region".to_owned(),
|
||||
required: true,
|
||||
default_value: None,
|
||||
transform: None,
|
||||
condition: None,
|
||||
notes: None,
|
||||
}],
|
||||
};
|
||||
registry
|
||||
.create_operation(&workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.expect("the Read Operation is created through the registry API");
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &time::OffsetDateTime::now_utc(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.expect("the Read Operation exact version publishes");
|
||||
}
|
||||
|
||||
/// One published Resource template with an enumerated `region` parameter.
|
||||
async fn publish_template(
|
||||
registry: &PostgresRegistry,
|
||||
resource_id: &ResourceId,
|
||||
uri_template: &str,
|
||||
size: usize,
|
||||
) {
|
||||
let operation_name = "completion_lookup";
|
||||
let options: Vec<String> = if size == 3 {
|
||||
vec!["eu".into(), "us".into(), "apac".into()]
|
||||
} else {
|
||||
(0..size).map(|i| format!("item{i:04}")).collect()
|
||||
};
|
||||
let definition = json!({
|
||||
"operation_id": format!("op_{operation_name}"),
|
||||
"operation_version": 1,
|
||||
"uri_template": uri_template,
|
||||
"parameters": [{
|
||||
"name": "region",
|
||||
"input_path": "input.region",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"required": true,
|
||||
"enum_values": options
|
||||
}
|
||||
}],
|
||||
"cache": {"ttl_ms": 60_000, "vary": ["region"], "max_bytes": 65_536},
|
||||
"failure": {"outcome": "error"}
|
||||
});
|
||||
let source = TemplateSource::parse(&definition)
|
||||
.expect("the template fixture satisfies the Story 3.10 source contract");
|
||||
let definition_json =
|
||||
serde_json::to_string(&source).expect("the normalized definition serializes canonically");
|
||||
let digest = format!("{:x}", Sha256::digest(definition_json.as_bytes()));
|
||||
let draft = registry
|
||||
.create_resource_template_draft(CreateResourceTemplateDraftRequest {
|
||||
workspace_id: workspace_id(),
|
||||
resource_id: resource_id.clone(),
|
||||
uri: format!("crank://templates/{}", resource_id.as_str()),
|
||||
name: format!("{}_template", resource_id.as_str()),
|
||||
description: "Bound completion fixture template".to_owned(),
|
||||
kind: "text".to_owned(),
|
||||
mime_type: "application/json".to_owned(),
|
||||
sensitivity: "internal".to_owned(),
|
||||
definition_json: definition_json.clone(),
|
||||
definition_digest: digest.clone(),
|
||||
created_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("the template Draft is created through the registry API");
|
||||
registry
|
||||
.publish_resource_template_version(PublishResourceTemplateVersionRequest {
|
||||
workspace_id: workspace_id(),
|
||||
resource_id: resource_id.clone(),
|
||||
expected_updated_at: draft.snapshot.resource.updated_at,
|
||||
expected_version: draft.snapshot.version.version,
|
||||
expected_digest: digest,
|
||||
published_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.expect("the template exact version publishes");
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_catalog_follows_new_publication_without_reinitializing() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let list = json!({"jsonrpc":"2.0","id":60,"method":"tools/list"});
|
||||
let listed = frames(
|
||||
h.request("2025-11-25", Some(&session), &list)
|
||||
.send()
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
listed.last().unwrap()["result"]["tools"][0]["name"],
|
||||
"create_lead"
|
||||
);
|
||||
let workspace = crank_core::WorkspaceId::new("ws_default");
|
||||
let agent = crank_core::AgentId::new("agent_dispatch-observer");
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let version = crank_core::AgentVersion {
|
||||
agent_id: agent.clone(),
|
||||
version: 2,
|
||||
status: crank_core::AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: crank_core::ToolSelectionPolicy::default(),
|
||||
created_at: now,
|
||||
};
|
||||
let bindings = [crank_core::AgentOperationBinding {
|
||||
operation_id: crank_core::OperationId::new("op_create_lead"),
|
||||
operation_version: 1,
|
||||
tool_name: "new_publication_tool".into(),
|
||||
tool_title: "New Tool".into(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}];
|
||||
h.state
|
||||
.registry
|
||||
.create_agent_draft_version(crank_registry::CreateAgentDraftVersionRequest {
|
||||
workspace_id: &workspace,
|
||||
agent_id: &agent,
|
||||
version: &version,
|
||||
bindings: &bindings,
|
||||
updated_at: &now,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
h.state
|
||||
.registry
|
||||
.publish_agent(crank_registry::PublishAgentRequest {
|
||||
workspace_id: &workspace,
|
||||
agent_id: &agent,
|
||||
version: 2,
|
||||
published_at: &now,
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let listed = frames(
|
||||
h.request("2025-11-25", Some(&session), &list)
|
||||
.send()
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
listed.last().unwrap()["result"]["tools"][0]["name"],
|
||||
"new_publication_tool"
|
||||
);
|
||||
let mut message = call(61, "new-publication@example.test", false, false);
|
||||
message["params"]["name"] = json!("new_publication_tool");
|
||||
let result = frames(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
result.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_new-publication@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_identical_request_ids_in_two_sessions_cancel_independently() {
|
||||
let h = Harness::new().await;
|
||||
let a = h.initialize().await;
|
||||
let b = h.initialize().await;
|
||||
let message = call(62, "slow-same-id@example.test", false, true);
|
||||
let first = h
|
||||
.request("2025-11-25", Some(&a), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let second = h
|
||||
.request("2025-11-25", Some(&b), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
wait_count(&h.calls, 2).await;
|
||||
let cancel =
|
||||
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":62}});
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&a), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
wait_count(&h.active, 1).await;
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&b), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
wait_count(&h.active, 0).await;
|
||||
for response in [first, second] {
|
||||
assert_eq!(
|
||||
frames(response).await.last().unwrap()["error"]["message"],
|
||||
"request cancelled"
|
||||
);
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
// Independent QA: dropped legacy response bodies must not bypass the stream ceiling.
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_qa_legacy_disconnect_does_not_bypass_admission() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let mut admitted = Vec::new();
|
||||
for id in 100..117 {
|
||||
let response = h
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&call(id, "slow-qa-limit@example.test", false, true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
if response.status() == 429 {
|
||||
break;
|
||||
}
|
||||
assert_eq!(response.status(), 200);
|
||||
admitted.push(id);
|
||||
wait_count(&h.calls, admitted.len()).await;
|
||||
drop(response);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
}
|
||||
let active_before_cleanup = h.active.load(Ordering::SeqCst);
|
||||
let free_slots = h.state.session_slots.available_permits();
|
||||
let calls_before_cleanup = h.calls.load(Ordering::SeqCst);
|
||||
for id in &admitted {
|
||||
let cancel =
|
||||
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":id}});
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
}
|
||||
wait_count(&h.active, 0).await;
|
||||
assert!(
|
||||
admitted.len() <= 16 && calls_before_cleanup <= 16 && active_before_cleanup <= 16,
|
||||
"legacy disconnect bypassed admission: admitted={}, calls={calls_before_cleanup}, active={active_before_cleanup}, free_slots={free_slots}",
|
||||
admitted.len()
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while h.state.session_slots.available_permits() != 16 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned cancellation cleanup must release all admission permits");
|
||||
assert_eq!(h.state.session_slots.available_permits(), 16);
|
||||
|
||||
// After cleanup, normal admission and REST execution must work again.
|
||||
let calls_before = h.calls.load(Ordering::SeqCst);
|
||||
let message = call(199, "recovery-after-cleanup@example.test", false, false);
|
||||
let response = h
|
||||
.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let result = frames(response).await;
|
||||
assert_eq!(
|
||||
result.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_recovery-after-cleanup@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), calls_before + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_correlation_headers_preserve_each_concurrent_request() {
|
||||
let h = Harness::new().await;
|
||||
let first = call(50, "correlation-first@example.test", true, false);
|
||||
let second = call(51, "correlation-second@example.test", true, false);
|
||||
let (a, b) = tokio::join!(
|
||||
h.request("2026-07-28", None, &first)
|
||||
.header("x-request-id", "rmcp-oracle-first")
|
||||
.send(),
|
||||
h.request("2026-07-28", None, &second)
|
||||
.header("x-request-id", "rmcp-oracle-second")
|
||||
.send()
|
||||
);
|
||||
let a = a.unwrap();
|
||||
let b = b.unwrap();
|
||||
assert_eq!(a.headers()["x-request-id"], "rmcp-oracle-first");
|
||||
assert_eq!(b.headers()["x-request-id"], "rmcp-oracle-second");
|
||||
assert_eq!(
|
||||
frames(a).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_correlation-first@example.test"
|
||||
);
|
||||
assert_eq!(
|
||||
frames(b).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_correlation-second@example.test"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_stream_limit_rejects_without_dispatch_then_recovers() {
|
||||
let h = Harness::new().await;
|
||||
let permits = h
|
||||
.state
|
||||
.session_slots
|
||||
.clone()
|
||||
.acquire_many_owned(16)
|
||||
.await
|
||||
.unwrap();
|
||||
let message = call(40, "limit@example.test", true, false);
|
||||
assert_eq!(
|
||||
h.request("2026-07-28", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
429
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
drop(permits);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
frames(response).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_limit@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_rate_limit_is_existing_admission_not_sdk_default() {
|
||||
let h = Harness::with_rate(1).await;
|
||||
let list = json!({"jsonrpc":"2.0","id":41,"method":"tools/list","params":{"_meta":meta()}});
|
||||
assert_eq!(
|
||||
h.request("2026-07-28", None, &list)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
200
|
||||
);
|
||||
assert_eq!(
|
||||
h.request(
|
||||
"2026-07-28",
|
||||
None,
|
||||
&call(42, "rate@example.test", true, false)
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
429
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_current_durable_store_survives_recreation_but_is_not_sdk_store() {
|
||||
use crate::session::{PostgresTransportSessionStore, TransportSessionStore};
|
||||
let h = Harness::new().await;
|
||||
let store = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let id = store
|
||||
.create(
|
||||
"2025-11-25",
|
||||
"default",
|
||||
"dispatch-observer",
|
||||
false,
|
||||
now,
|
||||
Some(now + time::Duration::hours(24)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(store.mark_initialized(&id, now).await.unwrap());
|
||||
drop(store);
|
||||
let restarted = PostgresTransportSessionStore::from_pool(h.state.registry.pool().clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let persisted = restarted.get(&id).await.unwrap().unwrap();
|
||||
assert!(persisted.initialized);
|
||||
assert_eq!(persisted.workspace_slug, "default");
|
||||
assert_eq!(persisted.protocol_version, "2025-11-25");
|
||||
assert_eq!(
|
||||
h.request(
|
||||
"2025-11-25",
|
||||
Some(&id),
|
||||
&call(43, "durable@example.test", false, false)
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
404
|
||||
);
|
||||
assert!(
|
||||
restarted.get(&id).await.unwrap().is_some(),
|
||||
"SDK experiment never mutates existing storage"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_json_modern_and_legacy_sse_only_gap_are_observed() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let response = h
|
||||
.request(
|
||||
"2026-07-28",
|
||||
None,
|
||||
&call(44, "json@example.test", true, false),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("application/json")
|
||||
);
|
||||
assert_eq!(
|
||||
frames(response).await[0]["result"]["structuredContent"]["id"],
|
||||
"lead_for_json@example.test"
|
||||
);
|
||||
// Build a JSON-only request (not two contradictory Accept carriers).
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/dispatch-observer", h.server.1))
|
||||
.bearer_auth(&h.key)
|
||||
.header("Accept", "application/json")
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("MCP-Session-Id", &session)
|
||||
.json(&call(45, "legacy-json@example.test", false, false))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
406,
|
||||
"bundled SDK legacy mode requires SSE Accept; not transport parity"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_concurrent_keys_keep_outputs_and_cancel_ownership_separate() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
|
||||
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
|
||||
let first = call(46, "first@example.test", false, false);
|
||||
let second = call(47, "second@example.test", false, false);
|
||||
let (a, b) = tokio::join!(
|
||||
h.request("2025-11-25", Some(&session), &first).send(),
|
||||
h.request_as(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&second,
|
||||
"test-second-valid-key"
|
||||
)
|
||||
.send()
|
||||
);
|
||||
assert_eq!(
|
||||
frames(a.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_first@example.test"
|
||||
);
|
||||
assert_eq!(
|
||||
frames(b.unwrap()).await.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_second@example.test"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
//! Wire oracle for the isolated SDK adapter, with real registry and REST execution.
|
||||
#[path = "rmcp_http_limits.rs"]
|
||||
mod limits;
|
||||
use super::{AgentRoutePath, dispatch_observer::support, rmcp_http, rmcp_tools_tests::test_state};
|
||||
use axum::{Json, Router, extract::State, routing::post};
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
struct Server(tokio::task::JoinHandle<()>, String);
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
async fn serve(router: Router) -> Server {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
Server(
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
}),
|
||||
url,
|
||||
)
|
||||
}
|
||||
async fn upstream(State(count): State<Arc<AtomicUsize>>, Json(body): Json<Value>) -> Json<Value> {
|
||||
count.fetch_add(1, Ordering::SeqCst);
|
||||
if body["email"].as_str().unwrap().starts_with("slow") {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
|
||||
}
|
||||
Json(json!({"id": format!("lead_for_{}", body["email"].as_str().unwrap())}))
|
||||
}
|
||||
struct Harness {
|
||||
server: Server,
|
||||
_upstream: Server,
|
||||
key: String,
|
||||
calls: Arc<AtomicUsize>,
|
||||
state: Arc<super::AppState>,
|
||||
active: Arc<AtomicUsize>,
|
||||
}
|
||||
impl Harness {
|
||||
async fn new() -> Self {
|
||||
Self::with_rate(10_000).await
|
||||
}
|
||||
async fn with_rate(burst: u32) -> Self {
|
||||
let mut state = test_state().await;
|
||||
Arc::get_mut(&mut state).unwrap().api_rate_limiter = crank_runtime::RequestRateLimiter::new(
|
||||
crank_runtime::RequestRateLimitConfig::new(burst, burst).unwrap(),
|
||||
);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream = serve(
|
||||
Router::new()
|
||||
.route("/crm/leads", post(upstream))
|
||||
.with_state(calls.clone()),
|
||||
)
|
||||
.await;
|
||||
let mut operation = support::test_operation(&upstream.1);
|
||||
operation.execution_config.timeout_ms = 10_000;
|
||||
state
|
||||
.registry
|
||||
.create_operation(
|
||||
&crank_core::WorkspaceId::new("ws_default"),
|
||||
&operation,
|
||||
Some("alice"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
state
|
||||
.registry
|
||||
.publish_operation(crank_registry::PublishRequest {
|
||||
workspace_id: &crank_core::WorkspaceId::new("ws_default"),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &operation.created_at,
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
support::publish_agent(&state.registry, &operation).await;
|
||||
let key = support::create_api_key(&state.registry).await;
|
||||
let active = Arc::new(AtomicUsize::new(0));
|
||||
let server = serve(rmcp_http::router(
|
||||
state.clone(),
|
||||
AgentRoutePath {
|
||||
workspace_slug: "default".into(),
|
||||
agent_slug: "dispatch-observer".into(),
|
||||
},
|
||||
active.clone(),
|
||||
))
|
||||
.await;
|
||||
Self {
|
||||
server,
|
||||
_upstream: upstream,
|
||||
key,
|
||||
calls,
|
||||
state,
|
||||
active,
|
||||
}
|
||||
}
|
||||
fn request(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
message: &Value,
|
||||
) -> reqwest::RequestBuilder {
|
||||
self.request_as(revision, session, message, &self.key)
|
||||
}
|
||||
fn request_as(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
message: &Value,
|
||||
key: &str,
|
||||
) -> reqwest::RequestBuilder {
|
||||
let mut request = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/dispatch-observer", self.server.1))
|
||||
.bearer_auth(key)
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.header("MCP-Protocol-Version", revision);
|
||||
if let Some(session) = session {
|
||||
request = request.header("MCP-Session-Id", session);
|
||||
}
|
||||
if revision == "2026-07-28" {
|
||||
request = request.header("Mcp-Method", message["method"].as_str().unwrap());
|
||||
if let Some(name) = message.pointer("/params/name").and_then(Value::as_str) {
|
||||
request = request.header("Mcp-Name", name);
|
||||
}
|
||||
}
|
||||
request
|
||||
.json(message)
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
}
|
||||
async fn initialize(&self) -> String {
|
||||
let message = json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"oracle","version":"1"}}});
|
||||
let response = self
|
||||
.request("2025-11-25", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let session = response.headers()["MCP-Session-Id"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let body = frames(response).await;
|
||||
assert_eq!(
|
||||
body.last().unwrap()["result"]["protocolVersion"],
|
||||
"2025-11-25"
|
||||
);
|
||||
let response = self
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&json!({"jsonrpc":"2.0","method":"notifications/initialized"}),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 202);
|
||||
session
|
||||
}
|
||||
}
|
||||
fn meta() -> Value {
|
||||
json!({"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{},"io.modelcontextprotocol/clientInfo":{"name":"oracle","version":"1"}})
|
||||
}
|
||||
async fn frames(response: reqwest::Response) -> Vec<Value> {
|
||||
let status = response.status();
|
||||
let text = response.text().await.unwrap();
|
||||
if let Ok(json) = serde_json::from_str(&text) {
|
||||
return vec![json];
|
||||
}
|
||||
let values: Vec<Value> = text
|
||||
.lines()
|
||||
.filter_map(|line| line.strip_prefix("data:"))
|
||||
.filter_map(|data| serde_json::from_str(data.trim()).ok())
|
||||
.collect();
|
||||
assert!(!values.is_empty(), "{status}: {text}");
|
||||
values
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_positive_tools_both_revisions() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
|
||||
if session.is_none() {
|
||||
let discover = json!({"jsonrpc":"2.0","id":2,"method":"server/discover","params":{"_meta":meta()}});
|
||||
let response = h.request(revision, None, &discover).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(!frames(response).await[0]["result"].is_null());
|
||||
}
|
||||
let list = json!({"jsonrpc":"2.0","id":3,"method":"tools/list","params":{"_meta": if session.is_none() { meta() } else { json!({}) }}});
|
||||
let response = h.request(revision, session, &list).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(response.headers().contains_key("x-request-id"));
|
||||
let listed = frames(response).await;
|
||||
assert_eq!(
|
||||
listed.last().unwrap()["result"]["tools"][0]["name"],
|
||||
"create_lead"
|
||||
);
|
||||
let call = json!({"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":"rmcp@example.test"},"_meta":if session.is_none() { meta() } else { json!({}) }}});
|
||||
let response = h.request(revision, session, &call).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
if session.is_none() {
|
||||
assert!(!response.headers().contains_key("MCP-Session-Id"));
|
||||
}
|
||||
let result = frames(response).await;
|
||||
assert_eq!(
|
||||
result.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_rmcp@example.test",
|
||||
"{result:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
fn call(id: i64, email: &str, modern: bool, progress: bool) -> Value {
|
||||
let mut metadata = if modern { meta() } else { json!({}) };
|
||||
if progress {
|
||||
metadata["progressToken"] = json!("wire-progress");
|
||||
}
|
||||
json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"create_lead","arguments":{"email":email},"_meta":metadata}})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_credentials_revalidated_after_initialize_and_no_outbound() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let message = call(10, "denied@example.test", false, false);
|
||||
assert_eq!(
|
||||
h.request_as("2025-11-25", Some(&session), &message, "invalid-test-key")
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
// A scope change and expiry are authoritative registry changes, not session credentials.
|
||||
sqlx::query("update platform_api_keys set scopes_json = '[\"read\"]'::jsonb where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
403
|
||||
);
|
||||
sqlx::query("update platform_api_keys set scopes_json = '[\"read\",\"write\"]'::jsonb, expires_at = now() - interval '1 second' where id = 'pk_dispatch_observer'").execute(h.state.registry.pool()).await.unwrap();
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
sqlx::query("update platform_api_keys set expires_at = null where id = 'pk_dispatch_observer'")
|
||||
.execute(h.state.registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
h.state
|
||||
.registry
|
||||
.revoke_platform_api_key(
|
||||
&crank_core::WorkspaceId::new("ws_default"),
|
||||
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
&time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
for method in [reqwest::Method::GET, reqwest::Method::DELETE] {
|
||||
assert_eq!(
|
||||
reqwest::Client::new()
|
||||
.request(
|
||||
method,
|
||||
format!("{}/v1/default/dispatch-observer", h.server.1)
|
||||
)
|
||||
.bearer_auth(&h.key)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("Accept", "text/event-stream")
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_admission_rejects_foreign_route_revision_headers_and_body_overflow() {
|
||||
let h = Harness::new().await;
|
||||
let message = call(11, "denied@example.test", true, false);
|
||||
let response = h
|
||||
.request("2026-07-28", Some("foreign"), &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 400);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.header("Mcp-Name", "foreign")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 400);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.header("Origin", "https://evil.example")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 403);
|
||||
assert_eq!(
|
||||
h.request("2030-01-01", None, &message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
400
|
||||
);
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/foreign-agent", h.server.1))
|
||||
.bearer_auth(&h.key)
|
||||
.json(&message)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 404);
|
||||
let response = h
|
||||
.request("2026-07-28", None, &message)
|
||||
.body(vec![b' '; 2 * 1024 * 1024 + 1])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 413);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_progress_is_real_and_tool_error_is_not_protocol_error() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
for (revision, session) in [("2025-11-25", Some(session.as_str())), ("2026-07-28", None)] {
|
||||
let response = h
|
||||
.request(
|
||||
revision,
|
||||
session,
|
||||
&call(20, "progress@example.test", session.is_none(), true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(
|
||||
response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("text/event-stream")
|
||||
);
|
||||
let messages = frames(response).await;
|
||||
let progress: Vec<_> = messages
|
||||
.iter()
|
||||
.filter(|m| m["method"] == "notifications/progress")
|
||||
.collect();
|
||||
assert_eq!(progress.len(), 3, "{messages:?}");
|
||||
for (index, phase) in [
|
||||
"tool_resolved",
|
||||
"authorization_resolved",
|
||||
"execution_dispatched",
|
||||
]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
assert_eq!(progress[index]["params"]["message"], *phase);
|
||||
assert_eq!(
|
||||
progress[index]["params"]["progress"].as_f64(),
|
||||
Some((index + 1) as f64)
|
||||
);
|
||||
assert_eq!(progress[index]["params"]["progressToken"], "wire-progress");
|
||||
}
|
||||
let mut invalid = call(21, "unused", session.is_none(), false);
|
||||
invalid["params"]["arguments"] = json!({"email":123});
|
||||
let response = h.request(revision, session, &invalid).send().await.unwrap();
|
||||
let messages = frames(response).await;
|
||||
let result = messages.last().unwrap();
|
||||
assert_eq!(result["result"]["isError"], true, "{messages:?}");
|
||||
assert!(result.get("error").is_none());
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_reconnect_live_manager_and_restart_loses_sdk_session() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
for _ in 0..2 {
|
||||
let response = reqwest::Client::new()
|
||||
.get(format!("{}/v1/default/dispatch-observer", h.server.1))
|
||||
.bearer_auth(&h.key)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("Accept", "text/event-stream")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
drop(response);
|
||||
}
|
||||
let response = h
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&call(22, "reconnect@example.test", false, false),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
frames(response).await.last().unwrap()["result"]["structuredContent"]["id"],
|
||||
"lead_for_reconnect@example.test"
|
||||
);
|
||||
let fresh = serve(rmcp_http::router(
|
||||
h.state.clone(),
|
||||
AgentRoutePath {
|
||||
workspace_slug: "default".into(),
|
||||
agent_slug: "dispatch-observer".into(),
|
||||
},
|
||||
h.active.clone(),
|
||||
))
|
||||
.await;
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{}/v1/default/dispatch-observer", fresh.1))
|
||||
.bearer_auth(&h.key)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.header("MCP-Protocol-Version", "2025-11-25")
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.json(&call(23, "restart@example.test", false, false))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
404,
|
||||
"new SDK manager cannot restore from Crank session storage"
|
||||
);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
async fn wait_count(counter: &AtomicUsize, expected: usize) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(3), async {
|
||||
while counter.load(Ordering::SeqCst) != expected {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded wait for the real execution/cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_legacy_cancel_is_owned_by_session_and_current_key() {
|
||||
let h = Harness::new().await;
|
||||
let session = h.initialize().await;
|
||||
let foreign_session = h.initialize().await;
|
||||
// A second valid credential can use this Agent/session but does not own the first call.
|
||||
sqlx::query("insert into platform_api_keys (id, workspace_id, agent_id, key_kind, name, prefix, secret_hash, scopes_json, status, created_at) select 'pk_second', workspace_id, agent_id, key_kind, 'second', prefix, $1, scopes_json, status, created_at from platform_api_keys where id = 'pk_dispatch_observer'")
|
||||
.bind(crate::access::hash_access_secret("test-second-valid-key")).execute(h.state.registry.pool()).await.unwrap();
|
||||
let response = h
|
||||
.request(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&call(30, "slow-cancel@example.test", false, true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
wait_count(&h.calls, 1).await;
|
||||
let cancel =
|
||||
json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":30}});
|
||||
assert_eq!(
|
||||
h.request_as(
|
||||
"2025-11-25",
|
||||
Some(&session),
|
||||
&cancel,
|
||||
"test-second-valid-key"
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&foreign_session), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(75)).await;
|
||||
assert_eq!(
|
||||
h.active.load(Ordering::SeqCst),
|
||||
1,
|
||||
"foreign cancel must not stop call"
|
||||
);
|
||||
assert_eq!(
|
||||
h.request("2025-11-25", Some(&session), &cancel)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
wait_count(&h.active, 0).await;
|
||||
// Gap: SDK emits an error frame where the existing transport suppresses all terminal frames.
|
||||
let messages = frames(response).await;
|
||||
assert_eq!(
|
||||
messages.last().unwrap()["error"]["message"],
|
||||
"request cancelled"
|
||||
);
|
||||
assert!(!messages.iter().any(|m| m.get("result").is_some()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_http_modern_disconnect_drops_execution_and_releases_stream_slot() {
|
||||
let h = Harness::new().await;
|
||||
let response = h
|
||||
.request(
|
||||
"2026-07-28",
|
||||
None,
|
||||
&call(31, "slow-disconnect@example.test", true, true),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
wait_count(&h.calls, 1).await;
|
||||
assert_eq!(h.active.load(Ordering::SeqCst), 1);
|
||||
drop(response);
|
||||
wait_count(&h.active, 0).await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
while h.state.session_slots.available_permits() != 16 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
h.calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"disconnect never retries a possible external effect"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
//! #107 logging: existing HTTP wire control before the independent SDK marker assertions.
|
||||
use super::{CounterSet, observe, support::*};
|
||||
use axum::{body::Body, response::Response};
|
||||
use futures_util::{StreamExt, stream};
|
||||
use serde_json::{Value, json};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
#[path = "rmcp_logging_test_support.rs"]
|
||||
mod fixtures;
|
||||
use fixtures::Endpoint;
|
||||
const LEVEL: &str = "io.modelcontextprotocol/logLevel";
|
||||
|
||||
fn counts(c: &CounterSet) -> [usize; 3] {
|
||||
[
|
||||
c.sdk_logging_dispatch(),
|
||||
c.sdk_logging_level(),
|
||||
c.sdk_logging_notification(),
|
||||
]
|
||||
}
|
||||
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = stream::unfold(
|
||||
(body.into_data_stream(), counters),
|
||||
|(mut body, c)| async move {
|
||||
observe(c.clone(), body.next())
|
||||
.await
|
||||
.map(|chunk| (chunk, (body, c)))
|
||||
},
|
||||
);
|
||||
Response::from_parts(parts, Body::from_stream(body))
|
||||
}
|
||||
fn notification(level: &str, event: &str, outcome: &str) -> Value {
|
||||
json!({"jsonrpc":"2.0","method":"notifications/message","params":{
|
||||
"level":level,"logger":"crank-mcp-server","data":{"event":event,"outcome":outcome}}})
|
||||
}
|
||||
async fn reply(response: reqwest::Response) -> (Value, Vec<Value>, String) {
|
||||
let status = response.status();
|
||||
assert!(
|
||||
status == 200 || status == 404,
|
||||
"unexpected HTTP status {status}"
|
||||
);
|
||||
let sse = response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("text/event-stream");
|
||||
let raw = tokio::time::timeout(Duration::from_secs(5), response.text())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut frames: Vec<Value> = if sse {
|
||||
raw.lines()
|
||||
.filter_map(|line| {
|
||||
line.strip_prefix("data:")
|
||||
.map(|s| serde_json::from_str(s.trim()).unwrap())
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
vec![serde_json::from_str(&raw).unwrap()]
|
||||
};
|
||||
let terminal = frames.pop().expect("one terminal response");
|
||||
assert_eq!(terminal["jsonrpc"], "2.0");
|
||||
assert_eq!(terminal["id"], 51);
|
||||
assert_eq!(
|
||||
status.as_u16(),
|
||||
if terminal["error"]["code"] == -32601 {
|
||||
404
|
||||
} else {
|
||||
200
|
||||
}
|
||||
);
|
||||
if !frames.is_empty() {
|
||||
assert!(sse, "notifications require SSE");
|
||||
}
|
||||
assert!(frames.iter().all(|f| f.get("id").is_none()));
|
||||
(terminal, frames, raw)
|
||||
}
|
||||
fn opted(level: Value) -> Value {
|
||||
let mut meta = modern_meta();
|
||||
meta[LEVEL] = level;
|
||||
json!({"_meta":meta})
|
||||
}
|
||||
async fn control(revision: &str) -> [usize; 3] {
|
||||
let h = Endpoint::started().await;
|
||||
let session = h.session(revision).await;
|
||||
let session = session.as_deref();
|
||||
// Negative opt-in controls precede any accepted level; a silent response is not enough alone.
|
||||
let (body, events, _) = reply(h.post(revision, session, "tools/list", json!({})).await).await;
|
||||
assert!(body["result"]["tools"].is_array());
|
||||
assert!(events.is_empty());
|
||||
assert_eq!(counts(&h.counters), [0; 3]);
|
||||
if revision == MODERN_REVISION {
|
||||
for value in [
|
||||
json!("verbose"),
|
||||
json!(7),
|
||||
json!(null),
|
||||
json!("CANARY".repeat(1000)),
|
||||
] {
|
||||
let (body, events, _) =
|
||||
reply(h.post(revision, session, "tools/list", opted(value)).await).await;
|
||||
assert!(body["result"]["tools"].is_array());
|
||||
assert!(events.is_empty());
|
||||
assert_eq!(counts(&h.counters), [0; 3]);
|
||||
}
|
||||
let (body, events, _) = reply(
|
||||
h.post(
|
||||
revision,
|
||||
session,
|
||||
"logging/setLevel",
|
||||
json!({"level":"notice"}),
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body["error"]["code"], -32601);
|
||||
assert!(events.is_empty());
|
||||
assert_eq!(counts(&h.counters), [0; 3]);
|
||||
} else {
|
||||
for params in [
|
||||
json!({}),
|
||||
json!({"level":7}),
|
||||
json!({"level":"CANARY".repeat(1000)}),
|
||||
] {
|
||||
let (body, events, raw) =
|
||||
reply(h.post(revision, session, "logging/setLevel", params).await).await;
|
||||
assert_eq!(body["error"]["code"], -32602);
|
||||
assert_eq!(body["error"]["message"], "invalid logging level");
|
||||
assert!(body.get("result").is_none());
|
||||
assert!(!raw.contains("CANARY"));
|
||||
assert!(raw.len() < 4096);
|
||||
assert!(events.is_empty());
|
||||
assert_eq!(h.counters.sdk_logging_level(), 0);
|
||||
assert_eq!(h.counters.sdk_logging_notification(), 0);
|
||||
}
|
||||
}
|
||||
let before = counts(&h.counters);
|
||||
let (method, params, expected) = if revision == LEGACY_REVISION {
|
||||
(
|
||||
"logging/setLevel",
|
||||
json!({"level":"notice"}),
|
||||
notification("notice", "mcp.logging.level_applied", "applied"),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
"tools/list",
|
||||
opted(json!("info")),
|
||||
notification("info", "mcp.request.served", "ok"),
|
||||
)
|
||||
};
|
||||
let (body, events, raw) = reply(h.post(revision, session, method, params).await).await;
|
||||
assert!(body.get("error").is_none(), "{body}");
|
||||
if revision == LEGACY_REVISION {
|
||||
assert_eq!(body["result"], json!({}));
|
||||
} else {
|
||||
assert!(body["result"]["tools"].is_array());
|
||||
}
|
||||
assert_eq!(events, vec![expected]);
|
||||
assert!(raw.len() < 4096);
|
||||
let after = counts(&h.counters);
|
||||
let delta = std::array::from_fn(|i| after[i] - before[i]);
|
||||
// Legacy severity persists, modern opt-in does not leak into the next request.
|
||||
let (body, events, _) = reply(h.post(revision, session, "tools/list", json!({})).await).await;
|
||||
assert!(body["result"]["tools"].is_array());
|
||||
assert!(events.is_empty());
|
||||
h.registry
|
||||
.revoke_platform_api_key(
|
||||
&workspace_id(),
|
||||
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
&time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let before = counts(&h.counters);
|
||||
let era = h.counters.snapshot();
|
||||
let response = h
|
||||
.post(
|
||||
revision,
|
||||
session,
|
||||
method,
|
||||
if revision == LEGACY_REVISION {
|
||||
json!({"level":"notice"})
|
||||
} else {
|
||||
opted(json!("info"))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 401);
|
||||
let raw = response.text().await.unwrap();
|
||||
assert!(!raw.contains("notifications/message"));
|
||||
assert_eq!(counts(&h.counters), before);
|
||||
let after = h.counters.snapshot();
|
||||
assert_eq!(after.era_modern, era.era_modern);
|
||||
assert_eq!(after.era_legacy, era.era_legacy);
|
||||
eprintln!(
|
||||
"existing GREEN logging HTTP {revision}: safe SSE, result, opt-in/invalid/era/auth controls passed; SDK positive delta={delta:?}"
|
||||
);
|
||||
delta
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_logging_wire_legacy() {
|
||||
control(LEGACY_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_logging_wire_modern() {
|
||||
control(MODERN_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_logging_sdk_legacy() {
|
||||
assert_eq!(
|
||||
control(LEGACY_REVISION).await,
|
||||
[1, 1, 1],
|
||||
"missing SDK legacy request/level/notification after successful HTTP wire assertions"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_logging_sdk_modern() {
|
||||
assert_eq!(
|
||||
control(MODERN_REVISION).await,
|
||||
[0, 1, 1],
|
||||
"missing SDK modern level/notification after successful HTTP wire assertions"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_logging_observer_scope_control() {
|
||||
use super::{
|
||||
sdk_logging_dispatch_enter, sdk_logging_level_parsed, sdk_logging_notification_serialized,
|
||||
};
|
||||
let a = Arc::new(CounterSet::default());
|
||||
let b = Arc::new(CounterSet::default());
|
||||
sdk_logging_dispatch_enter();
|
||||
sdk_logging_level_parsed();
|
||||
sdk_logging_notification_serialized();
|
||||
assert_eq!(counts(&a), [0; 3]);
|
||||
tokio::join!(
|
||||
observe(a.clone(), async {
|
||||
sdk_logging_dispatch_enter();
|
||||
tokio::task::yield_now().await;
|
||||
sdk_logging_level_parsed();
|
||||
}),
|
||||
observe(b.clone(), async {
|
||||
sdk_logging_notification_serialized();
|
||||
})
|
||||
);
|
||||
assert_eq!(counts(&a), [1, 1, 0]);
|
||||
assert_eq!(counts(&b), [0, 0, 1]);
|
||||
let body = Body::from_stream(stream::once(async {
|
||||
sdk_logging_notification_serialized();
|
||||
Ok::<_, std::convert::Infallible>("data: terminal\n\n")
|
||||
}));
|
||||
let response = observe_body(Response::new(body), a.clone());
|
||||
assert_eq!(counts(&a), [1, 1, 0]);
|
||||
assert_eq!(
|
||||
&axum::body::to_bytes(response.into_body(), 100)
|
||||
.await
|
||||
.unwrap()[..],
|
||||
b"data: terminal\n\n"
|
||||
);
|
||||
assert_eq!(counts(&a), [1, 1, 1]);
|
||||
assert_eq!(counts(&b), [0, 0, 1]);
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
//! Minimal logging HTTP fixture; shared registry/auth/lifecycle fixtures remain authoritative.
|
||||
use super::*;
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
|
||||
};
|
||||
|
||||
pub(super) struct Endpoint {
|
||||
pub registry: crank_registry::PostgresRegistry,
|
||||
pub counters: Arc<CounterSet>,
|
||||
key: String,
|
||||
url: String,
|
||||
server: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
impl Drop for Endpoint {
|
||||
fn drop(&mut self) {
|
||||
self.server.abort();
|
||||
}
|
||||
}
|
||||
impl Endpoint {
|
||||
pub async fn started() -> Self {
|
||||
crank_test_support::postgres_schema_url_for_task("logging_owner", "t_c28c3800").await;
|
||||
let registry = test_registry().await;
|
||||
let operation = test_operation("http://127.0.0.1:9");
|
||||
registry
|
||||
.create_operation(&workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(crank_registry::PublishRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent(®istry, &operation).await;
|
||||
let key = create_api_key(®istry).await;
|
||||
let counters = Arc::new(CounterSet::default());
|
||||
let app = crate::build_app(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Some("https://crank.example.com".into()),
|
||||
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
|
||||
crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
Arc::new(crate::session::InMemorySessionStore::default()),
|
||||
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
|
||||
);
|
||||
let scope = counters.clone();
|
||||
let app = app.layer(axum::middleware::from_fn(
|
||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||
let counters = scope.clone();
|
||||
async move {
|
||||
let response = observe(counters.clone(), next.run(request)).await;
|
||||
observe_body(response, counters)
|
||||
}
|
||||
},
|
||||
));
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!(
|
||||
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
|
||||
listener.local_addr().unwrap()
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Self {
|
||||
registry,
|
||||
counters,
|
||||
key,
|
||||
url,
|
||||
server,
|
||||
}
|
||||
}
|
||||
pub async fn post(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
method: &str,
|
||||
mut params: Value,
|
||||
) -> reqwest::Response {
|
||||
let mut request = reqwest::Client::new()
|
||||
.post(&self.url)
|
||||
.bearer_auth(&self.key)
|
||||
.header("Accept", "text/event-stream, application/json")
|
||||
.header("MCP-Protocol-Version", revision);
|
||||
if let Some(session) = session {
|
||||
request = request.header("MCP-Session-Id", session);
|
||||
}
|
||||
if revision == MODERN_REVISION {
|
||||
request = request.header("Mcp-Method", method);
|
||||
if params.get("_meta").is_none() {
|
||||
params["_meta"] = modern_meta();
|
||||
}
|
||||
}
|
||||
request
|
||||
.json(&json!({"jsonrpc":"2.0", "id":51, "method":method, "params":params}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
pub async fn session(&self, revision: &str) -> Option<String> {
|
||||
if revision == MODERN_REVISION {
|
||||
return None;
|
||||
}
|
||||
let response = self
|
||||
.post(
|
||||
revision,
|
||||
None,
|
||||
"initialize",
|
||||
json!({"protocolVersion":revision,"capabilities":{}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let session = response.headers()["MCP-Session-Id"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
assert_eq!(
|
||||
reply(response).await.0["result"]["protocolVersion"],
|
||||
revision
|
||||
);
|
||||
let response = reqwest::Client::new()
|
||||
.post(&self.url)
|
||||
.bearer_auth(&self.key)
|
||||
.header("Accept", "application/json, text/event-stream")
|
||||
.header("MCP-Protocol-Version", revision)
|
||||
.header("MCP-Session-Id", &session)
|
||||
.json(&initialized_notification())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 202);
|
||||
response.bytes().await.unwrap();
|
||||
Some(session)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
//! #107 Prompts: existing wire GREEN and an independent SDK typed-path RED oracle.
|
||||
use super::{CounterSet, observe, support::*};
|
||||
use axum::{body::Body, response::Response};
|
||||
use futures_util::{StreamExt, stream};
|
||||
use serde_json::{Value, json};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
#[path = "rmcp_prompts_test_support.rs"]
|
||||
mod fixtures;
|
||||
use fixtures::Endpoint;
|
||||
|
||||
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = stream::unfold(
|
||||
(body.into_data_stream(), counters),
|
||||
|(mut body, counters)| async move {
|
||||
observe(counters.clone(), body.next())
|
||||
.await
|
||||
.map(|chunk| (chunk, (body, counters)))
|
||||
},
|
||||
);
|
||||
Response::from_parts(parts, Body::from_stream(body))
|
||||
}
|
||||
fn counts(c: &CounterSet) -> [usize; 3] {
|
||||
[
|
||||
c.sdk_prompts_dispatch(),
|
||||
c.sdk_prompts_list_result(),
|
||||
c.sdk_prompts_get_result(),
|
||||
]
|
||||
}
|
||||
fn request(revision: &str, method: &str, mut params: Value) -> Value {
|
||||
if revision == MODERN_REVISION {
|
||||
params["_meta"] = modern_meta();
|
||||
}
|
||||
json!({"jsonrpc":"2.0","id":9007199254740993_u64,"method":method,"params":params})
|
||||
}
|
||||
async fn wire(response: reqwest::Response, id: &Value, sse: bool) -> Value {
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(response.headers()["x-request-id"], "sdk-prompts-wire");
|
||||
assert_eq!(
|
||||
response.headers()["x-trace-id"],
|
||||
"11111111111111111111111111111111"
|
||||
);
|
||||
let content_type = response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let body: Value = if sse {
|
||||
assert!(
|
||||
content_type.starts_with("text/event-stream"),
|
||||
"{content_type}: {text}"
|
||||
);
|
||||
let frames: Vec<Value> = text
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
line.strip_prefix("data:")
|
||||
.map(|s| serde_json::from_str(s.trim()).unwrap())
|
||||
})
|
||||
.collect();
|
||||
let terminal: Vec<_> = frames.iter().filter(|f| f.get("id") == Some(id)).collect();
|
||||
assert_eq!(terminal.len(), 1, "{frames:?}");
|
||||
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
|
||||
terminal[0].clone()
|
||||
} else {
|
||||
assert!(
|
||||
content_type.starts_with("application/json"),
|
||||
"{content_type}: {text}"
|
||||
);
|
||||
serde_json::from_str(&text).unwrap()
|
||||
};
|
||||
assert_eq!(body["jsonrpc"], "2.0");
|
||||
assert_eq!(&body["id"], id);
|
||||
body
|
||||
}
|
||||
fn era(result: &Value, revision: &str, method: &str) {
|
||||
if revision == MODERN_REVISION {
|
||||
assert_eq!(result["resultType"], "complete", "{result}");
|
||||
if method == "prompts/list" {
|
||||
assert_eq!(result["ttlMs"], 0);
|
||||
assert_eq!(result["cacheScope"], "private");
|
||||
} else {
|
||||
assert!(result.get("ttlMs").is_none(), "{result}");
|
||||
assert!(result.get("cacheScope").is_none(), "{result}");
|
||||
}
|
||||
} else {
|
||||
for key in ["resultType", "ttlMs", "cacheScope"] {
|
||||
assert!(result.get(key).is_none(), "{result}");
|
||||
}
|
||||
}
|
||||
assert!(result.get("_meta").is_none(), "{result}");
|
||||
}
|
||||
async fn success(
|
||||
h: &Endpoint,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
method: &str,
|
||||
params: Value,
|
||||
sse: bool,
|
||||
) -> (Value, [usize; 3]) {
|
||||
let req = request(revision, method, params);
|
||||
let before = counts(&h.counters);
|
||||
let body = wire(h.post(revision, session, &req, sse).await, &req["id"], sse).await;
|
||||
assert!(body.get("error").is_none(), "{body}");
|
||||
era(&body["result"], revision, method);
|
||||
let after = counts(&h.counters);
|
||||
(
|
||||
body["result"].clone(),
|
||||
std::array::from_fn(|i| after[i] - before[i]),
|
||||
)
|
||||
}
|
||||
async fn control(revision: &str) -> Vec<([usize; 3], [usize; 3])> {
|
||||
let h = Endpoint::started().await;
|
||||
let session = h.session(revision).await;
|
||||
let session = session.as_deref();
|
||||
let mut deltas = Vec::new();
|
||||
let mut first_cursor = None;
|
||||
// Both revisions' JSON contract, plus modern terminal SSE through the same normal app.
|
||||
let modes: &[bool] = if revision == MODERN_REVISION {
|
||||
&[false, true]
|
||||
} else {
|
||||
&[false]
|
||||
};
|
||||
for &sse in modes {
|
||||
let mut cursor = None;
|
||||
for (page, name) in ["greeting", "archived-prompt"].iter().enumerate() {
|
||||
let mut params = json!({"limit":1});
|
||||
if let Some(ref cursor) = cursor {
|
||||
params["cursor"] = json!(cursor);
|
||||
}
|
||||
let (result, delta) = success(&h, revision, session, "prompts/list", params, sse).await;
|
||||
let prompts = result["prompts"].as_array().unwrap();
|
||||
assert_eq!(prompts.len(), 1, "{result}");
|
||||
assert_eq!(
|
||||
prompts[0],
|
||||
json!({
|
||||
"name":name, "description":format!("Published {name}"),
|
||||
"arguments":[
|
||||
{"name":"name","schema":{"type":"string"},"required":true},
|
||||
{"name":"count","schema":{"type":"integer"},"required":false,"default":2},
|
||||
{"name":"enabled","schema":{"type":"boolean"},"required":false,"default":true},
|
||||
{"name":"tone","schema":{"type":"string","enum":["warm","dry"]},"required":false,"default":"warm"}
|
||||
]
|
||||
})
|
||||
);
|
||||
cursor = result
|
||||
.get("nextCursor")
|
||||
.map(|v| v.as_str().unwrap().to_owned());
|
||||
assert_eq!(cursor.is_some(), page == 0, "{result}");
|
||||
if page == 0 {
|
||||
assert_ne!(cursor.as_deref(), Some("1"));
|
||||
first_cursor = cursor.clone();
|
||||
}
|
||||
deltas.push((delta, [1, 1, 0]));
|
||||
}
|
||||
for (name, args, text) in [
|
||||
(
|
||||
"greeting",
|
||||
json!({"name":"Ada"}),
|
||||
"Hello Ada count=2 enabled=true tone=warm",
|
||||
),
|
||||
(
|
||||
"greeting",
|
||||
json!({"name":"Ada","count":7,"enabled":false,"tone":"dry"}),
|
||||
"Hello Ada count=7 enabled=false tone=dry",
|
||||
),
|
||||
(
|
||||
"archived-prompt",
|
||||
json!({"name":"Ada"}),
|
||||
"Hello Ada count=2 enabled=true tone=warm",
|
||||
),
|
||||
] {
|
||||
let mut previous = None;
|
||||
for _ in 0..2 {
|
||||
let (result, delta) = success(
|
||||
&h,
|
||||
revision,
|
||||
session,
|
||||
"prompts/get",
|
||||
json!({"name":name,"arguments":args}),
|
||||
sse,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result["description"], format!("Published {name}"));
|
||||
assert_eq!(
|
||||
result["messages"],
|
||||
json!([
|
||||
{"role":"system","content":{"type":"text","text":"Be brief."}},
|
||||
{"role":"user","content":{"type":"text","text":text}}
|
||||
])
|
||||
);
|
||||
if let Some(ref previous) = previous {
|
||||
assert_eq!(&result, previous);
|
||||
}
|
||||
previous = Some(result);
|
||||
deltas.push((delta, [1, 0, 1]));
|
||||
}
|
||||
}
|
||||
for (arguments, reason, argument) in [
|
||||
(json!({}), "missing_argument", "name"),
|
||||
(
|
||||
json!({"name":"Ada","extra":"x"}),
|
||||
"unknown_argument",
|
||||
"extra",
|
||||
),
|
||||
(json!({"name":5}), "argument_invalid", "name"),
|
||||
(
|
||||
json!({"name":"Ada","count":"7"}),
|
||||
"argument_invalid",
|
||||
"count",
|
||||
),
|
||||
(
|
||||
json!({"name":"Ada","enabled":"false"}),
|
||||
"argument_invalid",
|
||||
"enabled",
|
||||
),
|
||||
(
|
||||
json!({"name":"Ada","tone":"loud"}),
|
||||
"argument_invalid",
|
||||
"tone",
|
||||
),
|
||||
(
|
||||
json!({"name":"x".repeat(crank_core::prompt::MAX_TEXT_BYTES + 1)}),
|
||||
"argument_too_large",
|
||||
"name",
|
||||
),
|
||||
] {
|
||||
let req = request(
|
||||
revision,
|
||||
"prompts/get",
|
||||
json!({"name":"greeting","arguments":arguments}),
|
||||
);
|
||||
let body = wire(h.post(revision, session, &req, sse).await, &req["id"], sse).await;
|
||||
assert_eq!(body["error"]["code"], -32602, "{body}");
|
||||
assert_eq!(body["error"]["data"]["reason"], reason, "{body}");
|
||||
assert_eq!(body["error"]["data"]["argument"], argument);
|
||||
assert!(body.get("result").is_none(), "{body}");
|
||||
for leaked in [
|
||||
"Be brief.",
|
||||
"Hello ",
|
||||
"Changed ",
|
||||
"Published unbound-prompt",
|
||||
] {
|
||||
assert!(!body.to_string().contains(leaked), "{body}");
|
||||
}
|
||||
}
|
||||
let mut unavailable = None;
|
||||
for name in [
|
||||
"draft-prompt",
|
||||
"unbound-prompt",
|
||||
"other-agent-prompt",
|
||||
"never-existed",
|
||||
"prm_a",
|
||||
] {
|
||||
let req = request(revision, "prompts/get", json!({"name":name}));
|
||||
let body = wire(h.post(revision, session, &req, sse).await, &req["id"], sse).await;
|
||||
assert_eq!(body["error"]["code"], -32602, "{body}");
|
||||
assert_eq!(
|
||||
body["error"]["message"],
|
||||
"the requested prompt is not available"
|
||||
);
|
||||
assert!(body.get("result").is_none());
|
||||
if let Some(ref previous) = unavailable {
|
||||
assert_eq!(&body, previous);
|
||||
}
|
||||
unavailable = Some(body);
|
||||
}
|
||||
}
|
||||
for params in [json!({"cursor":"invalid"}), json!({"limit":"invalid"})] {
|
||||
let req = request(revision, "prompts/list", params);
|
||||
let body = wire(
|
||||
h.post(revision, session, &req, false).await,
|
||||
&req["id"],
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body["error"]["code"], -32602, "{body}");
|
||||
}
|
||||
let other = if revision == MODERN_REVISION {
|
||||
LEGACY_REVISION
|
||||
} else {
|
||||
MODERN_REVISION
|
||||
};
|
||||
let other_session = h.session(other).await;
|
||||
let req = request(
|
||||
other,
|
||||
"prompts/list",
|
||||
json!({"cursor":first_cursor.unwrap()}),
|
||||
);
|
||||
let body = wire(
|
||||
h.post(other, other_session.as_deref(), &req, false).await,
|
||||
&req["id"],
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
body["error"]["code"], -32602,
|
||||
"cross-revision cursor: {body}"
|
||||
);
|
||||
// Archive keeps serving v1 above, but disallows new publications and bindings.
|
||||
use crank_core::ids::PromptId;
|
||||
use crank_registry::model::{
|
||||
AgentPromptBinding, PublishPromptRequest, SaveAgentCatalogConfigRequest,
|
||||
};
|
||||
let publication = h
|
||||
.registry
|
||||
.publish_prompt_draft(PublishPromptRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new("prm_b"),
|
||||
expected_updated_at: h.archived_at,
|
||||
published_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(
|
||||
publication,
|
||||
Err(crank_registry::RegistryError::PromptNotDraft { .. })
|
||||
),
|
||||
"{publication:?}"
|
||||
);
|
||||
let binding = h
|
||||
.registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
agent_version: 1,
|
||||
bindings: &[],
|
||||
tool_selection_policy: &crank_core::ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[],
|
||||
expected_state: None,
|
||||
},
|
||||
&[AgentPromptBinding {
|
||||
prompt_id: PromptId::new("prm_b"),
|
||||
prompt_version: 1,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(
|
||||
binding,
|
||||
Err(crank_registry::RegistryError::PromptArchived { .. })
|
||||
),
|
||||
"{binding:?}"
|
||||
);
|
||||
h.registry
|
||||
.revoke_platform_api_key(
|
||||
&workspace_id(),
|
||||
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
&time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
for (method, params) in [
|
||||
("prompts/list", json!({})),
|
||||
(
|
||||
"prompts/get",
|
||||
json!({"name":"greeting","arguments":{"name":"Ada"}}),
|
||||
),
|
||||
] {
|
||||
assert_eq!(
|
||||
h.post(revision, session, &request(revision, method, params), false)
|
||||
.await
|
||||
.status(),
|
||||
401
|
||||
);
|
||||
}
|
||||
eprintln!(
|
||||
"existing GREEN Prompts wire {revision}: populated pagination, typed metadata/values, exact immutable rendering, archive, structured refusals/non-disclosure, correlation/ID/cache, revoked401; SDK actual/expected={deltas:?}"
|
||||
);
|
||||
deltas
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_prompts_wire_legacy() {
|
||||
control(LEGACY_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_prompts_wire_modern() {
|
||||
control(MODERN_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_prompts_sdk_legacy() {
|
||||
let deltas = control(LEGACY_REVISION).await;
|
||||
assert!(
|
||||
deltas.iter().all(|(a, e)| a == e),
|
||||
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_prompts_sdk_modern() {
|
||||
let deltas = control(MODERN_REVISION).await;
|
||||
assert!(
|
||||
deltas.iter().all(|(a, e)| a == e),
|
||||
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_prompts_observer_scope_control() {
|
||||
use super::{
|
||||
sdk_prompts_dispatch_enter, sdk_prompts_get_result_serialized,
|
||||
sdk_prompts_list_result_serialized,
|
||||
};
|
||||
let a = Arc::new(CounterSet::default());
|
||||
let b = Arc::new(CounterSet::default());
|
||||
sdk_prompts_dispatch_enter();
|
||||
assert_eq!(counts(&a), [0; 3]);
|
||||
let snapshot = a.snapshot();
|
||||
observe(a.clone(), async {
|
||||
sdk_prompts_dispatch_enter();
|
||||
sdk_prompts_list_result_serialized();
|
||||
sdk_prompts_get_result_serialized();
|
||||
})
|
||||
.await;
|
||||
observe(b.clone(), async {
|
||||
sdk_prompts_get_result_serialized();
|
||||
})
|
||||
.await;
|
||||
assert_eq!(counts(&a), [1; 3]);
|
||||
assert_eq!(counts(&b), [0, 0, 1]);
|
||||
assert_eq!(a.snapshot(), snapshot);
|
||||
let body = Body::from_stream(stream::once(async {
|
||||
sdk_prompts_list_result_serialized();
|
||||
Ok::<_, std::convert::Infallible>("terminal")
|
||||
}));
|
||||
let response = observe_body(Response::new(body), b.clone());
|
||||
assert_eq!(
|
||||
&axum::body::to_bytes(response.into_body(), 100)
|
||||
.await
|
||||
.unwrap()[..],
|
||||
b"terminal"
|
||||
);
|
||||
assert_eq!(counts(&b), [0, 1, 1]);
|
||||
assert_eq!(counts(&a), [1; 3]);
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
//! Published Prompts fixture using the existing observed HTTP app and disposable PostgreSQL.
|
||||
use super::*;
|
||||
use crank_core::prompt::{PromptDefinition, PromptSensitivity};
|
||||
use crank_core::{Agent, AgentStatus, AgentVersion, ToolSelectionPolicy, ids::PromptId};
|
||||
use crank_registry::model::{
|
||||
AgentPromptBinding, ArchivePromptRequest, CreatePromptDraftRequest, PublishPromptRequest,
|
||||
SaveAgentCatalogConfigRequest, UpdatePromptDraftRequest,
|
||||
};
|
||||
use crank_registry::{CreateAgentRequest, PublishAgentRequest};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
|
||||
};
|
||||
|
||||
pub(super) fn definition(name: &str, content: &str) -> PromptDefinition {
|
||||
PromptDefinition::parse(&json!({
|
||||
"name":name, "description":format!("Published {name}"),
|
||||
"arguments":[
|
||||
{"name":"name","schema":{"type":"string"},"required":true},
|
||||
{"name":"count","schema":{"type":"integer"},"required":false,"default":2},
|
||||
{"name":"enabled","schema":{"type":"boolean"},"required":false,"default":true},
|
||||
{"name":"tone","schema":{"type":"string","enum":["warm","dry"]},"required":false,"default":"warm"}
|
||||
],
|
||||
"messages":[{"role":"system","content":"Be brief."},{"role":"user","content":content}]
|
||||
})).unwrap()
|
||||
}
|
||||
|
||||
pub(super) struct Endpoint {
|
||||
pub registry: crank_registry::PostgresRegistry,
|
||||
pub counters: Arc<CounterSet>,
|
||||
pub key: String,
|
||||
pub url: String,
|
||||
pub archived_at: time::OffsetDateTime,
|
||||
server: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
impl Drop for Endpoint {
|
||||
fn drop(&mut self) {
|
||||
self.server.abort();
|
||||
}
|
||||
}
|
||||
impl Endpoint {
|
||||
pub async fn started() -> Self {
|
||||
crank_test_support::postgres_schema_url_for_task("prompts_owner", "t_160a763a").await;
|
||||
let registry = test_registry().await;
|
||||
let mut archived_at = test_time();
|
||||
for (id, name, publish) in [
|
||||
("prm_a", "greeting", true),
|
||||
("prm_b", "archived-prompt", true),
|
||||
("prm_c", "unbound-prompt", true),
|
||||
("prm_d", "draft-prompt", false),
|
||||
("prm_e", "other-agent-prompt", true),
|
||||
] {
|
||||
let row = registry
|
||||
.create_prompt_draft(CreatePromptDraftRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new(id),
|
||||
sensitivity: PromptSensitivity::Internal,
|
||||
definition: definition(
|
||||
name,
|
||||
"Hello {{name}} count={{count}} enabled={{enabled}} tone={{tone}}",
|
||||
),
|
||||
created_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if publish {
|
||||
let row = registry
|
||||
.publish_prompt_draft(PublishPromptRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new(id),
|
||||
expected_updated_at: row.updated_at,
|
||||
published_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if id == "prm_b" {
|
||||
archived_at = row.updated_at;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (id, slug) in [
|
||||
(agent_id(), AGENT_SLUG),
|
||||
(
|
||||
crank_core::AgentId::new("agent_other_prompts"),
|
||||
"other-prompts",
|
||||
),
|
||||
] {
|
||||
registry
|
||||
.create_agent(CreateAgentRequest {
|
||||
agent: &Agent {
|
||||
id: id.clone(),
|
||||
workspace_id: workspace_id(),
|
||||
slug: slug.into(),
|
||||
display_name: "SDK Prompts".into(),
|
||||
description: String::new(),
|
||||
status: AgentStatus::Draft,
|
||||
current_draft_version: 1,
|
||||
latest_published_version: None,
|
||||
created_at: test_time(),
|
||||
updated_at: test_time(),
|
||||
published_at: None,
|
||||
},
|
||||
version: &AgentVersion {
|
||||
agent_id: id,
|
||||
version: 1,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: test_time(),
|
||||
},
|
||||
bindings: &[],
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let other_agent = crank_core::AgentId::new("agent_other_prompts");
|
||||
registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &other_agent,
|
||||
agent_version: 1,
|
||||
bindings: &[],
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[],
|
||||
expected_state: None,
|
||||
},
|
||||
&[AgentPromptBinding {
|
||||
prompt_id: PromptId::new("prm_e"),
|
||||
prompt_version: 1,
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &other_agent,
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.save_agent_catalog_config_editable_with_prompts(
|
||||
SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
agent_version: 1,
|
||||
bindings: &[],
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[],
|
||||
expected_state: None,
|
||||
},
|
||||
&[
|
||||
AgentPromptBinding {
|
||||
prompt_id: PromptId::new("prm_a"),
|
||||
prompt_version: 1,
|
||||
},
|
||||
AgentPromptBinding {
|
||||
prompt_id: PromptId::new("prm_b"),
|
||||
prompt_version: 1,
|
||||
},
|
||||
],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let archived_at = registry
|
||||
.archive_prompt_draft(ArchivePromptRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new("prm_b"),
|
||||
expected_updated_at: archived_at,
|
||||
archived_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.updated_at;
|
||||
// A mutable v2 draft must not change the bound v1 rendering or discovery metadata.
|
||||
let draft = registry
|
||||
.get_prompt_draft(&workspace_id(), &PromptId::new("prm_a"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.update_prompt_draft(UpdatePromptDraftRequest {
|
||||
workspace_id: workspace_id(),
|
||||
prompt_id: PromptId::new("prm_a"),
|
||||
sensitivity: PromptSensitivity::Internal,
|
||||
definition: definition("changed-draft", "Changed {{name}}"),
|
||||
expected_updated_at: draft.updated_at,
|
||||
updated_at: time::OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let key = create_api_key(®istry).await;
|
||||
let counters = Arc::new(CounterSet::default());
|
||||
let app = crate::build_app(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
None,
|
||||
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
|
||||
crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
Arc::new(crate::session::InMemorySessionStore::default()),
|
||||
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
|
||||
);
|
||||
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
|
||||
let counters = counters.clone();
|
||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||
let counters = counters.clone();
|
||||
async move { observe_body(next.run(request).await, counters) }
|
||||
}
|
||||
}));
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!(
|
||||
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
|
||||
listener.local_addr().unwrap()
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Self {
|
||||
registry,
|
||||
counters,
|
||||
key,
|
||||
url,
|
||||
archived_at,
|
||||
server,
|
||||
}
|
||||
}
|
||||
pub async fn post(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
body: &Value,
|
||||
sse: bool,
|
||||
) -> reqwest::Response {
|
||||
let mut request = reqwest::Client::new()
|
||||
.post(&self.url)
|
||||
.bearer_auth(&self.key)
|
||||
.header(
|
||||
"Accept",
|
||||
if sse {
|
||||
"text/event-stream, application/json"
|
||||
} else {
|
||||
"application/json"
|
||||
},
|
||||
)
|
||||
.header("x-request-id", "sdk-prompts-wire")
|
||||
.header(
|
||||
"traceparent",
|
||||
"00-11111111111111111111111111111111-2222222222222222-01",
|
||||
)
|
||||
.header("MCP-Protocol-Version", revision);
|
||||
if let Some(session) = session {
|
||||
request = request.header("MCP-Session-Id", session);
|
||||
}
|
||||
if revision == MODERN_REVISION {
|
||||
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
|
||||
if let Some(name) = body["params"]["name"].as_str() {
|
||||
request = request.header("Mcp-Name", name);
|
||||
}
|
||||
}
|
||||
request.json(body).send().await.unwrap()
|
||||
}
|
||||
pub async fn session(&self, revision: &str) -> Option<String> {
|
||||
if revision == MODERN_REVISION {
|
||||
return None;
|
||||
}
|
||||
let response = self
|
||||
.post(revision, None, &legacy_initialize(1), false)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let session = response.headers()["MCP-Session-Id"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
assert_eq!(
|
||||
json_body(response).await["result"]["protocolVersion"],
|
||||
revision
|
||||
);
|
||||
assert_eq!(
|
||||
self.post(revision, Some(&session), &initialized_notification(), false)
|
||||
.await
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
Some(session)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
//! #107 Resources: existing wire GREEN is independent of the missing SDK typed-path oracle.
|
||||
use super::{CounterSet, observe, support::*};
|
||||
use axum::{body::Body, response::Response};
|
||||
use futures_util::{StreamExt, stream};
|
||||
use serde_json::{Value, json};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
#[path = "rmcp_resources_test_support.rs"]
|
||||
mod fixtures;
|
||||
use fixtures::{Endpoint, TEXT, URI};
|
||||
|
||||
// As in rmcp_result_runtime_tests: deferred body polling keeps this request's observer.
|
||||
fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = stream::unfold(
|
||||
(body.into_data_stream(), counters),
|
||||
|(mut body, counters)| async move {
|
||||
observe(counters.clone(), body.next())
|
||||
.await
|
||||
.map(|chunk| (chunk, (body, counters)))
|
||||
},
|
||||
);
|
||||
Response::from_parts(parts, Body::from_stream(body))
|
||||
}
|
||||
fn counts(c: &CounterSet) -> [usize; 4] {
|
||||
[
|
||||
c.sdk_resources_dispatch(),
|
||||
c.sdk_resources_list_result(),
|
||||
c.sdk_resources_read_result(),
|
||||
c.sdk_resource_templates_list_result(),
|
||||
]
|
||||
}
|
||||
fn request(revision: &str, method: &str, mut params: Value) -> Value {
|
||||
if revision == MODERN_REVISION {
|
||||
params["_meta"] = modern_meta();
|
||||
}
|
||||
json!({"jsonrpc":"2.0", "id":9007199254740993_u64, "method":method, "params":params})
|
||||
}
|
||||
async fn wire(response: reqwest::Response, id: &Value, sse: bool) -> Value {
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(response.headers()["x-request-id"], "sdk-resources-wire");
|
||||
assert_eq!(
|
||||
response.headers()["x-trace-id"],
|
||||
"11111111111111111111111111111111"
|
||||
);
|
||||
let content_type = response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let body: Value = if sse {
|
||||
assert!(
|
||||
content_type.starts_with("text/event-stream"),
|
||||
"{content_type}: {text}"
|
||||
);
|
||||
let frames: Vec<Value> = text
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
line.strip_prefix("data:")
|
||||
.map(|s| serde_json::from_str(s.trim()).unwrap())
|
||||
})
|
||||
.collect();
|
||||
let terminal: Vec<_> = frames.iter().filter(|f| f.get("id") == Some(id)).collect();
|
||||
assert_eq!(terminal.len(), 1, "{frames:?}");
|
||||
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
|
||||
terminal[0].clone()
|
||||
} else {
|
||||
assert!(
|
||||
content_type.starts_with("application/json"),
|
||||
"{content_type}: {text}"
|
||||
);
|
||||
serde_json::from_str(&text).unwrap()
|
||||
};
|
||||
assert_eq!(body["jsonrpc"], "2.0");
|
||||
assert_eq!(&body["id"], id);
|
||||
body
|
||||
}
|
||||
fn era(result: &Value, revision: &str) {
|
||||
if revision == MODERN_REVISION {
|
||||
assert_eq!(result["resultType"], "complete", "{result}");
|
||||
assert_eq!(result["ttlMs"], 0);
|
||||
assert_eq!(result["cacheScope"], "private");
|
||||
} else {
|
||||
for key in ["resultType", "ttlMs", "cacheScope"] {
|
||||
assert!(result.get(key).is_none(), "{result}");
|
||||
}
|
||||
}
|
||||
assert!(result.get("_meta").is_none(), "{result}");
|
||||
}
|
||||
async fn success(
|
||||
h: &Endpoint,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
method: &str,
|
||||
params: Value,
|
||||
sse_accept: bool,
|
||||
) -> (Value, [usize; 4]) {
|
||||
let req = request(revision, method, params);
|
||||
let before = counts(&h.counters);
|
||||
let body = wire(
|
||||
h.post(revision, session, &req, sse_accept).await,
|
||||
&req["id"],
|
||||
sse_accept,
|
||||
)
|
||||
.await;
|
||||
assert!(body.get("error").is_none(), "{body}");
|
||||
era(&body["result"], revision);
|
||||
let after = counts(&h.counters);
|
||||
(
|
||||
body["result"].clone(),
|
||||
std::array::from_fn(|i| after[i] - before[i]),
|
||||
)
|
||||
}
|
||||
async fn control(revision: &str) -> Vec<([usize; 4], [usize; 4])> {
|
||||
let h = Endpoint::started().await;
|
||||
let session = h.session(revision).await;
|
||||
let session = session.as_deref();
|
||||
let mut deltas = Vec::new();
|
||||
// Preference order negotiates JSON or terminal SSE in both Resources revisions.
|
||||
for sse in [false, true] {
|
||||
let mut cursor = None;
|
||||
let mut names = Vec::new();
|
||||
for page in 0..2 {
|
||||
let mut params = json!({"limit":1});
|
||||
if let Some(ref cursor) = cursor {
|
||||
params["cursor"] = json!(cursor);
|
||||
}
|
||||
let (result, delta) =
|
||||
success(&h, revision, session, "resources/list", params, sse).await;
|
||||
let resources = result["resources"].as_array().unwrap();
|
||||
assert_eq!(resources.len(), 1, "{result}");
|
||||
let item = &resources[0];
|
||||
assert_eq!(item["mimeType"], "text/plain");
|
||||
assert!(
|
||||
item.get("description").is_none(),
|
||||
"static descriptors omit description: {item}"
|
||||
);
|
||||
names.push((item["name"].clone(), item["uri"].clone()));
|
||||
cursor = result
|
||||
.get("nextCursor")
|
||||
.map(|v| v.as_str().unwrap().to_owned());
|
||||
assert_eq!(cursor.is_some(), page == 0, "{result}");
|
||||
if let Some(ref cursor) = cursor {
|
||||
assert_ne!(cursor, "1", "opaque cursor required");
|
||||
}
|
||||
deltas.push((delta, [1, 1, 0, 0]));
|
||||
}
|
||||
names.sort_by_key(|n| n.0.as_str().unwrap().to_owned());
|
||||
assert_eq!(
|
||||
names,
|
||||
vec![
|
||||
(json!("sdk_a"), json!(URI)),
|
||||
(json!("sdk_b"), json!("crank://resources/sdk-b"))
|
||||
]
|
||||
);
|
||||
let (result, delta) = success(
|
||||
&h,
|
||||
revision,
|
||||
session,
|
||||
"resources/read",
|
||||
json!({"uri":URI}),
|
||||
sse,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
result["contents"],
|
||||
json!([{"uri":URI,"mimeType":"text/plain","text":TEXT}])
|
||||
);
|
||||
deltas.push((delta, [1, 0, 1, 0]));
|
||||
let (result, delta) = success(
|
||||
&h,
|
||||
revision,
|
||||
session,
|
||||
"resources/templates/list",
|
||||
json!({}),
|
||||
sse,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result["resourceTemplates"], json!([]));
|
||||
assert!(result.get("nextCursor").is_none());
|
||||
deltas.push((delta, [1, 0, 0, 1]));
|
||||
}
|
||||
// Malformed parameters/cursors and indistinguishable unavailable URIs remain command errors.
|
||||
for (method, params) in [
|
||||
("resources/list", json!({"cursor":"invalid"})),
|
||||
("resources/list", json!({"limit":"invalid"})),
|
||||
("resources/templates/list", json!({"cursor":"invalid"})),
|
||||
("resources/templates/list", json!({"limit":"invalid"})),
|
||||
("resources/read", json!({"uri":"crank://resources/unbound"})),
|
||||
] {
|
||||
let req = request(revision, method, params);
|
||||
let body = wire(
|
||||
h.post(revision, session, &req, false).await,
|
||||
&req["id"],
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body["error"]["code"], -32602, "{body}");
|
||||
assert!(body.get("result").is_none(), "{body}");
|
||||
}
|
||||
// An existing session does not preserve revoked authorization.
|
||||
h.registry
|
||||
.revoke_platform_api_key(
|
||||
&workspace_id(),
|
||||
&crank_core::PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
&time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
for (method, params) in [
|
||||
("resources/list", json!({})),
|
||||
("resources/read", json!({"uri":URI})),
|
||||
("resources/templates/list", json!({})),
|
||||
] {
|
||||
let req = request(revision, method, params);
|
||||
assert_eq!(h.post(revision, session, &req, false).await.status(), 401);
|
||||
}
|
||||
eprintln!(
|
||||
"existing GREEN Resources wire {revision}: pagination, exact content URI/bytes, empty templates, JSON/SSE negotiation, InvalidParams, revoked401; actual/expected SDK deltas={deltas:?}"
|
||||
);
|
||||
deltas
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_resources_wire_legacy() {
|
||||
control(LEGACY_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_resources_wire_modern() {
|
||||
control(MODERN_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_resources_sdk_legacy() {
|
||||
let deltas = control(LEGACY_REVISION).await;
|
||||
assert!(
|
||||
deltas.iter().all(|(actual, expected)| actual == expected),
|
||||
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_resources_sdk_modern() {
|
||||
let deltas = control(MODERN_REVISION).await;
|
||||
assert!(
|
||||
deltas.iter().all(|(actual, expected)| actual == expected),
|
||||
"SDK typed dispatch/result missing after successful HTTP: {deltas:?}"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_resources_observer_scope_control() {
|
||||
use super::{
|
||||
sdk_resource_templates_list_result_serialized, sdk_resources_dispatch_enter,
|
||||
sdk_resources_list_result_serialized, sdk_resources_read_result_serialized,
|
||||
};
|
||||
let a = Arc::new(CounterSet::default());
|
||||
let b = Arc::new(CounterSet::default());
|
||||
sdk_resources_dispatch_enter();
|
||||
assert_eq!(counts(&a), [0; 4]);
|
||||
let snapshot = a.snapshot();
|
||||
observe(a.clone(), async {
|
||||
sdk_resources_dispatch_enter();
|
||||
sdk_resources_list_result_serialized();
|
||||
sdk_resources_read_result_serialized();
|
||||
sdk_resource_templates_list_result_serialized();
|
||||
})
|
||||
.await;
|
||||
observe(b.clone(), async {
|
||||
sdk_resources_read_result_serialized();
|
||||
})
|
||||
.await;
|
||||
assert_eq!(counts(&a), [1; 4]);
|
||||
assert_eq!(counts(&b), [0, 0, 1, 0]);
|
||||
assert_eq!(a.snapshot(), snapshot, "old Snapshot stays unchanged");
|
||||
let body = Body::from_stream(stream::once(async {
|
||||
sdk_resource_templates_list_result_serialized();
|
||||
Ok::<_, std::convert::Infallible>("terminal")
|
||||
}));
|
||||
let response = observe_body(Response::new(body), b.clone());
|
||||
assert_eq!(
|
||||
&axum::body::to_bytes(response.into_body(), 100)
|
||||
.await
|
||||
.unwrap()[..],
|
||||
b"terminal"
|
||||
);
|
||||
assert_eq!(counts(&b), [0, 0, 1, 1]);
|
||||
assert_eq!(counts(&a), [1; 4]);
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
//! Small Resources fixture: existing observer helpers, exact artifact publications, normal app.
|
||||
use super::*;
|
||||
use crank_artifacts::ArtifactStore;
|
||||
use crank_core::{Agent, AgentStatus, AgentVersion, ToolSelectionPolicy, ids::ResourceId};
|
||||
use crank_registry::{
|
||||
AgentResourceBinding, CreateAgentRequest, PublishAgentRequest, SaveAgentCatalogConfigRequest,
|
||||
};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(super) const URI: &str = "crank://resources/sdk-a";
|
||||
pub(super) const TEXT: &str = "published v1\n";
|
||||
|
||||
pub(super) struct Endpoint {
|
||||
pub registry: crank_registry::PostgresRegistry,
|
||||
pub counters: Arc<CounterSet>,
|
||||
pub key: String,
|
||||
pub url: String,
|
||||
server: tokio::task::JoinHandle<()>,
|
||||
root: std::path::PathBuf,
|
||||
}
|
||||
impl Drop for Endpoint {
|
||||
fn drop(&mut self) {
|
||||
self.server.abort();
|
||||
let _ = std::fs::remove_dir_all(&self.root);
|
||||
}
|
||||
}
|
||||
impl Endpoint {
|
||||
pub async fn started() -> Self {
|
||||
crank_test_support::postgres_schema_url_for_task("resources_owner", "t_73561633").await;
|
||||
let registry = test_registry().await;
|
||||
let root = std::env::temp_dir().join(format!("rmcp-resources-{}", uuid::Uuid::now_v7()));
|
||||
std::fs::create_dir(&root).unwrap();
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
let store = ArtifactStore::open(&root).unwrap();
|
||||
// Same immutable row shapes and real ArtifactStore::put as static_resources/fixtures.rs.
|
||||
for (id, uri, name, bytes) in [
|
||||
("res_sdk_a", URI, "sdk_a", TEXT),
|
||||
(
|
||||
"res_sdk_b",
|
||||
"crank://resources/sdk-b",
|
||||
"sdk_b",
|
||||
"second published resource\n",
|
||||
),
|
||||
] {
|
||||
let stored = store.put(bytes.as_bytes()).unwrap();
|
||||
let digest = format!("{:x}", Sha256::digest(bytes.as_bytes()));
|
||||
let source = format!("src_{id}");
|
||||
sqlx::query("insert into artifact_blobs (digest, artifact_ref, size_bytes, storage_lifecycle, created_at, updated_at) values ($1,$2,$3,'available',now(),now())")
|
||||
.bind(&digest).bind(format!("sha256:{digest}")).bind(stored.size_bytes as i64).execute(registry.pool()).await.unwrap();
|
||||
sqlx::query("insert into artifact_sources (workspace_id,source_id,blob_digest,mime_type,sensitivity,lifecycle,created_at,updated_at) values ('ws_default',$1,$2,'text/plain','internal','active',now(),now())")
|
||||
.bind(&source).bind(&digest).execute(registry.pool()).await.unwrap();
|
||||
sqlx::query("insert into resources (workspace_id,resource_id,uri,name,description,status,kind,current_draft_version,created_at,updated_at) values ('ws_default',$1,$2,$3,'','published','text',1,now(),now())")
|
||||
.bind(id).bind(uri).bind(name).execute(registry.pool()).await.unwrap();
|
||||
sqlx::query("insert into resource_versions (workspace_id,resource_id,version,kind,mime_type,size_bytes,digest,sensitivity,artifact_source_id,created_at,superseded_at) values ('ws_default',$1,1,'text','text/plain',$2,$3,'internal',$4,now(),null)")
|
||||
.bind(id).bind(stored.size_bytes as i64).bind(&digest).bind(&source).execute(registry.pool()).await.unwrap();
|
||||
sqlx::query("insert into resource_publications (workspace_id,resource_id,version,uri,name,description,kind,mime_type,size_bytes,digest,sensitivity,artifact_source_id,published_at) values ('ws_default',$1,1,$2,$3,'','text','text/plain',$4,$5,'internal',$6,now())")
|
||||
.bind(id).bind(uri).bind(name).bind(stored.size_bytes as i64).bind(&digest).bind(&source).execute(registry.pool()).await.unwrap();
|
||||
}
|
||||
registry
|
||||
.create_agent(CreateAgentRequest {
|
||||
agent: &Agent {
|
||||
id: agent_id(),
|
||||
workspace_id: workspace_id(),
|
||||
slug: AGENT_SLUG.into(),
|
||||
display_name: "SDK Resources".into(),
|
||||
description: String::new(),
|
||||
status: AgentStatus::Draft,
|
||||
current_draft_version: 1,
|
||||
latest_published_version: None,
|
||||
created_at: test_time(),
|
||||
updated_at: test_time(),
|
||||
published_at: None,
|
||||
},
|
||||
version: &AgentVersion {
|
||||
agent_id: agent_id(),
|
||||
version: 1,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: test_time(),
|
||||
},
|
||||
bindings: &[],
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.save_agent_catalog_config(SaveAgentCatalogConfigRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
agent_version: 1,
|
||||
bindings: &[],
|
||||
tool_selection_policy: &ToolSelectionPolicy::default(),
|
||||
resource_bindings: &[
|
||||
AgentResourceBinding {
|
||||
resource_id: ResourceId::new("res_sdk_a"),
|
||||
resource_version: 1,
|
||||
},
|
||||
AgentResourceBinding {
|
||||
resource_id: ResourceId::new("res_sdk_b"),
|
||||
resource_version: 1,
|
||||
},
|
||||
],
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let key = create_api_key(®istry).await;
|
||||
let counters = Arc::new(CounterSet::default());
|
||||
// Normal build_app composition with its sanctioned artifact root, not the isolated SDK router.
|
||||
let app = crate::build_app_with_artifacts(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
None,
|
||||
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
|
||||
crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
Arc::new(crate::session::InMemorySessionStore::default()),
|
||||
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
|
||||
16,
|
||||
store,
|
||||
);
|
||||
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
|
||||
let counters = counters.clone();
|
||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||
let counters = counters.clone();
|
||||
async move {
|
||||
let response = next.run(request).await;
|
||||
observe_body(response, counters)
|
||||
}
|
||||
}
|
||||
}));
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!(
|
||||
"http://{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
|
||||
listener.local_addr().unwrap()
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Self {
|
||||
registry,
|
||||
counters,
|
||||
key,
|
||||
url,
|
||||
server,
|
||||
root,
|
||||
}
|
||||
}
|
||||
pub async fn post(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
body: &Value,
|
||||
sse: bool,
|
||||
) -> reqwest::Response {
|
||||
let mut request = reqwest::Client::new()
|
||||
.post(&self.url)
|
||||
.bearer_auth(&self.key)
|
||||
.header(
|
||||
"Accept",
|
||||
if sse {
|
||||
"text/event-stream, application/json"
|
||||
} else {
|
||||
"application/json"
|
||||
},
|
||||
)
|
||||
.header("x-request-id", "sdk-resources-wire")
|
||||
.header(
|
||||
"traceparent",
|
||||
"00-11111111111111111111111111111111-2222222222222222-01",
|
||||
)
|
||||
.header("MCP-Protocol-Version", revision);
|
||||
if let Some(session) = session {
|
||||
request = request.header("MCP-Session-Id", session);
|
||||
}
|
||||
if revision == MODERN_REVISION {
|
||||
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
|
||||
if let Some(uri) = body["params"]["uri"].as_str() {
|
||||
request = request.header("Mcp-Name", uri);
|
||||
}
|
||||
}
|
||||
request.json(body).send().await.unwrap()
|
||||
}
|
||||
pub async fn session(&self, revision: &str) -> Option<String> {
|
||||
if revision == MODERN_REVISION {
|
||||
return None;
|
||||
}
|
||||
let response = self
|
||||
.post(revision, None, &legacy_initialize(1), false)
|
||||
.await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let session = response.headers()["MCP-Session-Id"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
assert_eq!(
|
||||
json_body(response).await["result"]["protocolVersion"],
|
||||
revision
|
||||
);
|
||||
assert_eq!(
|
||||
self.post(revision, Some(&session), &initialized_notification(), false)
|
||||
.await
|
||||
.status(),
|
||||
202
|
||||
);
|
||||
Some(session)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
//! #107 result serialization: wire controls run independently of the structural SDK oracle.
|
||||
use super::*;
|
||||
use axum::{
|
||||
Json,
|
||||
body::Body,
|
||||
extract::State,
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures_util::{StreamExt, stream};
|
||||
|
||||
const ERROR_EMAIL: &str = "upstream-error@example.test";
|
||||
const REQUEST_ID: &str = "sdk-result-wire";
|
||||
const TRACE_ID: &str = "11111111111111111111111111111111";
|
||||
|
||||
pub(super) async fn upstream(state: State<Arc<AtomicUsize>>, payload: Json<Value>) -> Response {
|
||||
if payload.0["email"] == ERROR_EMAIL {
|
||||
state.0.fetch_add(1, Ordering::SeqCst);
|
||||
return (
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
Json(json!({"error":"fixture rejection"})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
create_lead(state, payload).await.into_response()
|
||||
}
|
||||
|
||||
// The HTTP body is polled after next.run returns. Carry only the test observer, not a new lifecycle.
|
||||
pub(super) fn observe_body(response: Response, counters: Arc<CounterSet>) -> Response {
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = stream::unfold(
|
||||
(body.into_data_stream(), counters),
|
||||
|(mut body, counters)| async move {
|
||||
observe(counters.clone(), body.next())
|
||||
.await
|
||||
.map(|chunk| (chunk, (body, counters)))
|
||||
},
|
||||
);
|
||||
Response::from_parts(parts, Body::from_stream(body))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_result_http_pagination_both_revisions() {
|
||||
let h = Endpoint::started().await;
|
||||
let mut operation = test_operation(&h._upstream.1);
|
||||
operation.id = crank_core::OperationId::new("op_second_lead");
|
||||
operation.name = "second_lead".to_owned();
|
||||
h.registry
|
||||
.create_operation(&workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
h.registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
h.registry
|
||||
.create_agent_draft_version(CreateAgentDraftVersionRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
version: &AgentVersion {
|
||||
agent_id: agent_id(),
|
||||
version: 2,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: now,
|
||||
},
|
||||
bindings: &[
|
||||
AgentOperationBinding {
|
||||
operation_id: crank_core::OperationId::new("op_create_lead"),
|
||||
operation_version: 1,
|
||||
tool_name: TOOL_NAME.into(),
|
||||
tool_title: "Create Lead".into(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
},
|
||||
AgentOperationBinding {
|
||||
operation_id: operation.id,
|
||||
operation_version: 1,
|
||||
tool_name: "second_lead".into(),
|
||||
tool_title: "Second Lead".into(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
},
|
||||
],
|
||||
updated_at: &now,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
h.registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
version: 2,
|
||||
published_at: &now,
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
for revision in [LEGACY_REVISION, MODERN_REVISION] {
|
||||
let session = h.session(revision).await;
|
||||
let mut names = Vec::new();
|
||||
for (cursor, expected_next, expected_len) in [
|
||||
(None, Some("1"), 1),
|
||||
(Some("1"), None, 1),
|
||||
(Some("2"), None, 0),
|
||||
] {
|
||||
let id = json!(format!("page-{cursor:?}"));
|
||||
let mut request =
|
||||
json!({"jsonrpc":"2.0","id":id,"method":"tools/list","params":{"limit":1}});
|
||||
if let Some(cursor) = cursor {
|
||||
request["params"]["cursor"] = json!(cursor);
|
||||
}
|
||||
if revision == MODERN_REVISION {
|
||||
request["params"]["_meta"] = modern_meta();
|
||||
}
|
||||
let before = h.counters.sdk_tools_list_result();
|
||||
let body = wire_body(
|
||||
h.post(revision, session.as_deref(), &request).await,
|
||||
&id,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_era(&body["result"], revision, true);
|
||||
assert_eq!(
|
||||
body["result"].get("nextCursor"),
|
||||
expected_next.map(|s| json!(s)).as_ref()
|
||||
);
|
||||
let tools = body["result"]["tools"].as_array().unwrap();
|
||||
assert_eq!(tools.len(), expected_len);
|
||||
names.extend(
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| tool["name"].as_str().unwrap().to_owned()),
|
||||
);
|
||||
assert_eq!(h.counters.sdk_tools_list_result() - before, 1);
|
||||
}
|
||||
names.sort();
|
||||
let mut expected = vec![TOOL_NAME.to_owned(), "second_lead".to_owned()];
|
||||
expected.sort();
|
||||
assert_eq!(names, expected);
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
fn assert_era(result: &Value, revision: &str, list: bool) {
|
||||
if revision == MODERN_REVISION {
|
||||
assert_eq!(result["resultType"], "complete", "{result}");
|
||||
if list {
|
||||
assert_eq!(result["ttlMs"], 0);
|
||||
assert_eq!(result["cacheScope"], "private");
|
||||
}
|
||||
} else {
|
||||
assert!(result.get("resultType").is_none(), "{result}");
|
||||
assert!(result.get("ttlMs").is_none(), "{result}");
|
||||
assert!(result.get("cacheScope").is_none(), "{result}");
|
||||
}
|
||||
assert!(result.get("_meta").is_none(), "{result}");
|
||||
}
|
||||
|
||||
async fn wire_body(response: reqwest::Response, id: &Value, sse: bool) -> Value {
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(response.headers()["x-request-id"], REQUEST_ID);
|
||||
assert_eq!(response.headers()["x-trace-id"], TRACE_ID);
|
||||
let content_type = response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let text = tokio::time::timeout(Duration::from_secs(5), response.text())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let body: Value = if sse {
|
||||
assert!(
|
||||
content_type.starts_with("text/event-stream"),
|
||||
"{content_type}"
|
||||
);
|
||||
let frames: Vec<Value> = text
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
line.strip_prefix("data:")
|
||||
.map(|data| serde_json::from_str(data.trim()).unwrap())
|
||||
})
|
||||
.collect();
|
||||
let terminal: Vec<_> = frames
|
||||
.iter()
|
||||
.filter(|frame| frame.get("id") == Some(id))
|
||||
.collect();
|
||||
assert_eq!(terminal.len(), 1, "{frames:?}");
|
||||
assert_eq!(frames.last(), Some(terminal[0]), "terminal must be last");
|
||||
assert!(
|
||||
frames
|
||||
.iter()
|
||||
.any(|frame| frame["method"] == "notifications/progress"
|
||||
&& frame["params"]["progressToken"] == "result-progress"),
|
||||
"{frames:?}"
|
||||
);
|
||||
terminal[0].clone()
|
||||
} else {
|
||||
assert!(
|
||||
content_type.starts_with("application/json"),
|
||||
"{content_type}"
|
||||
);
|
||||
serde_json::from_str(&text).unwrap()
|
||||
};
|
||||
assert_eq!(body["jsonrpc"], "2.0");
|
||||
assert_eq!(&body["id"], id);
|
||||
body
|
||||
}
|
||||
|
||||
fn assert_call_result(body: &Value, revision: &str, email: &str) {
|
||||
assert!(body.get("error").is_none(), "{body}");
|
||||
let result = &body["result"];
|
||||
assert_era(result, revision, false);
|
||||
let (output, text, is_error) = if email == ERROR_EMAIL {
|
||||
let contract = crate::tool_error::tool_error_contract_from_runtime(
|
||||
&crank_runtime::RuntimeError::ProtocolAdapter(
|
||||
crank_core::ProtocolAdapterError::UnexpectedStatus {
|
||||
status: 422,
|
||||
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
||||
},
|
||||
),
|
||||
REQUEST_ID,
|
||||
TRACE_ID,
|
||||
crank_core::ExecutionLocale::En,
|
||||
);
|
||||
(
|
||||
json!({"error":crate::tool_error::tool_error_value(&contract)}),
|
||||
crate::tool_error::tool_error_text(&contract),
|
||||
true,
|
||||
)
|
||||
} else {
|
||||
let output = json!({"id":format!("lead_for_{email}")});
|
||||
let text = serde_json::to_string_pretty(&output).unwrap();
|
||||
(output, text, false)
|
||||
};
|
||||
assert_eq!(result["structuredContent"], output, "{body}");
|
||||
assert_eq!(result["content"], json!([{"type":"text", "text":text}]));
|
||||
assert_eq!(result["isError"], is_error);
|
||||
assert_eq!(
|
||||
result.as_object().unwrap().len(),
|
||||
if revision == MODERN_REVISION { 4 } else { 3 },
|
||||
"no unexpected optional fields: {result}"
|
||||
);
|
||||
}
|
||||
|
||||
async fn result_control(revision: &str) -> Vec<usize> {
|
||||
let h = Endpoint::started().await;
|
||||
let session = h.session(revision).await;
|
||||
let session = session.as_deref();
|
||||
let mut counts = Vec::new();
|
||||
// Actual published descriptor and schema, then the empty page at the catalog boundary.
|
||||
for (cursor, expected_tools, id) in [
|
||||
(
|
||||
None,
|
||||
json!([{"name":TOOL_NAME,"title":"Create Lead","description":"Creates a CRM lead","inputSchema":{"type":"object","properties":{"email":{"type":"string"}},"required":["email"]}}]),
|
||||
json!("list-string-id"),
|
||||
),
|
||||
(Some("1"), json!([]), json!(9007199254740993_u64)),
|
||||
] {
|
||||
let mut request =
|
||||
json!({"jsonrpc":"2.0","id":id,"method":"tools/list","params":{"limit":1}});
|
||||
if revision == MODERN_REVISION {
|
||||
request["params"]["_meta"] = modern_meta();
|
||||
}
|
||||
if let Some(cursor) = cursor {
|
||||
request["params"]["cursor"] = json!(cursor);
|
||||
}
|
||||
let before = h.counters.sdk_tools_list_result();
|
||||
let before_call = h.counters.sdk_tools_call_result();
|
||||
let before_dispatch = h.counters.sdk_tools_dispatch();
|
||||
let body = wire_body(h.post(revision, session, &request).await, &id, false).await;
|
||||
assert!(body.get("error").is_none(), "{body}");
|
||||
assert_eq!(body["result"]["tools"], expected_tools);
|
||||
assert!(body["result"].get("nextCursor").is_none(), "{body}");
|
||||
assert_era(&body["result"], revision, true);
|
||||
assert_eq!(h.counters.sdk_tools_call_result(), before_call);
|
||||
assert_eq!(h.counters.sdk_tools_dispatch() - before_dispatch, 1);
|
||||
counts.push(h.counters.sdk_tools_list_result() - before);
|
||||
}
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), 0);
|
||||
for (email, id, sse) in [
|
||||
(
|
||||
"result-success@example.test",
|
||||
json!(9007199254740993_u64),
|
||||
false,
|
||||
),
|
||||
(ERROR_EMAIL, json!("tool-error-string-id"), false),
|
||||
("result-sse@example.test", json!("sse-string-id"), true),
|
||||
(ERROR_EMAIL, json!(92), true),
|
||||
] {
|
||||
if sse && revision != MODERN_REVISION {
|
||||
continue;
|
||||
}
|
||||
let mut request = if revision == MODERN_REVISION {
|
||||
modern_tool_call(90, email)
|
||||
} else {
|
||||
legacy_tool_call(90, email)
|
||||
};
|
||||
request["id"] = id.clone();
|
||||
if sse {
|
||||
request["params"]["_meta"]["progressToken"] = json!("result-progress");
|
||||
}
|
||||
let before = h.counters.sdk_tools_call_result();
|
||||
let before_list = h.counters.sdk_tools_list_result();
|
||||
let before_dispatch = h.counters.sdk_tools_dispatch();
|
||||
let arrivals = h.calls.load(Ordering::SeqCst);
|
||||
let response = h
|
||||
.post_accept(
|
||||
revision,
|
||||
session,
|
||||
&request,
|
||||
if sse {
|
||||
"application/json, text/event-stream"
|
||||
} else {
|
||||
"application/json"
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let body = wire_body(response, &id, sse).await;
|
||||
assert_call_result(&body, revision, email);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst) - arrivals, 1);
|
||||
assert_eq!(h.counters.sdk_tools_list_result(), before_list);
|
||||
assert_eq!(h.counters.sdk_tools_dispatch() - before_dispatch, 1);
|
||||
counts.push(h.counters.sdk_tools_call_result() - before);
|
||||
}
|
||||
// Unknown tool is a command InvalidParams, not an execution tool-error result.
|
||||
let mut invalid = if revision == MODERN_REVISION {
|
||||
modern_tool_call(93, "unused@example.test")
|
||||
} else {
|
||||
legacy_tool_call(93, "unused@example.test")
|
||||
};
|
||||
invalid["params"]["name"] = json!("missing_tool");
|
||||
invalid["id"] = json!("invalid-params-id");
|
||||
let before = h.counters.sdk_tools_call_result();
|
||||
let arrivals = h.calls.load(Ordering::SeqCst);
|
||||
let body = wire_body(
|
||||
h.post(revision, session, &invalid).await,
|
||||
&invalid["id"],
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(body["error"]["code"], -32602, "{body}");
|
||||
assert!(body.get("result").is_none(), "{body}");
|
||||
assert_eq!(body["error"]["data"]["request_id"], REQUEST_ID);
|
||||
assert_eq!(body["error"]["data"]["trace_id"], TRACE_ID);
|
||||
assert_eq!(h.counters.sdk_tools_call_result(), before);
|
||||
assert_eq!(h.calls.load(Ordering::SeqCst), arrivals);
|
||||
eprintln!(
|
||||
"existing GREEN result wire {revision}: published list/schema + empty page, exact success/error JSON{}; InvalidParams/no result/no marker/no outbound; SDK result deltas={counts:?}",
|
||||
if revision == MODERN_REVISION {
|
||||
" + success/error terminal SSE"
|
||||
} else {
|
||||
" (legacy JSON-only)"
|
||||
}
|
||||
);
|
||||
counts
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_result_wire_modern() {
|
||||
result_control(MODERN_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_result_wire_legacy() {
|
||||
result_control(LEGACY_REVISION).await;
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_result_sdk_serialization_modern() {
|
||||
assert_eq!(
|
||||
result_control(MODERN_REVISION).await,
|
||||
vec![1; 6],
|
||||
"structural oracle: SDK list/final-empty/success/error/SSE-success/SSE-error must each serialize once after successful real HTTP wire controls"
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn rmcp_result_sdk_serialization_legacy() {
|
||||
assert_eq!(
|
||||
result_control(LEGACY_REVISION).await,
|
||||
vec![1; 4],
|
||||
"structural oracle: SDK list/final-empty/success/error must each serialize once after successful real HTTP wire controls"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_result_observer_scope_control() {
|
||||
use crate::app::dispatch_observer::{
|
||||
sdk_tools_call_result_serialized, sdk_tools_list_result_serialized,
|
||||
};
|
||||
let a = Arc::new(CounterSet::default());
|
||||
let b = Arc::new(CounterSet::default());
|
||||
sdk_tools_list_result_serialized();
|
||||
sdk_tools_call_result_serialized();
|
||||
assert_eq!(
|
||||
(a.sdk_tools_list_result(), a.sdk_tools_call_result()),
|
||||
(0, 0)
|
||||
);
|
||||
observe(a.clone(), async {
|
||||
sdk_tools_list_result_serialized();
|
||||
sdk_tools_call_result_serialized();
|
||||
})
|
||||
.await;
|
||||
// A different request cannot alter a's result counters.
|
||||
observe(b.clone(), async {
|
||||
sdk_tools_list_result_serialized();
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
(a.sdk_tools_list_result(), a.sdk_tools_call_result()),
|
||||
(1, 1)
|
||||
);
|
||||
assert_eq!(
|
||||
(b.sdk_tools_list_result(), b.sdk_tools_call_result()),
|
||||
(1, 0)
|
||||
);
|
||||
// Prove the stream-body scope actually records deferred work after the header scope ended.
|
||||
let stream_counters = b.clone();
|
||||
let body = Body::from_stream(stream::once(async move {
|
||||
sdk_tools_call_result_serialized();
|
||||
Ok::<_, std::convert::Infallible>("terminal")
|
||||
}));
|
||||
let response = observe_body(Response::new(body), stream_counters);
|
||||
let bytes = axum::body::to_bytes(response.into_body(), 100)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(&bytes[..], b"terminal");
|
||||
assert_eq!(b.sdk_tools_call_result(), 1);
|
||||
assert_eq!(a.sdk_tools_call_result(), 1);
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
//! #107: real build_app endpoint controls plus a separate SDK structural seam oracle.
|
||||
//! No SDK transport router is used. The structural oracle alone does not prove wire parity.
|
||||
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{Router, routing::post};
|
||||
use crank_core::{
|
||||
AgentOperationBinding, AgentStatus, AgentVersion, PlatformApiKeyId, ToolSelectionPolicy,
|
||||
};
|
||||
use crank_registry::{
|
||||
CreateAgentDraftVersionRequest, PostgresRegistry, PublishAgentRequest, PublishRequest,
|
||||
};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{CounterSet, observe, sdk_tools_dispatch_enter, support::*};
|
||||
|
||||
#[path = "rmcp_result_runtime_tests.rs"]
|
||||
mod result_serialization;
|
||||
|
||||
struct Server(tokio::task::JoinHandle<()>, String);
|
||||
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn serve(app: Router) -> Server {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
Server(
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
}),
|
||||
format!("http://{address}"),
|
||||
)
|
||||
}
|
||||
|
||||
struct Endpoint {
|
||||
server: Server,
|
||||
_upstream: Server,
|
||||
registry: PostgresRegistry,
|
||||
key: String,
|
||||
counters: Arc<CounterSet>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Endpoint {
|
||||
async fn started() -> Self {
|
||||
// First caller owns the process-wide fixture and its atexit cleanup, including RED panics.
|
||||
let _ =
|
||||
crank_test_support::postgres_schema_url_for_task("rmcp_runtime_owner", "t_c559abb1")
|
||||
.await;
|
||||
let registry = test_registry().await;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let upstream = serve(
|
||||
Router::new()
|
||||
.route("/crm/leads", post(result_serialization::upstream))
|
||||
.with_state(calls.clone()),
|
||||
)
|
||||
.await;
|
||||
let operation = test_operation(&upstream.1);
|
||||
registry
|
||||
.create_operation(&workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &test_time(),
|
||||
published_by: Some("alice"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent(®istry, &operation).await;
|
||||
let key = create_api_key(®istry).await;
|
||||
let counters = Arc::new(CounterSet::default());
|
||||
let app = crate::build_app(
|
||||
registry.clone(),
|
||||
Duration::ZERO,
|
||||
Some("https://crank.example.com".into()),
|
||||
SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(),
|
||||
crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
Arc::new(crate::session::InMemorySessionStore::default()),
|
||||
Arc::new(crate::auth::CommunityMachineCredentialVerifier),
|
||||
);
|
||||
let app = observe_requests(app, counters.clone(), None).layer(axum::middleware::from_fn({
|
||||
let counters = counters.clone();
|
||||
move |request: axum::extract::Request, next: axum::middleware::Next| {
|
||||
let counters = counters.clone();
|
||||
async move {
|
||||
let response = next.run(request).await;
|
||||
result_serialization::observe_body(response, counters)
|
||||
}
|
||||
}
|
||||
}));
|
||||
let server = serve(app).await;
|
||||
Self {
|
||||
server,
|
||||
_upstream: upstream,
|
||||
registry,
|
||||
key,
|
||||
counters,
|
||||
calls,
|
||||
}
|
||||
}
|
||||
|
||||
async fn post(&self, revision: &str, session: Option<&str>, body: &Value) -> reqwest::Response {
|
||||
self.post_accept(revision, session, body, "application/json")
|
||||
.await
|
||||
}
|
||||
|
||||
async fn post_accept(
|
||||
&self,
|
||||
revision: &str,
|
||||
session: Option<&str>,
|
||||
body: &Value,
|
||||
accept: &str,
|
||||
) -> reqwest::Response {
|
||||
let mut request = reqwest::Client::new()
|
||||
.post(format!(
|
||||
"{}/v1/{WORKSPACE_SLUG}/{AGENT_SLUG}",
|
||||
self.server.1
|
||||
))
|
||||
.bearer_auth(&self.key)
|
||||
// Also guards the legacy JSON-only contract; no SSE parsing fallback here.
|
||||
.header("Accept", accept)
|
||||
.header("x-request-id", "sdk-result-wire")
|
||||
.header(
|
||||
"traceparent",
|
||||
"00-11111111111111111111111111111111-2222222222222222-01",
|
||||
)
|
||||
.header("MCP-Protocol-Version", revision);
|
||||
if let Some(session) = session {
|
||||
request = request.header("MCP-Session-Id", session);
|
||||
}
|
||||
if revision == MODERN_REVISION {
|
||||
request = request.header("Mcp-Method", body["method"].as_str().unwrap());
|
||||
if let Some(name) = body["params"]["name"].as_str() {
|
||||
request = request.header("Mcp-Name", name);
|
||||
}
|
||||
}
|
||||
request.json(body).send().await.unwrap()
|
||||
}
|
||||
|
||||
async fn session(&self, revision: &str) -> Option<String> {
|
||||
if revision == MODERN_REVISION {
|
||||
return None;
|
||||
}
|
||||
let response = self.post(revision, None, &legacy_initialize(1)).await;
|
||||
assert_eq!(response.status(), 200);
|
||||
let session = response.headers()["MCP-Session-Id"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let body = json_body(response).await;
|
||||
assert_eq!(body["result"]["protocolVersion"], revision);
|
||||
assert!(body["result"]["capabilities"]["tools"].is_object());
|
||||
let response = self
|
||||
.post(revision, Some(&session), &initialized_notification())
|
||||
.await;
|
||||
assert_eq!(response.status(), 202);
|
||||
Some(session)
|
||||
}
|
||||
|
||||
async fn list(&self, revision: &str, session: Option<&str>, expected_name: &str) -> usize {
|
||||
let before = self.counters.sdk_tools_dispatch();
|
||||
let mut body = json!({"jsonrpc":"2.0", "id":10, "method":"tools/list", "params":{}});
|
||||
if revision == MODERN_REVISION {
|
||||
body["params"]["_meta"] = modern_meta();
|
||||
}
|
||||
let calls = self.calls.load(Ordering::SeqCst);
|
||||
let response = self.post(revision, session, &body).await;
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(
|
||||
response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("application/json")
|
||||
);
|
||||
let body = json_body(response).await;
|
||||
assert_eq!(body["id"], 10);
|
||||
assert_eq!(body["result"]["tools"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(body["result"]["tools"][0]["name"], expected_name);
|
||||
assert!(body["error"].is_null(), "{body}");
|
||||
assert_eq!(
|
||||
self.calls.load(Ordering::SeqCst),
|
||||
calls,
|
||||
"listing must not execute upstream"
|
||||
);
|
||||
self.counters.sdk_tools_dispatch() - before
|
||||
}
|
||||
|
||||
async fn call(&self, revision: &str, session: Option<&str>, name: &str, email: &str) -> usize {
|
||||
let before = self.counters.sdk_tools_dispatch();
|
||||
let arrivals = self.calls.load(Ordering::SeqCst);
|
||||
let mut body = if revision == MODERN_REVISION {
|
||||
modern_tool_call(11, email)
|
||||
} else {
|
||||
legacy_tool_call(11, email)
|
||||
};
|
||||
body["params"]["name"] = json!(name);
|
||||
let response = self.post(revision, session, &body).await;
|
||||
assert_eq!(response.status(), 200);
|
||||
assert!(
|
||||
response.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("application/json")
|
||||
);
|
||||
let body = json_body(response).await;
|
||||
assert_eq!(body["id"], 11);
|
||||
assert!(body["error"].is_null(), "{body}");
|
||||
assert_ne!(body["result"]["isError"], true, "{body}");
|
||||
assert_scoped_output(revision, &body, email);
|
||||
if revision == MODERN_REVISION {
|
||||
assert_eq!(body["result"]["resultType"], "complete");
|
||||
}
|
||||
assert_eq!(self.calls.load(Ordering::SeqCst) - arrivals, 1);
|
||||
self.counters.sdk_tools_dispatch() - before
|
||||
}
|
||||
|
||||
async fn publish_renamed_tool(&self) {
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let version = AgentVersion {
|
||||
agent_id: agent_id(),
|
||||
version: 2,
|
||||
status: AgentStatus::Draft,
|
||||
instructions: json!({}),
|
||||
tool_selection_policy: ToolSelectionPolicy::default(),
|
||||
created_at: now,
|
||||
};
|
||||
let bindings = [AgentOperationBinding {
|
||||
operation_id: crank_core::OperationId::new("op_create_lead"),
|
||||
operation_version: 1,
|
||||
tool_name: "new_publication_tool".into(),
|
||||
tool_title: "New Tool".into(),
|
||||
tool_description_override: None,
|
||||
enabled: true,
|
||||
}];
|
||||
self.registry
|
||||
.create_agent_draft_version(CreateAgentDraftVersionRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
version: &version,
|
||||
bindings: &bindings,
|
||||
updated_at: &now,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
self.registry
|
||||
.publish_agent(PublishAgentRequest {
|
||||
workspace_id: &workspace_id(),
|
||||
agent_id: &agent_id(),
|
||||
version: 2,
|
||||
published_at: &now,
|
||||
published_by: Some("alice"),
|
||||
expected_state: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
async fn endpoint_control(revision: &str) -> [usize; 2] {
|
||||
let h = Endpoint::started().await;
|
||||
let session = h.session(revision).await;
|
||||
let session = session.as_deref();
|
||||
let list_dispatches = h.list(revision, session, TOOL_NAME).await;
|
||||
let call_dispatches = h
|
||||
.call(
|
||||
revision,
|
||||
session,
|
||||
TOOL_NAME,
|
||||
&format!("{revision}@example.test"),
|
||||
)
|
||||
.await;
|
||||
// A fresh publication must be visible without reinitialize or a new endpoint.
|
||||
h.publish_renamed_tool().await;
|
||||
h.list(revision, session, "new_publication_tool").await;
|
||||
h.call(
|
||||
revision,
|
||||
session,
|
||||
"new_publication_tool",
|
||||
"new-publication@example.test",
|
||||
)
|
||||
.await;
|
||||
h.registry
|
||||
.revoke_platform_api_key(
|
||||
&workspace_id(),
|
||||
&PlatformApiKeyId::new("pk_dispatch_observer"),
|
||||
&time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let before_sdk = h.counters.sdk_tools_dispatch();
|
||||
let arrivals = h.calls.load(Ordering::SeqCst);
|
||||
let mut denied = if revision == MODERN_REVISION {
|
||||
modern_tool_call(12, "denied@example.test")
|
||||
} else {
|
||||
legacy_tool_call(12, "denied@example.test")
|
||||
};
|
||||
denied["params"]["name"] = json!("new_publication_tool");
|
||||
assert_eq!(h.post(revision, session, &denied).await.status(), 401);
|
||||
assert_eq!(
|
||||
h.calls.load(Ordering::SeqCst),
|
||||
arrivals,
|
||||
"revoked credential must not execute upstream"
|
||||
);
|
||||
assert_eq!(
|
||||
h.counters.sdk_tools_dispatch(),
|
||||
before_sdk,
|
||||
"revocation must reject before SDK dispatch"
|
||||
);
|
||||
eprintln!(
|
||||
"existing GREEN {revision}: JSON list/call, scoped output, upstream=2, publication refresh, revoked=401/no outbound; SDK list/call deltas={:?}",
|
||||
[list_dispatches, call_dispatches]
|
||||
);
|
||||
[list_dispatches, call_dispatches]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_runtime_existing_wire_modern() {
|
||||
endpoint_control(MODERN_REVISION).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_runtime_existing_wire_legacy() {
|
||||
endpoint_control(LEGACY_REVISION).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_runtime_sdk_dispatch_modern() {
|
||||
let counts = endpoint_control(MODERN_REVISION).await;
|
||||
assert_eq!(
|
||||
counts,
|
||||
[1, 1],
|
||||
"real endpoint tools/list and tools/call must each enter SDK typed dispatch once (structural seam oracle)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_runtime_sdk_dispatch_legacy() {
|
||||
let counts = endpoint_control(LEGACY_REVISION).await;
|
||||
assert_eq!(
|
||||
counts,
|
||||
[1, 1],
|
||||
"real endpoint tools/list and tools/call must each enter SDK typed dispatch once (structural seam oracle)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_runtime_observer_is_request_scoped() {
|
||||
let counters = Arc::new(CounterSet::default());
|
||||
sdk_tools_dispatch_enter();
|
||||
assert_eq!(counters.sdk_tools_dispatch(), 0);
|
||||
observe(counters.clone(), async {
|
||||
sdk_tools_dispatch_enter();
|
||||
})
|
||||
.await;
|
||||
assert_eq!(counters.sdk_tools_dispatch(), 1);
|
||||
let other = Arc::new(CounterSet::default());
|
||||
observe(other.clone(), async {
|
||||
sdk_tools_dispatch_enter();
|
||||
})
|
||||
.await;
|
||||
assert_eq!(other.sdk_tools_dispatch(), 1);
|
||||
assert_eq!(counters.sdk_tools_dispatch(), 1);
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! Behavioural ATDD for the rmcp 3.7.0 Tools adapter over the existing typed Tools service.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crank_core::{
|
||||
AdmissionSchedulerConfig, CoordinationStateStore, CorrelationContext, ExecutionLocale,
|
||||
MachineAccessMode, OperationSecurityLevel, PlatformApiKeyScope,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, MigrationAuthority, PostgresRegistry, PostgresTaskStore};
|
||||
use crank_runtime::{
|
||||
InMemoryCoordinationStateStore, RequestRateLimitConfig, RequestRateLimiter, SecretCrypto,
|
||||
TaskService,
|
||||
};
|
||||
|
||||
use rmcp::model::{CallToolRequestParams, PaginatedRequestParams, RequestId};
|
||||
use rmcp::service::{RequestContext, serve_directly};
|
||||
use rmcp::{RoleServer, ServerHandler};
|
||||
|
||||
use super::{
|
||||
AppState,
|
||||
rmcp_tools::{RmcpRequestIdentity, RmcpTools},
|
||||
};
|
||||
use crate::app::TaskApplication;
|
||||
use crate::auth::{CommunityMachineCredentialVerifier, VerifiedMachineCredential};
|
||||
use crate::catalog::PublishedToolCatalog;
|
||||
use crate::session::{ActiveSessionMetrics, InMemorySessionStore};
|
||||
use crate::transport::AllowedOrigins;
|
||||
|
||||
pub(super) async fn test_state() -> Arc<AppState> {
|
||||
let database_url =
|
||||
crank_test_support::postgres_schema_url_for_task("rmcp_tools_tests", "t_fdcc7df8").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
||||
|
||||
let coordination: Arc<dyn CoordinationStateStore> =
|
||||
Arc::new(InMemoryCoordinationStateStore::default());
|
||||
let sessions = Arc::new(InMemorySessionStore::default());
|
||||
let session_metrics = ActiveSessionMetrics::start(sessions.clone());
|
||||
let catalog = PublishedToolCatalog::new(registry.clone(), Duration::ZERO, coordination.clone());
|
||||
|
||||
let tasks_store = PostgresTaskStore::new(
|
||||
registry.pool().clone(),
|
||||
AdmissionLimits::default(),
|
||||
AdmissionSchedulerConfig::default(),
|
||||
)
|
||||
.expect("default admission limits and scheduler weights are valid");
|
||||
let tasks = Arc::new(TaskApplication::new(
|
||||
registry.clone(),
|
||||
Duration::from_secs(60),
|
||||
coordination.clone(),
|
||||
TaskService::new(Arc::new(tasks_store)),
|
||||
));
|
||||
|
||||
Arc::new(AppState {
|
||||
registry,
|
||||
catalog,
|
||||
runtime: crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
api_rate_limiter: RequestRateLimiter::new(RequestRateLimitConfig {
|
||||
requests_per_second: 10_000,
|
||||
burst: 10_000,
|
||||
}),
|
||||
artifact_store: None,
|
||||
secret_crypto: SecretCrypto::new("01234567890123456789012345678901")
|
||||
.expect("test master key"),
|
||||
sessions,
|
||||
session_metrics,
|
||||
session_slots: Arc::new(tokio::sync::Semaphore::new(16)),
|
||||
credential_verifier: Arc::new(CommunityMachineCredentialVerifier),
|
||||
allowed_origins: AllowedOrigins::new(None),
|
||||
logging_levels: Arc::new(crate::protocol::LogLevelStore::new()),
|
||||
signal_streams: Arc::new(crate::protocol::StreamRegistry::new()),
|
||||
in_flight: Arc::new(super::lifecycle::InFlightCalls::new(
|
||||
super::lifecycle::MAX_IN_FLIGHT_CALLS,
|
||||
)),
|
||||
tasks,
|
||||
})
|
||||
}
|
||||
|
||||
fn verified_credential() -> VerifiedMachineCredential {
|
||||
VerifiedMachineCredential {
|
||||
machine_access_mode: MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: OperationSecurityLevel::Standard,
|
||||
scopes: vec![PlatformApiKeyScope::Write],
|
||||
platform_api_key_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn rmcp_tools(state: Arc<AppState>) -> RmcpTools {
|
||||
RmcpTools {
|
||||
state,
|
||||
active: Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn request_context(adapter: &RmcpTools) -> RequestContext<RoleServer> {
|
||||
let (client_io, server_io) = tokio::io::duplex(1024);
|
||||
let (server_reader, server_writer) = tokio::io::split(server_io);
|
||||
let running = serve_directly::<RoleServer, _, _, _, _>(
|
||||
adapter.clone(),
|
||||
(server_reader, server_writer),
|
||||
None,
|
||||
);
|
||||
// keep client side alive for transport lifetime
|
||||
let _ = client_io;
|
||||
let mut context = RequestContext::new(RequestId::Number(1), running.peer().clone());
|
||||
let (mut parts, _) = axum::http::Request::new(()).into_parts();
|
||||
parts.extensions.insert(RmcpRequestIdentity {
|
||||
workspace_slug: "missing-workspace".into(),
|
||||
agent_slug: "missing-agent".into(),
|
||||
credential: verified_credential(),
|
||||
correlation: CorrelationContext::generate(),
|
||||
continuation: crate::protocol::ContinuationSupport::from_supported(false),
|
||||
locale: ExecutionLocale::En,
|
||||
session_id: None,
|
||||
progress_meta: None,
|
||||
sse_acceptable: false,
|
||||
});
|
||||
context.extensions.insert(parts);
|
||||
context
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_adapter_rejects_missing_request_identity() {
|
||||
let adapter = rmcp_tools(test_state().await);
|
||||
let mut context = request_context(&adapter).await;
|
||||
context.extensions.remove::<axum::http::request::Parts>();
|
||||
let result = <RmcpTools as ServerHandler>::list_tools(&adapter, None, context).await;
|
||||
assert!(
|
||||
result
|
||||
.unwrap_err()
|
||||
.message
|
||||
.contains("missing authenticated request context")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_adapter_list_tools_reports_catalog_unavailable_for_unknown_agent() {
|
||||
let state = test_state().await;
|
||||
let adapter = rmcp_tools(state);
|
||||
let ctx = request_context(&adapter).await;
|
||||
|
||||
match <RmcpTools as ServerHandler>::list_tools(
|
||||
&adapter,
|
||||
Some(PaginatedRequestParams::default()),
|
||||
ctx,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(err) => assert!(
|
||||
err.message.contains("crate catalog unavailable"),
|
||||
"unexpected error: {}",
|
||||
err.message
|
||||
),
|
||||
Ok(_) => panic!("expected error outcome for unknown agent"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rmcp_adapter_call_tool_reports_catalog_unavailable_for_unknown_agent() {
|
||||
let state = test_state().await;
|
||||
let adapter = rmcp_tools(state);
|
||||
let ctx = request_context(&adapter).await;
|
||||
|
||||
let request = CallToolRequestParams::new("missing_tool");
|
||||
match <RmcpTools as ServerHandler>::call_tool(&adapter, request, ctx).await {
|
||||
Err(err) => assert!(
|
||||
err.message.contains("crate catalog unavailable"),
|
||||
"unexpected error: {}",
|
||||
err.message
|
||||
),
|
||||
Ok(_) => panic!("expected error outcome for unknown agent"),
|
||||
}
|
||||
}
|
||||
@@ -75,11 +75,16 @@ pub mod domain {
|
||||
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
|
||||
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use crate::task::{
|
||||
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
|
||||
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES, TASK_IDENTITY_MAX_BYTES,
|
||||
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
|
||||
AttemptResolution, BeginAttempt, TASK_APPROVAL_REFERENCE_MAX_BYTES,
|
||||
TASK_CANCEL_REASON_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
|
||||
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES,
|
||||
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
|
||||
TASK_IDENTITY_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
|
||||
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
|
||||
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskSupportIds,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
|
||||
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind,
|
||||
TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskSupportIds,
|
||||
};
|
||||
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
|
||||
pub use crate::tool_quality::{
|
||||
@@ -202,11 +207,17 @@ pub use product_event::{
|
||||
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
|
||||
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
|
||||
pub use task::{
|
||||
TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
|
||||
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
|
||||
TaskState, TaskStore, TaskSupportIds, redact_task_envelope, sensitive_task_key,
|
||||
validate_task_envelope,
|
||||
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
|
||||
AttemptResolution, BeginAttempt, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
|
||||
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
|
||||
TASK_INPUT_PROMPT_CODE_MAX_BYTES, TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
TASK_INPUT_WAIT_ID_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
|
||||
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest,
|
||||
TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
|
||||
TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
|
||||
TaskRetryPolicy, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
|
||||
redact_task_envelope, sensitive_task_key, validate_task_envelope,
|
||||
};
|
||||
pub use tool_catalog::{
|
||||
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
//! Durable attempt, retry-snapshot and cancellation vocabulary of a Task (Story 4.4).
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{TASK_IDENTITY_MAX_BYTES, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskRecord};
|
||||
|
||||
/// Upper bound of the retry budget a Task may pin at acceptance (Story 4.4).
|
||||
pub const TASK_MAX_ATTEMPTS: u32 = 5;
|
||||
/// Upper bound of the pinned backoff step between two durable attempts (Story 4.4).
|
||||
pub const TASK_MAX_BACKOFF_STEP_MS: i64 = 1_000;
|
||||
/// Bound of a stored cancellation reason.
|
||||
pub const TASK_CANCEL_REASON_MAX_BYTES: usize = 160;
|
||||
|
||||
/// Immutable retry snapshot of one accepted Task.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct TaskRetryPolicy {
|
||||
/// Total durable outbound attempts (1 = a single attempt, never retried).
|
||||
pub max_attempts: u32,
|
||||
/// Pinned base backoff between two attempts; the delay grows by this step per attempt.
|
||||
pub backoff_step_ms: i64,
|
||||
}
|
||||
|
||||
impl TaskRetryPolicy {
|
||||
/// A Task that may be dispatched exactly once (legacy Tasks and unverified writes).
|
||||
pub const fn single_attempt() -> Self {
|
||||
Self {
|
||||
max_attempts: 1,
|
||||
backoff_step_ms: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(max_attempts: u32, backoff_step_ms: i64) -> Self {
|
||||
Self {
|
||||
max_attempts,
|
||||
backoff_step_ms,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if !(1..=TASK_MAX_ATTEMPTS).contains(&self.max_attempts) {
|
||||
return Err(TaskError::invalid("task retry budget is invalid"));
|
||||
}
|
||||
if !(0..=TASK_MAX_BACKOFF_STEP_MS).contains(&self.backoff_step_ms) {
|
||||
return Err(TaskError::invalid("task retry backoff is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delay before attempt number `next_ordinal` (2..=max_attempts), capped so a pinned snapshot can
|
||||
/// never stretch the accepted execution deadline on its own.
|
||||
pub fn backoff_before(&self, next_ordinal: u32) -> i64 {
|
||||
let steps = i64::from(next_ordinal.saturating_sub(1)).max(1);
|
||||
self.backoff_step_ms
|
||||
.saturating_mul(steps)
|
||||
.min(self.backoff_step_ms.saturating_mul(4))
|
||||
}
|
||||
}
|
||||
|
||||
/// Certainty of one durable attempt outcome.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AttemptCertainty {
|
||||
/// The outcome of the outbound operation is proven (success or a definite failure).
|
||||
Known,
|
||||
/// The operation may have been applied upstream; the effect cannot be proven locally.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AttemptCertainty {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Known => "known",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"known" => Ok(Self::Known),
|
||||
"unknown" => Ok(Self::Unknown),
|
||||
_ => Err(TaskError::invalid("unknown attempt certainty")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Phase of one durable attempt at the moment it was resolved.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AttemptPhase {
|
||||
/// The durable checkpoint exists but no outbound send was observed.
|
||||
BeforeDispatch,
|
||||
/// The request was handed to the transport; its effect may exist upstream.
|
||||
Dispatched,
|
||||
}
|
||||
|
||||
impl AttemptPhase {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::BeforeDispatch => "before_dispatch",
|
||||
Self::Dispatched => "dispatched",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"before_dispatch" => Ok(Self::BeforeDispatch),
|
||||
"dispatched" => Ok(Self::Dispatched),
|
||||
_ => Err(TaskError::invalid("unknown attempt phase")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Closed outcome vocabulary of one resolved attempt.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AttemptOutcomeClass {
|
||||
Completed,
|
||||
Failed,
|
||||
/// A bounded further attempt was scheduled instead of a terminal outcome.
|
||||
RetryScheduled,
|
||||
OutcomeUnknown,
|
||||
}
|
||||
|
||||
impl AttemptOutcomeClass {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::RetryScheduled => "retry_scheduled",
|
||||
Self::OutcomeUnknown => "outcome_unknown",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
||||
match value {
|
||||
"completed" => Ok(Self::Completed),
|
||||
"failed" => Ok(Self::Failed),
|
||||
"retry_scheduled" => Ok(Self::RetryScheduled),
|
||||
"outcome_unknown" => Ok(Self::OutcomeUnknown),
|
||||
_ => Err(TaskError::invalid("unknown attempt outcome")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One durable outbound attempt. The row is created by the dispatch checkpoint before any network
|
||||
/// use and becomes immutable once resolved, so a stale worker can never append to the history.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskAttempt {
|
||||
pub attempt_id: String,
|
||||
pub task_id: TaskId,
|
||||
pub ordinal: u32,
|
||||
pub owner: String,
|
||||
pub fence: u64,
|
||||
/// Canonical fingerprint of the sealed executable input this attempt was admitted for.
|
||||
pub input_fingerprint: String,
|
||||
/// Digest of the stable upstream idempotency key (never the raw key or headers).
|
||||
pub upstream_key_digest: Option<String>,
|
||||
/// Whether a retry of this attempt is safe (read or server-verified idempotent write).
|
||||
pub retry_allowed: bool,
|
||||
pub checkpoint_at_unix_ms: i64,
|
||||
pub resolved_at_unix_ms: Option<i64>,
|
||||
pub outcome: Option<AttemptOutcomeClass>,
|
||||
pub certainty: Option<AttemptCertainty>,
|
||||
pub phase: Option<AttemptPhase>,
|
||||
}
|
||||
|
||||
/// Committed dispatch intent of one durable attempt (Story 4.4). The store owns the transaction: the
|
||||
/// attempt row, the incremented budget and the new Task version commit together before the network is
|
||||
/// touched, so an uncommitted checkpoint can never produce an outbound call.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct BeginAttempt {
|
||||
pub task_id: TaskId,
|
||||
pub owner: String,
|
||||
pub fence: u64,
|
||||
/// Version the worker read from the lease; a concurrent change rejects the checkpoint.
|
||||
pub expected_version: u32,
|
||||
pub attempt_id: String,
|
||||
pub input_fingerprint: String,
|
||||
pub upstream_key_digest: Option<String>,
|
||||
/// The runtime's decision that a retry of this attempt is safe; the store still enforces the
|
||||
/// pinned budget and the absolute deadline.
|
||||
pub retry_allowed: bool,
|
||||
}
|
||||
|
||||
impl BeginAttempt {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.owner.is_empty() || self.owner.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task attempt owner is invalid"));
|
||||
}
|
||||
if self.attempt_id.is_empty() || self.attempt_id.len() > TASK_IDENTITY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task attempt identity is invalid"));
|
||||
}
|
||||
if self.input_fingerprint.len() != 64 {
|
||||
return Err(TaskError::invalid("task attempt fingerprint is invalid"));
|
||||
}
|
||||
if let Some(digest) = self.upstream_key_digest.as_deref()
|
||||
&& (digest.is_empty() || digest.len() > TASK_IDENTITY_MAX_BYTES)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task attempt upstream key digest is invalid",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Leased, committed view of one in-flight attempt and the Task version it was admitted under.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct AttemptLease {
|
||||
pub attempt: TaskAttempt,
|
||||
pub task: TaskRecord,
|
||||
}
|
||||
|
||||
/// Typed outcome of one attempt, handed to the store for a single atomic resolution.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AttemptOutcome {
|
||||
/// Proven success; the Task becomes `completed` with this result.
|
||||
Completed(TaskEnvelope),
|
||||
/// Proven failure. `retryable` only schedules another attempt when the store's pinned budget,
|
||||
/// the attempt's safe-retry decision and the absolute deadline all still allow it; otherwise the
|
||||
/// Task fails with this error.
|
||||
Failed {
|
||||
error: TaskEnvelope,
|
||||
retryable: bool,
|
||||
phase: AttemptPhase,
|
||||
},
|
||||
/// The outbound effect cannot be proven. `repeatable` only schedules another bounded attempt when
|
||||
/// repeating it is safe and the pinned budget/deadline still allow it; otherwise the Task becomes
|
||||
/// the non-terminal `outcome_unknown`, never a plain failure.
|
||||
Uncertain {
|
||||
error: TaskEnvelope,
|
||||
repeatable: bool,
|
||||
phase: AttemptPhase,
|
||||
},
|
||||
}
|
||||
|
||||
/// Single-winner result of resolving one attempt.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum AttemptResolution {
|
||||
/// A terminal transition committed (completed or failed).
|
||||
Finished(TaskRecord),
|
||||
/// The attempt failed in a known way and another bounded attempt was armed.
|
||||
RetryAt {
|
||||
task: TaskRecord,
|
||||
next_attempt_at_unix_ms: i64,
|
||||
},
|
||||
/// The attempt resolved to the non-terminal uncertain state.
|
||||
Unknown(TaskRecord),
|
||||
}
|
||||
|
||||
/// Owner-side cancellation request. `expected_version` is optional so a repeated cancel of the same
|
||||
/// accepted Task stays idempotent while a stale version is reported as a conflict.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskCancelRequest {
|
||||
pub access: TaskAccess,
|
||||
pub task_id: TaskId,
|
||||
pub expected_version: Option<u32>,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
impl TaskCancelRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
self.access.validate()?;
|
||||
if self.reason.is_empty()
|
||||
|| self.reason.len() > TASK_CANCEL_REASON_MAX_BYTES
|
||||
|| self.reason.bytes().any(|byte| byte.is_ascii_control())
|
||||
{
|
||||
return Err(TaskError::invalid("task cancellation reason is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed result of an owner-scoped cancellation.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum TaskCancelOutcome {
|
||||
/// Cancelled before the dispatch checkpoint: no outbound call can have happened.
|
||||
Cancelled(TaskRecord),
|
||||
/// The Task already had a checkpointed attempt: non-terminal uncertain, never a plain cancel.
|
||||
OutcomeUnknown(TaskRecord),
|
||||
/// The Task was already in a terminal state; its accepted result is unchanged.
|
||||
AlreadyTerminal(TaskRecord),
|
||||
/// The caller's expected version is stale; `current` is the authorized current state.
|
||||
Conflict(TaskRecord),
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
//! Continuation of an accepted Task through a validated additional input (Story 4.5, FR-25).
|
||||
//!
|
||||
//! A Task that cannot run because the pinned operation still lacks a required input item is
|
||||
//! suspended as `input_required` before any outbound dispatch. The durable request keeps the exact
|
||||
//! immutable schema snapshot, its JSON projection, safe prompt metadata, the absolute TTL and the
|
||||
//! update fingerprint contract. A valid update moves the Task back to `working` exactly once; an
|
||||
//! identical replay returns the committed outcome and a conflicting or stale reply is typed.
|
||||
//!
|
||||
//! Deliberately absent: raw input in the request/receipt/audit read models, a second cipher store and
|
||||
//! any public transport. The regular executable-input ciphertext carries the continuation binding.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::task::{TaskAccess, TaskError, TaskId};
|
||||
|
||||
/// Version of the persisted input-continuation contract. A stored snapshot always declares the
|
||||
/// version it was written with, so a later binary never reinterprets it silently.
|
||||
pub const TASK_INPUT_SCHEMA_FORMAT_VERSION: u32 = 1;
|
||||
/// Bound of a wait identity before it is persisted.
|
||||
pub const TASK_INPUT_WAIT_ID_MAX_BYTES: usize = 64;
|
||||
/// Bound of the safe prompt code (never the prompt text or an input value).
|
||||
pub const TASK_INPUT_PROMPT_CODE_MAX_BYTES: usize = 64;
|
||||
/// Bound of the missing-field path metadata.
|
||||
pub const TASK_INPUT_MAX_MISSING_FIELDS: usize = 32;
|
||||
/// Bound of one missing-field path.
|
||||
pub const TASK_INPUT_FIELD_PATH_MAX_BYTES: usize = 160;
|
||||
|
||||
/// Durable request of additional input for one suspended Task.
|
||||
///
|
||||
/// `schema_snapshot` is the native serialized schema subset the server validates against; the JSON
|
||||
/// projection is presentation metadata only and never the validation authority. `expected_version`
|
||||
/// is the Task version a caller must present with the answer.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TaskInputRequest {
|
||||
pub wait_id: String,
|
||||
pub expected_version: u32,
|
||||
pub schema_revision: u32,
|
||||
pub schema_format_version: u32,
|
||||
pub schema_snapshot: Value,
|
||||
pub projected_schema: Value,
|
||||
pub prompt_code: String,
|
||||
pub missing_fields: Vec<String>,
|
||||
pub expires_at_unix_ms: i64,
|
||||
pub fingerprint_format_version: u32,
|
||||
}
|
||||
|
||||
impl TaskInputRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
if self.wait_id.is_empty()
|
||||
|| self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES
|
||||
|| self.wait_id.bytes().any(|byte| byte.is_ascii_control())
|
||||
{
|
||||
return Err(TaskError::invalid("task input wait identity is invalid"));
|
||||
}
|
||||
if self.expected_version == 0 || self.schema_revision == 0 {
|
||||
return Err(TaskError::invalid(
|
||||
"task input request generation is invalid",
|
||||
));
|
||||
}
|
||||
if self.schema_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
|
||||
|| self.fingerprint_format_version < TASK_INPUT_SCHEMA_FORMAT_VERSION
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task input request format version is invalid",
|
||||
));
|
||||
}
|
||||
if self.schema_snapshot.as_object().is_none() || self.projected_schema.as_object().is_none()
|
||||
{
|
||||
return Err(TaskError::invalid("task input schema snapshot is invalid"));
|
||||
}
|
||||
if self.prompt_code.is_empty() || self.prompt_code.len() > TASK_INPUT_PROMPT_CODE_MAX_BYTES
|
||||
{
|
||||
return Err(TaskError::invalid("task input prompt code is invalid"));
|
||||
}
|
||||
if self.missing_fields.len() > TASK_INPUT_MAX_MISSING_FIELDS
|
||||
|| self
|
||||
.missing_fields
|
||||
.iter()
|
||||
.any(|field| field.is_empty() || field.len() > TASK_INPUT_FIELD_PATH_MAX_BYTES)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task input missing-field metadata is invalid",
|
||||
));
|
||||
}
|
||||
if self.expires_at_unix_ms <= 0 {
|
||||
return Err(TaskError::invalid("task input TTL is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Authoritative write of one validated answer. The transaction owns schema-generation, TTL,
|
||||
/// fingerprint and CAS checks; this request only carries the committed values.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TaskInputUpdateRequest {
|
||||
pub access: TaskAccess,
|
||||
pub task_id: TaskId,
|
||||
pub wait_id: String,
|
||||
pub expected_version: u32,
|
||||
/// Digest of the scoped update key; the caller key is never persisted.
|
||||
pub update_key_digest: String,
|
||||
/// Canonical digest of the validated full executable input.
|
||||
pub canonical_fingerprint: String,
|
||||
/// Effective execution fingerprint of the resumed payload (never the acceptance fingerprint).
|
||||
pub effective_input_fingerprint: String,
|
||||
pub executable_input_ciphertext: String,
|
||||
pub executable_input_key_version: String,
|
||||
pub executable_input_master_key_epoch: i64,
|
||||
}
|
||||
|
||||
impl TaskInputUpdateRequest {
|
||||
pub fn validate(&self) -> Result<(), TaskError> {
|
||||
self.access.validate()?;
|
||||
if self.wait_id.is_empty() || self.wait_id.len() > TASK_INPUT_WAIT_ID_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task input wait identity is invalid"));
|
||||
}
|
||||
if self.expected_version == 0 {
|
||||
return Err(TaskError::invalid("task input version is invalid"));
|
||||
}
|
||||
if !valid_digest(&self.update_key_digest)
|
||||
|| !valid_digest(&self.canonical_fingerprint)
|
||||
|| !valid_digest(&self.effective_input_fingerprint)
|
||||
{
|
||||
return Err(TaskError::invalid("task input digest is invalid"));
|
||||
}
|
||||
if self.executable_input_ciphertext.is_empty()
|
||||
|| self.executable_input_ciphertext.len()
|
||||
> crate::task::TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task continuation ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
if self.executable_input_key_version.is_empty()
|
||||
|| self.executable_input_key_version.len() > 64
|
||||
|| self.executable_input_master_key_epoch < 1
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task continuation ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Committed outcome of one input update. An identical replay returns the same stored outcome with
|
||||
/// `replayed = true` instead of repeating the CAS, audit or dispatch.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskInputUpdateOutcome {
|
||||
pub task_id: TaskId,
|
||||
pub wait_id: String,
|
||||
pub state: crate::task::TaskState,
|
||||
pub version: u32,
|
||||
pub replayed: bool,
|
||||
}
|
||||
|
||||
fn valid_digest(value: &str) -> bool {
|
||||
value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
}
|
||||
|
||||
/// Durable input-continuation operations of the Task store.
|
||||
///
|
||||
/// Split out of the general [`crate::task::TaskStore`] port so the lifecycle store stays readable and
|
||||
/// the continuation contract (suspend, answer, receipt, expiry) is implemented and audited as one
|
||||
/// cohesive surface. [`crate::task::TaskStore`] extends this trait, so an owned `Arc<dyn TaskStore>`
|
||||
/// exposes both halves through one object.
|
||||
#[async_trait::async_trait]
|
||||
pub trait TaskInputStore: Send + Sync {
|
||||
/// Suspends a leased Task that still lacks a required input of its pinned operation. The
|
||||
/// transaction persists the immutable request and its audit row, moves `working` to
|
||||
/// `input_required` exactly once, releases the worker lease and keeps the existing admission
|
||||
/// permit. No outbound dispatch may have happened in this generation.
|
||||
async fn suspend_for_input(
|
||||
&self,
|
||||
lease: &crate::task::TaskLease,
|
||||
request: TaskInputRequest,
|
||||
) -> Result<crate::task::TaskRecord, TaskError>;
|
||||
|
||||
/// Owner-scoped read of the durable input request of one Task.
|
||||
async fn input_request(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<TaskInputRequest, TaskError>;
|
||||
|
||||
/// Owner-scoped read of the executable-input binding that a committed continuation may extend.
|
||||
/// It is not lease-authorized: it only serves the update transaction and never dispatch.
|
||||
async fn read_executable_input_for_update(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<(String, String, i64), TaskError>;
|
||||
|
||||
/// Applies exactly one validated answer under a row lock: authority recheck, receipt lookup,
|
||||
/// generation/TTL check, atomic ciphertext/effective-fingerprint/receipt/`working` transition and
|
||||
/// audit. An identical replay returns the stored outcome without a second transition.
|
||||
async fn apply_input(
|
||||
&self,
|
||||
request: TaskInputUpdateRequest,
|
||||
) -> Result<TaskInputUpdateOutcome, TaskError>;
|
||||
|
||||
/// Bounded sweep of `input_required` Tasks whose TTL elapsed: `failed` with an audit reason and
|
||||
/// the release of the admission permit, without any dispatch.
|
||||
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn request() -> TaskInputRequest {
|
||||
TaskInputRequest {
|
||||
wait_id: "wait-1".to_owned(),
|
||||
expected_version: 2,
|
||||
schema_revision: 1,
|
||||
schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
schema_snapshot: json!({"type": "object", "fields": {}}),
|
||||
projected_schema: json!({"type": "object", "properties": {}}),
|
||||
prompt_code: "required_input_missing".to_owned(),
|
||||
missing_fields: vec!["$.email".to_owned()],
|
||||
expires_at_unix_ms: 1_900_000_000_000,
|
||||
fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION,
|
||||
}
|
||||
}
|
||||
|
||||
fn update() -> TaskInputUpdateRequest {
|
||||
TaskInputUpdateRequest {
|
||||
access: crate::task::TaskAccess {
|
||||
workspace_id: "ws_default".to_owned(),
|
||||
owner_kind: crate::task::TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
},
|
||||
task_id: crate::task::TaskId::generate(),
|
||||
wait_id: "wait-1".to_owned(),
|
||||
expected_version: 2,
|
||||
update_key_digest: "a".repeat(64),
|
||||
canonical_fingerprint: "b".repeat(64),
|
||||
effective_input_fingerprint: "c".repeat(64),
|
||||
executable_input_ciphertext: "sealed".to_owned(),
|
||||
executable_input_key_version: "key-v1".to_owned(),
|
||||
executable_input_master_key_epoch: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_request_holds_generation_schema_ttl_and_bounds() {
|
||||
assert!(request().validate().is_ok());
|
||||
|
||||
let mut no_wait = request();
|
||||
no_wait.wait_id.clear();
|
||||
assert!(no_wait.validate().is_err());
|
||||
|
||||
let mut no_generation = request();
|
||||
no_generation.expected_version = 0;
|
||||
assert!(no_generation.validate().is_err());
|
||||
|
||||
let mut bad_schema = request();
|
||||
bad_schema.schema_snapshot = json!("not-an-object");
|
||||
assert!(bad_schema.validate().is_err());
|
||||
|
||||
let mut no_ttl = request();
|
||||
no_ttl.expires_at_unix_ms = 0;
|
||||
assert!(no_ttl.validate().is_err());
|
||||
|
||||
let mut too_many_fields = request();
|
||||
too_many_fields.missing_fields = vec!["$.x".to_owned(); TASK_INPUT_MAX_MISSING_FIELDS + 1];
|
||||
assert!(too_many_fields.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_request_rejects_invalid_digests_and_ciphertext_bounds() {
|
||||
assert!(update().validate().is_ok());
|
||||
|
||||
let mut short_digest = update();
|
||||
short_digest.canonical_fingerprint = "abc".to_owned();
|
||||
assert!(short_digest.validate().is_err());
|
||||
|
||||
let mut upper_digest = update();
|
||||
upper_digest.update_key_digest = "A".repeat(64);
|
||||
assert!(upper_digest.validate().is_err());
|
||||
|
||||
let mut no_ciphertext = update();
|
||||
no_ciphertext.executable_input_ciphertext.clear();
|
||||
assert!(no_ciphertext.validate().is_err());
|
||||
|
||||
let mut no_epoch = update();
|
||||
no_epoch.executable_input_master_key_epoch = 0;
|
||||
assert!(no_epoch.validate().is_err());
|
||||
}
|
||||
}
|
||||
@@ -9,16 +9,25 @@
|
||||
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
|
||||
//! Task was executed. A terminal state is written by a conditional transition only.
|
||||
|
||||
mod attempt;
|
||||
mod input;
|
||||
use async_trait::async_trait;
|
||||
pub use attempt::*;
|
||||
pub use input::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::correlation::{RequestId, TraceContext, TraceId};
|
||||
use crate::ids::PlatformApiKeyId;
|
||||
use crate::operation::ToolExecutionMode;
|
||||
|
||||
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
|
||||
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
|
||||
/// Maximum byte budget of the decrypted executable Task input.
|
||||
pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES;
|
||||
/// Maximum byte budget of the stored encrypted executable Task input ciphertext.
|
||||
pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304;
|
||||
/// Maximum nesting depth of a persisted Task envelope.
|
||||
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
|
||||
/// Maximum number of JSON members/nodes of a persisted Task envelope.
|
||||
@@ -122,9 +131,9 @@ impl TaskState {
|
||||
| Self::Cancelled
|
||||
),
|
||||
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
|
||||
Self::OutcomeUnknown => {
|
||||
matches!(next, Self::Completed | Self::Failed | Self::Cancelled)
|
||||
}
|
||||
// Story 4.4: an uncertain outbound outcome is never resolved by a generic transition.
|
||||
// Reconciliation is Story 4.6, so no ordinary finish/cancel/retry may leave this state.
|
||||
Self::OutcomeUnknown => false,
|
||||
Self::Completed | Self::Failed | Self::Cancelled => false,
|
||||
}
|
||||
}
|
||||
@@ -201,6 +210,9 @@ pub struct TaskPins {
|
||||
pub retain_until_unix_ms: i64,
|
||||
/// Verified approval evidence reference (identifier only, never a bearer token).
|
||||
pub approval_reference: Option<String>,
|
||||
/// Bounded retry budget pinned at acceptance (Story 4.4). A later publication, a restart or a
|
||||
/// new binary never changes the numbers an accepted Task was admitted with.
|
||||
pub retry: TaskRetryPolicy,
|
||||
}
|
||||
|
||||
impl TaskPins {
|
||||
@@ -228,6 +240,7 @@ impl TaskPins {
|
||||
{
|
||||
return Err(TaskError::invalid("task approval reference is invalid"));
|
||||
}
|
||||
self.retry.validate()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -351,6 +364,16 @@ pub fn redact_task_envelope(value: &Value) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
/// Recursive check used to fail closed on sensitive fields before encryption.
|
||||
pub fn contains_sensitive_task_keys(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Object(object) => object
|
||||
.iter()
|
||||
.any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)),
|
||||
Value::Array(values) => values.iter().any(contains_sensitive_task_keys),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
|
||||
/// queue capacity commit together, and no acknowledgement exists before that commit.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@@ -360,6 +383,10 @@ pub struct TaskAcceptRequest {
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
/// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the
|
||||
/// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively.
|
||||
/// `None` means the accepting principal has no machine key (administrator) and fail-closes.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
pub pins: TaskPins,
|
||||
/// Digest of the caller key over the full owner/authorization/Tool scope.
|
||||
@@ -368,6 +395,10 @@ pub struct TaskAcceptRequest {
|
||||
pub input_fingerprint: String,
|
||||
pub input: TaskEnvelope,
|
||||
pub support: TaskSupportIds,
|
||||
/// Bounded encrypted executable input, persisted alongside the redacted input projection.
|
||||
pub executable_input_ciphertext: Option<String>,
|
||||
pub executable_input_key_version: Option<String>,
|
||||
pub executable_input_master_key_epoch: Option<i64>,
|
||||
}
|
||||
|
||||
impl TaskAcceptRequest {
|
||||
@@ -382,6 +413,13 @@ impl TaskAcceptRequest {
|
||||
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
||||
}
|
||||
}
|
||||
if let Some(key_id) = self.platform_api_key_id.as_ref()
|
||||
&& (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES)
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task platform api key reference is invalid",
|
||||
));
|
||||
}
|
||||
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
|
||||
return Err(TaskError::invalid("task scoped key digest is invalid"));
|
||||
}
|
||||
@@ -394,7 +432,40 @@ impl TaskAcceptRequest {
|
||||
return Err(TaskError::invalid("task input fingerprint is invalid"));
|
||||
}
|
||||
self.pins.validate()?;
|
||||
self.support.validate()
|
||||
self.support.validate()?;
|
||||
let has_ciphertext = self.executable_input_ciphertext.is_some();
|
||||
let has_key = self.executable_input_key_version.is_some();
|
||||
let has_epoch = self.executable_input_master_key_epoch.is_some();
|
||||
match (has_ciphertext, has_key, has_epoch) {
|
||||
(false, false, false) => {}
|
||||
(true, true, true) => {
|
||||
let ciphertext = self.executable_input_ciphertext.as_deref().unwrap();
|
||||
if ciphertext.is_empty()
|
||||
|| ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
|
||||
{
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
let key_version = self.executable_input_key_version.as_deref().unwrap();
|
||||
if key_version.is_empty() || key_version.len() > 64 {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
if self.executable_input_master_key_epoch.unwrap_or_default() < 1 {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -448,9 +519,16 @@ pub struct TaskRecord {
|
||||
pub owner_kind: TaskOwnerKind,
|
||||
pub owner_id: String,
|
||||
pub authorization_scope: String,
|
||||
/// Typed immutable identity of the Platform API key that authorized the acceptance; the worker
|
||||
/// revalidates this exact key against the registry before any outbound dispatch.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
|
||||
pub input_fingerprint: String,
|
||||
/// Effective fingerprint of the payload the next attempt will execute. It equals the acceptance
|
||||
/// fingerprint until a committed input continuation replaces the effective payload; the
|
||||
/// acceptance fingerprint above is never overwritten, so an original accept replay stays replay.
|
||||
pub effective_input_fingerprint: String,
|
||||
pub pins: TaskPins,
|
||||
pub support: TaskSupportIds,
|
||||
pub input: TaskEnvelope,
|
||||
@@ -458,6 +536,22 @@ pub struct TaskRecord {
|
||||
pub result: Option<TaskEnvelope>,
|
||||
pub error: Option<TaskEnvelope>,
|
||||
pub attempts: u32,
|
||||
/// Monotonic lease/fencing token. It never resets and protects Task state mutations.
|
||||
pub fence: u64,
|
||||
/// Random process-instance lease owner, if a worker currently owns the Task.
|
||||
pub lease_owner: Option<String>,
|
||||
/// Lease expiry epoch milliseconds, if the Task is claimed.
|
||||
pub lease_until_unix_ms: Option<i64>,
|
||||
/// Current dispatch attempt identity, committed before any outbound adapter call.
|
||||
pub current_attempt_id: Option<String>,
|
||||
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
|
||||
pub dispatch_started_at_unix_ms: Option<i64>,
|
||||
/// Earliest moment the next durable attempt may be dispatched (pinned backoff).
|
||||
pub next_attempt_at_unix_ms: Option<i64>,
|
||||
/// Durable cancellation request that arrived while an attempt was already checkpointed.
|
||||
pub cancel_requested_at_unix_ms: Option<i64>,
|
||||
/// Bounded reason of the cancellation request.
|
||||
pub cancel_reason: Option<String>,
|
||||
/// Monotonic lifecycle version used by the terminal CAS.
|
||||
pub version: u32,
|
||||
pub created_at_unix_ms: i64,
|
||||
@@ -519,6 +613,8 @@ pub enum TaskError {
|
||||
FingerprintConflict,
|
||||
#[error("task lifecycle transition was rejected")]
|
||||
StateConflict,
|
||||
#[error("task input request has expired")]
|
||||
InputExpired,
|
||||
#[error("background task capacity is exhausted")]
|
||||
Overloaded { retry_after_ms: u64 },
|
||||
}
|
||||
@@ -542,9 +638,18 @@ impl TaskError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Leased view of a claimed Task: its durable record plus lease fencing identity.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TaskLease {
|
||||
pub record: TaskRecord,
|
||||
pub owner: String,
|
||||
pub fence: u64,
|
||||
pub lease_until: i64,
|
||||
}
|
||||
|
||||
/// Authoritative durable Task port.
|
||||
#[async_trait]
|
||||
pub trait TaskStore: Send + Sync {
|
||||
pub trait TaskStore: TaskInputStore + Send + Sync {
|
||||
/// Proves the durable Task state is usable (schema and operations reachable).
|
||||
async fn health(&self) -> Result<(), TaskError>;
|
||||
|
||||
@@ -566,9 +671,78 @@ pub trait TaskStore: Send + Sync {
|
||||
/// Applies exactly one conditional terminal transition and releases its intent atomically.
|
||||
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Bounded retention: fails guaranteed-undispatched queued Tasks past their deadline and removes
|
||||
/// expired terminal records. Live and non-terminal work is never deleted.
|
||||
/// Reads encrypted executable input only under a valid, unexpired current lease.
|
||||
async fn read_executable_input(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
) -> Result<(String, String, i64), TaskError>;
|
||||
|
||||
/// Bounded retention...
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
|
||||
|
||||
/// Claims a bounded batch of queued or safely recoverable Tasks under leases.
|
||||
async fn claim_batch(
|
||||
&self,
|
||||
owner: &str,
|
||||
batch_limit: u32,
|
||||
lease_ms: i64,
|
||||
) -> Result<Vec<TaskLease>, TaskError>;
|
||||
|
||||
/// Renews a lease owned by `owner` when fence and expiry still match.
|
||||
async fn renew_lease(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
lease_ms: i64,
|
||||
) -> Result<TaskLease, TaskError>;
|
||||
|
||||
/// Persists redacted progress under a valid lease.
|
||||
async fn write_progress(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
progress: TaskEnvelope,
|
||||
) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Commits one durable outbound attempt before network use: attempt row, incremented budget and
|
||||
/// the new Task version commit in one transaction. A returned lease is the only authority to
|
||||
/// dispatch that attempt; an unknown commit means zero outbound calls.
|
||||
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError>;
|
||||
|
||||
/// Resolves exactly one committed attempt under its lease/fence/version. Only the current attempt
|
||||
/// of the current owner may be resolved, and the attempt becomes immutable afterwards.
|
||||
async fn resolve_attempt(
|
||||
&self,
|
||||
lease: &AttemptLease,
|
||||
outcome: AttemptOutcome,
|
||||
) -> Result<AttemptResolution, TaskError>;
|
||||
|
||||
/// Owner-scoped atomic cancellation. It serializes with claim and checkpoint on the same Task.
|
||||
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError>;
|
||||
|
||||
/// Owner-scoped bounded history of the durable attempts of one Task.
|
||||
async fn attempts(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<Vec<TaskAttempt>, TaskError>;
|
||||
|
||||
/// Applies a terminal transition only under a valid lease and releases intent atomically.
|
||||
async fn finish_leased(
|
||||
&self,
|
||||
request: TaskFinishRequest,
|
||||
owner: &str,
|
||||
fence: u64,
|
||||
) -> Result<TaskRecord, TaskError>;
|
||||
|
||||
/// Resolves a leased Task whose next attempt was refused before its dispatch checkpoint (elapsed
|
||||
/// deadline, rejected authority). A Task that still carries a proven ambiguity is never downgraded
|
||||
/// to a plain failure: it ends non-terminal `outcome_unknown` with `code`; otherwise `failed`.
|
||||
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -583,6 +757,7 @@ mod tests {
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 3,
|
||||
@@ -590,6 +765,7 @@ mod tests {
|
||||
execution_deadline_unix_ms: 1_790_000_000_000,
|
||||
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
|
||||
approval_reference: Some("appr_1".to_owned()),
|
||||
retry: TaskRetryPolicy::new(3, 1),
|
||||
},
|
||||
scoped_key: "a".repeat(64),
|
||||
input_fingerprint: "b".repeat(64),
|
||||
@@ -599,6 +775,9 @@ mod tests {
|
||||
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
||||
.unwrap(),
|
||||
),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -694,6 +873,61 @@ mod tests {
|
||||
let mut bad_retention = request();
|
||||
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
|
||||
assert!(bad_retention.validate().is_err());
|
||||
|
||||
// The persisted authority reference is bounded and never empty.
|
||||
let mut empty_key_reference = request();
|
||||
empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new(""));
|
||||
assert!(empty_key_reference.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_unknown_has_no_generic_successor() {
|
||||
// Story 4.4: reconciliation of an uncertain outcome is Story 4.6. Until then no ordinary
|
||||
// finish/cancel/retry may leave the non-terminal uncertain state.
|
||||
for next in [
|
||||
TaskState::Queued,
|
||||
TaskState::Working,
|
||||
TaskState::InputRequired,
|
||||
TaskState::OutcomeUnknown,
|
||||
TaskState::Completed,
|
||||
TaskState::Failed,
|
||||
TaskState::Cancelled,
|
||||
] {
|
||||
assert!(
|
||||
!TaskState::OutcomeUnknown.allows_transition_to(next),
|
||||
"outcome_unknown must not transition to {next:?}"
|
||||
);
|
||||
}
|
||||
assert!(!TaskState::OutcomeUnknown.is_terminal());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_policy_bounds_are_pinned_and_bounded() {
|
||||
assert!(TaskRetryPolicy::single_attempt().validate().is_ok());
|
||||
assert!(
|
||||
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS)
|
||||
.validate()
|
||||
.is_ok()
|
||||
);
|
||||
assert!(TaskRetryPolicy::new(0, 0).validate().is_err());
|
||||
assert!(
|
||||
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS + 1, 0)
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
TaskRetryPolicy::new(1, TASK_MAX_BACKOFF_STEP_MS + 1)
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
// The pinned step grows the delay but never beyond four steps.
|
||||
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(2), 2);
|
||||
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(5), 8);
|
||||
assert_eq!(TaskRetryPolicy::single_attempt().backoff_before(2), 0);
|
||||
// A pin whose retry snapshot is invalid is rejected at acceptance.
|
||||
let mut invalid = request();
|
||||
invalid.pins.retry = TaskRetryPolicy::new(0, 0);
|
||||
assert!(invalid.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -6,7 +6,12 @@ mod artifact_cleanup_indexes_v13;
|
||||
mod artifact_metadata_v12;
|
||||
mod authority;
|
||||
mod baseline_v1;
|
||||
mod durable_task_input_v30;
|
||||
mod durable_tasks_v25;
|
||||
mod durable_tasks_v26;
|
||||
mod durable_tasks_v27;
|
||||
mod durable_tasks_v28;
|
||||
mod durable_tasks_v29;
|
||||
mod execution_admission_v24;
|
||||
mod execution_outcome_v5;
|
||||
mod import_plan_v14;
|
||||
@@ -39,6 +44,11 @@ mod schema_guard_v22;
|
||||
mod schema_guard_v23;
|
||||
mod schema_guard_v24;
|
||||
mod schema_guard_v25;
|
||||
mod schema_guard_v26;
|
||||
mod schema_guard_v27;
|
||||
mod schema_guard_v28;
|
||||
mod schema_guard_v29;
|
||||
mod schema_guard_v30;
|
||||
mod schema_guard_v7;
|
||||
mod schema_guard_v8;
|
||||
mod schema_guard_v9;
|
||||
|
||||
@@ -4,7 +4,12 @@ use super::agent_resource_bindings_v18;
|
||||
use super::approval_side_effects_v10;
|
||||
use super::artifact_cleanup_indexes_v13;
|
||||
use super::artifact_metadata_v12;
|
||||
use super::durable_task_input_v30;
|
||||
use super::durable_tasks_v25;
|
||||
use super::durable_tasks_v26;
|
||||
use super::durable_tasks_v27;
|
||||
use super::durable_tasks_v28;
|
||||
use super::durable_tasks_v29;
|
||||
use super::execution_admission_v24;
|
||||
use super::execution_outcome_v5;
|
||||
use super::import_plan_v14;
|
||||
@@ -41,9 +46,10 @@ use preflight::*;
|
||||
use contract::baseline_source_digest;
|
||||
use contract::validate_descriptors;
|
||||
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
|
||||
const CURRENT_VERSION: i64 = 25;
|
||||
const CURRENT_VERSION: i64 = 30;
|
||||
const IMPLEMENTED_VERSIONS: &[i64] = &[
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
|
||||
27, 28, 29, 30,
|
||||
];
|
||||
const BASELINE_SOURCE_SHA256: &str =
|
||||
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
|
||||
@@ -344,6 +350,36 @@ impl MigrationAuthority {
|
||||
durable_tasks_v25::SOURCE_SHA256,
|
||||
24,
|
||||
),
|
||||
expand_descriptor(
|
||||
26,
|
||||
durable_tasks_v26::NAME,
|
||||
durable_tasks_v26::SOURCE_SHA256,
|
||||
25,
|
||||
),
|
||||
expand_descriptor(
|
||||
27,
|
||||
durable_tasks_v27::NAME,
|
||||
durable_tasks_v27::SOURCE_SHA256,
|
||||
26,
|
||||
),
|
||||
expand_descriptor(
|
||||
28,
|
||||
durable_tasks_v28::NAME,
|
||||
durable_tasks_v28::SOURCE_SHA256,
|
||||
27,
|
||||
),
|
||||
expand_descriptor(
|
||||
29,
|
||||
durable_tasks_v29::NAME,
|
||||
durable_tasks_v29::SOURCE_SHA256,
|
||||
28,
|
||||
),
|
||||
expand_descriptor(
|
||||
30,
|
||||
durable_task_input_v30::NAME,
|
||||
durable_task_input_v30::SOURCE_SHA256,
|
||||
29,
|
||||
),
|
||||
]
|
||||
}
|
||||
pub fn validate_sequence() -> Result<(), MigrationError> {
|
||||
@@ -509,6 +545,21 @@ impl MigrationAuthority {
|
||||
if from < 25 {
|
||||
durable_tasks_v25::apply(&mut transaction, &Self::sequence()[24]).await?;
|
||||
}
|
||||
if from < 26 {
|
||||
durable_tasks_v26::apply(&mut transaction, &Self::sequence()[25]).await?;
|
||||
}
|
||||
if from < 27 {
|
||||
durable_tasks_v27::apply(&mut transaction, &Self::sequence()[26]).await?;
|
||||
}
|
||||
if from < 28 {
|
||||
durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?;
|
||||
}
|
||||
if from < 29 {
|
||||
durable_tasks_v29::apply(&mut transaction, &Self::sequence()[28]).await?;
|
||||
}
|
||||
if from < 30 {
|
||||
durable_task_input_v30::apply(&mut transaction, &Self::sequence()[29]).await?;
|
||||
}
|
||||
match inspect(&mut transaction).await? {
|
||||
MigrationPreflight::Current {
|
||||
version: CURRENT_VERSION,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-task-input-v30";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_task_input_v30.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"84a51f712fd0bc439acba2ab3849232777c29e3b181586bbb4b06fd129843a38";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_task_input_v30",
|
||||
Some(30),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_task_input_v30",
|
||||
Some(30),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
-- V30: continuation through validated input (Story 4.5).
|
||||
alter table tasks add column if not exists effective_input_fingerprint text;
|
||||
update tasks set effective_input_fingerprint = input_fingerprint where effective_input_fingerprint is null;
|
||||
alter table tasks alter column effective_input_fingerprint set not null;
|
||||
alter table tasks add constraint tasks_effective_input_fingerprint_check check (
|
||||
length(effective_input_fingerprint) = 64 and effective_input_fingerprint ~ '^[0-9a-f]{64}$'
|
||||
);
|
||||
|
||||
create table if not exists task_input_requests (
|
||||
task_id uuid primary key references tasks (task_id) on delete cascade,
|
||||
wait_id text not null,
|
||||
expected_version integer not null,
|
||||
schema_revision integer not null,
|
||||
schema_format_version integer not null,
|
||||
fingerprint_format_version integer not null,
|
||||
schema_snapshot jsonb not null,
|
||||
projected_schema jsonb not null,
|
||||
prompt_code text not null,
|
||||
missing_fields jsonb not null default '[]'::jsonb,
|
||||
expires_at timestamptz not null,
|
||||
created_at timestamptz not null default now(),
|
||||
accepted_at timestamptz,
|
||||
update_key_digest text,
|
||||
canonical_fingerprint text,
|
||||
effective_input_fingerprint text,
|
||||
committed_state text,
|
||||
committed_version integer,
|
||||
constraint task_input_requests_wait_unique unique (wait_id),
|
||||
constraint task_input_requests_prompt_check check (octet_length(prompt_code) between 1 and 64),
|
||||
constraint task_input_requests_receipt_check check (
|
||||
(accepted_at is null and update_key_digest is null and canonical_fingerprint is null
|
||||
and effective_input_fingerprint is null and committed_state is null and committed_version is null)
|
||||
or (accepted_at is not null and update_key_digest is not null and canonical_fingerprint is not null
|
||||
and effective_input_fingerprint is not null and committed_state is not null and committed_version is not null)
|
||||
)
|
||||
);
|
||||
-- Not unique on purpose: several waiting Tasks may share one absolute TTL value (the same pinned
|
||||
-- policy deadline), and an expired/cancelled wait keeps its row. Per-Task uniqueness comes from the
|
||||
-- primary key and task_input_requests_wait_unique.
|
||||
create index if not exists task_input_requests_active_ttl_idx
|
||||
on task_input_requests (expires_at) where accepted_at is null;
|
||||
|
||||
create table if not exists task_input_audit (
|
||||
id bigint generated always as identity primary key,
|
||||
task_id uuid not null references tasks (task_id) on delete cascade,
|
||||
wait_id text not null,
|
||||
event text not null,
|
||||
from_state text,
|
||||
to_state text,
|
||||
from_version integer,
|
||||
to_version integer,
|
||||
actor text,
|
||||
reason_code text,
|
||||
created_at timestamptz not null default now(),
|
||||
constraint task_input_audit_event_check check (event in ('input_required','input_accepted','input_expired')),
|
||||
constraint task_input_audit_actor_check check (actor is null or octet_length(actor) between 1 and 160),
|
||||
constraint task_input_audit_reason_check check (reason_code is null or octet_length(reason_code) between 1 and 64)
|
||||
);
|
||||
create index if not exists task_input_audit_task_idx on task_input_audit (task_id, id);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v26";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v26.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"6dcbf9990deabb3984a333acbfc3cb490cfc886c2ec30093e3b46d9113616089";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v26",
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v26",
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
-- V26: PostgreSQL lease/fencing/CAS additions for durable background Tasks (Story 4.3).
|
||||
alter table tasks add column if not exists fence bigint not null default 0;
|
||||
alter table tasks add column if not exists lease_owner text;
|
||||
alter table tasks add column if not exists lease_until timestamptz;
|
||||
alter table tasks add column if not exists current_attempt_id text;
|
||||
alter table tasks add column if not exists dispatch_started_at timestamptz;
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v27";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v27.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"a3c5335cbf81c244359faaee993487ae132d8b3be6b9c63fa322788021273dc8";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v27",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v27",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
-- V27: encrypted executable Task input (Story 4.3).
|
||||
alter table tasks add column if not exists executable_input_ciphertext text;
|
||||
alter table tasks add column if not exists executable_input_key_version text;
|
||||
alter table tasks add column if not exists executable_input_master_key_epoch bigint;
|
||||
|
||||
alter table tasks add constraint tasks_executable_input_key_version_check check (
|
||||
executable_input_ciphertext is null
|
||||
or (executable_input_key_version is not null and octet_length(executable_input_key_version) between 1 and 64)
|
||||
);
|
||||
|
||||
alter table tasks add column if not exists target_executable_input_ciphertext text;
|
||||
alter table tasks add column if not exists target_executable_input_key_version text;
|
||||
alter table tasks add column if not exists target_executable_input_master_key_epoch bigint;
|
||||
|
||||
alter table tasks add constraint tasks_executable_input_epoch_check check (
|
||||
executable_input_ciphertext is null
|
||||
or (executable_input_master_key_epoch is not null
|
||||
and executable_input_master_key_epoch >= 1)
|
||||
);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v28";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v28.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"4dc77739c2ea1e32a4b00abd70df0f00fd82b7cf7933372f869f2c1e997a3676";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v28",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v28",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
-- V28: typed Platform API key authority reference of a durable Task (Story 4.3).
|
||||
-- Only the immutable key identifier is stored, never the bearer secret.
|
||||
alter table tasks add column if not exists platform_api_key_id text;
|
||||
|
||||
alter table tasks add constraint tasks_platform_api_key_id_check check (
|
||||
platform_api_key_id is null
|
||||
or octet_length(platform_api_key_id) between 1 and 160
|
||||
);
|
||||
@@ -0,0 +1,45 @@
|
||||
use sqlx::{Postgres, Transaction, query};
|
||||
|
||||
use super::authority::{MigrationDescriptor, MigrationError};
|
||||
|
||||
pub(super) const NAME: &str = "durable-tasks-v29";
|
||||
pub(super) const SOURCE: &str = include_str!("durable_tasks_v29.sql");
|
||||
pub(super) const SOURCE_SHA256: &str =
|
||||
"0666c7d2aa440e6aae2170ceaf2fce2549118612e1505ef78817be60436a75dd";
|
||||
|
||||
pub(super) async fn apply(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
descriptor: &MigrationDescriptor,
|
||||
) -> Result<(), MigrationError> {
|
||||
sqlx::raw_sql(SOURCE)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v29",
|
||||
Some(29),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(descriptor.version)
|
||||
.bind(descriptor.name)
|
||||
.bind(&descriptor.checksum)
|
||||
.bind(descriptor.phase)
|
||||
.bind(descriptor.compatibility)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
MigrationError::new(
|
||||
"apply_failed",
|
||||
"apply.durable_tasks_v29",
|
||||
Some(29),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
-- V29: durable per-outbound attempt history, pinned retry policy and cancellation (Story 4.4).
|
||||
-- An accepted Task keeps the retry snapshot it was admitted with; legacy Tasks get a single attempt.
|
||||
alter table tasks add column if not exists max_attempts integer not null default 1;
|
||||
alter table tasks add column if not exists retry_backoff_step_ms bigint not null default 0;
|
||||
alter table tasks add column if not exists next_attempt_at timestamptz;
|
||||
alter table tasks add column if not exists cancel_requested_at timestamptz;
|
||||
alter table tasks add column if not exists cancel_reason text;
|
||||
|
||||
alter table tasks add constraint tasks_max_attempts_check check (max_attempts between 1 and 5);
|
||||
alter table tasks add constraint tasks_retry_backoff_step_check
|
||||
check (retry_backoff_step_ms between 0 and 1000);
|
||||
alter table tasks add constraint tasks_cancel_reason_check check (
|
||||
cancel_reason is null or octet_length(cancel_reason) between 1 and 160
|
||||
);
|
||||
|
||||
-- One row per committed outbound attempt. The authoritative outcome of a Task stays on `tasks`;
|
||||
-- this history only records what was attempted and how each attempt resolved.
|
||||
create table if not exists task_attempts (
|
||||
attempt_id text primary key,
|
||||
task_id uuid not null references tasks (task_id) on delete cascade,
|
||||
ordinal integer not null,
|
||||
owner text not null,
|
||||
fence bigint not null default 0,
|
||||
input_fingerprint text not null,
|
||||
upstream_key_digest text,
|
||||
retry_allowed boolean not null default false,
|
||||
checkpoint_at timestamptz not null default now(),
|
||||
resolved_at timestamptz,
|
||||
outcome text,
|
||||
certainty text,
|
||||
phase text,
|
||||
constraint task_attempts_task_ordinal_unique unique (task_id, ordinal),
|
||||
constraint task_attempts_ordinal_check check (ordinal between 1 and 5),
|
||||
constraint task_attempts_outcome_check check (
|
||||
outcome is null
|
||||
or outcome in ('completed', 'failed', 'retry_scheduled', 'outcome_unknown')
|
||||
),
|
||||
constraint task_attempts_certainty_check check (
|
||||
certainty is null or certainty in ('known', 'unknown')
|
||||
),
|
||||
constraint task_attempts_phase_check check (
|
||||
phase is null or phase in ('before_dispatch', 'dispatched')
|
||||
),
|
||||
-- A resolved attempt is complete: an unresolved row carries no resolution at all, so a stale
|
||||
-- writer can never append a second outcome to the same attempt.
|
||||
constraint task_attempts_resolution_check check (
|
||||
(resolved_at is null and outcome is null and certainty is null and phase is null)
|
||||
or (resolved_at is not null and outcome is not null and certainty is not null)
|
||||
)
|
||||
);
|
||||
|
||||
create index if not exists task_attempts_task_id_idx on task_attempts (task_id, ordinal);
|
||||
@@ -480,8 +480,79 @@ pub(super) async fn validate_schema_fingerprint(
|
||||
if !super::schema_guard_v25::validate_v25_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 26 {
|
||||
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 27 {
|
||||
super::schema_guard_v26::validate_v26_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 28 {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v27::validate_v27_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 29 {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v28::validate_v28_durable_tasks(connection).await?;
|
||||
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else if current_version < 30 {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v29::validate_v29_durable_attempts(connection).await?;
|
||||
if !super::schema_guard_v30::validate_v30_absent(connection).await? {
|
||||
return Err(schema_error(current_version));
|
||||
}
|
||||
} else {
|
||||
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
|
||||
super::schema_guard_v30::validate_v30_input_continuation(connection).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//! V26 schema guard: lease/fencing columns for durable background Tasks (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
const TASK_COLUMNS: &[(&str, &str, bool)] = &[
|
||||
("task_id", "uuid", false),
|
||||
("workspace_id", "text", false),
|
||||
("owner_kind", "text", false),
|
||||
("owner_id", "text", false),
|
||||
("authorization_scope", "text", false),
|
||||
("operation_id", "text", false),
|
||||
("operation_version", "integer", false),
|
||||
("scoped_key", "text", false),
|
||||
("input_fingerprint", "text", false),
|
||||
("request_id", "text", false),
|
||||
("trace_id", "text", false),
|
||||
("traceparent", "text", false),
|
||||
("trace_sampled", "boolean", false),
|
||||
("policy_mode", "text", false),
|
||||
("approval_reference", "text", true),
|
||||
("state", "text", false),
|
||||
("version", "integer", false),
|
||||
("attempts", "integer", false),
|
||||
("input_envelope", "jsonb", false),
|
||||
("progress_envelope", "jsonb", true),
|
||||
("result_envelope", "jsonb", true),
|
||||
("error_envelope", "jsonb", true),
|
||||
("execution_deadline", "timestamp with time zone", false),
|
||||
("retain_until", "timestamp with time zone", false),
|
||||
("created_at", "timestamp with time zone", false),
|
||||
("updated_at", "timestamp with time zone", false),
|
||||
("fence", "bigint", false),
|
||||
("lease_owner", "text", true),
|
||||
("lease_until", "timestamp with time zone", true),
|
||||
("current_attempt_id", "text", true),
|
||||
("dispatch_started_at", "timestamp with time zone", true),
|
||||
];
|
||||
|
||||
fn v26_error(stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(26),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
// delegate to v25 absent for relation mostly; V26 columns must not exist.
|
||||
for column in [
|
||||
"fence",
|
||||
"lease_owner",
|
||||
"lease_until",
|
||||
"current_attempt_id",
|
||||
"dispatch_started_at",
|
||||
] {
|
||||
if super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_base_columns(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
if !relation_exists(connection, TASKS).await? {
|
||||
return Err(v26_error("v26.relations"));
|
||||
}
|
||||
for (column, data_type, nullable) in TASK_COLUMNS {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(*data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if *nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(v26_error("v26.columns"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v26_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
validate_v26_base_columns(connection).await?;
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != i64::try_from(TASK_COLUMNS.len()).unwrap_or_default() {
|
||||
return Err(v26_error("v26.columns"));
|
||||
}
|
||||
// Constraints/index checks inherited conceptually from V25; V26 adds columns only.
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
//! V27 schema guard: encrypted executable Task input columns (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
use super::schema_guard_v26::validate_v26_base_columns;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
pub(super) async fn validate_v27_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
for column in [
|
||||
"executable_input_ciphertext",
|
||||
"executable_input_key_version",
|
||||
"executable_input_master_key_epoch",
|
||||
] {
|
||||
if super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v27_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
validate_v26_base_columns(connection).await?;
|
||||
// Additional V27 columns.
|
||||
let extra = [
|
||||
("executable_input_ciphertext", "text", true),
|
||||
("executable_input_key_version", "text", true),
|
||||
("executable_input_master_key_epoch", "bigint", true),
|
||||
("target_executable_input_ciphertext", "text", true),
|
||||
("target_executable_input_key_version", "text", true),
|
||||
("target_executable_input_master_key_epoch", "bigint", true),
|
||||
];
|
||||
for (column, data_type, nullable) in extra {
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind(column)
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some(data_type)
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref()
|
||||
== Some(if nullable { "YES" } else { "NO" })
|
||||
});
|
||||
if !valid {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v27.columns",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
}
|
||||
// Exact total column count for version 27: 43 V26 columns + 3 V27 columns.
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != 37_i64 {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v27.columns",
|
||||
Some(27),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
//! V28 schema guard: typed Platform API key authority reference of a durable Task (Story 4.3).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::relation_exists;
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
|
||||
pub(super) async fn validate_v28_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await?
|
||||
&& super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await?
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v28_durable_tasks(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
// The V27 column set is still required and unchanged; V28 only appends one column.
|
||||
validate_v27_base_columns(connection).await?;
|
||||
let row = query(
|
||||
"select data_type, is_nullable from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1 and column_name = $2",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.bind("platform_api_key_id")
|
||||
.fetch_optional(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
let valid = row.is_some_and(|row| {
|
||||
row.try_get::<String, _>("data_type").ok().as_deref() == Some("text")
|
||||
&& row.try_get::<String, _>("is_nullable").ok().as_deref() == Some("YES")
|
||||
});
|
||||
if !valid {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v28.columns",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
// Exact total column count for version 28: the 37 V27 columns + 1 V28 column.
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != 38_i64 {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v28.columns",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// V27 columns must still be exactly present before V28 is considered applied.
|
||||
async fn validate_v27_base_columns(connection: &mut PgConnection) -> Result<(), MigrationError> {
|
||||
for column in [
|
||||
"executable_input_ciphertext",
|
||||
"executable_input_key_version",
|
||||
"executable_input_master_key_epoch",
|
||||
"target_executable_input_ciphertext",
|
||||
"target_executable_input_key_version",
|
||||
"target_executable_input_master_key_epoch",
|
||||
] {
|
||||
if !super::schema_guard::column_exists(connection, TASKS, column).await? {
|
||||
return Err(MigrationError::new(
|
||||
"partial_sequence",
|
||||
"v28.columns",
|
||||
Some(28),
|
||||
"restore_known_good_backup",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! V29 schema guard: durable per-outbound attempt history, pinned retry policy and cancellation
|
||||
//! (Story 4.4).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::{column_exists, relation_exists};
|
||||
|
||||
const TASKS: &str = "tasks";
|
||||
const ATTEMPTS: &str = "task_attempts";
|
||||
|
||||
pub(super) async fn validate_v29_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, TASKS).await? {
|
||||
if column_exists(connection, TASKS, "max_attempts").await? {
|
||||
return Ok(false);
|
||||
}
|
||||
if relation_exists(connection, ATTEMPTS).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
pub(super) async fn validate_v29_durable_attempts(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
// The V28 column set is still required and unchanged; V29 only appends columns and one table.
|
||||
for column in [
|
||||
"platform_api_key_id",
|
||||
"max_attempts",
|
||||
"retry_backoff_step_ms",
|
||||
"next_attempt_at",
|
||||
"cancel_requested_at",
|
||||
"cancel_reason",
|
||||
] {
|
||||
if !column_exists(connection, TASKS, column).await? {
|
||||
return Err(invalid(29, "v29.columns"));
|
||||
}
|
||||
}
|
||||
// Exact total column count for version 29: the 38 V28 columns + 5 V29 columns.
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind(TASKS)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != 43_i64 {
|
||||
return Err(invalid(29, "v29.columns"));
|
||||
}
|
||||
if !relation_exists(connection, ATTEMPTS).await? {
|
||||
return Err(invalid(29, "v29.attempts"));
|
||||
}
|
||||
for column in [
|
||||
"attempt_id",
|
||||
"task_id",
|
||||
"ordinal",
|
||||
"owner",
|
||||
"fence",
|
||||
"input_fingerprint",
|
||||
"upstream_key_digest",
|
||||
"retry_allowed",
|
||||
"checkpoint_at",
|
||||
"resolved_at",
|
||||
"outcome",
|
||||
"certainty",
|
||||
"phase",
|
||||
] {
|
||||
if !column_exists(connection, ATTEMPTS, column).await? {
|
||||
return Err(invalid(29, "v29.attempts"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn invalid(version: i64, stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(version),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
//! V30 schema guard: continuation through validated input (Story 4.5).
|
||||
|
||||
use sqlx::{PgConnection, Row, query};
|
||||
|
||||
use super::authority::MigrationError;
|
||||
use super::schema_guard::{column_exists, relation_exists};
|
||||
|
||||
pub(super) async fn validate_v30_absent(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<bool, MigrationError> {
|
||||
if relation_exists(connection, "tasks").await?
|
||||
&& column_exists(connection, "tasks", "effective_input_fingerprint").await?
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
if relation_exists(connection, "task_input_requests").await? {
|
||||
return Ok(false);
|
||||
}
|
||||
if relation_exists(connection, "task_input_audit").await? {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn validate_v30_input_continuation(
|
||||
connection: &mut PgConnection,
|
||||
) -> Result<(), MigrationError> {
|
||||
// V30 builds on the applied V29 shape. The exact V29 column count (43) no longer holds once V30
|
||||
// appends its column (44), so the V29-specific columns are re-checked here by name instead of
|
||||
// calling the exact-count V29 guard, which is only valid before V30 is applied.
|
||||
for column in [
|
||||
"platform_api_key_id",
|
||||
"max_attempts",
|
||||
"retry_backoff_step_ms",
|
||||
"next_attempt_at",
|
||||
"cancel_requested_at",
|
||||
"cancel_reason",
|
||||
] {
|
||||
if !column_exists(connection, "tasks", column).await? {
|
||||
return Err(invalid(30, "v30.columns"));
|
||||
}
|
||||
}
|
||||
|
||||
// V30 appends one column to tasks.
|
||||
if !column_exists(connection, "tasks", "effective_input_fingerprint").await? {
|
||||
return Err(invalid(30, "v30.columns"));
|
||||
}
|
||||
let count: i64 = query(
|
||||
"select count(*)::bigint as count from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = $1",
|
||||
)
|
||||
.bind("tasks")
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get("count")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if count != 44_i64 {
|
||||
return Err(invalid(30, "v30.columns"));
|
||||
}
|
||||
|
||||
// Tables.
|
||||
for table in ["task_input_requests", "task_input_audit"] {
|
||||
if !relation_exists(connection, table).await? {
|
||||
return Err(invalid(30, "v30.tables"));
|
||||
}
|
||||
}
|
||||
|
||||
// task_input_requests columns.
|
||||
for column in [
|
||||
"task_id",
|
||||
"wait_id",
|
||||
"expected_version",
|
||||
"schema_revision",
|
||||
"schema_format_version",
|
||||
"fingerprint_format_version",
|
||||
"schema_snapshot",
|
||||
"projected_schema",
|
||||
"prompt_code",
|
||||
"missing_fields",
|
||||
"expires_at",
|
||||
"created_at",
|
||||
"accepted_at",
|
||||
"update_key_digest",
|
||||
"canonical_fingerprint",
|
||||
"effective_input_fingerprint",
|
||||
"committed_state",
|
||||
"committed_version",
|
||||
] {
|
||||
if !column_exists(connection, "task_input_requests", column).await? {
|
||||
return Err(invalid(30, "v30.task_input_requests"));
|
||||
}
|
||||
}
|
||||
|
||||
// task_input_audit columns.
|
||||
for column in [
|
||||
"id",
|
||||
"task_id",
|
||||
"wait_id",
|
||||
"event",
|
||||
"from_state",
|
||||
"to_state",
|
||||
"from_version",
|
||||
"to_version",
|
||||
"actor",
|
||||
"reason_code",
|
||||
"created_at",
|
||||
] {
|
||||
if !column_exists(connection, "task_input_audit", column).await? {
|
||||
return Err(invalid(30, "v30.task_input_audit"));
|
||||
}
|
||||
}
|
||||
|
||||
// Constraints.
|
||||
for constraint in [
|
||||
"task_input_requests_pkey",
|
||||
"task_input_requests_wait_unique",
|
||||
"task_input_requests_prompt_check",
|
||||
"task_input_requests_receipt_check",
|
||||
"task_input_audit_event_check",
|
||||
"task_input_audit_actor_check",
|
||||
"task_input_audit_reason_check",
|
||||
"tasks_effective_input_fingerprint_check",
|
||||
] {
|
||||
let table = if constraint.starts_with("task_input_audit")
|
||||
|| constraint.starts_with("task_input_audit_")
|
||||
{
|
||||
"task_input_audit"
|
||||
} else if constraint == "tasks_effective_input_fingerprint_check" {
|
||||
"tasks"
|
||||
} else {
|
||||
"task_input_requests"
|
||||
};
|
||||
if !named_constraint_exists(connection, table, constraint).await? {
|
||||
return Err(invalid(30, "v30.constraints"));
|
||||
}
|
||||
}
|
||||
|
||||
// Indexes.
|
||||
for index in [
|
||||
"task_input_requests_active_ttl_idx",
|
||||
"task_input_audit_task_idx",
|
||||
] {
|
||||
let present = query(
|
||||
"select exists (select 1 from pg_catalog.pg_indexes
|
||||
where schemaname = current_schema() and indexname = $1) as present",
|
||||
)
|
||||
.bind(index)
|
||||
.fetch_one(&mut *connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?;
|
||||
if !present {
|
||||
return Err(invalid(30, "v30.indexes"));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn invalid(version: i64, stage: &'static str) -> MigrationError {
|
||||
MigrationError::new(
|
||||
"partial_sequence",
|
||||
stage,
|
||||
Some(version),
|
||||
"restore_known_good_backup",
|
||||
)
|
||||
}
|
||||
|
||||
// helper
|
||||
async fn named_constraint_exists(
|
||||
connection: &mut PgConnection,
|
||||
table: &str,
|
||||
constraint: &str,
|
||||
) -> Result<bool, MigrationError> {
|
||||
query(
|
||||
"select exists (
|
||||
select 1 from pg_catalog.pg_constraint c
|
||||
join pg_catalog.pg_class t on t.oid = c.conrelid
|
||||
join pg_catalog.pg_namespace n on n.oid = t.relnamespace
|
||||
where n.nspname = current_schema()
|
||||
and t.relname = $1
|
||||
and c.conname = $2
|
||||
) as present",
|
||||
)
|
||||
.bind(table)
|
||||
.bind(constraint)
|
||||
.fetch_one(connection)
|
||||
.await
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))?
|
||||
.try_get::<bool, _>("present")
|
||||
.map_err(|_| MigrationError::storage("preflight.schema"))
|
||||
}
|
||||
@@ -577,6 +577,145 @@ pub(super) async fn insert_admission_entry(
|
||||
Ok(u64::try_from(ticket_id).unwrap_or_default())
|
||||
}
|
||||
|
||||
/// Shared grant step used by the durable Task store: grants one reserved Task intent if capacity allows,
|
||||
/// marks it `executing`, persists the weighted round-robin cursor, and returns its Task id. It reuses
|
||||
/// the same selector/capacity logic as synchronous admission so no second policy exists.
|
||||
pub(super) async fn grant_next_task(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &str,
|
||||
cursor: &AdmissionCursor,
|
||||
limits: &AdmissionLimits,
|
||||
scheduler: AdmissionSchedulerConfig,
|
||||
) -> Result<(Option<Uuid>, AdmissionCursor), AttemptError> {
|
||||
let rows = sqlx::query(
|
||||
"select ticket_id, agent_key, operation_id, task_id
|
||||
from execution_admission_entries
|
||||
where state = 'reserved' and task_id is not null
|
||||
order by ticket_id
|
||||
limit $1
|
||||
for update skip locked",
|
||||
)
|
||||
.bind(QUEUE_READ_LIMIT)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
if rows.is_empty() {
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
struct Candidate {
|
||||
entry: AdmissionQueueEntry,
|
||||
task_id: Uuid,
|
||||
}
|
||||
let mut candidates = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let ticket_id: i64 = row.try_get("ticket_id").map_err(AttemptError::from_sql)?;
|
||||
let task_id: Uuid = row.try_get("task_id").map_err(AttemptError::from_sql)?;
|
||||
candidates.push(Candidate {
|
||||
entry: AdmissionQueueEntry {
|
||||
ticket_id: u64::try_from(ticket_id).unwrap_or_default(),
|
||||
sequence: 0,
|
||||
agent_key: row.try_get("agent_key").map_err(AttemptError::from_sql)?,
|
||||
operation_id: row
|
||||
.try_get("operation_id")
|
||||
.map_err(AttemptError::from_sql)?,
|
||||
},
|
||||
task_id,
|
||||
});
|
||||
}
|
||||
let queue: Vec<AdmissionQueueEntry> = candidates.iter().map(|c| c.entry.clone()).collect();
|
||||
let Some(selection) = select_next_grant(&queue, cursor, scheduler) else {
|
||||
return Ok((None, cursor.clone()));
|
||||
};
|
||||
let head_index = selection.index;
|
||||
let head = &candidates[head_index];
|
||||
|
||||
let global_executing: i64 = sqlx::query(
|
||||
"select count(*)::bigint as count from execution_admission_entries where state = 'executing'",
|
||||
)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?
|
||||
.try_get("count")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let agent_counts = sqlx::query(
|
||||
"select agent_key as key, count(*)::bigint as count
|
||||
from execution_admission_entries where state = 'executing' group by agent_key",
|
||||
)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let mut agent_map = std::collections::BTreeMap::new();
|
||||
for row in agent_counts {
|
||||
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
|
||||
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
|
||||
agent_map.insert(key, u64::try_from(count).unwrap_or_default());
|
||||
}
|
||||
let tool_counts = sqlx::query(
|
||||
"select operation_id as key, count(*)::bigint as count
|
||||
from execution_admission_entries where state = 'executing' group by operation_id",
|
||||
)
|
||||
.fetch_all(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let mut tool_map = std::collections::BTreeMap::new();
|
||||
for row in tool_counts {
|
||||
let key: String = row.try_get("key").map_err(AttemptError::from_sql)?;
|
||||
let count: i64 = row.try_get("count").map_err(AttemptError::from_sql)?;
|
||||
tool_map.insert(key, u64::try_from(count).unwrap_or_default());
|
||||
}
|
||||
|
||||
if u64::try_from(global_executing).unwrap_or(u64::MAX)
|
||||
>= u64::try_from(limits.global_limit).unwrap_or(u64::MAX)
|
||||
{
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
let agent_executing = agent_map
|
||||
.get(head.entry.agent_key.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if agent_executing >= u64::try_from(limits.agent_limit).unwrap_or(u64::MAX) {
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
let tool_executing = tool_map
|
||||
.get(head.entry.operation_id.as_str())
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if tool_executing >= u64::try_from(limits.tool_limit).unwrap_or(u64::MAX) {
|
||||
return Ok((None, cursor.clone()));
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"update execution_admission_entries
|
||||
set state = 'executing', granted_at = clock_timestamp()
|
||||
where ticket_id = $1 and state = 'reserved'",
|
||||
)
|
||||
.bind(i64::try_from(head.entry.ticket_id).unwrap_or(i64::MAX))
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
sqlx::query(
|
||||
"update execution_admission_controllers
|
||||
set cursor_agent = $2,
|
||||
cursor_agent_credit = $3,
|
||||
cursor_tool = $4,
|
||||
cursor_tool_credit = $5,
|
||||
updated_at = now()
|
||||
where workspace_id = $1",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(selection.cursor.agent.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.agent_credit).unwrap_or_default())
|
||||
.bind(selection.cursor.tool.as_deref())
|
||||
.bind(i32::try_from(selection.cursor.tool_credit).unwrap_or_default())
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
Ok((Some(head.task_id), selection.cursor))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum AttemptError {
|
||||
/// Lock or statement timeout: the transaction made no decision and the waiter retries.
|
||||
Contended,
|
||||
|
||||
@@ -170,6 +170,69 @@ impl PostgresRegistry {
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Authoritative revalidation of the Platform API key that authorized a durable Task (Story 4.3).
|
||||
///
|
||||
/// The key is returned only when the exact immutable ID is still active, unexpired and bound to
|
||||
/// the same workspace and agent; otherwise `None`, so a worker fails closed. Only the immutable
|
||||
/// identity is supplied here — the bearer secret is never stored or re-read.
|
||||
pub async fn revalidate_platform_api_key(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
agent_id: &AgentId,
|
||||
key_id: &PlatformApiKeyId,
|
||||
) -> Result<Option<PlatformApiKeyRecord>, RegistryError> {
|
||||
let row = sqlx::query(
|
||||
"select
|
||||
k.id,
|
||||
k.workspace_id,
|
||||
k.agent_id,
|
||||
k.key_kind,
|
||||
k.name,
|
||||
k.prefix,
|
||||
k.scopes_json,
|
||||
k.status,
|
||||
k.created_at,
|
||||
k.last_used_at,
|
||||
k.expires_at,
|
||||
k.allowed_origins_json
|
||||
from platform_api_keys k
|
||||
where k.id = $1
|
||||
and k.workspace_id = $2
|
||||
and k.agent_id = $3
|
||||
and k.key_kind = 'mcp_client'
|
||||
and k.status = 'active'
|
||||
and (k.expires_at is null or k.expires_at > now())
|
||||
limit 1",
|
||||
)
|
||||
.bind(key_id.as_str())
|
||||
.bind(workspace_id.as_str())
|
||||
.bind(agent_id.as_str())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
row.map(|row| {
|
||||
Ok(PlatformApiKeyRecord {
|
||||
api_key: PlatformApiKey {
|
||||
id: PlatformApiKeyId::new(row.get::<String, _>("id")),
|
||||
workspace_id: WorkspaceId::new(row.get::<String, _>("workspace_id")),
|
||||
agent_id: row.get::<Option<String>, _>("agent_id").map(AgentId::new),
|
||||
key_kind: deserialize_enum_text(&row.get::<String, _>("key_kind"), "key_kind")?,
|
||||
name: row.get("name"),
|
||||
prefix: row.get("prefix"),
|
||||
scopes: deserialize_json_value(row.get::<Value, _>("scopes_json"))?,
|
||||
status: deserialize_enum_text(&row.get::<String, _>("status"), "status")?,
|
||||
created_at: row.get("created_at"),
|
||||
last_used_at: row.get("last_used_at"),
|
||||
expires_at: row.get("expires_at"),
|
||||
allowed_origins: deserialize_json_value(
|
||||
row.get::<Value, _>("allowed_origins_json"),
|
||||
)?,
|
||||
},
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub async fn get_approval_api_key_by_secret_for_agent_slug(
|
||||
&self,
|
||||
workspace_slug: &str,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::*;
|
||||
|
||||
const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
|
||||
pub(super) const MASTER_KEY_PAGE_LIMIT_MAX: i64 = 10_000;
|
||||
|
||||
impl PostgresRegistry {
|
||||
pub async fn verify_or_register_master_key_identity(
|
||||
@@ -356,12 +356,19 @@ impl PostgresRegistry {
|
||||
return Err(RegistryError::MasterKeyRotationConflict);
|
||||
}
|
||||
|
||||
let total = sqlx::query_scalar::<_, i64>(
|
||||
let secret_total = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from secret_versions where master_key_epoch = $1",
|
||||
)
|
||||
.bind(source_epoch)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
let task_total = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from tasks where executable_input_master_key_epoch = $1",
|
||||
)
|
||||
.bind(source_epoch)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
let total = secret_total + task_total;
|
||||
let rotation_id = format!("master-key-e{source_epoch}-to-e{target_epoch}");
|
||||
let aborted = sqlx::query(
|
||||
"select
|
||||
@@ -403,6 +410,16 @@ impl PostgresRegistry {
|
||||
.bind(target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set target_executable_input_ciphertext = null,
|
||||
target_executable_input_key_version = null,
|
||||
target_executable_input_master_key_epoch = null
|
||||
where target_executable_input_master_key_epoch = $1",
|
||||
)
|
||||
.bind(target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set state = 'running',
|
||||
@@ -662,9 +679,24 @@ impl PostgresRegistry {
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set executable_input_ciphertext = target_executable_input_ciphertext,
|
||||
executable_input_key_version = target_executable_input_key_version,
|
||||
executable_input_master_key_epoch = target_executable_input_master_key_epoch,
|
||||
target_executable_input_ciphertext = null,
|
||||
target_executable_input_key_version = null,
|
||||
target_executable_input_master_key_epoch = null
|
||||
where executable_input_master_key_epoch = $1
|
||||
and target_executable_input_master_key_epoch = $2",
|
||||
)
|
||||
.bind(rotation.source_epoch)
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set state = 'promoted',
|
||||
set state = 'promoted',
|
||||
updated_at = $2::timestamptz
|
||||
where id = $1",
|
||||
)
|
||||
@@ -697,6 +729,16 @@ impl PostgresRegistry {
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set target_executable_input_ciphertext = null,
|
||||
target_executable_input_key_version = null,
|
||||
target_executable_input_master_key_epoch = null
|
||||
where target_executable_input_master_key_epoch = $1",
|
||||
)
|
||||
.bind(rotation.target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set state = 'aborted',
|
||||
@@ -883,7 +925,7 @@ async fn select_active_master_key_identity_for_update(
|
||||
Ok(row.map(map_master_key_identity))
|
||||
}
|
||||
|
||||
async fn select_master_key_rotation_for_update(
|
||||
pub(super) async fn select_master_key_rotation_for_update(
|
||||
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
rotation_id: &str,
|
||||
) -> Result<MasterKeyRotationRecord, RegistryError> {
|
||||
@@ -915,16 +957,19 @@ async fn select_master_key_rotation_for_update(
|
||||
})
|
||||
}
|
||||
|
||||
async fn count_staged_rotation_targets(
|
||||
pub(super) async fn count_staged_rotation_targets(
|
||||
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
source_epoch: i64,
|
||||
target_epoch: i64,
|
||||
) -> Result<i64, RegistryError> {
|
||||
let count = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*)
|
||||
from secret_versions
|
||||
where master_key_epoch = $1
|
||||
and target_master_key_epoch = $2",
|
||||
"select
|
||||
(select count(*) from secret_versions
|
||||
where master_key_epoch = $1
|
||||
and target_master_key_epoch = $2) +
|
||||
(select count(*) from tasks
|
||||
where executable_input_master_key_epoch = $1
|
||||
and target_executable_input_master_key_epoch = $2)",
|
||||
)
|
||||
.bind(source_epoch)
|
||||
.bind(target_epoch)
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Task ciphertext paging/staging of the master-key rotation (Story 4.3).
|
||||
//!
|
||||
//! Extracted from `master_key.rs` without logic changes so that module stays within the
|
||||
//! code-health line limit; the operations and their SQL are unchanged.
|
||||
|
||||
use super::master_key::{
|
||||
MASTER_KEY_PAGE_LIMIT_MAX, count_staged_rotation_targets, select_master_key_rotation_for_update,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
fn map_master_key_task_version(
|
||||
row: sqlx::postgres::PgRow,
|
||||
) -> Result<MasterKeyTaskVersion, RegistryError> {
|
||||
Ok(MasterKeyTaskVersion {
|
||||
task_id: row.get::<String, _>("task_id"),
|
||||
ciphertext: row
|
||||
.get::<Option<String>, _>("executable_input_ciphertext")
|
||||
.unwrap_or_default(),
|
||||
key_version: row
|
||||
.get::<Option<String>, _>("executable_input_key_version")
|
||||
.unwrap_or_default(),
|
||||
epoch: row
|
||||
.get::<Option<i64>, _>("executable_input_master_key_epoch")
|
||||
.unwrap_or_default(),
|
||||
target_ciphertext: row.get::<Option<String>, _>("target_executable_input_ciphertext"),
|
||||
target_key_version: row.get::<Option<String>, _>("target_executable_input_key_version"),
|
||||
target_epoch: row.get::<Option<i64>, _>("target_executable_input_master_key_epoch"),
|
||||
})
|
||||
}
|
||||
|
||||
impl PostgresRegistry {
|
||||
pub async fn list_tasks_for_master_key_epoch_page(
|
||||
&self,
|
||||
epoch: i64,
|
||||
after_task_id: Option<&str>,
|
||||
limit: i64,
|
||||
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
|
||||
if epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
|
||||
return Err(RegistryError::InvalidMasterKeyIdentity);
|
||||
}
|
||||
let rows = sqlx::query(
|
||||
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
|
||||
)
|
||||
.bind(epoch)
|
||||
.bind(after_task_id)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
rows.into_iter().map(map_master_key_task_version).collect()
|
||||
}
|
||||
|
||||
pub async fn list_target_tasks_for_master_key_rotation_page(
|
||||
&self,
|
||||
target_epoch: i64,
|
||||
after_task_id: Option<&str>,
|
||||
limit: i64,
|
||||
) -> Result<Vec<MasterKeyTaskVersion>, RegistryError> {
|
||||
if target_epoch < 1 || !(1..=MASTER_KEY_PAGE_LIMIT_MAX).contains(&limit) {
|
||||
return Err(RegistryError::InvalidMasterKeyIdentity);
|
||||
}
|
||||
let rows = sqlx::query(
|
||||
"select task_id::text as task_id, executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch, target_executable_input_ciphertext, target_executable_input_key_version, target_executable_input_master_key_epoch from tasks where target_executable_input_master_key_epoch = $1 and ($2::text is null or task_id::text > $2::text) order by task_id asc limit $3",
|
||||
)
|
||||
.bind(target_epoch)
|
||||
.bind(after_task_id)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
rows.into_iter().map(map_master_key_task_version).collect()
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn stage_master_key_rotation_task_ciphertext(
|
||||
&self,
|
||||
rotation_id: &str,
|
||||
task_id: &str,
|
||||
source_epoch: i64,
|
||||
target_ciphertext: &str,
|
||||
target_key_version: &str,
|
||||
target_epoch: i64,
|
||||
now: &OffsetDateTime,
|
||||
) -> Result<MasterKeyRotationRecord, RegistryError> {
|
||||
let mut tx = self.pool.begin().await?;
|
||||
let rotation = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
|
||||
if rotation.state != "running"
|
||||
|| rotation.source_epoch != source_epoch
|
||||
|| rotation.target_epoch != target_epoch
|
||||
{
|
||||
return Err(RegistryError::MasterKeyRotationConflict);
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"update tasks
|
||||
set target_executable_input_ciphertext = $3,
|
||||
target_executable_input_key_version = $4,
|
||||
target_executable_input_master_key_epoch = $5
|
||||
where task_id::text = $1
|
||||
and executable_input_master_key_epoch = $2
|
||||
and (
|
||||
target_executable_input_master_key_epoch is null
|
||||
or target_executable_input_master_key_epoch = $5
|
||||
)
|
||||
and (
|
||||
target_executable_input_ciphertext is null
|
||||
or target_executable_input_ciphertext = $3
|
||||
)",
|
||||
)
|
||||
.bind(task_id)
|
||||
.bind(source_epoch)
|
||||
.bind(target_ciphertext)
|
||||
.bind(target_key_version)
|
||||
.bind(target_epoch)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
if result.rows_affected() != 1 {
|
||||
return Err(RegistryError::MasterKeyRotationConflict);
|
||||
}
|
||||
let processed = count_staged_rotation_targets(&mut tx, source_epoch, target_epoch).await?;
|
||||
sqlx::query(
|
||||
"update master_key_rotations
|
||||
set checkpoint_secret_id = checkpoint_secret_id,
|
||||
processed_secret_versions = $2,
|
||||
updated_at = $3::timestamptz
|
||||
where id = $1",
|
||||
)
|
||||
.bind(rotation_id)
|
||||
.bind(processed)
|
||||
.bind(now)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let updated = select_master_key_rotation_for_update(&mut tx, rotation_id).await?;
|
||||
tx.commit().await?;
|
||||
Ok(updated)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MasterKeyTaskVersion {
|
||||
pub task_id: String,
|
||||
pub ciphertext: String,
|
||||
pub key_version: String,
|
||||
pub epoch: i64,
|
||||
pub target_ciphertext: Option<String>,
|
||||
pub target_key_version: Option<String>,
|
||||
pub target_epoch: Option<i64>,
|
||||
}
|
||||
@@ -10,6 +10,7 @@ mod auth;
|
||||
mod connection;
|
||||
mod import_job;
|
||||
mod master_key;
|
||||
mod master_key_tasks;
|
||||
mod observability;
|
||||
mod onboarding;
|
||||
mod onboarding_import;
|
||||
|
||||
@@ -1,586 +0,0 @@
|
||||
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
|
||||
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
|
||||
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
|
||||
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
|
||||
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
|
||||
//!
|
||||
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
|
||||
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
|
||||
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest,
|
||||
TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::admission::{
|
||||
AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked,
|
||||
insert_admission_entry, lock_admission_in_tx,
|
||||
};
|
||||
|
||||
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
|
||||
authorization_scope, operation_id, input_fingerprint, operation_version, policy_mode,
|
||||
approval_reference,
|
||||
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
|
||||
result_envelope, error_envelope, attempts, version,
|
||||
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
|
||||
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
|
||||
(extract(epoch from created_at) * 1000)::bigint as created_ms,
|
||||
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PostgresTaskStore {
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: crank_core::AdmissionSchedulerConfig,
|
||||
) -> Result<Self, TaskError> {
|
||||
limits
|
||||
.validate()
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
scheduler
|
||||
.validate()
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
Ok(Self { pool, limits })
|
||||
}
|
||||
|
||||
fn unavailable(error: impl std::fmt::Display) -> TaskError {
|
||||
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
|
||||
}
|
||||
|
||||
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
|
||||
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local lock_timeout = '3000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local statement_timeout = '5000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
Ok(transaction)
|
||||
}
|
||||
|
||||
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
|
||||
/// query, so a foreign row is never fetched.
|
||||
async fn stored_acceptance(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(&request.scoped_key)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let Some(row) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
|
||||
Ok(Some((
|
||||
acceptance_from_record(&record),
|
||||
record.input_fingerprint,
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
|
||||
TaskAcceptance {
|
||||
task_id: record.task_id,
|
||||
state: record.state,
|
||||
operation_version: record.pins.operation_version,
|
||||
support: record.support.clone(),
|
||||
created_at_unix_ms: record.created_at_unix_ms,
|
||||
retain_until_unix_ms: record.pins.retain_until_unix_ms,
|
||||
replayed: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
|
||||
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
|
||||
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
|
||||
sqlx::query("select count(*)::bigint from tasks where false")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresTaskStore::unavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One transaction's decision of the acceptance path.
|
||||
enum Accepted {
|
||||
Created(TaskAcceptance),
|
||||
Replayed(TaskAcceptance),
|
||||
Conflict,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskStore for PostgresTaskStore {
|
||||
async fn health(&self) -> Result<(), TaskError> {
|
||||
sqlx::query("select count(*)::bigint from tasks where false")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
super::admission::admission_health(&self.pool)
|
||||
.await
|
||||
.map_err(|error| TaskError::unavailable(error.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
let outcome = self.accept_locked(&mut transaction, &request).await;
|
||||
match outcome {
|
||||
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
|
||||
// The acknowledgement of a created Task is only possible after a real COMMIT.
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(TaskAcceptance {
|
||||
replayed: acceptance.replayed,
|
||||
..acceptance
|
||||
})
|
||||
}
|
||||
Ok(Accepted::Conflict) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::FingerprintConflict)
|
||||
}
|
||||
Err(AttemptError::Overloaded) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::Overloaded {
|
||||
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
})
|
||||
}
|
||||
Err(AttemptError::Contended) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::unavailable(
|
||||
"durable task acceptance could not take the admission locks",
|
||||
))
|
||||
}
|
||||
Err(AttemptError::UniqueViolation) => {
|
||||
// Roll back the aborted transaction, then reread the scoped key once outside it.
|
||||
transaction.rollback().await.ok();
|
||||
self.reread_after_unique(&request).await
|
||||
}
|
||||
Err(AttemptError::Failure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(error)
|
||||
}
|
||||
Err(AttemptError::AdmissionFailure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::unavailable(error.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5"
|
||||
)))
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
record_from_row(&row).map_err(Self::unavailable)
|
||||
}
|
||||
|
||||
async fn find(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
operation_id: &str,
|
||||
scoped_key: &str,
|
||||
) -> Result<Option<TaskRecord>, TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(operation_id)
|
||||
.bind(scoped_key)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
row.map(|row| record_from_row(&row)).transpose()
|
||||
}
|
||||
|
||||
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
let access = &request.access;
|
||||
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = $1,
|
||||
result_envelope = $2,
|
||||
error_envelope = $3,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
||||
and owner_id = $7 and authorization_scope = $8
|
||||
and state = $9 and version = $10
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(request.target_state.as_str())
|
||||
.bind(request.result.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.error.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(request.expected_state.as_str())
|
||||
.bind(i64::from(request.expected_version))
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = update else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(self.classify_missed_finish(&request).await);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(request.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
|
||||
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
|
||||
let mut transaction = self.start().await?;
|
||||
let expired = sqlx::query(
|
||||
"with expired as (
|
||||
update tasks
|
||||
set state = 'failed',
|
||||
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id in (
|
||||
select task_id from tasks
|
||||
where state = 'queued' and execution_deadline <= now()
|
||||
order by task_id
|
||||
limit $1
|
||||
)
|
||||
returning task_id
|
||||
), released as (
|
||||
delete from execution_admission_entries
|
||||
where task_id in (select task_id from expired)
|
||||
returning task_id
|
||||
)
|
||||
select (select count(*) from expired)::bigint as expired_queued",
|
||||
)
|
||||
.bind(batch)
|
||||
.fetch_one(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let expired_queued: i64 = expired
|
||||
.try_get("expired_queued")
|
||||
.map_err(Self::unavailable)?;
|
||||
let purged = sqlx::query(
|
||||
"delete from tasks
|
||||
where task_id in (
|
||||
select task_id from tasks
|
||||
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
|
||||
order by task_id
|
||||
limit $1
|
||||
)",
|
||||
)
|
||||
.bind(batch)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(TaskPurgeReport {
|
||||
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
|
||||
expired_terminal: purged.rows_affected(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
async fn accept_locked(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<Accepted, AttemptError> {
|
||||
// Replay first: an already accepted Task is returned without capacity mutation, so it works
|
||||
// with a full queue and after the original execution deadline while the record is retained.
|
||||
if let Some((acceptance, fingerprint)) =
|
||||
self.stored_acceptance(transaction, request).await?
|
||||
{
|
||||
return Ok(if fingerprint == request.input_fingerprint {
|
||||
Accepted::Replayed(acceptance)
|
||||
} else {
|
||||
Accepted::Conflict
|
||||
});
|
||||
}
|
||||
|
||||
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
|
||||
lock_admission_in_tx(transaction, &request.workspace_id).await?;
|
||||
|
||||
// The competitor may have committed while this transaction waited for the controller lock.
|
||||
if let Some((acceptance, fingerprint)) =
|
||||
self.stored_acceptance(transaction, request).await?
|
||||
{
|
||||
return Ok(if fingerprint == request.input_fingerprint {
|
||||
Accepted::Replayed(acceptance)
|
||||
} else {
|
||||
Accepted::Conflict
|
||||
});
|
||||
}
|
||||
|
||||
cleanup_expired_unlinked(transaction).await?;
|
||||
let queued = bounded_queue_len(transaction).await?;
|
||||
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
|
||||
return Err(AttemptError::Overloaded);
|
||||
}
|
||||
|
||||
let created_ms: i64 = sqlx::query(
|
||||
"insert into tasks (
|
||||
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
|
||||
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
|
||||
trace_sampled, policy_mode, approval_reference, state, input_envelope,
|
||||
execution_deadline, retain_until
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued',
|
||||
$16, to_timestamp($17::double precision / 1000.0),
|
||||
to_timestamp($18::double precision / 1000.0)
|
||||
)
|
||||
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
|
||||
.bind(&request.scoped_key)
|
||||
.bind(&request.input_fingerprint)
|
||||
.bind(request.support.request_id.as_str())
|
||||
.bind(request.support.trace_id.as_str())
|
||||
.bind(&request.support.traceparent)
|
||||
.bind(request.support.trace_sampled)
|
||||
.bind(policy_mode_str(request.pins.policy_mode))
|
||||
.bind(request.pins.approval_reference.as_deref())
|
||||
.bind(request.input.value().clone())
|
||||
.bind(request.pins.execution_deadline_unix_ms)
|
||||
.bind(request.pins.retain_until_unix_ms)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(task_sql_error)?
|
||||
.try_get("created_ms")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
let admission_request = AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: request.workspace_id.clone(),
|
||||
agent_key: request.owner_id.clone(),
|
||||
operation_id: request.operation_id.clone(),
|
||||
operation_version: request.pins.operation_version,
|
||||
},
|
||||
mode: AdmissionMode::Background,
|
||||
policy_mode: request.pins.policy_mode,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
// Never reached: the intent row is inserted by this transaction, not by `admit`.
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: request.input_fingerprint.clone(),
|
||||
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
|
||||
};
|
||||
let owner_token = format!("task_{}", request.task_id);
|
||||
insert_admission_entry(
|
||||
transaction,
|
||||
&admission_request,
|
||||
&owner_token,
|
||||
"reserved",
|
||||
Some(request.task_id.as_uuid()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Accepted::Created(TaskAcceptance {
|
||||
task_id: request.task_id,
|
||||
state: TaskState::Queued,
|
||||
operation_version: request.pins.operation_version,
|
||||
support: request.support.clone(),
|
||||
created_at_unix_ms: created_ms,
|
||||
retain_until_unix_ms: request.pins.retain_until_unix_ms,
|
||||
replayed: false,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
|
||||
async fn reread_after_unique(
|
||||
&self,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<TaskAcceptance, TaskError> {
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(&request.scoped_key)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::unavailable(
|
||||
"durable task acceptance ended with an unresolved unique conflict",
|
||||
));
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
if record.input_fingerprint != request.input_fingerprint {
|
||||
return Err(TaskError::FingerprintConflict);
|
||||
}
|
||||
Ok(acceptance_from_record(&record))
|
||||
}
|
||||
|
||||
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
|
||||
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
|
||||
async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
|
||||
let access = &request.access;
|
||||
match sqlx::query(
|
||||
"select state, version from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => TaskError::StateConflict,
|
||||
Ok(None) => TaskError::NotFound,
|
||||
Err(error) => Self::unavailable(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
|
||||
match mode {
|
||||
ToolExecutionMode::Synchronous => "synchronous",
|
||||
ToolExecutionMode::Background => "background",
|
||||
ToolExecutionMode::Adaptive => "adaptive",
|
||||
}
|
||||
}
|
||||
|
||||
fn task_sql_error(error: sqlx::Error) -> AttemptError {
|
||||
if error
|
||||
.as_database_error()
|
||||
.and_then(|database| database.code())
|
||||
.as_deref()
|
||||
== Some("23505")
|
||||
{
|
||||
return AttemptError::UniqueViolation;
|
||||
}
|
||||
AttemptError::from_sql(error)
|
||||
}
|
||||
|
||||
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
|
||||
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
|
||||
let state: String = row.try_get("state").map_err(unavailable_row)?;
|
||||
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
|
||||
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
|
||||
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
|
||||
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
|
||||
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
|
||||
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
|
||||
value.map(TaskEnvelope::from_persisted).transpose()
|
||||
};
|
||||
Ok(TaskRecord {
|
||||
task_id: TaskId::parse(&task_id.to_string())?,
|
||||
state: TaskState::parse(&state)?,
|
||||
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
|
||||
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
|
||||
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
|
||||
authorization_scope: row
|
||||
.try_get("authorization_scope")
|
||||
.map_err(unavailable_row)?,
|
||||
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
|
||||
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
|
||||
pins: TaskPins {
|
||||
operation_version: u32::try_from(
|
||||
row.try_get::<i32, _>("operation_version")
|
||||
.map_err(unavailable_row)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
policy_mode: match policy_mode.as_str() {
|
||||
"background" => ToolExecutionMode::Background,
|
||||
"adaptive" => ToolExecutionMode::Adaptive,
|
||||
_ => ToolExecutionMode::Synchronous,
|
||||
},
|
||||
execution_deadline_unix_ms: row
|
||||
.try_get("execution_deadline_ms")
|
||||
.map_err(unavailable_row)?,
|
||||
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
|
||||
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
|
||||
},
|
||||
support: TaskSupportIds {
|
||||
request_id: crank_core::RequestId::parse(&request_id)
|
||||
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
|
||||
trace_id: crank_core::TraceId::parse(&trace_id)
|
||||
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
|
||||
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
|
||||
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
|
||||
},
|
||||
input: envelope("input_envelope")?
|
||||
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
|
||||
progress: envelope("progress_envelope")?,
|
||||
result: envelope("result_envelope")?,
|
||||
error: envelope("error_envelope")?,
|
||||
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
|
||||
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn unavailable_row(error: sqlx::Error) -> TaskError {
|
||||
PostgresTaskStore::unavailable(error)
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
//! Acceptance half of the PostgreSQL Task store (Story 4.2/4.3): the one-transaction acceptance
|
||||
//! path, its bounded unique-conflict reread and the typed miss classification.
|
||||
|
||||
use crank_core::{
|
||||
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
|
||||
PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskError, TaskFinishRequest, TaskState,
|
||||
};
|
||||
use sqlx::{Postgres, Row, Transaction};
|
||||
|
||||
use super::super::admission::{
|
||||
AttemptError, bounded_queue_len, cleanup_expired_unlinked, insert_admission_entry,
|
||||
lock_admission_in_tx,
|
||||
};
|
||||
use super::{
|
||||
Accepted, PostgresTaskStore, TASK_PROJECTION, acceptance_from_record, policy_mode_str,
|
||||
record_from_row, task_sql_error,
|
||||
};
|
||||
|
||||
impl PostgresTaskStore {
|
||||
pub(super) async fn accept_locked(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<Accepted, AttemptError> {
|
||||
// Replay first: an already accepted Task is returned without capacity mutation, so it works
|
||||
// with a full queue and after the original execution deadline while the record is retained.
|
||||
if let Some((acceptance, fingerprint)) =
|
||||
self.stored_acceptance(transaction, request).await?
|
||||
{
|
||||
return Ok(if fingerprint == request.input_fingerprint {
|
||||
Accepted::Replayed(acceptance)
|
||||
} else {
|
||||
Accepted::Conflict
|
||||
});
|
||||
}
|
||||
|
||||
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
|
||||
lock_admission_in_tx(transaction, &request.workspace_id).await?;
|
||||
|
||||
// The competitor may have committed while this transaction waited for the controller lock.
|
||||
if let Some((acceptance, fingerprint)) =
|
||||
self.stored_acceptance(transaction, request).await?
|
||||
{
|
||||
return Ok(if fingerprint == request.input_fingerprint {
|
||||
Accepted::Replayed(acceptance)
|
||||
} else {
|
||||
Accepted::Conflict
|
||||
});
|
||||
}
|
||||
|
||||
cleanup_expired_unlinked(transaction).await?;
|
||||
let queued = bounded_queue_len(transaction).await?;
|
||||
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
|
||||
return Err(AttemptError::Overloaded);
|
||||
}
|
||||
|
||||
let created_ms: i64 = sqlx::query(
|
||||
"insert into tasks (
|
||||
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
|
||||
operation_version, scoped_key, input_fingerprint, effective_input_fingerprint,
|
||||
request_id, trace_id, traceparent,
|
||||
trace_sampled, policy_mode, approval_reference, state, input_envelope,
|
||||
execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version,
|
||||
executable_input_master_key_epoch, platform_api_key_id, max_attempts, retry_backoff_step_ms
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
|
||||
to_timestamp($17::double precision / 1000.0),
|
||||
to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22, $23, $24
|
||||
)
|
||||
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
|
||||
.bind(&request.scoped_key)
|
||||
.bind(&request.input_fingerprint)
|
||||
.bind(request.support.request_id.as_str())
|
||||
.bind(request.support.trace_id.as_str())
|
||||
.bind(&request.support.traceparent)
|
||||
.bind(request.support.trace_sampled)
|
||||
.bind(policy_mode_str(request.pins.policy_mode))
|
||||
.bind(request.pins.approval_reference.as_deref())
|
||||
.bind(request.input.value().clone())
|
||||
.bind(request.pins.execution_deadline_unix_ms)
|
||||
.bind(request.pins.retain_until_unix_ms)
|
||||
.bind(request.executable_input_ciphertext.as_deref())
|
||||
.bind(request.executable_input_key_version.as_deref())
|
||||
.bind(request.executable_input_master_key_epoch)
|
||||
.bind(
|
||||
request
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(PlatformApiKeyId::as_str),
|
||||
)
|
||||
.bind(i32::try_from(request.pins.retry.max_attempts).unwrap_or(1))
|
||||
.bind(request.pins.retry.backoff_step_ms)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(task_sql_error)?
|
||||
.try_get("created_ms")
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
|
||||
let admission_request = AdmissionRequest {
|
||||
scope: AdmissionScope {
|
||||
workspace_id: request.workspace_id.clone(),
|
||||
agent_key: request.owner_id.clone(),
|
||||
operation_id: request.operation_id.clone(),
|
||||
operation_version: request.pins.operation_version,
|
||||
},
|
||||
mode: AdmissionMode::Background,
|
||||
policy_mode: request.pins.policy_mode,
|
||||
no_tasks_fallback: NoTasksFallback::Unsupported,
|
||||
// Never reached: the intent row is inserted by this transaction, not by `admit`.
|
||||
approval: ApprovalEvidence::NotRequired,
|
||||
fingerprint: request.input_fingerprint.clone(),
|
||||
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
|
||||
};
|
||||
let owner_token = format!("task_{}", request.task_id);
|
||||
insert_admission_entry(
|
||||
transaction,
|
||||
&admission_request,
|
||||
&owner_token,
|
||||
"reserved",
|
||||
Some(request.task_id.as_uuid()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Accepted::Created(TaskAcceptance {
|
||||
task_id: request.task_id,
|
||||
state: TaskState::Queued,
|
||||
operation_version: request.pins.operation_version,
|
||||
support: request.support.clone(),
|
||||
created_at_unix_ms: created_ms,
|
||||
retain_until_unix_ms: request.pins.retain_until_unix_ms,
|
||||
replayed: false,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
|
||||
pub(super) async fn reread_after_unique(
|
||||
&self,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<TaskAcceptance, TaskError> {
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(&request.scoped_key)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::unavailable(
|
||||
"durable task acceptance ended with an unresolved unique conflict",
|
||||
));
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
if record.input_fingerprint != request.input_fingerprint {
|
||||
return Err(TaskError::FingerprintConflict);
|
||||
}
|
||||
Ok(acceptance_from_record(&record))
|
||||
}
|
||||
|
||||
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
|
||||
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
|
||||
pub(super) async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
|
||||
let access = &request.access;
|
||||
match sqlx::query(
|
||||
"select state, version from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
{
|
||||
Ok(Some(_)) => TaskError::StateConflict,
|
||||
Ok(None) => TaskError::NotFound,
|
||||
Err(error) => Self::unavailable(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,585 @@
|
||||
//! Durable per-outbound attempt persistence of a Task (Story 4.4): the dispatch checkpoint, the
|
||||
//! single-winner attempt resolution and the owner-scoped cancellation live here so the acceptance
|
||||
//! and retention half of the store stays readable.
|
||||
|
||||
use crank_core::{
|
||||
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
|
||||
AttemptResolution, BeginAttempt, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest,
|
||||
TaskEnvelope, TaskError, TaskId, TaskRecord, TaskState,
|
||||
};
|
||||
use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row, unavailable_row};
|
||||
|
||||
const ATTEMPT_PROJECTION: &str = "attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
|
||||
upstream_key_digest, retry_allowed,
|
||||
(extract(epoch from checkpoint_at) * 1000)::bigint as checkpoint_at_ms,
|
||||
(extract(epoch from resolved_at) * 1000)::bigint as resolved_at_ms,
|
||||
outcome, certainty, phase";
|
||||
|
||||
/// Bounded attempt-resolution helper: `retry` may only be armed when the attempt itself is marked
|
||||
/// safe-to-retry, the pinned budget still has room and the accepted absolute deadline still does.
|
||||
impl PostgresTaskStore {
|
||||
fn retry_delay(
|
||||
&self,
|
||||
record: &TaskRecord,
|
||||
lease: &AttemptLease,
|
||||
safe: bool,
|
||||
now: i64,
|
||||
) -> Option<i64> {
|
||||
let delay = record.pins.retry.backoff_before(lease.attempt.ordinal + 1);
|
||||
(safe
|
||||
&& lease.attempt.retry_allowed
|
||||
&& lease.attempt.ordinal < record.pins.retry.max_attempts
|
||||
&& now.saturating_add(delay) < record.pins.execution_deadline_unix_ms)
|
||||
.then_some(delay)
|
||||
}
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
/// Authoritative word on whether an earlier resolved attempt really reached upstream despite
|
||||
/// the current one not having dispatched. Returns Some(phase) only when the latest resolved
|
||||
/// attempt that actually dispatched (phase = 'dispatched') is still of unknown certainty.
|
||||
pub(super) async fn unproven_dispatch_phase(
|
||||
&self,
|
||||
task_id: TaskId,
|
||||
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
) -> Result<Option<String>, TaskError> {
|
||||
let row = sqlx::query(
|
||||
"select certainty, phase from task_attempts where task_id = $1 and resolved_at is not null and phase = 'dispatched' order by ordinal desc limit 1",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
let certainty: Option<String> = row.try_get("certainty").map_err(Self::unavailable)?;
|
||||
if certainty.as_deref() != Some(AttemptCertainty::as_str(AttemptCertainty::Unknown)) {
|
||||
return Ok(None);
|
||||
}
|
||||
let phase: Option<String> = row.try_get("phase").map_err(Self::unavailable)?;
|
||||
Ok(Some(phase.unwrap_or_else(|| "dispatched".to_owned())))
|
||||
}
|
||||
}
|
||||
|
||||
fn attempt_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskAttempt, TaskError> {
|
||||
let attempt_id: String = row.try_get("attempt_id").map_err(unavailable_row)?;
|
||||
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
|
||||
let outcome: Option<String> = row.try_get("outcome").map_err(unavailable_row)?;
|
||||
let certainty: Option<String> = row.try_get("certainty").map_err(unavailable_row)?;
|
||||
let phase: Option<String> = row.try_get("phase").map_err(unavailable_row)?;
|
||||
Ok(TaskAttempt {
|
||||
attempt_id,
|
||||
task_id: TaskId::parse(&task_id.to_string())?,
|
||||
ordinal: u32::try_from(row.try_get::<i32, _>("ordinal").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
owner: row.try_get("owner").map_err(unavailable_row)?,
|
||||
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
|
||||
upstream_key_digest: row
|
||||
.try_get("upstream_key_digest")
|
||||
.map_err(unavailable_row)?,
|
||||
retry_allowed: row.try_get("retry_allowed").map_err(unavailable_row)?,
|
||||
checkpoint_at_unix_ms: row.try_get("checkpoint_at_ms").map_err(unavailable_row)?,
|
||||
resolved_at_unix_ms: row.try_get("resolved_at_ms").map_err(unavailable_row)?,
|
||||
outcome: outcome
|
||||
.as_deref()
|
||||
.map(AttemptOutcomeClass::parse)
|
||||
.transpose()?,
|
||||
certainty: certainty
|
||||
.as_deref()
|
||||
.map(AttemptCertainty::parse)
|
||||
.transpose()?,
|
||||
phase: phase.as_deref().map(AttemptPhase::parse).transpose()?,
|
||||
})
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
pub(super) async fn checkpoint_dispatch_attempt(
|
||||
&self,
|
||||
request: BeginAttempt,
|
||||
) -> Result<AttemptLease, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
// One row lock serializes this checkpoint with claim, cancellation and the resolution of the
|
||||
// same Task, so a cancel that wins first can never be followed by an outbound call.
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
|
||||
)))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
// The checkpoint must bind the payload this attempt actually executes: after a committed
|
||||
// input continuation that is the effective fingerprint, not the unchanged acceptance one.
|
||||
if record.effective_input_fingerprint != request.input_fingerprint {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::FingerprintConflict);
|
||||
}
|
||||
let now = crank_core::current_unix_ms();
|
||||
let eligible = record.state == TaskState::Working
|
||||
&& record.lease_owner.as_deref() == Some(request.owner.as_str())
|
||||
&& record.fence == request.fence
|
||||
&& record.version == request.expected_version
|
||||
&& record.current_attempt_id.is_none()
|
||||
&& record.cancel_requested_at_unix_ms.is_none()
|
||||
&& record.lease_until_unix_ms.is_some_and(|until| until > now)
|
||||
&& record.attempts < record.pins.retry.max_attempts
|
||||
&& record.pins.execution_deadline_unix_ms > now
|
||||
&& record.next_attempt_at_unix_ms.is_none_or(|at| at <= now);
|
||||
if !eligible {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
let ordinal = record.attempts.saturating_add(1);
|
||||
sqlx::query(
|
||||
"insert into task_attempts (
|
||||
attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
|
||||
upstream_key_digest, retry_allowed
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8)",
|
||||
)
|
||||
.bind(&request.attempt_id)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(i32::try_from(ordinal).unwrap_or(i32::MAX))
|
||||
.bind(&request.owner)
|
||||
.bind(i64::try_from(request.fence).unwrap_or_default())
|
||||
.bind(&request.input_fingerprint)
|
||||
.bind(request.upstream_key_digest.as_deref())
|
||||
.bind(request.retry_allowed)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set current_attempt_id = $1,
|
||||
dispatch_started_at = now(),
|
||||
attempts = attempts + 1,
|
||||
next_attempt_at = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $2 and current_attempt_id is null and version = $3
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(&request.attempt_id)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(i64::from(record.version))
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let task = record_from_row(&row)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
let attempt = TaskAttempt {
|
||||
attempt_id: request.attempt_id,
|
||||
task_id: request.task_id,
|
||||
ordinal,
|
||||
owner: request.owner,
|
||||
fence: request.fence,
|
||||
input_fingerprint: request.input_fingerprint,
|
||||
upstream_key_digest: request.upstream_key_digest,
|
||||
retry_allowed: request.retry_allowed,
|
||||
checkpoint_at_unix_ms: task.dispatch_started_at_unix_ms.unwrap_or(now),
|
||||
resolved_at_unix_ms: None,
|
||||
outcome: None,
|
||||
certainty: None,
|
||||
phase: None,
|
||||
};
|
||||
Ok(AttemptLease { attempt, task })
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_attempt_outcome(
|
||||
&self,
|
||||
lease: &AttemptLease,
|
||||
outcome: AttemptOutcome,
|
||||
) -> Result<AttemptResolution, TaskError> {
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
|
||||
)))
|
||||
.bind(lease.attempt.task_id.as_uuid())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
let now = crank_core::current_unix_ms();
|
||||
let prior_uncertain_phase = self
|
||||
.unproven_dispatch_phase(lease.attempt.task_id, &mut transaction)
|
||||
.await?;
|
||||
// Only the current attempt of the current owner, under the version it was admitted with, may
|
||||
// be resolved. A stale worker can never append a second outcome to the history or the Task.
|
||||
let current = record.state == TaskState::Working
|
||||
&& record.lease_owner.as_deref() == Some(lease.attempt.owner.as_str())
|
||||
&& record.fence == lease.attempt.fence
|
||||
&& record.version == lease.task.version
|
||||
&& record.current_attempt_id.as_deref() == Some(lease.attempt.attempt_id.as_str())
|
||||
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
|
||||
if !current {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
let (class, certainty, phase) = match &outcome {
|
||||
AttemptOutcome::Completed(_) => (
|
||||
AttemptOutcomeClass::Completed,
|
||||
AttemptCertainty::Known,
|
||||
AttemptPhase::Dispatched,
|
||||
),
|
||||
AttemptOutcome::Failed { phase, .. } => {
|
||||
(AttemptOutcomeClass::Failed, AttemptCertainty::Known, *phase)
|
||||
}
|
||||
AttemptOutcome::Uncertain { phase, .. } => (
|
||||
AttemptOutcomeClass::OutcomeUnknown,
|
||||
AttemptCertainty::Unknown,
|
||||
*phase,
|
||||
),
|
||||
};
|
||||
// A failed or uncertain attempt that may be safely repeated keeps the lease and arms the
|
||||
// pinned backoff. Exhausted budget or deadline never turns uncertainty into a plain failure.
|
||||
let retry = match &outcome {
|
||||
AttemptOutcome::Failed {
|
||||
retryable,
|
||||
phase: _,
|
||||
error: _,
|
||||
} => self.retry_delay(&record, lease, *retryable, now),
|
||||
AttemptOutcome::Uncertain {
|
||||
repeatable,
|
||||
phase: _,
|
||||
error: _,
|
||||
} => self.retry_delay(&record, lease, *repeatable, now),
|
||||
AttemptOutcome::Completed(_) => None,
|
||||
};
|
||||
let attempt_class = if retry.is_some() {
|
||||
AttemptOutcomeClass::RetryScheduled
|
||||
} else {
|
||||
class
|
||||
};
|
||||
let closed = sqlx::query(
|
||||
"update task_attempts
|
||||
set resolved_at = now(), outcome = $1, certainty = $2, phase = $3
|
||||
where attempt_id = $4 and resolved_at is null",
|
||||
)
|
||||
.bind(attempt_class.as_str())
|
||||
.bind(certainty.as_str())
|
||||
.bind(phase.as_str())
|
||||
.bind(&lease.attempt.attempt_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
if closed.rows_affected() != 1 {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
let (state, result, error, keep_lease, next_attempt_at) = match outcome {
|
||||
AttemptOutcome::Completed(result) => {
|
||||
(TaskState::Completed, Some(result), None, false, None)
|
||||
}
|
||||
AttemptOutcome::Failed {
|
||||
error,
|
||||
retryable: _,
|
||||
phase,
|
||||
} => match retry {
|
||||
Some(delay) => (
|
||||
TaskState::Working,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
Some(now.saturating_add(delay)),
|
||||
),
|
||||
None => {
|
||||
if phase == AttemptPhase::BeforeDispatch {
|
||||
if let Some(prior) = prior_uncertain_phase.as_ref() {
|
||||
let mut value = error.value().clone();
|
||||
if let serde_json::Value::Object(map) = &mut value {
|
||||
map.insert(
|
||||
"phase".to_string(),
|
||||
serde_json::Value::String(prior.clone()),
|
||||
);
|
||||
}
|
||||
let replaced = TaskEnvelope::try_redacted(&value)?;
|
||||
(TaskState::OutcomeUnknown, None, Some(replaced), false, None)
|
||||
} else {
|
||||
(TaskState::Failed, None, Some(error), false, None)
|
||||
}
|
||||
} else {
|
||||
(TaskState::Failed, None, Some(error), false, None)
|
||||
}
|
||||
}
|
||||
},
|
||||
AttemptOutcome::Uncertain {
|
||||
error,
|
||||
repeatable: _,
|
||||
phase: _,
|
||||
} => match retry {
|
||||
Some(delay) => (
|
||||
TaskState::Working,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
Some(now.saturating_add(delay)),
|
||||
),
|
||||
None => (TaskState::OutcomeUnknown, None, Some(error), false, None),
|
||||
},
|
||||
};
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = $1,
|
||||
result_envelope = $2,
|
||||
error_envelope = $3,
|
||||
lease_owner = case when $4 then lease_owner else null end,
|
||||
lease_until = case when $4 then lease_until else null end,
|
||||
-- A retry-scheduled attempt is already resolved, so it no longer holds the Task:
|
||||
-- the next durable attempt needs its own checkpoint under the same lease.
|
||||
current_attempt_id = case when $8 then null else current_attempt_id end,
|
||||
next_attempt_at = case when $5::bigint is null then null
|
||||
else to_timestamp($5::double precision / 1000.0) end,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $6 and version = $7
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(state.as_str())
|
||||
.bind(result.as_ref().map(|value| value.value().clone()))
|
||||
.bind(error.as_ref().map(|value| value.value().clone()))
|
||||
.bind(keep_lease)
|
||||
.bind(next_attempt_at)
|
||||
.bind(lease.attempt.task_id.as_uuid())
|
||||
.bind(i64::from(record.version))
|
||||
.bind(retry.is_some())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let task = record_from_row(&row)?;
|
||||
if !keep_lease {
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(lease.attempt.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
}
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(match retry {
|
||||
Some(_) => AttemptResolution::RetryAt {
|
||||
next_attempt_at_unix_ms: next_attempt_at.unwrap_or(now),
|
||||
task,
|
||||
},
|
||||
None if state == TaskState::OutcomeUnknown => AttemptResolution::Unknown(task),
|
||||
None => AttemptResolution::Finished(task),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn cancel_task(
|
||||
&self,
|
||||
request: TaskCancelRequest,
|
||||
) -> Result<TaskCancelOutcome, TaskError> {
|
||||
request.validate()?;
|
||||
let access = &request.access;
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5
|
||||
for update"
|
||||
)))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
if let Some(expected) = request.expected_version
|
||||
&& record.version != expected
|
||||
{
|
||||
transaction.rollback().await.ok();
|
||||
return Ok(TaskCancelOutcome::Conflict(record));
|
||||
}
|
||||
if record.state.is_terminal() {
|
||||
transaction.rollback().await.ok();
|
||||
return Ok(TaskCancelOutcome::AlreadyTerminal(record));
|
||||
}
|
||||
if record.state == TaskState::OutcomeUnknown {
|
||||
transaction.rollback().await.ok();
|
||||
return Ok(TaskCancelOutcome::OutcomeUnknown(record));
|
||||
}
|
||||
// A checkpointed attempt may already have reached the upstream: the honest answer is the
|
||||
// non-terminal uncertain state, never a plain cancellation and never a retry.
|
||||
let checkpointed = record.current_attempt_id.clone();
|
||||
// Preserve an uncertainty recorded by a previously resolved attempt even after RetryAt
|
||||
// cleared current_attempt_id: the request already reached the upstream in that case.
|
||||
let prior_uncertain = self
|
||||
.unproven_dispatch_phase(request.task_id, &mut transaction)
|
||||
.await?
|
||||
.is_some();
|
||||
let (state, code) = match (&checkpointed, prior_uncertain) {
|
||||
(Some(_), _) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
|
||||
(None, true) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
|
||||
(None, false) => (TaskState::Cancelled, "cancelled_before_dispatch"),
|
||||
};
|
||||
if let Some(attempt_id) = checkpointed {
|
||||
let closed = sqlx::query(
|
||||
"update task_attempts
|
||||
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
|
||||
phase = 'dispatched'
|
||||
where task_id = $1 and attempt_id = $2 and resolved_at is null",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&attempt_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let _ = closed;
|
||||
}
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = $1,
|
||||
error_envelope = jsonb_build_object('code', $2),
|
||||
cancel_requested_at = now(),
|
||||
cancel_reason = $3,
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $4 and version = $5
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(state.as_str())
|
||||
.bind(code)
|
||||
.bind(&request.reason)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(i64::from(record.version))
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let task = record_from_row(&row)?;
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(request.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(match state {
|
||||
TaskState::Cancelled => TaskCancelOutcome::Cancelled(task),
|
||||
_ => TaskCancelOutcome::OutcomeUnknown(task),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn refuse_attempt_resolution(
|
||||
&self,
|
||||
lease: &crank_core::TaskLease,
|
||||
code: &str,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks\n where task_id = $1\n for update"
|
||||
)))
|
||||
.bind(lease.record.task_id.as_uuid())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
let now = crank_core::current_unix_ms();
|
||||
let eligible = matches!(record.state, TaskState::Queued | TaskState::Working)
|
||||
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
|
||||
&& record.fence == lease.fence
|
||||
&& record.version == lease.record.version
|
||||
&& record.current_attempt_id.is_none()
|
||||
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
|
||||
if !eligible {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
let prior_phase = self
|
||||
.unproven_dispatch_phase(lease.record.task_id, &mut transaction)
|
||||
.await?;
|
||||
let uncertain = prior_phase.is_some();
|
||||
let state = if uncertain {
|
||||
TaskState::OutcomeUnknown
|
||||
} else {
|
||||
TaskState::Failed
|
||||
};
|
||||
let phase = prior_phase.unwrap_or_else(|| "before_dispatch".to_owned());
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks\n set state = $1,\n error_envelope = jsonb_build_object('code', $2, 'phase', $3),\n lease_owner = null,\n lease_until = null,\n next_attempt_at = null,\n version = version + 1,\n updated_at = now()\n where task_id = $4 and version = $5\n returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(state.as_str())
|
||||
.bind(code)
|
||||
.bind(&phase)
|
||||
.bind(lease.record.task_id.as_uuid())
|
||||
.bind(i64::from(record.version))
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let task = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(lease.record.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(task)
|
||||
}
|
||||
|
||||
pub(super) async fn attempt_history(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<Vec<TaskAttempt>, TaskError> {
|
||||
access.validate()?;
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {ATTEMPT_PROJECTION} from task_attempts
|
||||
where task_id = $1 and exists (
|
||||
select 1 from tasks
|
||||
where tasks.task_id = task_attempts.task_id and tasks.workspace_id = $2
|
||||
and tasks.owner_kind = $3 and tasks.owner_id = $4
|
||||
and tasks.authorization_scope = $5
|
||||
)
|
||||
order by ordinal"
|
||||
)))
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
rows.iter().map(attempt_from_row).collect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,659 @@
|
||||
//! Input-continuation half of the PostgreSQL Task store (Story 4.5, FR-25).
|
||||
//!
|
||||
//! One transaction owns each transition: suspend (`working` -> `input_required`), the committed
|
||||
//! answer (`input_required` -> `working`) and the TTL expiry (`input_required` -> `failed`). Authority,
|
||||
//! generation, TTL and receipt checks run under the same Task row lock, so a revocation, a competing
|
||||
//! answer, a sweeper or a cancellation has exactly one winner and never a duplicated side effect.
|
||||
//!
|
||||
//! Raw input never enters a request, receipt, audit row or error text: the validated payload is
|
||||
//! encrypted by the runtime service before this module sees it.
|
||||
|
||||
use crank_core::{
|
||||
TaskAccess, TaskError, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
|
||||
TaskInputUpdateRequest, TaskRecord, TaskState,
|
||||
};
|
||||
use sqlx::{Postgres, Row, Transaction};
|
||||
|
||||
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row};
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TaskInputStore for PostgresTaskStore {
|
||||
async fn suspend_for_input(
|
||||
&self,
|
||||
lease: &crank_core::TaskLease,
|
||||
request: TaskInputRequest,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
self.suspend_input_locked(lease, request).await
|
||||
}
|
||||
|
||||
async fn input_request(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: crank_core::TaskId,
|
||||
) -> Result<TaskInputRequest, TaskError> {
|
||||
self.read_input_request(access, task_id).await
|
||||
}
|
||||
|
||||
async fn read_executable_input_for_update(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: crank_core::TaskId,
|
||||
) -> Result<(String, String, i64), TaskError> {
|
||||
self.read_binding_for_update(access, task_id).await
|
||||
}
|
||||
|
||||
async fn apply_input(
|
||||
&self,
|
||||
request: TaskInputUpdateRequest,
|
||||
) -> Result<TaskInputUpdateOutcome, TaskError> {
|
||||
self.apply_input_locked(request).await
|
||||
}
|
||||
|
||||
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError> {
|
||||
self.expire_input_waits_locked(batch).await
|
||||
}
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
/// One transactional suspend of a leased Task before any dispatch checkpoint.
|
||||
pub(super) async fn suspend_input_locked(
|
||||
&self,
|
||||
lease: &crank_core::TaskLease,
|
||||
request: TaskInputRequest,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
request.validate()?;
|
||||
if request.expected_version != lease.record.version.saturating_add(1) {
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
|
||||
)))
|
||||
.bind(lease.record.task_id.as_uuid())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
let current = record.state == TaskState::Working
|
||||
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
|
||||
&& record.fence == lease.fence
|
||||
&& record.version == lease.record.version
|
||||
&& record.current_attempt_id.is_none()
|
||||
&& record.attempts == 0
|
||||
&& record.cancel_requested_at_unix_ms.is_none()
|
||||
&& record
|
||||
.lease_until_unix_ms
|
||||
.is_some_and(|until| until > crank_core::current_unix_ms())
|
||||
&& record.pins.execution_deadline_unix_ms > crank_core::current_unix_ms();
|
||||
// A generation that already dispatched may not be suspended: that would hide a possible side
|
||||
// effect behind an apparent "waiting for input".
|
||||
if !current {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
if request.expires_at_unix_ms > record.pins.execution_deadline_unix_ms {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::invalid(
|
||||
"task input TTL exceeds the execution deadline",
|
||||
));
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"insert into task_input_requests (
|
||||
task_id, wait_id, expected_version, schema_revision, schema_format_version,
|
||||
fingerprint_format_version, schema_snapshot, projected_schema, prompt_code,
|
||||
missing_fields, expires_at
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
||||
to_timestamp($11::double precision / 1000.0)
|
||||
)",
|
||||
)
|
||||
.bind(lease.record.task_id.as_uuid())
|
||||
.bind(&request.wait_id)
|
||||
.bind(i32::try_from(request.expected_version).unwrap_or(i32::MAX))
|
||||
.bind(i32::try_from(request.schema_revision).unwrap_or(i32::MAX))
|
||||
.bind(i32::try_from(request.schema_format_version).unwrap_or(i32::MAX))
|
||||
.bind(i32::try_from(request.fingerprint_format_version).unwrap_or(i32::MAX))
|
||||
.bind(request.schema_snapshot.clone())
|
||||
.bind(request.projected_schema.clone())
|
||||
.bind(&request.prompt_code)
|
||||
.bind(serde_json::Value::Array(
|
||||
request
|
||||
.missing_fields
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(serde_json::Value::String)
|
||||
.collect(),
|
||||
))
|
||||
.bind(request.expires_at_unix_ms)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
match error
|
||||
.as_database_error()
|
||||
.and_then(|db| db.code())
|
||||
.as_deref()
|
||||
{
|
||||
Some("23505") => TaskError::StateConflict,
|
||||
_ => Self::unavailable(error),
|
||||
}
|
||||
})?;
|
||||
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = 'input_required',
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $1 and state = 'working' and version = $2
|
||||
and lease_owner = $3 and fence = $4 and current_attempt_id is null
|
||||
and lease_until > clock_timestamp()
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(lease.record.task_id.as_uuid())
|
||||
.bind(i64::from(record.version))
|
||||
.bind(&lease.owner)
|
||||
.bind(i64::try_from(lease.fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let suspended = record_from_row(&row)?;
|
||||
insert_audit(
|
||||
&mut transaction,
|
||||
lease.record.task_id,
|
||||
&request.wait_id,
|
||||
"input_required",
|
||||
TaskState::Working,
|
||||
TaskState::InputRequired,
|
||||
record.version,
|
||||
suspended.version,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(suspended)
|
||||
}
|
||||
|
||||
/// Owner-scoped read of the durable request; a foreign Task is one opaque not-found.
|
||||
pub(super) async fn read_input_request(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: crank_core::TaskId,
|
||||
) -> Result<TaskInputRequest, TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(
|
||||
"select r.wait_id, r.expected_version, r.schema_revision, r.schema_format_version,
|
||||
r.fingerprint_format_version, r.schema_snapshot, r.projected_schema,
|
||||
r.prompt_code, r.missing_fields,
|
||||
(extract(epoch from r.expires_at) * 1000)::bigint as expires_at_ms
|
||||
from task_input_requests r
|
||||
join tasks t on t.task_id = r.task_id
|
||||
where r.task_id = $1 and t.workspace_id = $2 and t.owner_kind = $3
|
||||
and t.owner_id = $4 and t.authorization_scope = $5",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let missing_fields: Vec<String> = serde_json::from_value(
|
||||
row.try_get::<serde_json::Value, _>("missing_fields")
|
||||
.map_err(Self::unavailable)?,
|
||||
)
|
||||
.map_err(|_| TaskError::unavailable("stored input request metadata is invalid"))?;
|
||||
Ok(TaskInputRequest {
|
||||
wait_id: row.try_get("wait_id").map_err(Self::unavailable)?,
|
||||
expected_version: u32::try_from(
|
||||
row.try_get::<i32, _>("expected_version")
|
||||
.map_err(Self::unavailable)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
schema_revision: u32::try_from(
|
||||
row.try_get::<i32, _>("schema_revision")
|
||||
.map_err(Self::unavailable)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
schema_format_version: u32::try_from(
|
||||
row.try_get::<i32, _>("schema_format_version")
|
||||
.map_err(Self::unavailable)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
schema_snapshot: row.try_get("schema_snapshot").map_err(Self::unavailable)?,
|
||||
projected_schema: row.try_get("projected_schema").map_err(Self::unavailable)?,
|
||||
prompt_code: row.try_get("prompt_code").map_err(Self::unavailable)?,
|
||||
missing_fields,
|
||||
expires_at_unix_ms: row.try_get("expires_at_ms").map_err(Self::unavailable)?,
|
||||
fingerprint_format_version: u32::try_from(
|
||||
row.try_get::<i32, _>("fingerprint_format_version")
|
||||
.map_err(Self::unavailable)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Owner-scoped read of the binding a continuation may extend. Never lease-authorized.
|
||||
///
|
||||
/// Deliberately not filtered by lifecycle state: an identical replay of an already committed
|
||||
/// answer (including after completion or TTL expiry) must reach the receipt lookup, which is the
|
||||
/// single authority for that outcome.
|
||||
pub(super) async fn read_binding_for_update(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: crank_core::TaskId,
|
||||
) -> Result<(String, String, i64), TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(
|
||||
"select executable_input_ciphertext, executable_input_key_version,
|
||||
executable_input_master_key_epoch
|
||||
from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let ciphertext: Option<String> = row
|
||||
.try_get("executable_input_ciphertext")
|
||||
.map_err(Self::unavailable)?;
|
||||
let key_version: Option<String> = row
|
||||
.try_get("executable_input_key_version")
|
||||
.map_err(Self::unavailable)?;
|
||||
let epoch: Option<i64> = row
|
||||
.try_get("executable_input_master_key_epoch")
|
||||
.map_err(Self::unavailable)?;
|
||||
match (ciphertext, key_version, epoch) {
|
||||
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
|
||||
Ok((ciphertext, key_version, epoch))
|
||||
}
|
||||
_ => Err(TaskError::NotFound),
|
||||
}
|
||||
}
|
||||
|
||||
/// One transactional answer application: authority, receipt, generation, TTL and CAS.
|
||||
pub(super) async fn apply_input_locked(
|
||||
&self,
|
||||
request: TaskInputUpdateRequest,
|
||||
) -> Result<TaskInputUpdateOutcome, TaskError> {
|
||||
request.validate()?;
|
||||
let access = &request.access;
|
||||
let mut transaction = self.start().await?;
|
||||
super::super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
super::super::master_key::ensure_active_master_key_epoch(
|
||||
&mut transaction,
|
||||
request.executable_input_master_key_epoch,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
|
||||
// Authority is revalidated inside this transaction, before any receipt lookup and under the
|
||||
// same lock order: master key check, platform key share lock, Task row lock.
|
||||
let key_row = sqlx::query(
|
||||
"select k.id from platform_api_keys k
|
||||
join tasks t on t.platform_api_key_id = k.id
|
||||
where t.task_id = $1 and t.owner_kind = 'agent'
|
||||
and k.workspace_id = $2 and k.agent_id = $3
|
||||
and k.key_kind = 'mcp_client' and k.status = 'active'
|
||||
and (k.expires_at is null or k.expires_at > clock_timestamp())
|
||||
and (k.scopes_json ? 'write' or k.scopes_json ? 'deploy')
|
||||
for share of k",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(&access.owner_id)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
if key_row.is_none() {
|
||||
transaction.rollback().await.ok();
|
||||
// A revoked key, a foreign scope and an absent Task share one opaque answer.
|
||||
return Err(TaskError::NotFound);
|
||||
}
|
||||
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5
|
||||
for update"
|
||||
)))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
let record = record_from_row(&row)?;
|
||||
|
||||
// Receipt lookup precedes the generation and TTL checks: an authorized identical replay keeps
|
||||
// returning the committed outcome even after completion, failure or TTL expiry.
|
||||
let receipt = sqlx::query(
|
||||
"select wait_id, update_key_digest, canonical_fingerprint, committed_state,
|
||||
committed_version
|
||||
from task_input_requests where task_id = $1 and accepted_at is not null",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
if let Some(receipt) = receipt {
|
||||
let wait_id: String = receipt.try_get("wait_id").map_err(Self::unavailable)?;
|
||||
if wait_id != request.wait_id {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
let stored_key: String = receipt
|
||||
.try_get("update_key_digest")
|
||||
.map_err(Self::unavailable)?;
|
||||
if stored_key != request.update_key_digest {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
let stored_fingerprint: String = receipt
|
||||
.try_get("canonical_fingerprint")
|
||||
.map_err(Self::unavailable)?;
|
||||
if stored_fingerprint != request.canonical_fingerprint {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::FingerprintConflict);
|
||||
}
|
||||
let state: String = receipt
|
||||
.try_get("committed_state")
|
||||
.map_err(Self::unavailable)?;
|
||||
let version: i32 = receipt
|
||||
.try_get("committed_version")
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
return Ok(TaskInputUpdateOutcome {
|
||||
task_id: request.task_id,
|
||||
wait_id,
|
||||
state: TaskState::parse(&state)?,
|
||||
version: u32::try_from(version).unwrap_or_default(),
|
||||
replayed: true,
|
||||
});
|
||||
}
|
||||
|
||||
let pending = sqlx::query(
|
||||
"select wait_id, expected_version,
|
||||
(extract(epoch from expires_at) * 1000)::bigint as expires_at_ms,
|
||||
(extract(epoch from clock_timestamp()) * 1000)::bigint as now_ms
|
||||
from task_input_requests where task_id = $1 and accepted_at is null",
|
||||
)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(pending) = pending else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let wait_id: String = pending.try_get("wait_id").map_err(Self::unavailable)?;
|
||||
let expected_version: i32 = pending
|
||||
.try_get("expected_version")
|
||||
.map_err(Self::unavailable)?;
|
||||
let expires_at_ms: i64 = pending
|
||||
.try_get("expires_at_ms")
|
||||
.map_err(Self::unavailable)?;
|
||||
// PostgreSQL clock, read after the row lock was taken: never a client timestamp.
|
||||
let now_ms: i64 = pending.try_get("now_ms").map_err(Self::unavailable)?;
|
||||
if wait_id != request.wait_id
|
||||
|| u32::try_from(expected_version).unwrap_or_default() != request.expected_version
|
||||
|| record.version != request.expected_version
|
||||
|| record.state != TaskState::InputRequired
|
||||
{
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
if record.cancel_requested_at_unix_ms.is_some() {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
if now_ms >= expires_at_ms {
|
||||
// The same transactional helper the sweeper uses: the Task ends failed with an audit
|
||||
// reason and no dispatch. The update never revives it.
|
||||
expire_locked(&mut transaction, &record, &wait_id, now_ms).await?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
return Err(TaskError::InputExpired);
|
||||
}
|
||||
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = 'working',
|
||||
effective_input_fingerprint = $1,
|
||||
executable_input_ciphertext = $2,
|
||||
executable_input_key_version = $3,
|
||||
executable_input_master_key_epoch = $4,
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $5 and state = 'input_required' and version = $6
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(&request.effective_input_fingerprint)
|
||||
.bind(&request.executable_input_ciphertext)
|
||||
.bind(&request.executable_input_key_version)
|
||||
.bind(request.executable_input_master_key_epoch)
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(i64::from(record.version))
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let accepted = record_from_row(&row)?;
|
||||
let updated = sqlx::query(
|
||||
"update task_input_requests
|
||||
set accepted_at = now(),
|
||||
update_key_digest = $1,
|
||||
canonical_fingerprint = $2,
|
||||
effective_input_fingerprint = $3,
|
||||
committed_state = 'working',
|
||||
committed_version = $4
|
||||
where task_id = $5 and wait_id = $6 and accepted_at is null",
|
||||
)
|
||||
.bind(&request.update_key_digest)
|
||||
.bind(&request.canonical_fingerprint)
|
||||
.bind(&request.effective_input_fingerprint)
|
||||
.bind(i64::from(accepted.version))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&request.wait_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
if updated.rows_affected() != 1 {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
insert_audit(
|
||||
&mut transaction,
|
||||
request.task_id,
|
||||
&request.wait_id,
|
||||
"input_accepted",
|
||||
TaskState::InputRequired,
|
||||
TaskState::Working,
|
||||
record.version,
|
||||
accepted.version,
|
||||
Some(access.owner_id.as_str()),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(TaskInputUpdateOutcome {
|
||||
task_id: request.task_id,
|
||||
wait_id: request.wait_id,
|
||||
state: TaskState::Working,
|
||||
version: accepted.version,
|
||||
replayed: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Bounded TTL sweep, safe to call on every poll while all worker slots are busy.
|
||||
pub(super) async fn expire_input_waits_locked(&self, batch: u32) -> Result<u64, TaskError> {
|
||||
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
|
||||
let mut transaction = self.start().await?;
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks t
|
||||
where t.state = 'input_required'
|
||||
and exists (
|
||||
select 1 from task_input_requests r
|
||||
where r.task_id = t.task_id and r.accepted_at is null
|
||||
and r.expires_at <= clock_timestamp()
|
||||
)
|
||||
order by t.task_id
|
||||
limit $1
|
||||
for update of t skip locked"
|
||||
)))
|
||||
.bind(batch)
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let mut expired = 0u64;
|
||||
for row in rows {
|
||||
let record = record_from_row(&row)?;
|
||||
let Some(wait_id) = sqlx::query_scalar::<_, String>(
|
||||
"select wait_id from task_input_requests
|
||||
where task_id = $1 and accepted_at is null and expires_at <= clock_timestamp()",
|
||||
)
|
||||
.bind(record.task_id.as_uuid())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if expire_locked(
|
||||
&mut transaction,
|
||||
&record,
|
||||
&wait_id,
|
||||
crank_core::current_unix_ms(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
expired += 1;
|
||||
}
|
||||
}
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(expired)
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one locked `input_required` Task failed, releases its admission permit and writes the audit
|
||||
/// reason. Returns whether this call performed the transition.
|
||||
async fn expire_locked(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
record: &TaskRecord,
|
||||
wait_id: &str,
|
||||
_now_ms: i64,
|
||||
) -> Result<bool, TaskError> {
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = 'failed',
|
||||
error_envelope = jsonb_build_object('reason', 'input_ttl_expired'),
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $1 and state = 'input_required' and version = $2
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(record.task_id.as_uuid())
|
||||
.bind(i64::from(record.version))
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(PostgresTaskStore::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Ok(false);
|
||||
};
|
||||
let failed = record_from_row(&row)?;
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(record.task_id.as_uuid())
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(PostgresTaskStore::unavailable)?;
|
||||
insert_audit(
|
||||
transaction,
|
||||
record.task_id,
|
||||
wait_id,
|
||||
"input_expired",
|
||||
TaskState::InputRequired,
|
||||
TaskState::Failed,
|
||||
record.version,
|
||||
failed.version,
|
||||
None,
|
||||
Some("input_ttl_expired"),
|
||||
)
|
||||
.await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn insert_audit(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
task_id: crank_core::TaskId,
|
||||
wait_id: &str,
|
||||
event: &str,
|
||||
from_state: TaskState,
|
||||
to_state: TaskState,
|
||||
from_version: u32,
|
||||
to_version: u32,
|
||||
actor: Option<&str>,
|
||||
reason_code: Option<&str>,
|
||||
) -> Result<(), TaskError> {
|
||||
sqlx::query(
|
||||
"insert into task_input_audit (
|
||||
task_id, wait_id, event, from_state, to_state, from_version, to_version, actor,
|
||||
reason_code
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(wait_id)
|
||||
.bind(event)
|
||||
.bind(from_state.as_str())
|
||||
.bind(to_state.as_str())
|
||||
.bind(i32::try_from(from_version).unwrap_or(i32::MAX))
|
||||
.bind(i32::try_from(to_version).unwrap_or(i32::MAX))
|
||||
.bind(actor)
|
||||
.bind(reason_code)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(PostgresTaskStore::unavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,953 @@
|
||||
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
|
||||
//!
|
||||
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
|
||||
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
|
||||
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
|
||||
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
|
||||
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
|
||||
//!
|
||||
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
|
||||
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
|
||||
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{
|
||||
AttemptLease, AttemptOutcome, AttemptResolution, BeginAttempt, PlatformApiKeyId,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
|
||||
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskLease,
|
||||
TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use sqlx::{PgPool, Postgres, Row, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::admission::{AdmissionLimits, AttemptError};
|
||||
|
||||
mod acceptance;
|
||||
mod attempts;
|
||||
mod input;
|
||||
|
||||
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
|
||||
authorization_scope, platform_api_key_id, operation_id, input_fingerprint,
|
||||
effective_input_fingerprint, operation_version,
|
||||
policy_mode,
|
||||
approval_reference,
|
||||
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
|
||||
result_envelope, error_envelope, attempts, fence, lease_owner, lease_until,
|
||||
(extract(epoch from lease_until) * 1000)::bigint as lease_until_ms,
|
||||
current_attempt_id, dispatch_started_at,
|
||||
(extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version,
|
||||
max_attempts, retry_backoff_step_ms,
|
||||
(extract(epoch from next_attempt_at) * 1000)::bigint as next_attempt_at_ms,
|
||||
(extract(epoch from cancel_requested_at) * 1000)::bigint as cancel_requested_at_ms,
|
||||
cancel_reason,
|
||||
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
|
||||
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
|
||||
(extract(epoch from created_at) * 1000)::bigint as created_ms,
|
||||
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PostgresTaskStore {
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: crank_core::AdmissionSchedulerConfig,
|
||||
}
|
||||
|
||||
impl PostgresTaskStore {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
limits: AdmissionLimits,
|
||||
scheduler: crank_core::AdmissionSchedulerConfig,
|
||||
) -> Result<Self, TaskError> {
|
||||
limits
|
||||
.validate()
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
scheduler
|
||||
.validate()
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
Ok(Self {
|
||||
pool,
|
||||
limits,
|
||||
scheduler,
|
||||
})
|
||||
}
|
||||
|
||||
fn unavailable(error: impl std::fmt::Display) -> TaskError {
|
||||
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
|
||||
}
|
||||
|
||||
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
|
||||
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local lock_timeout = '3000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
sqlx::query("set local statement_timeout = '5000ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
Ok(transaction)
|
||||
}
|
||||
|
||||
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
|
||||
/// query, so a foreign row is never fetched.
|
||||
async fn stored_acceptance(
|
||||
&self,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
request: &TaskAcceptRequest,
|
||||
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&request.workspace_id)
|
||||
.bind(request.owner_kind.as_str())
|
||||
.bind(&request.owner_id)
|
||||
.bind(&request.authorization_scope)
|
||||
.bind(&request.operation_id)
|
||||
.bind(&request.scoped_key)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(AttemptError::from_sql)?;
|
||||
let Some(row) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
|
||||
Ok(Some((
|
||||
acceptance_from_record(&record),
|
||||
record.input_fingerprint,
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
|
||||
TaskAcceptance {
|
||||
task_id: record.task_id,
|
||||
state: record.state,
|
||||
operation_version: record.pins.operation_version,
|
||||
support: record.support.clone(),
|
||||
created_at_unix_ms: record.created_at_unix_ms,
|
||||
retain_until_unix_ms: record.pins.retain_until_unix_ms,
|
||||
replayed: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
|
||||
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
|
||||
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
|
||||
sqlx::query("select count(*)::bigint from tasks where false")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(PostgresTaskStore::unavailable)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One transaction's decision of the acceptance path.
|
||||
enum Accepted {
|
||||
Created(TaskAcceptance),
|
||||
Replayed(TaskAcceptance),
|
||||
Conflict,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TaskStore for PostgresTaskStore {
|
||||
async fn health(&self) -> Result<(), TaskError> {
|
||||
sqlx::query("select count(*)::bigint from tasks where false")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
super::admission::admission_health(&self.pool)
|
||||
.await
|
||||
.map_err(|error| TaskError::unavailable(error.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
if let Some(epoch) = request.executable_input_master_key_epoch {
|
||||
super::master_key::ensure_active_master_key_epoch(&mut transaction, epoch)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
}
|
||||
let outcome = self.accept_locked(&mut transaction, &request).await;
|
||||
match outcome {
|
||||
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
|
||||
// The acknowledgement of a created Task is only possible after a real COMMIT.
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(TaskAcceptance {
|
||||
replayed: acceptance.replayed,
|
||||
..acceptance
|
||||
})
|
||||
}
|
||||
Ok(Accepted::Conflict) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::FingerprintConflict)
|
||||
}
|
||||
Err(AttemptError::Overloaded) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::Overloaded {
|
||||
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
||||
})
|
||||
}
|
||||
Err(AttemptError::Contended) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::unavailable(
|
||||
"durable task acceptance could not take the admission locks",
|
||||
))
|
||||
}
|
||||
Err(AttemptError::UniqueViolation) => {
|
||||
// Roll back the aborted transaction, then reread the scoped key once outside it.
|
||||
transaction.rollback().await.ok();
|
||||
self.reread_after_unique(&request).await
|
||||
}
|
||||
Err(AttemptError::Failure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(error)
|
||||
}
|
||||
Err(AttemptError::AdmissionFailure(error)) => {
|
||||
transaction.rollback().await.ok();
|
||||
Err(TaskError::unavailable(error.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
||||
and owner_id = $4 and authorization_scope = $5"
|
||||
)))
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::NotFound);
|
||||
};
|
||||
record_from_row(&row).map_err(Self::unavailable)
|
||||
}
|
||||
|
||||
async fn find(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
operation_id: &str,
|
||||
scoped_key: &str,
|
||||
) -> Result<Option<TaskRecord>, TaskError> {
|
||||
access.validate()?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"select {TASK_PROJECTION} from tasks
|
||||
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
||||
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
||||
)))
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(operation_id)
|
||||
.bind(scoped_key)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
row.map(|row| record_from_row(&row)).transpose()
|
||||
}
|
||||
|
||||
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
let access = &request.access;
|
||||
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = $1,
|
||||
result_envelope = $2,
|
||||
error_envelope = $3,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
||||
and owner_id = $7 and authorization_scope = $8
|
||||
and state = $9 and version = $10
|
||||
and (lease_owner is null or fence = 0)
|
||||
and current_attempt_id is null
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(request.target_state.as_str())
|
||||
.bind(request.result.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.error.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(request.expected_state.as_str())
|
||||
.bind(i64::from(request.expected_version))
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = update else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(self.classify_missed_finish(&request).await);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(request.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
|
||||
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
|
||||
let mut transaction = self.start().await?;
|
||||
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
||||
_ => TaskError::StateConflict,
|
||||
})?;
|
||||
let expired = sqlx::query(
|
||||
"with expired as (
|
||||
update tasks
|
||||
set state = 'failed',
|
||||
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id in (
|
||||
select task_id from tasks
|
||||
where state = 'queued' and execution_deadline <= now()
|
||||
and lease_owner is null
|
||||
order by task_id
|
||||
limit $1
|
||||
)
|
||||
returning task_id
|
||||
), released as (
|
||||
delete from execution_admission_entries
|
||||
where task_id in (select task_id from expired)
|
||||
returning task_id
|
||||
)
|
||||
select (select count(*) from expired)::bigint as expired_queued",
|
||||
)
|
||||
.bind(batch)
|
||||
.fetch_one(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let expired_queued: i64 = expired
|
||||
.try_get("expired_queued")
|
||||
.map_err(Self::unavailable)?;
|
||||
let purged = sqlx::query(
|
||||
"delete from tasks
|
||||
where task_id in (
|
||||
select task_id from tasks
|
||||
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
|
||||
order by task_id
|
||||
limit $1
|
||||
)",
|
||||
)
|
||||
.bind(batch)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(TaskPurgeReport {
|
||||
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
|
||||
expired_terminal: purged.rows_affected(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn claim_batch(
|
||||
&self,
|
||||
owner: &str,
|
||||
batch_limit: u32,
|
||||
lease_ms: i64,
|
||||
) -> Result<Vec<TaskLease>, TaskError> {
|
||||
let batch = (batch_limit.clamp(1, 64)) as usize;
|
||||
let mut transaction = self.start().await?;
|
||||
// A checkpointed attempt is never redispatched blindly: once its lease expires another worker
|
||||
// must decide per Task whether the pinned safe-retry evidence, the budget and the deadline
|
||||
// still allow exactly one bounded new attempt (Story 4.4). Everything else resolves to the
|
||||
// non-terminal uncertain state and releases its admission slot.
|
||||
let candidates = sqlx::query(
|
||||
"select t.task_id, t.current_attempt_id, t.attempts, t.max_attempts,
|
||||
t.execution_deadline > clock_timestamp() as deadline_left,
|
||||
coalesce(a.retry_allowed, false) as retry_allowed,
|
||||
a.resolved_at is null as attempt_open
|
||||
from tasks t
|
||||
left join task_attempts a
|
||||
on a.task_id = t.task_id and a.attempt_id = t.current_attempt_id
|
||||
where t.state = 'working' and t.lease_owner is not null
|
||||
and t.current_attempt_id is not null
|
||||
and t.lease_until < clock_timestamp()
|
||||
order by t.task_id
|
||||
limit 64
|
||||
for update of t skip locked",
|
||||
)
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
for candidate in candidates {
|
||||
let task_id: Uuid = candidate.try_get("task_id").map_err(Self::unavailable)?;
|
||||
let attempt_id: String = candidate
|
||||
.try_get("current_attempt_id")
|
||||
.map_err(Self::unavailable)?;
|
||||
let attempts: i32 = candidate.try_get("attempts").map_err(Self::unavailable)?;
|
||||
let max_attempts: i32 = candidate
|
||||
.try_get("max_attempts")
|
||||
.map_err(Self::unavailable)?;
|
||||
let deadline_left: bool = candidate
|
||||
.try_get("deadline_left")
|
||||
.map_err(Self::unavailable)?;
|
||||
let retry_allowed: bool = candidate
|
||||
.try_get("retry_allowed")
|
||||
.map_err(Self::unavailable)?;
|
||||
let attempt_open: bool = candidate
|
||||
.try_get("attempt_open")
|
||||
.map_err(Self::unavailable)?;
|
||||
let safe_recovery =
|
||||
retry_allowed && attempt_open && attempts < max_attempts && deadline_left;
|
||||
let close = sqlx::query(
|
||||
"update task_attempts
|
||||
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
|
||||
phase = 'dispatched'
|
||||
where task_id = $1 and attempt_id = $2 and resolved_at is null",
|
||||
)
|
||||
.bind(task_id)
|
||||
.bind(&attempt_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let _ = close;
|
||||
if safe_recovery {
|
||||
// Interrupted, not failed: re-arm the same admission intent and let the bounded claim
|
||||
// hand the Task to a new owner with a new fence and exactly one new attempt.
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set state = 'queued',
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
current_attempt_id = null,
|
||||
next_attempt_at = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $1 and state = 'working'
|
||||
and lease_until < clock_timestamp()",
|
||||
)
|
||||
.bind(task_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
sqlx::query(
|
||||
"update execution_admission_entries
|
||||
set state = 'reserved', granted_at = null
|
||||
where task_id = $1 and state = 'executing'",
|
||||
)
|
||||
.bind(task_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
} else {
|
||||
sqlx::query(
|
||||
"update tasks
|
||||
set state = 'outcome_unknown',
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
error_envelope = jsonb_build_object('code', 'outcome_unknown', 'phase', 'dispatched'),
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id = $1 and state = 'working'
|
||||
and lease_until < clock_timestamp()",
|
||||
)
|
||||
.bind(task_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(task_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
}
|
||||
}
|
||||
|
||||
// A crash before the durable dispatch checkpoint left the Task queued for execution: its
|
||||
// admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is
|
||||
// bounded by the <=30s lease expiry, inside the 60s story budget. The expiry condition is
|
||||
// re-checked in the UPDATE so a concurrent renewal can never be clobbered by recovery.
|
||||
sqlx::query(
|
||||
"with candidates as (
|
||||
select task_id from tasks
|
||||
where state = 'working' and lease_owner is not null
|
||||
and current_attempt_id is null
|
||||
and lease_until < clock_timestamp()
|
||||
order by task_id
|
||||
limit 64
|
||||
for update skip locked
|
||||
), requeued as (
|
||||
update tasks
|
||||
set state = 'queued',
|
||||
lease_owner = null,
|
||||
lease_until = null,
|
||||
next_attempt_at = null,
|
||||
version = version + 1,
|
||||
updated_at = now()
|
||||
where task_id in (select task_id from candidates)
|
||||
and state = 'working'
|
||||
and current_attempt_id is null
|
||||
and lease_until < clock_timestamp()
|
||||
returning task_id
|
||||
)
|
||||
update execution_admission_entries
|
||||
set state = 'reserved', granted_at = null
|
||||
where task_id in (select task_id from requeued) and state = 'executing'",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
|
||||
let mut leases = Vec::new();
|
||||
// Continuation claim: after a committed answer the Task is `working` without a lease and still
|
||||
// owns its executing admission permit. It is handed to exactly one new fence here, without a
|
||||
// second admission grant, so waiting capacity is never double-counted.
|
||||
let mut transaction = self.start().await?;
|
||||
let resumed = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set lease_owner = $1,
|
||||
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
|
||||
fence = fence + 1,
|
||||
updated_at = now()
|
||||
where task_id in (
|
||||
select t.task_id
|
||||
from tasks t
|
||||
join task_input_requests r on r.task_id = t.task_id and r.accepted_at is not null
|
||||
join execution_admission_entries e
|
||||
on e.task_id = t.task_id and e.state = 'executing'
|
||||
where t.state = 'working' and t.lease_owner is null
|
||||
and t.current_attempt_id is null
|
||||
order by t.task_id
|
||||
limit $3
|
||||
for update of t skip locked
|
||||
)
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(owner)
|
||||
.bind(lease_ms)
|
||||
.bind(i64::try_from(batch.clamp(1, 64)).unwrap_or(64))
|
||||
.fetch_all(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
for row in resumed {
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
leases.push(TaskLease {
|
||||
owner: owner.to_owned(),
|
||||
fence: record.fence,
|
||||
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
|
||||
record,
|
||||
});
|
||||
}
|
||||
|
||||
let remaining = batch.saturating_sub(leases.len());
|
||||
for _ in 0..remaining {
|
||||
let mut transaction = self.start().await?;
|
||||
let workspace_cursor =
|
||||
match super::admission::lock_admission_in_tx(&mut transaction, "*").await {
|
||||
Ok(cursor) => cursor,
|
||||
Err(error) => {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::unavailable(format!("{error:?}")));
|
||||
}
|
||||
};
|
||||
let (task_id, _updated_cursor) = match super::admission::grant_next_task(
|
||||
&mut transaction,
|
||||
"*",
|
||||
&workspace_cursor,
|
||||
&self.limits,
|
||||
self.scheduler,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::unavailable(format!("{error:?}")));
|
||||
}
|
||||
};
|
||||
// Persist the updated controller cursor even when no task is granted.
|
||||
let Some(task_id) = task_id else {
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
break;
|
||||
};
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = 'working',
|
||||
lease_owner = $1,
|
||||
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
|
||||
fence = fence + 1,
|
||||
updated_at = now()
|
||||
where task_id = $3
|
||||
and (
|
||||
(state = 'queued' and lease_owner is null)
|
||||
or (state = 'working' and lease_owner is not null
|
||||
and lease_until < clock_timestamp() and current_attempt_id is null)
|
||||
)
|
||||
and (next_attempt_at is null or next_attempt_at <= clock_timestamp())
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(owner)
|
||||
.bind(lease_ms)
|
||||
.bind(task_id)
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
// Persist cursor update (grant_next_task already did it) regardless.
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
break;
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
leases.push(TaskLease {
|
||||
owner: owner.to_owned(),
|
||||
fence: record.fence,
|
||||
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
|
||||
record,
|
||||
});
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
}
|
||||
Ok(leases)
|
||||
}
|
||||
|
||||
async fn renew_lease(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
lease_ms: i64,
|
||||
) -> Result<TaskLease, TaskError> {
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set lease_until = clock_timestamp() + ($1::bigint || ' milliseconds')::interval,
|
||||
updated_at = now()
|
||||
where task_id = $2 and lease_owner = $3 and fence = $4
|
||||
and lease_until > clock_timestamp()
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(lease_ms)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
let lease = TaskLease {
|
||||
owner: owner.to_owned(),
|
||||
fence: record.fence,
|
||||
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
|
||||
record,
|
||||
};
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(lease)
|
||||
}
|
||||
|
||||
async fn write_progress(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
progress: TaskEnvelope,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
let mut transaction = self.start().await?;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set progress_envelope = $1, updated_at = now()
|
||||
where task_id = $2 and lease_owner = $3 and fence = $4
|
||||
and lease_until > clock_timestamp()
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(progress.value().clone())
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn read_executable_input(
|
||||
&self,
|
||||
owner: &str,
|
||||
task_id: TaskId,
|
||||
fence: u64,
|
||||
) -> Result<(String, String, i64), TaskError> {
|
||||
let row = sqlx::query(
|
||||
"select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch
|
||||
from tasks
|
||||
where task_id = $1 and state = 'working' and lease_owner = $2 and fence = $3
|
||||
and lease_until > clock_timestamp()",
|
||||
)
|
||||
.bind(task_id.as_uuid())
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let ciphertext: Option<String> = row
|
||||
.try_get("executable_input_ciphertext")
|
||||
.map_err(Self::unavailable)?;
|
||||
let key_version: Option<String> = row
|
||||
.try_get("executable_input_key_version")
|
||||
.map_err(Self::unavailable)?;
|
||||
let epoch: Option<i64> = row
|
||||
.try_get("executable_input_master_key_epoch")
|
||||
.map_err(Self::unavailable)?;
|
||||
if let Some(epoch) = epoch {
|
||||
let active = sqlx::query_scalar::<_, i64>(
|
||||
"select epoch from master_key_identities where status = 'active' order by epoch desc limit 1",
|
||||
)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
if let Some(active) = active
|
||||
&& active != epoch
|
||||
{
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
}
|
||||
match (ciphertext, key_version, epoch) {
|
||||
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
|
||||
Ok((ciphertext, key_version, epoch))
|
||||
}
|
||||
_ => Err(TaskError::NotFound),
|
||||
}
|
||||
}
|
||||
|
||||
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError> {
|
||||
self.checkpoint_dispatch_attempt(request).await
|
||||
}
|
||||
|
||||
async fn resolve_attempt(
|
||||
&self,
|
||||
lease: &AttemptLease,
|
||||
outcome: AttemptOutcome,
|
||||
) -> Result<AttemptResolution, TaskError> {
|
||||
self.resolve_attempt_outcome(lease, outcome).await
|
||||
}
|
||||
|
||||
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError> {
|
||||
self.cancel_task(request).await
|
||||
}
|
||||
|
||||
async fn attempts(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<Vec<TaskAttempt>, TaskError> {
|
||||
self.attempt_history(access, task_id).await
|
||||
}
|
||||
|
||||
async fn finish_leased(
|
||||
&self,
|
||||
request: TaskFinishRequest,
|
||||
owner: &str,
|
||||
fence: u64,
|
||||
) -> Result<TaskRecord, TaskError> {
|
||||
request.validate()?;
|
||||
let mut transaction = self.start().await?;
|
||||
let access = &request.access;
|
||||
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"update tasks
|
||||
set state = $1,
|
||||
result_envelope = $2,
|
||||
error_envelope = $3,
|
||||
version = version + 1,
|
||||
updated_at = now(),
|
||||
lease_owner = null,
|
||||
lease_until = null
|
||||
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
||||
and owner_id = $7 and authorization_scope = $8
|
||||
and state = $9 and version = $10
|
||||
and lease_owner = $11 and fence = $12
|
||||
and lease_until > clock_timestamp()
|
||||
and current_attempt_id is null
|
||||
returning {TASK_PROJECTION}"
|
||||
)))
|
||||
.bind(request.target_state.as_str())
|
||||
.bind(request.result.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.error.as_ref().map(|value| value.value().clone()))
|
||||
.bind(request.task_id.as_uuid())
|
||||
.bind(&access.workspace_id)
|
||||
.bind(access.owner_kind.as_str())
|
||||
.bind(&access.owner_id)
|
||||
.bind(&access.authorization_scope)
|
||||
.bind(request.expected_state.as_str())
|
||||
.bind(i64::from(request.expected_version))
|
||||
.bind(owner)
|
||||
.bind(i64::try_from(fence).unwrap_or_default())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
let Some(row) = row else {
|
||||
transaction.rollback().await.ok();
|
||||
return Err(TaskError::StateConflict);
|
||||
};
|
||||
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
||||
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
||||
.bind(request.task_id.as_uuid())
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(Self::unavailable)?;
|
||||
transaction.commit().await.map_err(Self::unavailable)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError> {
|
||||
self.refuse_attempt_resolution(lease, code).await
|
||||
}
|
||||
}
|
||||
|
||||
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
|
||||
match mode {
|
||||
ToolExecutionMode::Synchronous => "synchronous",
|
||||
ToolExecutionMode::Background => "background",
|
||||
ToolExecutionMode::Adaptive => "adaptive",
|
||||
}
|
||||
}
|
||||
|
||||
fn task_sql_error(error: sqlx::Error) -> AttemptError {
|
||||
if error
|
||||
.as_database_error()
|
||||
.and_then(|database| database.code())
|
||||
.as_deref()
|
||||
== Some("23505")
|
||||
{
|
||||
return AttemptError::UniqueViolation;
|
||||
}
|
||||
AttemptError::from_sql(error)
|
||||
}
|
||||
|
||||
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
|
||||
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
|
||||
let state: String = row.try_get("state").map_err(unavailable_row)?;
|
||||
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
|
||||
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
|
||||
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
|
||||
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
|
||||
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
|
||||
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
|
||||
value.map(TaskEnvelope::from_persisted).transpose()
|
||||
};
|
||||
Ok(TaskRecord {
|
||||
task_id: TaskId::parse(&task_id.to_string())?,
|
||||
state: TaskState::parse(&state)?,
|
||||
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
|
||||
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
|
||||
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
|
||||
authorization_scope: row
|
||||
.try_get("authorization_scope")
|
||||
.map_err(unavailable_row)?,
|
||||
platform_api_key_id: row
|
||||
.try_get::<Option<String>, _>("platform_api_key_id")
|
||||
.map_err(unavailable_row)?
|
||||
.map(PlatformApiKeyId::new),
|
||||
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
|
||||
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
|
||||
effective_input_fingerprint: row
|
||||
.try_get("effective_input_fingerprint")
|
||||
.map_err(unavailable_row)?,
|
||||
pins: TaskPins {
|
||||
operation_version: u32::try_from(
|
||||
row.try_get::<i32, _>("operation_version")
|
||||
.map_err(unavailable_row)?,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
policy_mode: match policy_mode.as_str() {
|
||||
"background" => ToolExecutionMode::Background,
|
||||
"adaptive" => ToolExecutionMode::Adaptive,
|
||||
_ => ToolExecutionMode::Synchronous,
|
||||
},
|
||||
execution_deadline_unix_ms: row
|
||||
.try_get("execution_deadline_ms")
|
||||
.map_err(unavailable_row)?,
|
||||
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
|
||||
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
|
||||
retry: TaskRetryPolicy::new(
|
||||
u32::try_from(
|
||||
row.try_get::<i32, _>("max_attempts")
|
||||
.map_err(unavailable_row)?,
|
||||
)
|
||||
.unwrap_or(1),
|
||||
row.try_get::<i64, _>("retry_backoff_step_ms")
|
||||
.map_err(unavailable_row)?,
|
||||
),
|
||||
},
|
||||
support: TaskSupportIds {
|
||||
request_id: crank_core::RequestId::parse(&request_id)
|
||||
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
|
||||
trace_id: crank_core::TraceId::parse(&trace_id)
|
||||
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
|
||||
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
|
||||
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
|
||||
},
|
||||
input: envelope("input_envelope")?
|
||||
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
|
||||
progress: envelope("progress_envelope")?,
|
||||
result: envelope("result_envelope")?,
|
||||
error: envelope("error_envelope")?,
|
||||
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
lease_owner: row.try_get("lease_owner").map_err(unavailable_row)?,
|
||||
lease_until_unix_ms: row.try_get("lease_until_ms").map_err(unavailable_row)?,
|
||||
current_attempt_id: row.try_get("current_attempt_id").map_err(unavailable_row)?,
|
||||
dispatch_started_at_unix_ms: row
|
||||
.try_get("dispatch_started_at_ms")
|
||||
.map_err(unavailable_row)?,
|
||||
next_attempt_at_unix_ms: row.try_get("next_attempt_at_ms").map_err(unavailable_row)?,
|
||||
cancel_requested_at_unix_ms: row
|
||||
.try_get("cancel_requested_at_ms")
|
||||
.map_err(unavailable_row)?,
|
||||
cancel_reason: row.try_get("cancel_reason").map_err(unavailable_row)?,
|
||||
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
|
||||
.unwrap_or_default(),
|
||||
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
|
||||
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn unavailable_row(error: sqlx::Error) -> TaskError {
|
||||
PostgresTaskStore::unavailable(error)
|
||||
}
|
||||
@@ -2,6 +2,7 @@ use crank_registry::{MigrationAuthority, MigrationPreflight, PostgresRegistry};
|
||||
use sqlx::Row;
|
||||
|
||||
mod artifact_metadata;
|
||||
mod durable_tasks_guard;
|
||||
mod import_plan;
|
||||
mod legacy_adoption;
|
||||
mod preflight;
|
||||
@@ -72,6 +73,11 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
|
||||
(23, "prompt-dependencies-v23"),
|
||||
(24, "execution-admission-v24"),
|
||||
(25, "durable-tasks-v25"),
|
||||
(26, "durable-tasks-v26"),
|
||||
(27, "durable-tasks-v27"),
|
||||
(28, "durable-tasks-v28"),
|
||||
(29, "durable-tasks-v29"),
|
||||
(30, "durable-task-input-v30"),
|
||||
];
|
||||
assert_eq!(rows.len(), expected.len());
|
||||
for (row, (version, name)) in rows.iter().zip(expected) {
|
||||
@@ -182,7 +188,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(first.pool()).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
@@ -230,7 +236,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
|
||||
.expect("the V18-local guard must be restorable");
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"restoring the exact V18 contract must return the current verdict"
|
||||
);
|
||||
}
|
||||
@@ -356,7 +362,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -469,13 +475,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 2,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
let trace_column: bool = sqlx::query_scalar(
|
||||
"select exists (
|
||||
@@ -518,7 +524,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 3,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -578,7 +584,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 4,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -832,6 +838,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 11,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
|
||||
for (drift, restore) in [
|
||||
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
|
||||
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"restore: {restore}"
|
||||
);
|
||||
}
|
||||
|
||||
+8
-8
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"a rejected primary-key nullability change must leave the schema current"
|
||||
);
|
||||
|
||||
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
|
||||
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"the baseline before the drift matrix must be the current V16 schema"
|
||||
);
|
||||
let mut observations: Vec<String> = Vec::new();
|
||||
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
|
||||
.await
|
||||
.expect("restoring the exact definition must apply");
|
||||
let restored = MigrationAuthority::preflight(&pool).await;
|
||||
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 25 }));
|
||||
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 30 }));
|
||||
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
|
||||
// мнением того же guard-а, который принимает неправильные определения (F4).
|
||||
let restored_exact = catalog_definition_snapshot(&pool).await == before;
|
||||
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"baseline before inserting the shadow schema"
|
||||
);
|
||||
|
||||
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
// Тень не меняет вердикт guard-а по текущей схеме.
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"a same-named object in another schema must not change the verdict for the current schema"
|
||||
);
|
||||
|
||||
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 },
|
||||
MigrationPreflight::Current { version: 30 },
|
||||
"the exact restoration must return the current V16 schema"
|
||||
);
|
||||
|
||||
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
|
||||
assert!(
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
use super::*;
|
||||
use crank_registry::MigrationApplyResult;
|
||||
|
||||
#[tokio::test]
|
||||
async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_v43_clean_legacy").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
super::remove_v25_schema(&pool).await;
|
||||
// remove_v25_schema drops `tasks` and ledger >= 25, but V25 also added the
|
||||
// nullable dispatch-intent column/index/FK to execution_admission_entries;
|
||||
// restore those to the V24 shape so the legacy ledger is self-consistent.
|
||||
sqlx::raw_sql(
|
||||
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
|
||||
drop index if exists execution_admission_entries_task_uidx;
|
||||
alter table execution_admission_entries drop column if exists task_id;",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 24,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::apply(&pool).await.unwrap(),
|
||||
MigrationApplyResult::Applied { from: 24, to: 30 }
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_v27_on_legacy_ledger_is_rejected_by_preflight() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_v43_partial_v27").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
super::remove_v25_schema(&pool).await;
|
||||
sqlx::raw_sql(
|
||||
"alter table execution_admission_entries drop constraint if exists execution_admission_entries_task_fkey;
|
||||
drop index if exists execution_admission_entries_task_uidx;
|
||||
alter table execution_admission_entries drop column if exists task_id;",
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// Reintroduce a partial V27-shaped `tasks` table on top of the legacy (v24) ledger.
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(
|
||||
"create table tasks (
|
||||
task_id uuid primary key,
|
||||
executable_input_ciphertext text,
|
||||
executable_input_key_version text,
|
||||
executable_input_master_key_epoch bigint
|
||||
);",
|
||||
))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||
assert_eq!(error.code(), "partial_sequence");
|
||||
// The rejection today is attributed to V25's absent-schema guard because the
|
||||
// `tasks` relation already exists, so a behavioral RED pin for the missing
|
||||
// V27 guard line cannot be reached; this test fixes the rejection contract
|
||||
// instead of claiming new behavior.
|
||||
assert_eq!(error.version(), Some(24));
|
||||
assert_eq!(error.stage(), "preflight.schema");
|
||||
|
||||
let count: i64 =
|
||||
sqlx::query_scalar("select count(*)::bigint from __crank_migrations where version >= 25")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
count, 0,
|
||||
"nothing at or above V25 should have been applied over the partial fixture"
|
||||
);
|
||||
}
|
||||
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
let tables = sqlx::query(
|
||||
"select table_name
|
||||
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 13,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
|
||||
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
|
||||
assert_eq!(operation_count, 1);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
|
||||
assert!(supports_elicitation);
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 0,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+13
-7
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
|
||||
.fetch_one(&pool)
|
||||
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 18,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
MigrationAuthority::apply(&pool).await.unwrap(),
|
||||
MigrationApplyResult::Applied { from: 18, to: 25 }
|
||||
MigrationApplyResult::Applied { from: 18, to: 30 }
|
||||
);
|
||||
let after: Vec<(i64, String, String)> =
|
||||
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
|
||||
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
|
||||
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
|
||||
assert_eq!(
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::Current { version: 25 }
|
||||
MigrationPreflight::Current { version: 30 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -86,16 +86,22 @@ async fn a_database_newer_than_the_binary_fails_closed_at_startup() {
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
// The same `version > CURRENT_VERSION` branch an older binary takes when it meets a newer
|
||||
// database; a synthetic version above the current one exercises it without a second binary.
|
||||
let current: i64 = sqlx::query_scalar("select max(version) from __crank_migrations")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let future = current + 1;
|
||||
sqlx::query(
|
||||
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
|
||||
values (26, 'future-v26', repeat('a', 64), 'expand', 'n-minus-one-readable')",
|
||||
values ($1, 'future-synthetic', repeat('a', 64), 'expand', 'n-minus-one-readable')",
|
||||
)
|
||||
.bind(future)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let error = MigrationAuthority::preflight(&pool).await.unwrap_err();
|
||||
assert_eq!(error.code(), "future_version");
|
||||
assert_eq!(error.version(), Some(26));
|
||||
assert_eq!(error.version(), Some(future));
|
||||
assert_eq!(error.recovery(), "install_matching_application");
|
||||
assert!(
|
||||
MigrationAuthority::require_current(&pool).await.is_err(),
|
||||
@@ -155,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 18,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
|
||||
MigrationAuthority::preflight(&pool).await.unwrap(),
|
||||
MigrationPreflight::MigrationRequired {
|
||||
current: 11,
|
||||
target: 25,
|
||||
target: 30,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -330,9 +330,14 @@ pub(crate) async fn remove_v12_schema(pool: &sqlx::PgPool) {
|
||||
|
||||
/// Story 4.2: снять V25 (durable Tasks и Task-linked dispatch intent) перед возвратом к < 25.
|
||||
pub(crate) async fn remove_v25_schema(pool: &sqlx::PgPool) {
|
||||
// V25 created `tasks`; V26–V28 only altered it, so dropping the table removes their schema too.
|
||||
// V30 adds tables that reference `tasks`; they are dropped explicitly so the legacy preflight
|
||||
// sees an actually absent V30 shape instead of orphan continuation tables.
|
||||
sqlx::raw_sql(
|
||||
"drop table if exists tasks cascade;
|
||||
delete from __crank_migrations where version = 25;",
|
||||
"drop table if exists task_input_audit cascade;
|
||||
drop table if exists task_input_requests cascade;
|
||||
drop table if exists tasks cascade;
|
||||
delete from __crank_migrations where version >= 25;",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
|
||||
+241
-5
@@ -8,14 +8,19 @@
|
||||
use super::common;
|
||||
|
||||
use crank_core::{
|
||||
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, ApprovalEvidence,
|
||||
ExecutionAdmission, NoTasksFallback, RequestId, TaskAcceptRequest, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState, TaskStore,
|
||||
TaskSupportIds, ToolExecutionMode, TraceContext, current_unix_ms,
|
||||
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, AgentId, ApprovalEvidence,
|
||||
BeginAttempt, ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId,
|
||||
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest,
|
||||
TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins,
|
||||
TaskState, TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId,
|
||||
current_unix_ms,
|
||||
};
|
||||
use crank_registry::{
|
||||
AdmissionLimits, CreatePlatformApiKeyRequest, PostgresExecutionAdmission, PostgresTaskStore,
|
||||
};
|
||||
use crank_registry::{AdmissionLimits, PostgresExecutionAdmission, PostgresTaskStore};
|
||||
use serde_json::json;
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::{Duration as TimeDuration, OffsetDateTime};
|
||||
|
||||
const WORKSPACE: &str = "ws_default";
|
||||
const AGENT: &str = "agent_a";
|
||||
@@ -48,6 +53,7 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: agent.to_owned(),
|
||||
authorization_scope: format!("credential:pk_{agent}"),
|
||||
platform_api_key_id: Some(PlatformApiKeyId::new(format!("pk_{agent}"))),
|
||||
operation_id: "op_report".to_owned(),
|
||||
pins: TaskPins {
|
||||
operation_version: 3,
|
||||
@@ -55,11 +61,15 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
|
||||
execution_deadline_unix_ms: deadline,
|
||||
retain_until_unix_ms: deadline + 3_600_000,
|
||||
approval_reference: Some("appr_1".to_owned()),
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
},
|
||||
scoped_key: key.to_owned(),
|
||||
input_fingerprint: fingerprint.to_owned(),
|
||||
input: TaskEnvelope::try_redacted(&json!({"report": "q3", "api_key": "sk_live"})).unwrap(),
|
||||
support: unsampled_support(),
|
||||
executable_input_ciphertext: None,
|
||||
executable_input_key_version: None,
|
||||
executable_input_master_key_epoch: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -664,6 +674,80 @@ async fn invalid_bounded_envelope_is_rejected_before_persistence() {
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_worker_lease_and_fencing_contract() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("lease1"), &digest("f1"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let leases = store.claim_batch("worker_1", 4, 10_000).await.unwrap();
|
||||
assert_eq!(leases.len(), 1);
|
||||
assert_eq!(leases[0].record.task_id, created.task_id);
|
||||
assert_eq!(leases[0].record.state, TaskState::Working);
|
||||
assert_eq!(leases[0].owner, "worker_1");
|
||||
|
||||
let lease = &leases[0];
|
||||
let fenced_progress = store
|
||||
.write_progress(
|
||||
"worker_1",
|
||||
lease.record.task_id,
|
||||
lease.fence,
|
||||
TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(fenced_progress.progress.is_some());
|
||||
|
||||
// stale fence rejected
|
||||
let bad = store
|
||||
.renew_lease("worker_1", lease.record.task_id, lease.fence + 1, 10_000)
|
||||
.await;
|
||||
assert!(bad.is_err());
|
||||
|
||||
// checkpoint dispatch once
|
||||
let checkpointed = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker_1".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-1".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: true,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(checkpointed.attempt.attempt_id, "attempt-1");
|
||||
assert_eq!(checkpointed.attempt.ordinal, 1);
|
||||
assert!(checkpointed.task.dispatch_started_at_unix_ms.is_some());
|
||||
assert_eq!(
|
||||
checkpointed.task.current_attempt_id.as_deref(),
|
||||
Some("attempt-1")
|
||||
);
|
||||
|
||||
// second checkpoint rejected
|
||||
let second = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker_1".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: checkpointed.task.version,
|
||||
attempt_id: "attempt-2".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: true,
|
||||
})
|
||||
.await;
|
||||
assert!(second.is_err());
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_failure_fails_closed_without_records() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
@@ -680,3 +764,155 @@ async fn store_failure_fails_closed_without_records() {
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: the typed immutable Platform API key reference of an accepted Task is persisted,
|
||||
/// read back through the lease, and only an active, unexpired, exactly bound key is authoritatively
|
||||
/// revalidated. Revocation, expiry and a foreign workspace/agent binding all close; an administrator
|
||||
/// Task stores no machine key identity at all, so a worker fails closed on it.
|
||||
#[tokio::test]
|
||||
async fn accepted_task_authority_reference_is_persisted_and_revalidated() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let workspace = WorkspaceId::new(WORKSPACE);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
// `ws_default` is seeded by the baseline migration; the keys only need that workspace row.
|
||||
let agent = AgentId::new(AGENT);
|
||||
|
||||
let active_id = PlatformApiKeyId::new("pk_task_active");
|
||||
for key in [
|
||||
authority_key(
|
||||
"pk_task_active",
|
||||
&workspace,
|
||||
AGENT,
|
||||
PlatformApiKeyStatus::Active,
|
||||
None,
|
||||
),
|
||||
authority_key(
|
||||
"pk_task_revoked",
|
||||
&workspace,
|
||||
AGENT,
|
||||
PlatformApiKeyStatus::Revoked,
|
||||
None,
|
||||
),
|
||||
authority_key(
|
||||
"pk_task_expired",
|
||||
&workspace,
|
||||
AGENT,
|
||||
PlatformApiKeyStatus::Active,
|
||||
Some(now - TimeDuration::hours(1)),
|
||||
),
|
||||
] {
|
||||
let secret_hash = format!("secret-hash-{}", key.id.as_str());
|
||||
registry
|
||||
.create_platform_api_key(CreatePlatformApiKeyRequest {
|
||||
secret_hash: &secret_hash,
|
||||
api_key: &key,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Acceptance persists only the immutable typed reference, never the bearer secret.
|
||||
let mut accepted = request(&key("k_authority"), &digest("fa"), AGENT, 300_000);
|
||||
accepted.platform_api_key_id = Some(active_id.clone());
|
||||
let task_id = accepted.task_id;
|
||||
store.accept(accepted).await.expect("acceptance commits");
|
||||
|
||||
let leases = store
|
||||
.claim_batch("worker_authority", 8, 30_000)
|
||||
.await
|
||||
.expect("claim");
|
||||
let lease = leases
|
||||
.into_iter()
|
||||
.find(|lease| lease.record.task_id == task_id)
|
||||
.expect("accepted task is leased");
|
||||
assert_eq!(lease.record.platform_api_key_id.as_ref(), Some(&active_id));
|
||||
|
||||
// Positive: the exact active key of this workspace/agent is revalidated authoritatively.
|
||||
let revalidated = registry
|
||||
.revalidate_platform_api_key(&workspace, &agent, &active_id)
|
||||
.await
|
||||
.expect("authoritative read")
|
||||
.expect("active key is authoritative");
|
||||
assert!(
|
||||
revalidated
|
||||
.api_key
|
||||
.scopes
|
||||
.contains(&PlatformApiKeyScope::Write)
|
||||
);
|
||||
|
||||
// Negatives: revocation, expiry and a foreign workspace/agent binding all fail closed.
|
||||
let foreign_agent = AgentId::new("agent_b");
|
||||
let foreign_workspace = WorkspaceId::new("ws_elsewhere");
|
||||
for (other_workspace, other_agent, other_key) in [
|
||||
(&workspace, &agent, PlatformApiKeyId::new("pk_task_revoked")),
|
||||
(&workspace, &agent, PlatformApiKeyId::new("pk_task_expired")),
|
||||
(&workspace, &foreign_agent, active_id.clone()),
|
||||
(&foreign_workspace, &agent, active_id.clone()),
|
||||
] {
|
||||
let found = registry
|
||||
.revalidate_platform_api_key(other_workspace, other_agent, &other_key)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
found.is_none(),
|
||||
"authority must fail closed for {}",
|
||||
other_key.as_str()
|
||||
);
|
||||
}
|
||||
|
||||
// An administrator Task carries no machine key identity: the worker has nothing to revalidate.
|
||||
let mut admin_request = request(&key("k_admin"), &digest("fb"), AGENT, 300_000);
|
||||
admin_request.owner_kind = TaskOwnerKind::Admin;
|
||||
admin_request.owner_id = "user_admin".to_owned();
|
||||
admin_request.authorization_scope = "admin:user_admin".to_owned();
|
||||
admin_request.platform_api_key_id = None;
|
||||
let admin_task = admin_request.task_id;
|
||||
store
|
||||
.accept(admin_request)
|
||||
.await
|
||||
.expect("administrator acceptance commits");
|
||||
let admin_record = store
|
||||
.read(
|
||||
&TaskAccess {
|
||||
workspace_id: WORKSPACE.to_owned(),
|
||||
owner_kind: TaskOwnerKind::Admin,
|
||||
owner_id: "user_admin".to_owned(),
|
||||
authorization_scope: "admin:user_admin".to_owned(),
|
||||
},
|
||||
admin_task,
|
||||
)
|
||||
.await
|
||||
.expect("owner read");
|
||||
assert_eq!(admin_record.platform_api_key_id, None);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
fn authority_key(
|
||||
id: &str,
|
||||
workspace_id: &WorkspaceId,
|
||||
agent: &str,
|
||||
status: PlatformApiKeyStatus,
|
||||
expires_at: Option<OffsetDateTime>,
|
||||
) -> PlatformApiKey {
|
||||
PlatformApiKey {
|
||||
id: PlatformApiKeyId::new(id),
|
||||
workspace_id: workspace_id.clone(),
|
||||
agent_id: Some(AgentId::new(agent)),
|
||||
key_kind: PlatformApiKeyKind::McpClient,
|
||||
name: id.to_owned(),
|
||||
prefix: "crk_live".to_owned(),
|
||||
scopes: vec![PlatformApiKeyScope::Write],
|
||||
status,
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
last_used_at: None,
|
||||
expires_at,
|
||||
allowed_origins: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
mod task_attempts;
|
||||
mod task_worker;
|
||||
@@ -0,0 +1,534 @@
|
||||
use super::*;
|
||||
use crank_core::{
|
||||
AttemptOutcome, AttemptOutcomeClass, AttemptPhase, AttemptResolution, BeginAttempt,
|
||||
TaskCancelOutcome, TaskCancelRequest, TaskRetryPolicy,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_before_checkpoint_wins_race_with_claim_and_never_dispatches() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(
|
||||
&key("cancel_before_1"),
|
||||
&digest("fb1"),
|
||||
AGENT,
|
||||
30_000,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let outcome = store
|
||||
.cancel(TaskCancelRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_version: None,
|
||||
reason: "cancel before dispatch".to_owned(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
match outcome {
|
||||
TaskCancelOutcome::Cancelled(record) => assert_eq!(record.state, TaskState::Cancelled),
|
||||
other => panic!("expected Cancelled, got {other:?}"),
|
||||
}
|
||||
|
||||
let claims = store.claim_batch("worker", 4, 10_000).await.unwrap();
|
||||
assert!(claims.is_empty(), "cancelled task must not be claimed");
|
||||
|
||||
let attempts = store
|
||||
.attempts(&access(AGENT), created.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(attempts.is_empty(), "no attempts for cancelled task");
|
||||
|
||||
let intents: i64 = sqlx::query_scalar(
|
||||
"select count(*)::bigint from execution_admission_entries where task_id = $1",
|
||||
)
|
||||
.bind(created.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(intents, 0, "dispatch intent must be removed");
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_after_checkpoint_is_uncertain_and_never_cancelled_or_retried() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(
|
||||
&key("cancel_after_1"),
|
||||
&digest("fa1"),
|
||||
AGENT,
|
||||
30_000,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
|
||||
let lease = leases.into_iter().next().expect("claim");
|
||||
|
||||
let leased = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-cancel-after".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: true,
|
||||
})
|
||||
.await
|
||||
.expect("checkpoint");
|
||||
|
||||
let outcome = store
|
||||
.cancel(TaskCancelRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_version: None,
|
||||
reason: "cancel after checkpoint".to_owned(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
match outcome {
|
||||
TaskCancelOutcome::OutcomeUnknown(record) => {
|
||||
assert_eq!(record.state, TaskState::OutcomeUnknown)
|
||||
}
|
||||
other => panic!("expected OutcomeUnknown, got {other:?}"),
|
||||
}
|
||||
|
||||
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
||||
|
||||
let attempts = store
|
||||
.attempts(&access(AGENT), created.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts.len(), 1);
|
||||
assert_eq!(
|
||||
attempts[0].outcome,
|
||||
Some(AttemptOutcomeClass::OutcomeUnknown)
|
||||
);
|
||||
|
||||
// Repeated cancel stays uncertain.
|
||||
let outcome2 = store
|
||||
.cancel(TaskCancelRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_version: None,
|
||||
reason: "cancel after checkpoint".to_owned(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(outcome2, TaskCancelOutcome::OutcomeUnknown(_)));
|
||||
|
||||
// finish_leased by the original lease must be rejected.
|
||||
let finish = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_state: leased.task.state,
|
||||
expected_version: leased.task.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
error: None,
|
||||
},
|
||||
"worker",
|
||||
leased.attempt.fence,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(finish, Err(TaskError::StateConflict)));
|
||||
|
||||
let claims2 = store.claim_batch("worker2", 4, 10_000).await.unwrap();
|
||||
assert!(
|
||||
!claims2.iter().any(|l| l.record.task_id == created.task_id),
|
||||
"outcome unknown task must never be claimed"
|
||||
);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_and_resolve_are_cas_guarded_and_history_is_immutable() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("cas_1"), &digest("fc1"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
|
||||
let lease = leases.into_iter().next().expect("claim");
|
||||
let first_version = lease.record.version;
|
||||
|
||||
let committed = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-cas".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.expect("first checkpoint");
|
||||
let committed_lease = committed;
|
||||
|
||||
// Repeated checkpoint with the same old version -> StateConflict.
|
||||
let second = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: first_version,
|
||||
attempt_id: "attempt-cas-2".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await;
|
||||
assert!(matches!(second, Err(TaskError::StateConflict)));
|
||||
|
||||
// resolve with a modified lease task version -> StateConflict.
|
||||
let mut bad_lease = committed_lease.clone();
|
||||
bad_lease.task.version = first_version;
|
||||
let bad_resolve = store
|
||||
.resolve_attempt(
|
||||
&bad_lease,
|
||||
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(bad_resolve, Err(TaskError::StateConflict)));
|
||||
|
||||
// Correct resolve -> Finished(Completed).
|
||||
let ok = store
|
||||
.resolve_attempt(
|
||||
&committed_lease,
|
||||
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
)
|
||||
.await
|
||||
.expect("resolve");
|
||||
match ok {
|
||||
AttemptResolution::Finished(rec) => assert_eq!(rec.state, TaskState::Completed),
|
||||
other => panic!("expected Finished, got {other:?}"),
|
||||
}
|
||||
|
||||
// Repeated resolve -> StateConflict.
|
||||
let repeat = store
|
||||
.resolve_attempt(
|
||||
&committed_lease,
|
||||
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(repeat, Err(TaskError::StateConflict)));
|
||||
|
||||
let attempts = store
|
||||
.attempts(&access(AGENT), created.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts.len(), 1, "exactly one immutable attempt record");
|
||||
assert_eq!(attempts[0].outcome, Some(AttemptOutcomeClass::Completed));
|
||||
|
||||
let _ = created;
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_checkpointed_attempt_recovers_only_with_safe_retry_evidence() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
// Task A: retry policy allows 2 attempts, retry_allowed=true.
|
||||
let mut req_a = request(&key("rec_a"), &digest("fa"), AGENT, 30_000);
|
||||
req_a.pins.retry = TaskRetryPolicy::new(2, 0);
|
||||
let created_a = store.accept(req_a).await.unwrap();
|
||||
|
||||
// Task B: single attempt, retry_allowed=false.
|
||||
let created_b = store
|
||||
.accept(request(&key("rec_b"), &digest("fb"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let claims = store.claim_batch("worker_dead", 2, 150).await.unwrap();
|
||||
assert_eq!(claims.len(), 2);
|
||||
|
||||
let lease_a = claims
|
||||
.iter()
|
||||
.find(|l| l.record.task_id == created_a.task_id)
|
||||
.cloned()
|
||||
.expect("claim A");
|
||||
let lease_b = claims
|
||||
.iter()
|
||||
.find(|l| l.record.task_id == created_b.task_id)
|
||||
.cloned()
|
||||
.expect("claim B");
|
||||
|
||||
store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease_b.record.task_id,
|
||||
owner: "worker_dead".to_owned(),
|
||||
fence: lease_b.fence,
|
||||
expected_version: lease_b.record.version,
|
||||
attempt_id: "attempt-b".to_owned(),
|
||||
input_fingerprint: lease_b.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.expect("checkpoint B");
|
||||
|
||||
store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease_a.record.task_id,
|
||||
owner: "worker_dead".to_owned(),
|
||||
fence: lease_a.fence,
|
||||
expected_version: lease_a.record.version,
|
||||
attempt_id: "attempt-a".to_owned(),
|
||||
input_fingerprint: lease_a.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: true,
|
||||
})
|
||||
.await
|
||||
.expect("checkpoint A");
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
|
||||
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
|
||||
let lease_a2 = recovered
|
||||
.iter()
|
||||
.find(|l| l.record.task_id == created_a.task_id)
|
||||
.cloned()
|
||||
.expect("A recovered");
|
||||
assert!(lease_a2.fence > lease_a.fence, "recovery bumps fence");
|
||||
assert!(
|
||||
matches!(
|
||||
lease_a2.record.state,
|
||||
TaskState::Working | TaskState::Queued
|
||||
),
|
||||
"A state working/queued"
|
||||
);
|
||||
|
||||
assert!(
|
||||
!recovered
|
||||
.iter()
|
||||
.any(|l| l.record.task_id == created_b.task_id),
|
||||
"B must not be redispatched"
|
||||
);
|
||||
|
||||
let record_b = store.read(&access(AGENT), created_b.task_id).await.unwrap();
|
||||
assert_eq!(record_b.state, TaskState::OutcomeUnknown);
|
||||
|
||||
// A prior attempt resolved as uncertain.
|
||||
let attempts_a = store
|
||||
.attempts(&access(AGENT), created_a.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!attempts_a.is_empty());
|
||||
assert_eq!(
|
||||
attempts_a[0].outcome,
|
||||
Some(AttemptOutcomeClass::OutcomeUnknown)
|
||||
);
|
||||
|
||||
// Further scans do not hand B back.
|
||||
assert!(
|
||||
store
|
||||
.claim_batch("worker_live", 4, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|l| l.record.task_id != created_b.task_id)
|
||||
);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn before_dispatch_failed_attempt_does_not_erase_recorded_uncertainty() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let mut req = request(&key("bd_1"), &digest("fbd"), AGENT, 30_000);
|
||||
req.pins.retry = TaskRetryPolicy::new(2, 0);
|
||||
let created = store.accept(req).await.unwrap();
|
||||
|
||||
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
|
||||
let lease_a = leases.into_iter().next().expect("claim A");
|
||||
|
||||
let committed_a = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease_a.record.task_id,
|
||||
owner: "worker".to_owned(),
|
||||
fence: lease_a.fence,
|
||||
expected_version: lease_a.record.version,
|
||||
attempt_id: "attempt-a".to_owned(),
|
||||
input_fingerprint: lease_a.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: true,
|
||||
})
|
||||
.await
|
||||
.expect("checkpoint A");
|
||||
|
||||
// Attempt A resolves to a bounded retry (uncertain outcome).
|
||||
let retry = store
|
||||
.resolve_attempt(
|
||||
&committed_a,
|
||||
AttemptOutcome::Uncertain {
|
||||
error: TaskEnvelope::try_redacted(
|
||||
&json!({"code":"outcome_unknown","phase":"dispatched"}),
|
||||
)
|
||||
.unwrap(),
|
||||
repeatable: true,
|
||||
phase: AttemptPhase::Dispatched,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("resolve A");
|
||||
let retry_task = match retry {
|
||||
AttemptResolution::RetryAt { task, .. } => task,
|
||||
other => panic!("expected RetryAt, got {other:?}"),
|
||||
};
|
||||
assert_eq!(retry_task.state, TaskState::Working);
|
||||
assert!(retry_task.current_attempt_id.is_none());
|
||||
let retry_version = retry_task.version;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
|
||||
// The Task keeps its lease across the pinned backoff: the next attempt checkpoints under the
|
||||
// same owner/fence and the version the RetryAt record reported.
|
||||
let committed_b_result = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: retry_task.task_id,
|
||||
owner: "worker".to_owned(),
|
||||
fence: lease_a.fence,
|
||||
expected_version: retry_version,
|
||||
attempt_id: "attempt-b".to_owned(),
|
||||
input_fingerprint: retry_task.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await;
|
||||
let committed_b = committed_b_result.expect("checkpoint B");
|
||||
|
||||
// Attempt B never reached upstream: a before_dispatch failure must NOT overwrite
|
||||
// the recorded uncertainty of the previously dispatched attempt.
|
||||
let resolution = store
|
||||
.resolve_attempt(
|
||||
&committed_b,
|
||||
AttemptOutcome::Failed {
|
||||
error: TaskEnvelope::try_redacted(
|
||||
&json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}),
|
||||
)
|
||||
.unwrap(),
|
||||
retryable: false,
|
||||
phase: AttemptPhase::BeforeDispatch,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("resolve B");
|
||||
match resolution {
|
||||
AttemptResolution::Unknown(record) => assert_eq!(record.state, TaskState::OutcomeUnknown),
|
||||
other => panic!("expected Unknown, got {other:?}"),
|
||||
}
|
||||
|
||||
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
||||
assert!(record.result.is_none());
|
||||
let error = record.error.expect("error envelope present");
|
||||
assert_eq!(error.value()["code"], json!("shutdown_before_dispatch"));
|
||||
assert_eq!(error.value()["phase"], json!("dispatched"));
|
||||
assert_eq!(record.attempts, 2);
|
||||
|
||||
let attempts = store
|
||||
.attempts(&access(AGENT), created.task_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(attempts.len(), 2);
|
||||
assert_eq!(
|
||||
attempts[0].outcome,
|
||||
Some(AttemptOutcomeClass::RetryScheduled)
|
||||
);
|
||||
assert_eq!(
|
||||
attempts[0].certainty,
|
||||
Some(crank_core::AttemptCertainty::Unknown)
|
||||
);
|
||||
assert_eq!(attempts[0].phase, Some(AttemptPhase::Dispatched));
|
||||
assert_eq!(attempts[1].outcome, Some(AttemptOutcomeClass::Failed));
|
||||
assert_eq!(
|
||||
attempts[1].certainty,
|
||||
Some(crank_core::AttemptCertainty::Known)
|
||||
);
|
||||
assert_eq!(attempts[1].phase, Some(AttemptPhase::BeforeDispatch));
|
||||
|
||||
// Repeated read/cancel must not turn the state terminal.
|
||||
let again = store.read(&access(AGENT), created.task_id).await.unwrap();
|
||||
assert_eq!(again.state, TaskState::OutcomeUnknown);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn before_dispatch_failed_single_attempt_stays_failed() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("bd_single"), &digest("fbf"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
|
||||
let lease = leases.into_iter().next().expect("claim");
|
||||
let committed = store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-single".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.expect("checkpoint");
|
||||
|
||||
let resolution = store
|
||||
.resolve_attempt(
|
||||
&committed,
|
||||
AttemptOutcome::Failed {
|
||||
error: TaskEnvelope::try_redacted(
|
||||
&json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}),
|
||||
)
|
||||
.unwrap(),
|
||||
retryable: false,
|
||||
phase: AttemptPhase::BeforeDispatch,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("resolve");
|
||||
match resolution {
|
||||
AttemptResolution::Finished(record) => assert_eq!(record.state, TaskState::Failed),
|
||||
other => panic!("expected Finished, got {other:?}"),
|
||||
}
|
||||
|
||||
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::Failed);
|
||||
let error = record.error.expect("error envelope present");
|
||||
assert_eq!(error.value()["phase"], json!("before_dispatch"));
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//! Story 4.3 ATDD: durable Task leases, fencing, crash recovery and bounded claim behavior.
|
||||
//!
|
||||
//! Split from `tasks.rs` to keep both integration files inside the code-health limit.
|
||||
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
/// Story 4.3 ATDD: a worker crash before the durable dispatch checkpoint leaves only an expired
|
||||
/// lease. PostgreSQL is authoritative, so another worker recovers the Task and the dead owner can
|
||||
/// never commit a terminal outcome with its old fence.
|
||||
#[tokio::test]
|
||||
async fn task_worker_expired_pre_checkpoint_lease_is_recovered_and_stale_owner_cannot_commit() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("recover1"), &digest("f2"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let first = store.claim_batch("worker_dead", 1, 150).await.unwrap();
|
||||
assert_eq!(first.len(), 1);
|
||||
assert_eq!(first[0].record.task_id, created.task_id);
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
|
||||
// Recovery on the next bounded authoritative scan, with no Valkey wake-up.
|
||||
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
|
||||
let lease = recovered
|
||||
.into_iter()
|
||||
.find(|lease| lease.record.task_id == created.task_id)
|
||||
.expect("expired pre-checkpoint Task is recovered");
|
||||
assert!(lease.fence > first[0].fence, "recovery bumps the fence");
|
||||
|
||||
// The previous owner's terminal CAS is rejected; the live owner wins exactly once.
|
||||
let stale = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_state: first[0].record.state,
|
||||
expected_version: first[0].record.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"stale": true})).unwrap()),
|
||||
error: None,
|
||||
},
|
||||
"worker_dead",
|
||||
first[0].fence,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(stale, Err(TaskError::StateConflict)));
|
||||
|
||||
let finished = store
|
||||
.finish_leased(
|
||||
TaskFinishRequest {
|
||||
access: access(AGENT),
|
||||
task_id: created.task_id,
|
||||
expected_state: lease.record.state,
|
||||
expected_version: lease.record.version,
|
||||
target_state: TaskState::Completed,
|
||||
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
||||
error: None,
|
||||
},
|
||||
"worker_live",
|
||||
lease.fence,
|
||||
)
|
||||
.await
|
||||
.expect("current lease wins the terminal CAS");
|
||||
assert_eq!(finished.state, TaskState::Completed);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: after the dispatch checkpoint the outbound outcome is uncertain. An expired
|
||||
/// attempt resolves to `outcome_unknown`, is never redispatched automatically and releases the
|
||||
/// admission slot it held.
|
||||
#[tokio::test]
|
||||
async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_without_redispatch() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = store_with(registry.pool().clone(), 8);
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("uncertain1"), &digest("f3"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap();
|
||||
let lease = leases.into_iter().next().expect("claim");
|
||||
store
|
||||
.checkpoint_dispatch(BeginAttempt {
|
||||
task_id: lease.record.task_id,
|
||||
owner: "worker_dead".to_owned(),
|
||||
fence: lease.fence,
|
||||
expected_version: lease.record.version,
|
||||
attempt_id: "attempt-uncertain".to_owned(),
|
||||
input_fingerprint: lease.record.input_fingerprint.clone(),
|
||||
upstream_key_digest: None,
|
||||
retry_allowed: false,
|
||||
})
|
||||
.await
|
||||
.expect("durable dispatch checkpoint");
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
|
||||
// The authority scan resolves uncertainty instead of handing the attempt to another worker.
|
||||
let after = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
|
||||
assert!(
|
||||
after.is_empty(),
|
||||
"a checkpointed attempt must never be redispatched"
|
||||
);
|
||||
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
|
||||
assert_eq!(record.state, TaskState::OutcomeUnknown);
|
||||
assert_eq!(
|
||||
record.current_attempt_id.as_deref(),
|
||||
Some("attempt-uncertain")
|
||||
);
|
||||
let intents: i64 = sqlx::query_scalar(
|
||||
"select count(*)::bigint from execution_admission_entries where task_id = $1",
|
||||
)
|
||||
.bind(created.task_id.as_uuid())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(intents, 0, "uncertain terminal outcome releases capacity");
|
||||
|
||||
// Still no automatic retry on the following scans.
|
||||
assert!(
|
||||
store
|
||||
.claim_batch("worker_live", 4, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
/// Story 4.3 ATDD: concurrent claimers never hold the same Task at once.
|
||||
#[tokio::test]
|
||||
async fn task_worker_concurrent_claimers_never_double_lease_one_task() {
|
||||
let database = common::TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let store = std::sync::Arc::new(store_with(registry.pool().clone(), 8));
|
||||
|
||||
let created = store
|
||||
.accept(request(&key("concurrent1"), &digest("f4"), AGENT, 30_000))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut handles = Vec::new();
|
||||
for index in 0..4 {
|
||||
let store = store.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
store
|
||||
.claim_batch(&format!("worker_{index}"), 1, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
}));
|
||||
}
|
||||
let mut leased = Vec::new();
|
||||
for handle in handles {
|
||||
leased.extend(handle.await.unwrap());
|
||||
}
|
||||
assert_eq!(leased.len(), 1, "exactly one owner wins the claim");
|
||||
assert_eq!(leased[0].record.task_id, created.task_id);
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
@@ -22,9 +22,15 @@ use serde_json::{Map, Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
use tracing::{Instrument, Span, debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::task_attempt::{
|
||||
PrepareAttempt, TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
|
||||
TaskAttemptRefusal,
|
||||
};
|
||||
use crate::task_worker::TaskDispatchPermit;
|
||||
use crate::{
|
||||
AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError,
|
||||
AdapterResponse, ExecutionAuthorization, ExecutionRequestError, PreparedRequest, RuntimeError,
|
||||
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
|
||||
auth::apply_resolved_auth,
|
||||
request_preparation::{adapter_prepared_request, validate_selected_body_variant},
|
||||
@@ -34,6 +40,18 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
/// How many outbound sends one prepared request may perform.
|
||||
///
|
||||
/// `Legacy` keeps the existing synchronous `tools/call` retry behaviour inside the transport loop.
|
||||
/// `Single` performs exactly one send, because a durable Task attempt already has its own committed
|
||||
/// checkpoint and resolution around every send; it carries the pinned attempt ordinal and budget so
|
||||
/// the one `adapter_attempt` span still names the real durable attempt.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum SendMode {
|
||||
Legacy,
|
||||
Single { attempt: u32, max_attempts: u32 },
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeExecutor {
|
||||
adapters: AdapterRegistry,
|
||||
@@ -178,8 +196,305 @@ impl RuntimeExecutor {
|
||||
pub(crate) async fn execute_request(
|
||||
&self,
|
||||
request: RuntimeExecutionRequest<'_>,
|
||||
) -> Result<(Value, Value), RuntimeError> {
|
||||
self.execute_request_impl(request, None, SendMode::Legacy)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Drives the bounded, durable retry budget of one leased background Task (Story 4.4).
|
||||
///
|
||||
/// Each outbound send is preceded by its own committed attempt checkpoint and followed by one
|
||||
/// atomic resolution; the pinned budget, the safe-retry decision and the accepted absolute
|
||||
/// deadline are enforced by PostgreSQL, so a restart or a second worker cannot extend them.
|
||||
pub async fn execute_task_outcome(
|
||||
&self,
|
||||
context: &TaskAttemptContext,
|
||||
authority: &dyn TaskAttemptAuthority,
|
||||
) -> TaskAttemptOutcome {
|
||||
let store = context.store().clone();
|
||||
let lease = context.lease().clone();
|
||||
let mut record = lease.record.clone();
|
||||
loop {
|
||||
if context.is_cancelled() || context.is_stopping() {
|
||||
return TaskAttemptOutcome::Stopped;
|
||||
}
|
||||
let current_lease = crank_core::TaskLease {
|
||||
record: record.clone(),
|
||||
owner: lease.owner.clone(),
|
||||
fence: lease.fence,
|
||||
lease_until: lease.lease_until,
|
||||
};
|
||||
let plan = match authority.prepare_attempt(¤t_lease).await {
|
||||
Ok(PrepareAttempt::Ready(plan)) => *plan,
|
||||
Ok(PrepareAttempt::InputRequired(request)) => {
|
||||
// A missing required input is suspended before the dispatch checkpoint, so this
|
||||
// generation provably produced no outbound side effect. A CAS/store failure stops
|
||||
// without writing anything new; the lease expiry recovery makes it retryable.
|
||||
return match store.suspend_for_input(¤t_lease, request).await {
|
||||
Ok(_) => TaskAttemptOutcome::WaitingForInput,
|
||||
Err(crank_core::TaskError::StateConflict)
|
||||
| Err(crank_core::TaskError::NotFound)
|
||||
| Err(crank_core::TaskError::FingerprintConflict)
|
||||
| Err(crank_core::TaskError::InputExpired) => TaskAttemptOutcome::Stopped,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "input suspension failed; no outbound call");
|
||||
TaskAttemptOutcome::Stopped
|
||||
}
|
||||
};
|
||||
}
|
||||
Err(TaskAttemptRefusal::Stopped) => return TaskAttemptOutcome::Stopped,
|
||||
Err(TaskAttemptRefusal::Failed { code }) => {
|
||||
// A refused attempt never erases a proven ambiguity: if an earlier attempt of this
|
||||
// Task may already have reached the upstream, the store keeps the Task in the
|
||||
// non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
|
||||
return match store.refuse_attempt(¤t_lease, &code).await {
|
||||
Ok(record) if record.state == crank_core::TaskState::OutcomeUnknown => {
|
||||
TaskAttemptOutcome::OutcomeUnknown
|
||||
}
|
||||
_ => TaskAttemptOutcome::Failed { code },
|
||||
};
|
||||
}
|
||||
};
|
||||
let attempt_id = format!("attempt-{}", Uuid::now_v7());
|
||||
let begin = crank_core::BeginAttempt {
|
||||
task_id: record.task_id,
|
||||
owner: lease.owner.clone(),
|
||||
fence: lease.fence,
|
||||
expected_version: record.version,
|
||||
attempt_id: attempt_id.clone(),
|
||||
input_fingerprint: record.effective_input_fingerprint.clone(),
|
||||
upstream_key_digest: plan.upstream_key_digest.clone(),
|
||||
retry_allowed: plan.retry_allowed,
|
||||
};
|
||||
let attempt = match store.checkpoint_dispatch(begin).await {
|
||||
Ok(attempt) => attempt,
|
||||
// Cancellation, lease loss or a concurrent lifecycle change: nothing was dispatched.
|
||||
Err(crank_core::TaskError::StateConflict)
|
||||
| Err(crank_core::TaskError::NotFound)
|
||||
| Err(crank_core::TaskError::FingerprintConflict) => {
|
||||
return TaskAttemptOutcome::Stopped;
|
||||
}
|
||||
// An unproven commit is never retried by guess: no outbound call without a proven
|
||||
// durable checkpoint.
|
||||
Err(error) => {
|
||||
warn!(error = %error, "durable dispatch checkpoint failed; no outbound call");
|
||||
return TaskAttemptOutcome::Stopped;
|
||||
}
|
||||
};
|
||||
if context.is_stopping() {
|
||||
let code = "shutdown_before_dispatch".to_owned();
|
||||
// A stop that lands before the send is a proven failure of *this* attempt only: when
|
||||
// an earlier attempt of the same Task already lost its confirmation, the store keeps
|
||||
// the non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
|
||||
return match store
|
||||
.resolve_attempt(
|
||||
&attempt,
|
||||
crank_core::AttemptOutcome::Failed {
|
||||
error: bounded_attempt_error(
|
||||
&code,
|
||||
crank_core::AttemptPhase::BeforeDispatch,
|
||||
),
|
||||
retryable: false,
|
||||
phase: crank_core::AttemptPhase::BeforeDispatch,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(crank_core::AttemptResolution::Unknown(_)) => {
|
||||
TaskAttemptOutcome::OutcomeUnknown
|
||||
}
|
||||
_ => TaskAttemptOutcome::Failed { code },
|
||||
};
|
||||
}
|
||||
if context.is_cancelled() {
|
||||
let _ = store
|
||||
.resolve_attempt(
|
||||
&attempt,
|
||||
crank_core::AttemptOutcome::Uncertain {
|
||||
error: bounded_attempt_error(
|
||||
"outcome_unknown",
|
||||
crank_core::AttemptPhase::BeforeDispatch,
|
||||
),
|
||||
repeatable: false,
|
||||
phase: crank_core::AttemptPhase::BeforeDispatch,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return TaskAttemptOutcome::OutcomeUnknown;
|
||||
}
|
||||
// Each attempt owns exactly one `adapter_attempt` span, created around the single send
|
||||
// inside the transport loop so the accepted trace shape (durable attempt → runtime
|
||||
// execution → adapter attempt) is preserved.
|
||||
let result = self.execute_task_send(&plan, &attempt).await;
|
||||
let (outcome, code) = self.classify_task_attempt(&plan, result);
|
||||
match store.resolve_attempt(&attempt, outcome).await {
|
||||
Ok(crank_core::AttemptResolution::Finished(task)) => {
|
||||
return if task.state == crank_core::TaskState::Completed {
|
||||
TaskAttemptOutcome::Completed
|
||||
} else {
|
||||
TaskAttemptOutcome::Failed {
|
||||
code: code.unwrap_or_else(|| "task_failed".to_owned()),
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(crank_core::AttemptResolution::Unknown(_)) => {
|
||||
return TaskAttemptOutcome::OutcomeUnknown;
|
||||
}
|
||||
Ok(crank_core::AttemptResolution::RetryAt {
|
||||
task,
|
||||
next_attempt_at_unix_ms,
|
||||
}) => {
|
||||
record = task;
|
||||
let remaining =
|
||||
next_attempt_at_unix_ms.saturating_sub(crank_core::current_unix_ms());
|
||||
let delay = Duration::from_millis(u64::try_from(remaining).unwrap_or(0));
|
||||
if !self.wait_before_attempt(context, delay).await {
|
||||
return TaskAttemptOutcome::Stopped;
|
||||
}
|
||||
}
|
||||
// A stale fence or version lost the resolution CAS: another writer owns the Task now.
|
||||
Err(_) => return TaskAttemptOutcome::Stopped,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One outbound send of one committed durable attempt.
|
||||
async fn execute_task_send(
|
||||
&self,
|
||||
plan: &TaskAttemptPlan,
|
||||
attempt: &crank_core::AttemptLease,
|
||||
) -> Result<Value, RuntimeError> {
|
||||
let permit = TaskDispatchPermit::new(
|
||||
attempt.attempt.task_id,
|
||||
plan.operation.operation_id.as_str().to_owned(),
|
||||
plan.operation.operation_version,
|
||||
plan.workspace_id.as_str().to_owned(),
|
||||
plan.agent_id.as_ref().map(|id| id.as_str().to_owned()),
|
||||
attempt.attempt.owner.clone(),
|
||||
attempt.attempt.fence,
|
||||
attempt.attempt.attempt_id.clone(),
|
||||
);
|
||||
let request = match RuntimeExecutionRequest::try_new(
|
||||
&plan.workspace_id,
|
||||
plan.origin,
|
||||
plan.agent_id.as_ref(),
|
||||
&plan.operation,
|
||||
&plan.input,
|
||||
plan.authorization,
|
||||
plan.resolved_auth.as_ref(),
|
||||
&plan.request_context,
|
||||
plan.deadline,
|
||||
) {
|
||||
Ok(request) => request,
|
||||
Err(ExecutionRequestError::DeadlineElapsed) => {
|
||||
return Err(RuntimeError::ExecutionDeadlineElapsed {
|
||||
may_have_dispatched: false,
|
||||
});
|
||||
}
|
||||
Err(_) => return Err(RuntimeError::InvalidExecutionRequest),
|
||||
};
|
||||
let (output, _preview) = self
|
||||
.execute_request_impl(
|
||||
request,
|
||||
Some(permit),
|
||||
SendMode::Single {
|
||||
attempt: attempt.attempt.ordinal,
|
||||
max_attempts: attempt.task.pins.retry.max_attempts,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Maps one outbound result of a durable attempt onto its persisted attempt outcome.
|
||||
fn classify_task_attempt(
|
||||
&self,
|
||||
plan: &TaskAttemptPlan,
|
||||
result: Result<Value, RuntimeError>,
|
||||
) -> (crank_core::AttemptOutcome, Option<String>) {
|
||||
match result {
|
||||
Ok(output) => match crank_core::TaskEnvelope::try_redacted(&output) {
|
||||
Ok(envelope) => (crank_core::AttemptOutcome::Completed(envelope), None),
|
||||
Err(_) => {
|
||||
let code = "result_not_redactable".to_owned();
|
||||
(
|
||||
crank_core::AttemptOutcome::Failed {
|
||||
error: bounded_attempt_error(
|
||||
&code,
|
||||
crank_core::AttemptPhase::Dispatched,
|
||||
),
|
||||
retryable: false,
|
||||
phase: crank_core::AttemptPhase::Dispatched,
|
||||
},
|
||||
Some(code),
|
||||
)
|
||||
}
|
||||
},
|
||||
Err(error) => {
|
||||
let phase = if runtime_error_may_have_dispatched(&error) {
|
||||
crank_core::AttemptPhase::Dispatched
|
||||
} else {
|
||||
crank_core::AttemptPhase::BeforeDispatch
|
||||
};
|
||||
let correlation = crank_core::CorrelationContext::new(
|
||||
plan.request_context.request_id.clone(),
|
||||
plan.request_context.trace_context.clone(),
|
||||
);
|
||||
let failure = crate::normalize_runtime_error_for_operation(
|
||||
&error,
|
||||
&correlation,
|
||||
&plan.operation,
|
||||
);
|
||||
let code = format!("{:?}", failure.error_code());
|
||||
let safe_to_repeat = retry_is_allowed(&plan.operation, &error, plan.retry_allowed);
|
||||
let envelope = bounded_attempt_error(&code, phase);
|
||||
let outcome = match failure.outcome_certainty() {
|
||||
crank_core::OutcomeCertainty::OutcomeUnknown => {
|
||||
crank_core::AttemptOutcome::Uncertain {
|
||||
error: envelope,
|
||||
repeatable: safe_to_repeat,
|
||||
phase,
|
||||
}
|
||||
}
|
||||
_ => crank_core::AttemptOutcome::Failed {
|
||||
error: envelope,
|
||||
retryable: safe_to_repeat,
|
||||
phase,
|
||||
},
|
||||
};
|
||||
(outcome, Some(code))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded cooperative wait inside the pinned backoff window. The lease keeps being renewed by
|
||||
/// the worker, so a durable cancellation or lease loss still stops the local work here.
|
||||
async fn wait_before_attempt(&self, context: &TaskAttemptContext, delay: Duration) -> bool {
|
||||
let deadline = Instant::now() + delay;
|
||||
loop {
|
||||
if context.is_cancelled() || context.is_stopping() {
|
||||
return false;
|
||||
}
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
return true;
|
||||
}
|
||||
tokio::time::sleep((deadline - now).min(Duration::from_millis(50))).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_request_impl(
|
||||
&self,
|
||||
request: RuntimeExecutionRequest<'_>,
|
||||
permit: Option<TaskDispatchPermit>,
|
||||
send_mode: SendMode,
|
||||
) -> Result<(Value, Value), RuntimeError> {
|
||||
let operation = request.operation();
|
||||
// Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission.
|
||||
let trusted_task_dispatch = permit.is_some();
|
||||
// The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the
|
||||
// trace names the real persisted attempt instead of inventing one (Story 4.3 AC).
|
||||
let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned());
|
||||
let input = request.input();
|
||||
let request_context = request.request_context();
|
||||
debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized);
|
||||
@@ -205,7 +520,9 @@ impl RuntimeExecutor {
|
||||
let dispatch_started = Arc::new(AtomicBool::new(false));
|
||||
// True until the admission gate answered. A deadline that fires while the request is still
|
||||
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
|
||||
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
|
||||
let admission_pending = Arc::new(AtomicBool::new(
|
||||
!trusted_task_dispatch && self.admission.is_some(),
|
||||
));
|
||||
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
|
||||
metric_invocation_source(Some(request_context)),
|
||||
request_context
|
||||
@@ -222,14 +539,19 @@ impl RuntimeExecutor {
|
||||
// One admission gate for every executable path (Story 4.1): the published policy decides
|
||||
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
|
||||
// is held for the whole local execution future, including retries.
|
||||
let mut lease = crate::admission::acquire(
|
||||
self.admission.as_ref(),
|
||||
self.coordination_store.as_deref(),
|
||||
&request,
|
||||
execution_deadline,
|
||||
)
|
||||
.await?;
|
||||
admission_pending.store(false, Ordering::Release);
|
||||
let mut lease = if trusted_task_dispatch {
|
||||
None
|
||||
} else {
|
||||
let result = crate::admission::acquire(
|
||||
self.admission.as_ref(),
|
||||
self.coordination_store.as_deref(),
|
||||
&request,
|
||||
execution_deadline,
|
||||
)
|
||||
.await?;
|
||||
admission_pending.store(false, Ordering::Release);
|
||||
result
|
||||
};
|
||||
let _permit = self.acquire_unary_permit(operation)?;
|
||||
let _inflight = InFlightGuard::runtime();
|
||||
let mapping_span = Stage::RuntimeArgumentsMap.span();
|
||||
@@ -246,6 +568,8 @@ impl RuntimeExecutor {
|
||||
prepared_request,
|
||||
Some(request_context),
|
||||
Arc::clone(&dispatch_started),
|
||||
durable_attempt_id.as_deref(),
|
||||
send_mode,
|
||||
)
|
||||
.await;
|
||||
if let Some(lease) = lease.as_mut() {
|
||||
@@ -341,6 +665,7 @@ impl RuntimeExecutor {
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn execute_prepared(
|
||||
&self,
|
||||
operation: &RuntimeOperation,
|
||||
@@ -348,6 +673,8 @@ impl RuntimeExecutor {
|
||||
prepared_request: PreparedRequest,
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
dispatch_started: Arc<AtomicBool>,
|
||||
durable_attempt_id: Option<&str>,
|
||||
send_mode: SendMode,
|
||||
) -> Result<Value, RuntimeError> {
|
||||
let mut prepared_request = prepared_request;
|
||||
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
|
||||
@@ -397,7 +724,11 @@ impl RuntimeExecutor {
|
||||
approval_result?;
|
||||
}
|
||||
|
||||
let idempotency = if idempotency_applicable {
|
||||
// The synchronous coordination guard is the replay authority of one `tools/call`. A durable
|
||||
// Task attempt already has a PostgreSQL attempt authority around every send, so the guard is
|
||||
// skipped there: an ambiguous first response must not block the bounded durable retry. The
|
||||
// stable idempotency key is still delivered to the upstream by `prepare_idempotency` above.
|
||||
let idempotency = if idempotency_applicable && matches!(send_mode, SendMode::Legacy) {
|
||||
let idempotency_span = Stage::RuntimeIdempotency.span();
|
||||
let result = crate::idempotency::begin(
|
||||
self.coordination_store.as_deref(),
|
||||
@@ -464,6 +795,8 @@ impl RuntimeExecutor {
|
||||
operation,
|
||||
idempotency_key.as_deref(),
|
||||
),
|
||||
durable_attempt_id,
|
||||
send_mode,
|
||||
)
|
||||
.await;
|
||||
if let Ok(response) = &adapter_response {
|
||||
@@ -597,6 +930,7 @@ impl RuntimeExecutor {
|
||||
.map_err(|error| map_protocol_adapter_error(operation, error))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn execute_adapter_with_retry(
|
||||
&self,
|
||||
operation: &RuntimeOperation,
|
||||
@@ -604,10 +938,35 @@ impl RuntimeExecutor {
|
||||
request_context: Option<&RuntimeRequestContext>,
|
||||
dispatch_started: Arc<AtomicBool>,
|
||||
verified_idempotency: bool,
|
||||
durable_attempt_id: Option<&str>,
|
||||
send_mode: SendMode,
|
||||
) -> Result<AdapterResponse, RuntimeError> {
|
||||
let max_attempts = retry_attempts(operation);
|
||||
let mut attempt = 1;
|
||||
// `max_attempts` bounds the transport loop; `reported_max_attempts` is the pinned Task budget
|
||||
// the attempt span names. A durable attempt performs exactly one send, so its transport loop
|
||||
// always has a single iteration even when the Task may retry later.
|
||||
let (max_attempts, first_attempt, reported_max_attempts) = match send_mode {
|
||||
SendMode::Single {
|
||||
attempt,
|
||||
max_attempts,
|
||||
} => (1, attempt, max_attempts),
|
||||
SendMode::Legacy => {
|
||||
let attempts = retry_attempts(operation);
|
||||
(attempts, 1, attempts)
|
||||
}
|
||||
};
|
||||
let mut attempt = first_attempt;
|
||||
loop {
|
||||
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint. Only
|
||||
// paths without a durable attempt (synchronous tools/call) mint a synthetic identity.
|
||||
let attempt_id = durable_attempt_id
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7()));
|
||||
let attempt_span = tracing::info_span!(
|
||||
"adapter_attempt",
|
||||
attempt,
|
||||
max_attempts = reported_max_attempts,
|
||||
attempt_id = %attempt_id
|
||||
);
|
||||
let result = self
|
||||
.execute_adapter(
|
||||
operation,
|
||||
@@ -615,6 +974,7 @@ impl RuntimeExecutor {
|
||||
request_context,
|
||||
Arc::clone(&dispatch_started),
|
||||
)
|
||||
.instrument(attempt_span)
|
||||
.await;
|
||||
match result {
|
||||
Ok(response) => return Ok(response),
|
||||
@@ -756,6 +1116,22 @@ fn sanitized_request_preview(prepared: &PreparedRequest) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
/// Bounded persisted error envelope of one durable attempt: a closed code and the attempt phase.
|
||||
/// It never carries an upstream body, status text, URL or credential.
|
||||
fn bounded_attempt_error(code: &str, phase: crank_core::AttemptPhase) -> crank_core::TaskEnvelope {
|
||||
crank_core::TaskEnvelope::try_redacted(&json!({
|
||||
"code": code,
|
||||
"phase": phase.as_str(),
|
||||
}))
|
||||
.unwrap_or_else(|_| {
|
||||
crank_core::TaskEnvelope::try_redacted(&json!({
|
||||
"code": "task_error_not_recordable",
|
||||
"phase": phase.as_str(),
|
||||
}))
|
||||
.expect("bounded attempt error envelope is always valid")
|
||||
})
|
||||
}
|
||||
|
||||
fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
|
||||
@@ -15,6 +15,8 @@ mod rate_limit;
|
||||
mod request_context;
|
||||
mod request_preparation;
|
||||
mod response_cache;
|
||||
mod task_attempt;
|
||||
mod task_worker;
|
||||
mod tasks;
|
||||
pub fn validate_resource_template_parameter(
|
||||
schema: &crank_schema::Schema,
|
||||
@@ -115,4 +117,11 @@ pub use rate_limit::{
|
||||
};
|
||||
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
|
||||
pub use secret_crypto::SecretCrypto;
|
||||
pub use task_attempt::{
|
||||
PrepareAttempt, TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
|
||||
TaskAttemptRefusal, missing_required_input,
|
||||
};
|
||||
pub use task_worker::{
|
||||
TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
|
||||
};
|
||||
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
|
||||
@@ -0,0 +1,140 @@
|
||||
//! Durable per-outbound attempt execution of one background Task (Story 4.4).
|
||||
//!
|
||||
//! The authoritative state of a Task stays in PostgreSQL. This module only carries the owned
|
||||
//! execution plan of one attempt, the durable control (store + lease + cooperative stop) and the
|
||||
//! bounded seam the application layer implements to revalidate authority before each attempt.
|
||||
//! The single retry loop lives in [`crate::RuntimeExecutor::execute_task_outcome`]; there is no second
|
||||
//! loop in the worker and no retry inside the transport adapter.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::Instant;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{AgentId, ExecutionOrigin, TaskLease, TaskStore, WorkspaceId};
|
||||
|
||||
use crate::{ExecutionAuthorization, ResolvedAuth, RuntimeOperation, RuntimeRequestContext};
|
||||
|
||||
/// Owned execution plan of one durable attempt.
|
||||
///
|
||||
/// It is rebuilt by the application through [`TaskAttemptAuthority::prepare_attempt`] before every
|
||||
/// attempt, so a republished catalog, a revoked key or an elapsed deadline is revalidated between two
|
||||
/// attempts instead of being trusted from the first one.
|
||||
pub struct TaskAttemptPlan {
|
||||
pub workspace_id: WorkspaceId,
|
||||
pub origin: ExecutionOrigin,
|
||||
pub agent_id: Option<AgentId>,
|
||||
pub operation: RuntimeOperation,
|
||||
pub input: serde_json::Value,
|
||||
pub authorization: ExecutionAuthorization,
|
||||
pub resolved_auth: Option<ResolvedAuth>,
|
||||
pub request_context: RuntimeRequestContext,
|
||||
/// Absolute deadline pinned at acceptance; never re-derived from a later publication.
|
||||
pub deadline: Instant,
|
||||
/// Safe-to-retry decision of this attempt: a read, or a write whose upstream idempotency is
|
||||
/// backed by server-owned evidence. A client-supplied key or header never makes this true.
|
||||
pub retry_allowed: bool,
|
||||
/// Digest of the stable upstream idempotency key (never the raw key or a header value).
|
||||
pub upstream_key_digest: Option<String>,
|
||||
}
|
||||
|
||||
/// Application-owned seam: revalidate authority and prepare exactly one durable attempt.
|
||||
#[async_trait]
|
||||
pub trait TaskAttemptAuthority: Send + Sync {
|
||||
async fn prepare_attempt(
|
||||
&self,
|
||||
lease: &TaskLease,
|
||||
) -> Result<PrepareAttempt, TaskAttemptRefusal>;
|
||||
}
|
||||
|
||||
/// Typed preparation of one attempt: either it is ready to be checkpointed and dispatched, or the
|
||||
/// pinned operation still lacks a required input and the Task must be suspended before any side
|
||||
/// effect. The two are explicit so a real authority failure is never turned into a user question.
|
||||
pub enum PrepareAttempt {
|
||||
Ready(Box<TaskAttemptPlan>),
|
||||
InputRequired(crank_core::TaskInputRequest),
|
||||
}
|
||||
|
||||
/// Path of a required item the pinned schema still misses before dispatch.
|
||||
///
|
||||
/// This is the runtime-owned decision the application calls before checkpointing an attempt: only a
|
||||
/// typed missing required field becomes a user question, every other schema violation keeps the
|
||||
/// existing pre-dispatch attempt validation. Exposed as a plain value so the application layer never
|
||||
/// maps execution validation itself (execution boundary).
|
||||
pub fn missing_required_input(
|
||||
schema: &crank_schema::Schema,
|
||||
input: &serde_json::Value,
|
||||
) -> Option<String> {
|
||||
match schema.validate_shape(input) {
|
||||
Err(crank_schema::SchemaError::MissingField { path }) => Some(path),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Typed refusal of one attempt preparation.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum TaskAttemptRefusal {
|
||||
/// Proven refusal before any dispatch of this attempt; the bounded code is persisted.
|
||||
Failed { code: String },
|
||||
/// Durable control changed (cancellation, lease loss, shutdown): write nothing new from here.
|
||||
Stopped,
|
||||
}
|
||||
|
||||
/// Durable control of one leased Task: the store, the leased record and cooperative stop signals.
|
||||
#[derive(Clone)]
|
||||
pub struct TaskAttemptContext {
|
||||
pub(crate) store: Arc<dyn TaskStore>,
|
||||
pub(crate) lease: TaskLease,
|
||||
pub(crate) cancelled: Arc<AtomicBool>,
|
||||
pub(crate) stopping: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl TaskAttemptContext {
|
||||
pub(crate) fn new(
|
||||
store: Arc<dyn TaskStore>,
|
||||
lease: TaskLease,
|
||||
cancelled: Arc<AtomicBool>,
|
||||
stopping: Arc<AtomicBool>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
lease,
|
||||
cancelled,
|
||||
stopping,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn store(&self) -> &Arc<dyn TaskStore> {
|
||||
&self.store
|
||||
}
|
||||
|
||||
pub fn lease(&self) -> &TaskLease {
|
||||
&self.lease
|
||||
}
|
||||
|
||||
/// Cooperative stop: the lease renewal or an observed durable cancellation set this flag.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.cancelled.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Process shutdown was requested: no new outbound send may start after the current checkpoint.
|
||||
pub fn is_stopping(&self) -> bool {
|
||||
self.stopping.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of driving the whole pinned retry budget of one leased Task.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum TaskAttemptOutcome {
|
||||
/// The Task committed `completed` (exactly one terminal write).
|
||||
Completed,
|
||||
/// The Task committed `failed` without dispatch or with a proven failure.
|
||||
Failed { code: String },
|
||||
/// The Task is non-terminal `outcome_unknown`; it is never retried or cancelled generically.
|
||||
OutcomeUnknown,
|
||||
/// The Task was suspended as `input_required` before its first dispatch: no side effect happened
|
||||
/// and the worker future must stop; the answer is applied through the application entry.
|
||||
WaitingForInput,
|
||||
/// Nothing new was written by this worker: cancellation, lease loss or an unavailable store.
|
||||
Stopped,
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
//! Fenced durable Task worker loop for Story 4.3.
|
||||
//!
|
||||
//! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded,
|
||||
//! Valkey may only wake it up (never represented here as correctness), and every lease-protected
|
||||
//! write goes through CAS using owner + fencing token. The actual executable input and revalidation
|
||||
//! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a
|
||||
//! redacted projection and is never dispatched.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_core::{TaskError, TaskLease, TaskStore};
|
||||
use tokio::sync::watch;
|
||||
use tokio::task::JoinSet;
|
||||
use tracing::{Instrument, debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::task_attempt::{TaskAttemptContext, TaskAttemptOutcome};
|
||||
|
||||
/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct TaskWorkerConfig {
|
||||
/// Stable random-ish owner identity for this process instance. Required lease owner.
|
||||
pub owner: String,
|
||||
/// Maximum number of Tasks claimed in one bounded PG scan.
|
||||
pub batch_limit: u32,
|
||||
/// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used.
|
||||
pub lease_ms: i64,
|
||||
/// Authoritative PG polling interval when no wake-up hint is available.
|
||||
pub poll_interval: Duration,
|
||||
/// Renewal cadence while processing a lease.
|
||||
pub renewal_interval: Duration,
|
||||
/// Concurrent in-flight Tasks for this worker.
|
||||
pub max_inflight: usize,
|
||||
/// Maximum time to wait for active work during graceful shutdown.
|
||||
pub shutdown_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for TaskWorkerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
owner: format!("task-worker-{}", Uuid::now_v7()),
|
||||
batch_limit: 8,
|
||||
lease_ms: 30_000,
|
||||
poll_interval: Duration::from_millis(250),
|
||||
renewal_interval: Duration::from_millis(5_000),
|
||||
max_inflight: 4,
|
||||
shutdown_timeout: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Application-owned seam for pre-dispatch revalidation and execution.
|
||||
///
|
||||
/// The worker owns lease renewal/CAS/fencing and passes a durable [`TaskAttemptContext`]. The
|
||||
/// application implements [`crate::TaskAttemptAuthority`] and drives the pinned retry budget through
|
||||
/// [`crate::RuntimeExecutor::execute_task_outcome`]; it must not trust `TaskRecord.input` as
|
||||
/// executable payload.
|
||||
#[async_trait]
|
||||
pub trait TaskHandler: Send + Sync {
|
||||
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>;
|
||||
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome;
|
||||
}
|
||||
|
||||
/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and
|
||||
/// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and
|
||||
/// it binds one durable Task attempt to its execution request.
|
||||
pub struct TaskDispatchPermit {
|
||||
task_id: crank_core::TaskId,
|
||||
operation_id: String,
|
||||
operation_version: u32,
|
||||
workspace_id: String,
|
||||
agent_id: Option<String>,
|
||||
owner: String,
|
||||
fence: u64,
|
||||
attempt_id: String,
|
||||
consumed: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl TaskDispatchPermit {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn new(
|
||||
task_id: crank_core::TaskId,
|
||||
operation_id: String,
|
||||
operation_version: u32,
|
||||
workspace_id: String,
|
||||
agent_id: Option<String>,
|
||||
owner: String,
|
||||
fence: u64,
|
||||
attempt_id: String,
|
||||
) -> Self {
|
||||
Self {
|
||||
task_id,
|
||||
operation_id,
|
||||
operation_version,
|
||||
workspace_id,
|
||||
agent_id,
|
||||
owner,
|
||||
fence,
|
||||
attempt_id,
|
||||
consumed: std::sync::atomic::AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn task_id(&self) -> crank_core::TaskId {
|
||||
self.task_id
|
||||
}
|
||||
|
||||
pub fn operation_id(&self) -> &str {
|
||||
&self.operation_id
|
||||
}
|
||||
|
||||
pub fn operation_version(&self) -> u32 {
|
||||
self.operation_version
|
||||
}
|
||||
|
||||
pub fn workspace_id(&self) -> &str {
|
||||
&self.workspace_id
|
||||
}
|
||||
|
||||
pub fn agent_id(&self) -> Option<&str> {
|
||||
self.agent_id.as_deref()
|
||||
}
|
||||
|
||||
pub fn owner(&self) -> &str {
|
||||
&self.owner
|
||||
}
|
||||
|
||||
pub fn fence(&self) -> u64 {
|
||||
self.fence
|
||||
}
|
||||
|
||||
pub fn attempt_id(&self) -> &str {
|
||||
&self.attempt_id
|
||||
}
|
||||
|
||||
pub fn is_consumed(&self) -> bool {
|
||||
self.consumed.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn consume(&self) -> bool {
|
||||
!self
|
||||
.consumed
|
||||
.swap(true, std::sync::atomic::Ordering::AcqRel)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum TaskWorkerError {
|
||||
#[error("task store error: {0}")]
|
||||
Store(#[from] TaskError),
|
||||
#[error("invalid worker configuration: {0}")]
|
||||
Config(String),
|
||||
#[error("task terminal transition CAS failed")]
|
||||
CasRejected,
|
||||
}
|
||||
|
||||
pub struct TaskWorker {
|
||||
store: Arc<dyn TaskStore>,
|
||||
handler: Arc<dyn TaskHandler>,
|
||||
config: TaskWorkerConfig,
|
||||
stopping: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl TaskWorker {
|
||||
pub fn new(
|
||||
store: Arc<dyn TaskStore>,
|
||||
handler: Arc<dyn TaskHandler>,
|
||||
config: TaskWorkerConfig,
|
||||
) -> Result<Self, TaskWorkerError> {
|
||||
if config.owner.is_empty() {
|
||||
return Err(TaskWorkerError::Config(
|
||||
"owner must not be empty".to_owned(),
|
||||
));
|
||||
}
|
||||
if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 {
|
||||
return Err(TaskWorkerError::Config(
|
||||
"batch_limit must be 1..=64 and lease_ms must be positive".to_owned(),
|
||||
));
|
||||
}
|
||||
if config.lease_ms > 30_000
|
||||
|| config.poll_interval.is_zero()
|
||||
|| config.poll_interval > Duration::from_secs(30)
|
||||
|| config.renewal_interval.is_zero()
|
||||
|| config.renewal_interval >= Duration::from_millis(config.lease_ms as u64)
|
||||
|| config.max_inflight == 0
|
||||
|| config.shutdown_timeout.is_zero()
|
||||
|| config.shutdown_timeout > Duration::from_secs(30)
|
||||
{
|
||||
return Err(TaskWorkerError::Config(
|
||||
"lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds"
|
||||
.to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
store,
|
||||
handler,
|
||||
config,
|
||||
stopping: Arc::new(AtomicBool::new(false)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled.
|
||||
pub fn stop(&self) {
|
||||
self.stopping.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Returns a handle whose drop does not stop the worker; call `stop()` on the worker.
|
||||
pub fn stop_flag(&self) -> Arc<AtomicBool> {
|
||||
self.stopping.clone()
|
||||
}
|
||||
|
||||
/// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan.
|
||||
pub async fn run(&self) -> Result<(), TaskWorkerError> {
|
||||
self.run_inner(None).await
|
||||
}
|
||||
|
||||
/// Run until either `stop()` or the provided shutdown receiver becomes true.
|
||||
pub async fn run_until_shutdown(
|
||||
&self,
|
||||
mut shutdown: watch::Receiver<bool>,
|
||||
) -> Result<(), TaskWorkerError> {
|
||||
self.run_inner(Some(&mut shutdown)).await
|
||||
}
|
||||
|
||||
async fn run_inner(
|
||||
&self,
|
||||
mut shutdown: Option<&mut watch::Receiver<bool>>,
|
||||
) -> Result<(), TaskWorkerError> {
|
||||
let mut active: JoinSet<()> = JoinSet::new();
|
||||
loop {
|
||||
if self.stopping.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
if let Some(rx) = shutdown.as_mut()
|
||||
&& *rx.borrow()
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Reap finished attempts before claiming more.
|
||||
while active.try_join_next().is_some() {}
|
||||
|
||||
// The bounded TTL sweep runs on every poll, including when all local slots are busy: an
|
||||
// expired input wait must end `failed` with an audit reason even without free capacity.
|
||||
match self.store.expire_input_waits(self.config.batch_limit).await {
|
||||
Ok(expired) if expired > 0 => {
|
||||
debug!(expired, "expired input waits were failed")
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "input TTL sweep failed; retrying on next scan")
|
||||
}
|
||||
}
|
||||
|
||||
let available = self.config.max_inflight.saturating_sub(active.len());
|
||||
if available > 0 {
|
||||
let batch_limit = self
|
||||
.config
|
||||
.batch_limit
|
||||
.min(u32::try_from(available).unwrap_or(u32::MAX));
|
||||
match self
|
||||
.store
|
||||
.claim_batch(&self.config.owner, batch_limit, self.config.lease_ms)
|
||||
.await
|
||||
{
|
||||
Ok(leases) => {
|
||||
for lease in leases {
|
||||
let store = self.store.clone();
|
||||
let handler = self.handler.clone();
|
||||
let owner = self.config.owner.clone();
|
||||
let lease_ms = self.config.lease_ms;
|
||||
let renewal_interval = self.config.renewal_interval;
|
||||
let stopping = self.stopping.clone();
|
||||
active.spawn(async move {
|
||||
process_lease(
|
||||
store,
|
||||
handler,
|
||||
owner,
|
||||
lease,
|
||||
lease_ms,
|
||||
renewal_interval,
|
||||
stopping,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "bounded task claim failed; retrying on next scan");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(self.config.poll_interval) => {}
|
||||
_ = async {
|
||||
if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await }
|
||||
} => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded.
|
||||
// The same stop flag reaches cooperative cancellation and lease-renewal loops.
|
||||
self.stopping.store(true, Ordering::Release);
|
||||
let deadline = Instant::now() + self.config.shutdown_timeout;
|
||||
while !active.is_empty() {
|
||||
if Instant::now() >= deadline {
|
||||
warn!("graceful shutdown timeout reached; aborting remaining task attempts");
|
||||
active.abort_all();
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
_ = active.join_next() => {}
|
||||
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_lease(
|
||||
store: Arc<dyn TaskStore>,
|
||||
handler: Arc<dyn TaskHandler>,
|
||||
owner: String,
|
||||
lease: TaskLease,
|
||||
lease_ms: i64,
|
||||
renewal_interval: Duration,
|
||||
stopping: Arc<AtomicBool>,
|
||||
) {
|
||||
let task_id = lease.record.task_id;
|
||||
let fence = lease.fence;
|
||||
let cancelled = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Pre-dispatch revalidation under the fresh lease.
|
||||
if let Err(error) = handler.revalidate(&lease).await {
|
||||
// A transient authority-store outage is not evidence that the work is invalid: leave the
|
||||
// lease to expire so the bounded recovery scan can retry it without dispatching now.
|
||||
if matches!(
|
||||
&error,
|
||||
TaskWorkerError::Store(TaskError::Unavailable { .. })
|
||||
) {
|
||||
debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery");
|
||||
return;
|
||||
}
|
||||
debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch");
|
||||
finish_failed(&store, &lease, "revalidation_failed").await;
|
||||
return;
|
||||
}
|
||||
if stopping.load(Ordering::Acquire) {
|
||||
// Graceful shutdown before the durable dispatch checkpoint: no outbound call can have
|
||||
// happened, so the Task is left leased. The bounded recovery scan requeues it after the
|
||||
// lease expires instead of turning a never-started attempt into a user-visible failure.
|
||||
return;
|
||||
}
|
||||
|
||||
let attempt_id = format!("attempt-{}", Uuid::now_v7());
|
||||
|
||||
// Background lease renewal covers the whole bounded retry budget, including the pinned backoff.
|
||||
// A durable cancellation clears the lease, so the next renewal fails and the local work stops.
|
||||
let renewal_store = store.clone();
|
||||
let renewal_owner = owner.clone();
|
||||
let renewal_cancelled = cancelled.clone();
|
||||
let renewal_stopping = stopping.clone();
|
||||
let renewal_task_id = task_id;
|
||||
let renewal_handle = tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(renewal_interval);
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if renewal_cancelled.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
if renewal_stopping.load(Ordering::Acquire) {
|
||||
renewal_cancelled.store(true, Ordering::Release);
|
||||
break;
|
||||
}
|
||||
if renewal_store
|
||||
.renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
renewal_cancelled.store(true, Ordering::Release);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// The parent span names the leased Task generation; every individual outbound attempt adds its own
|
||||
// `adapter_attempt` child span with the durable attempt ID committed by its checkpoint.
|
||||
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, lease_ref = %attempt_id, fence);
|
||||
if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) {
|
||||
crank_trace::set_parent_from_trace_context(&span, &parent);
|
||||
}
|
||||
// Every durable write of this Task (checkpoint, attempt resolution, terminal state) happens inside
|
||||
// the executor's single retry loop under this lease/fence; the worker only renews and stops.
|
||||
let context = TaskAttemptContext::new(
|
||||
store.clone(),
|
||||
lease.clone(),
|
||||
cancelled.clone(),
|
||||
stopping.clone(),
|
||||
);
|
||||
let outcome = handler.execute(context).instrument(span).await;
|
||||
|
||||
cancelled.store(true, Ordering::Release);
|
||||
renewal_handle.abort();
|
||||
|
||||
match outcome {
|
||||
TaskAttemptOutcome::Completed => {
|
||||
debug!(task_id = %task_id, "durable task completed");
|
||||
}
|
||||
TaskAttemptOutcome::Failed { code } => {
|
||||
debug!(task_id = %task_id, code = %code, "durable task failed");
|
||||
}
|
||||
TaskAttemptOutcome::OutcomeUnknown => {
|
||||
warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry");
|
||||
}
|
||||
TaskAttemptOutcome::WaitingForInput => {
|
||||
debug!(task_id = %task_id, "durable task suspended until validated input arrives");
|
||||
}
|
||||
TaskAttemptOutcome::Stopped => {
|
||||
debug!(task_id = %task_id, "durable task attempt stopped without a new outcome");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_failed(store: &Arc<dyn TaskStore>, lease: &TaskLease, code: &str) {
|
||||
// The store owns the fate of a refused pre-dispatch attempt: a Task that already carries a proven
|
||||
// ambiguity still ends `outcome_unknown`, never a plain failure (Story 4.4).
|
||||
let _ = store.refuse_attempt(lease, code).await;
|
||||
}
|
||||
@@ -13,11 +13,14 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use crank_core::PlatformApiKeyId;
|
||||
use crank_core::{
|
||||
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_MAX_RETENTION_MS,
|
||||
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
|
||||
TaskError, TaskId, TaskOwnerKind, TaskPins, TaskRecord, TaskStore, TaskSupportIds,
|
||||
ToolExecutionMode, current_unix_ms, validate_task_envelope,
|
||||
CorrelationContext, TASK_DEFAULT_RETENTION_MS, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
|
||||
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_SCOPED_KEY_MAX_BYTES,
|
||||
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError, TaskId,
|
||||
TaskInputUpdateOutcome, TaskInputUpdateRequest, TaskLease, TaskOwnerKind, TaskPins, TaskRecord,
|
||||
TaskStore, TaskSupportIds, ToolExecutionMode, contains_sensitive_task_keys, current_unix_ms,
|
||||
validate_task_envelope,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -29,6 +32,9 @@ pub struct TaskAcceptDraft {
|
||||
pub owner_id: String,
|
||||
/// Server-derived authorization scope identity of the verified principal.
|
||||
pub authorization_scope: String,
|
||||
/// Typed immutable Platform API key identity of the verified machine credential. It is sealed
|
||||
/// into the executable-input binding and revalidated authoritatively by the worker.
|
||||
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
||||
pub operation_id: String,
|
||||
pub operation_version: u32,
|
||||
pub policy_mode: ToolExecutionMode,
|
||||
@@ -42,6 +48,8 @@ pub struct TaskAcceptDraft {
|
||||
/// Retention after the execution deadline.
|
||||
pub retention_ms: i64,
|
||||
pub approval_reference: Option<String>,
|
||||
/// Bounded retry snapshot pinned from the exact published operation at acceptance.
|
||||
pub retry: crank_core::TaskRetryPolicy,
|
||||
}
|
||||
|
||||
/// Projection of the durable status/result read. `result`/`error` are only present once a terminal
|
||||
@@ -68,11 +76,22 @@ impl TaskOutcome {
|
||||
#[derive(Clone)]
|
||||
pub struct TaskService {
|
||||
store: Arc<dyn TaskStore>,
|
||||
crypto: Option<Arc<crate::SecretCrypto>>,
|
||||
}
|
||||
|
||||
impl TaskService {
|
||||
pub fn new(store: Arc<dyn TaskStore>) -> Self {
|
||||
Self { store }
|
||||
Self {
|
||||
store,
|
||||
crypto: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_secret_crypto(store: Arc<dyn TaskStore>, crypto: Arc<crate::SecretCrypto>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
crypto: Some(crypto),
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness of the mandatory durable Task state.
|
||||
@@ -83,18 +102,61 @@ impl TaskService {
|
||||
/// Durable acceptance: the returned identity exists only after the store committed it.
|
||||
pub async fn accept(&self, draft: TaskAcceptDraft) -> Result<TaskAcceptance, TaskError> {
|
||||
validate_task_envelope(&draft.input)?;
|
||||
if contains_sensitive_task_keys(&draft.input) {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input contains sensitive fields",
|
||||
));
|
||||
}
|
||||
let (scoped_key, input_fingerprint) = (
|
||||
scoped_key_digest(&draft)?,
|
||||
canonical_input_fingerprint(&draft.input)?,
|
||||
);
|
||||
let execution_deadline_unix_ms = deadline_unix_ms(draft.deadline);
|
||||
let retention_ms = draft.retention_ms.clamp(0, TASK_MAX_RETENTION_MS);
|
||||
let task_id = TaskId::generate();
|
||||
let crypto = self.crypto.as_ref().ok_or_else(|| {
|
||||
TaskError::invalid("secret crypto is required for durable Task acceptance")
|
||||
})?;
|
||||
let binding = serde_json::json!({
|
||||
"task_id": task_id.to_string(),
|
||||
"workspace_id": draft.workspace_id,
|
||||
"owner_kind": draft.owner_kind.as_str(),
|
||||
"owner_id": draft.owner_id,
|
||||
"authorization_scope": draft.authorization_scope,
|
||||
"platform_api_key_id": draft
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(PlatformApiKeyId::as_str),
|
||||
"operation_id": draft.operation_id,
|
||||
"operation_version": draft.operation_version,
|
||||
"policy_mode": draft.policy_mode,
|
||||
"input_fingerprint": input_fingerprint,
|
||||
"input": draft.input,
|
||||
});
|
||||
let ciphertext = crypto
|
||||
.encrypt(&binding)
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
|
||||
return Err(TaskError::invalid(
|
||||
"encrypted task input exceeds its byte budget",
|
||||
));
|
||||
}
|
||||
let (
|
||||
executable_input_ciphertext,
|
||||
executable_input_key_version,
|
||||
executable_input_master_key_epoch,
|
||||
) = (
|
||||
Some(ciphertext),
|
||||
Some(crypto.key_version().to_owned()),
|
||||
Some(crypto.master_key_epoch()),
|
||||
);
|
||||
let request = TaskAcceptRequest {
|
||||
task_id: TaskId::generate(),
|
||||
task_id,
|
||||
workspace_id: draft.workspace_id,
|
||||
owner_kind: draft.owner_kind,
|
||||
owner_id: draft.owner_id,
|
||||
authorization_scope: draft.authorization_scope,
|
||||
platform_api_key_id: draft.platform_api_key_id,
|
||||
operation_id: draft.operation_id,
|
||||
pins: TaskPins {
|
||||
operation_version: draft.operation_version,
|
||||
@@ -102,6 +164,7 @@ impl TaskService {
|
||||
execution_deadline_unix_ms,
|
||||
retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms),
|
||||
approval_reference: draft.approval_reference,
|
||||
retry: draft.retry,
|
||||
},
|
||||
scoped_key,
|
||||
input_fingerprint,
|
||||
@@ -110,6 +173,9 @@ impl TaskService {
|
||||
draft.correlation.request_id().clone(),
|
||||
draft.correlation.trace_context(),
|
||||
),
|
||||
executable_input_ciphertext,
|
||||
executable_input_key_version,
|
||||
executable_input_master_key_epoch,
|
||||
};
|
||||
self.store.accept(request).await
|
||||
}
|
||||
@@ -134,6 +200,144 @@ impl TaskService {
|
||||
))
|
||||
}
|
||||
|
||||
/// Owner-scoped cancellation of one accepted Task. The store decides under a row lock whether the
|
||||
/// Task can still be cancelled or whether an already checkpointed attempt makes the outcome
|
||||
/// uncertain; the service only carries the request and a bounded reason.
|
||||
pub async fn cancel(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
expected_version: Option<u32>,
|
||||
reason: &str,
|
||||
) -> Result<crank_core::TaskCancelOutcome, TaskError> {
|
||||
let request = crank_core::TaskCancelRequest {
|
||||
access: access.clone(),
|
||||
task_id,
|
||||
expected_version,
|
||||
reason: reason.to_owned(),
|
||||
};
|
||||
request.validate()?;
|
||||
self.store.cancel(request).await
|
||||
}
|
||||
|
||||
/// Bounded owner-scoped attempt history of one Task.
|
||||
pub async fn attempts(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
) -> Result<Vec<crank_core::TaskAttempt>, TaskError> {
|
||||
self.store.attempts(access, task_id).await
|
||||
}
|
||||
|
||||
/// Decrypts and validates the executable input bound to a valid current lease.
|
||||
pub async fn executable_input(
|
||||
&self,
|
||||
lease: &TaskLease,
|
||||
) -> Result<serde_json::Value, TaskError> {
|
||||
let crypto = self.crypto.as_ref().ok_or_else(|| {
|
||||
TaskError::invalid("secret crypto is required for durable Task execution")
|
||||
})?;
|
||||
let (ciphertext, key_version, epoch) = self
|
||||
.store
|
||||
.read_executable_input(&lease.owner, lease.record.task_id, lease.fence)
|
||||
.await?;
|
||||
if ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input ciphertext is invalid",
|
||||
));
|
||||
}
|
||||
let binding = crypto
|
||||
.decrypt_for_epoch(&key_version, epoch, &ciphertext)
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
let object = binding
|
||||
.as_object()
|
||||
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
|
||||
let check_string = |field: &str, expected: &str| -> Result<(), TaskError> {
|
||||
match object.get(field).and_then(|value| value.as_str()) {
|
||||
Some(actual) if actual == expected => Ok(()),
|
||||
_ => Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
};
|
||||
check_string("task_id", &lease.record.task_id.to_string())?;
|
||||
check_string("workspace_id", &lease.record.workspace_id)?;
|
||||
check_string("owner_id", &lease.record.owner_id)?;
|
||||
check_string("authorization_scope", &lease.record.authorization_scope)?;
|
||||
let expected_key = lease
|
||||
.record
|
||||
.platform_api_key_id
|
||||
.as_ref()
|
||||
.map(PlatformApiKeyId::as_str);
|
||||
match (object.get("platform_api_key_id"), expected_key) {
|
||||
(None | Some(serde_json::Value::Null), None) => {}
|
||||
(Some(serde_json::Value::String(actual)), Some(expected)) if actual == expected => {}
|
||||
_ => {
|
||||
return Err(TaskError::invalid("task executable input binding mismatch"));
|
||||
}
|
||||
}
|
||||
check_string("operation_id", &lease.record.operation_id)?;
|
||||
match object.get("owner_kind").and_then(|value| value.as_str()) {
|
||||
Some(actual) if actual == lease.record.owner_kind.as_str() => {}
|
||||
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
match object
|
||||
.get("operation_version")
|
||||
.and_then(|value| value.as_u64())
|
||||
{
|
||||
Some(actual) if actual == u64::from(lease.record.pins.operation_version) => {}
|
||||
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
let expected_policy = match lease.record.pins.policy_mode {
|
||||
ToolExecutionMode::Synchronous => "synchronous",
|
||||
ToolExecutionMode::Background => "background",
|
||||
ToolExecutionMode::Adaptive => "adaptive",
|
||||
};
|
||||
match object.get("policy_mode").and_then(|value| value.as_str()) {
|
||||
Some(actual) if actual == expected_policy => {}
|
||||
_ => return Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
check_string("input_fingerprint", &lease.record.input_fingerprint)?;
|
||||
let base_input = object
|
||||
.get("input")
|
||||
.cloned()
|
||||
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
|
||||
if canonical_input_fingerprint(&base_input)? != lease.record.input_fingerprint {
|
||||
return Err(TaskError::invalid("task executable input binding mismatch"));
|
||||
}
|
||||
// A committed continuation replaces only the effective payload. The acceptance fingerprint and
|
||||
// the original input stay in the binding, so an original accept replay is still a replay.
|
||||
let effective_input = match object.get("continuation") {
|
||||
Some(continuation) => {
|
||||
let continuation = continuation
|
||||
.as_object()
|
||||
.ok_or_else(|| TaskError::invalid("task input continuation is invalid"))?;
|
||||
let effective = continuation
|
||||
.get("effective_input_fingerprint")
|
||||
.and_then(|value| value.as_str())
|
||||
.ok_or_else(|| TaskError::invalid("task input continuation is invalid"))?;
|
||||
if effective != lease.record.effective_input_fingerprint {
|
||||
return Err(TaskError::invalid(
|
||||
"task input continuation fingerprint mismatch",
|
||||
));
|
||||
}
|
||||
let input = continuation.get("input").cloned().ok_or_else(|| {
|
||||
TaskError::invalid("task input continuation is missing input")
|
||||
})?;
|
||||
if canonical_input_fingerprint(&input)? != effective {
|
||||
return Err(TaskError::invalid("task input continuation is invalid"));
|
||||
}
|
||||
input
|
||||
}
|
||||
None => {
|
||||
if lease.record.effective_input_fingerprint != lease.record.input_fingerprint {
|
||||
return Err(TaskError::invalid("task input continuation is missing"));
|
||||
}
|
||||
base_input
|
||||
}
|
||||
};
|
||||
validate_executable_payload(&effective_input)?;
|
||||
Ok(effective_input)
|
||||
}
|
||||
|
||||
/// Default retention pinned by the application layer when it does not choose one explicitly.
|
||||
pub fn default_retention_ms() -> i64 {
|
||||
TASK_DEFAULT_RETENTION_MS
|
||||
@@ -159,6 +363,105 @@ impl TaskService {
|
||||
pub fn input_fingerprint(input: &serde_json::Value) -> Result<String, TaskError> {
|
||||
canonical_input_fingerprint(input)
|
||||
}
|
||||
|
||||
/// Applies one additional validated input to a suspended Task (Story 4.5).
|
||||
///
|
||||
/// The stored native schema snapshot is the only validation authority; the canonical digest binds
|
||||
/// the effective payload, and the existing encrypted executable-input binding is extended with a
|
||||
/// continuation segment instead of a second cipher store. The store transaction owns authority,
|
||||
/// receipt, generation, TTL and CAS, so an identical replay returns the committed outcome.
|
||||
pub async fn update_input(
|
||||
&self,
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
wait_id: &str,
|
||||
expected_version: u32,
|
||||
update_key: &str,
|
||||
input: &serde_json::Value,
|
||||
) -> Result<TaskInputUpdateOutcome, TaskError> {
|
||||
access.validate()?;
|
||||
validate_task_envelope(input)?;
|
||||
if contains_sensitive_task_keys(input) {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input contains sensitive fields",
|
||||
));
|
||||
}
|
||||
let canonical_fingerprint = canonical_input_fingerprint(input)?;
|
||||
let pending = self.store.input_request(access, task_id).await?;
|
||||
if pending.wait_id != wait_id {
|
||||
return Err(TaskError::StateConflict);
|
||||
}
|
||||
// The immutable snapshot, never the current publication, is the schema authority.
|
||||
let schema: crank_schema::Schema = serde_json::from_value(pending.schema_snapshot.clone())
|
||||
.map_err(|_| TaskError::unavailable("stored task input schema is invalid"))?;
|
||||
schema
|
||||
.validate_shape(input)
|
||||
.map_err(|_| TaskError::invalid("task input does not match the pinned input schema"))?;
|
||||
let update_key_digest = input_update_key_digest(access, task_id, wait_id, update_key)?;
|
||||
let crypto = self.crypto.as_ref().ok_or_else(|| {
|
||||
TaskError::invalid("secret crypto is required for durable Task continuation")
|
||||
})?;
|
||||
let (ciphertext, key_version, epoch) = self
|
||||
.store
|
||||
.read_executable_input_for_update(access, task_id)
|
||||
.await?;
|
||||
let binding = crypto
|
||||
.decrypt_for_epoch(&key_version, epoch, &ciphertext)
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
let mut object = binding
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
|
||||
check_binding_string(&object, "task_id", &task_id.to_string())?;
|
||||
check_binding_string(&object, "workspace_id", &access.workspace_id)?;
|
||||
check_binding_string(&object, "owner_id", &access.owner_id)?;
|
||||
check_binding_string(&object, "authorization_scope", &access.authorization_scope)?;
|
||||
let base_input = object
|
||||
.get("input")
|
||||
.cloned()
|
||||
.ok_or_else(|| TaskError::invalid("task executable input binding is missing input"))?;
|
||||
let base_fingerprint = object
|
||||
.get("input_fingerprint")
|
||||
.and_then(|value| value.as_str())
|
||||
.ok_or_else(|| TaskError::invalid("task executable input binding is invalid"))?;
|
||||
if canonical_input_fingerprint(&base_input)? != base_fingerprint {
|
||||
return Err(TaskError::invalid("task executable input binding mismatch"));
|
||||
}
|
||||
object.insert(
|
||||
"binding_format_version".to_owned(),
|
||||
serde_json::json!(2_u32),
|
||||
);
|
||||
object.insert(
|
||||
"continuation".to_owned(),
|
||||
serde_json::json!({
|
||||
"wait_id": wait_id,
|
||||
"schema_revision": pending.schema_revision,
|
||||
"effective_input_fingerprint": canonical_fingerprint,
|
||||
"input": input,
|
||||
}),
|
||||
);
|
||||
let continuation = crypto
|
||||
.encrypt(&serde_json::Value::Object(object))
|
||||
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
||||
if continuation.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES {
|
||||
return Err(TaskError::invalid(
|
||||
"encrypted task input exceeds its byte budget",
|
||||
));
|
||||
}
|
||||
let request = TaskInputUpdateRequest {
|
||||
access: access.clone(),
|
||||
task_id,
|
||||
wait_id: wait_id.to_owned(),
|
||||
expected_version,
|
||||
update_key_digest,
|
||||
effective_input_fingerprint: canonical_fingerprint.clone(),
|
||||
canonical_fingerprint,
|
||||
executable_input_ciphertext: continuation,
|
||||
executable_input_key_version: crypto.key_version().to_owned(),
|
||||
executable_input_master_key_epoch: crypto.master_key_epoch(),
|
||||
};
|
||||
self.store.apply_input(request).await
|
||||
}
|
||||
}
|
||||
|
||||
fn deadline_unix_ms(deadline: Instant) -> i64 {
|
||||
@@ -209,6 +512,61 @@ fn hex_digest(bytes: &[u8]) -> String {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn check_binding_string(
|
||||
object: &serde_json::Map<String, serde_json::Value>,
|
||||
field: &str,
|
||||
expected: &str,
|
||||
) -> Result<(), TaskError> {
|
||||
match object.get(field).and_then(|value| value.as_str()) {
|
||||
Some(actual) if actual == expected => Ok(()),
|
||||
_ => Err(TaskError::invalid("task executable input binding mismatch")),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_executable_payload(input: &serde_json::Value) -> Result<(), TaskError> {
|
||||
validate_task_envelope(input)?;
|
||||
let encoded = serde_json::to_vec(input)
|
||||
.map_err(|_| TaskError::invalid("task executable input is not serializable"))?;
|
||||
if encoded.len() > TASK_EXECUTABLE_INPUT_MAX_BYTES {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input exceeds its byte budget",
|
||||
));
|
||||
}
|
||||
if contains_sensitive_task_keys(input) {
|
||||
return Err(TaskError::invalid(
|
||||
"task executable input contains sensitive fields",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Digest of the scoped update key. The key itself never leaves the caller, and the scope covers the
|
||||
/// exact Task generation so one key cannot be replayed across Tasks or waits.
|
||||
fn input_update_key_digest(
|
||||
access: &TaskAccess,
|
||||
task_id: TaskId,
|
||||
wait_id: &str,
|
||||
key: &str,
|
||||
) -> Result<String, TaskError> {
|
||||
if key.is_empty()
|
||||
|| key.len() > TASK_SCOPED_KEY_MAX_BYTES
|
||||
|| key.bytes().any(|byte| byte.is_ascii_control())
|
||||
{
|
||||
return Err(TaskError::invalid("task input update key is invalid"));
|
||||
}
|
||||
let material = format!(
|
||||
"{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
||||
access.workspace_id,
|
||||
access.owner_kind.as_str(),
|
||||
access.owner_id,
|
||||
access.authorization_scope,
|
||||
task_id,
|
||||
wait_id,
|
||||
key
|
||||
);
|
||||
Ok(hex_digest(material.as_bytes()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -220,6 +578,7 @@ mod tests {
|
||||
owner_kind: TaskOwnerKind::Agent,
|
||||
owner_id: "agent_a".to_owned(),
|
||||
authorization_scope: "credential:pk_1".to_owned(),
|
||||
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
|
||||
operation_id: "op_report".to_owned(),
|
||||
operation_version: 2,
|
||||
policy_mode: ToolExecutionMode::Background,
|
||||
@@ -229,6 +588,7 @@ mod tests {
|
||||
deadline: Instant::now() + std::time::Duration::from_secs(30),
|
||||
retention_ms: TASK_DEFAULT_RETENTION_MS,
|
||||
approval_reference: None,
|
||||
retry: crank_core::TaskRetryPolicy::single_attempt(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -106,6 +106,83 @@ async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
|
||||
assert_eq!(failure.stage(), ExecutionStage::Upstream);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adapter_retry_child_spans_are_distinct_without_a_durable_attempt() {
|
||||
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
|
||||
let capture = TraceCapture::default();
|
||||
let subscriber = tracing_subscriber::registry().with(capture.clone());
|
||||
|
||||
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let executor = RuntimeExecutorBuilder::new()
|
||||
.register_adapter(Arc::new(FlakyReadAdapter {
|
||||
call_count: Arc::clone(&call_count),
|
||||
}))
|
||||
.build();
|
||||
let mut operation: crank_runtime::RuntimeOperation = operation().into();
|
||||
let Target::Rest(target) = &mut operation.target;
|
||||
target.method = HttpMethod::Get;
|
||||
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
|
||||
|
||||
// The synchronous tools/call path has no durable Attempt: every adapter attempt gets its own
|
||||
// synthetic id. A Task dispatch instead reuses the durable id (mcp-server `attempt_trace`).
|
||||
let result = async {
|
||||
let request_root = tracing::info_span!("request_root");
|
||||
executor
|
||||
.execute(&operation, &json!({"name": "retry-read"}))
|
||||
.instrument(request_root)
|
||||
.await
|
||||
}
|
||||
.with_subscriber(subscriber)
|
||||
.await
|
||||
.expect("read retry should recover");
|
||||
|
||||
assert_eq!(result, json!({"accepted": true}));
|
||||
|
||||
let spans = capture.snapshot();
|
||||
let runtime_span = spans
|
||||
.iter()
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime.execute span");
|
||||
assert_eq!(
|
||||
runtime_span.parent_name,
|
||||
Some("request_root"),
|
||||
"the execution continues the caller's trace root"
|
||||
);
|
||||
let attempt_spans: Vec<_> = spans
|
||||
.iter()
|
||||
.filter(|span| span.name == "adapter_attempt")
|
||||
.collect();
|
||||
assert_eq!(attempt_spans.len(), 2);
|
||||
for span in &attempt_spans {
|
||||
assert_eq!(span.parent_name, Some("runtime.execute"));
|
||||
assert!(
|
||||
!span
|
||||
.fields
|
||||
.get("attempt_id")
|
||||
.unwrap_or(&"".to_string())
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!span
|
||||
.fields
|
||||
.get("attempt_id")
|
||||
.unwrap_or(&"".to_string())
|
||||
.contains("traceparent")
|
||||
);
|
||||
}
|
||||
let attempts: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|span| span.fields.get("attempt").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert!(attempts.contains(&"1".to_string()));
|
||||
assert!(attempts.contains(&"2".to_string()));
|
||||
let ids: Vec<String> = attempt_spans
|
||||
.iter()
|
||||
.map(|span| span.fields.get("attempt_id").cloned().unwrap_or_default())
|
||||
.collect();
|
||||
assert_ne!(ids[0], ids[1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_retry_revalidates_adapter_attempt_and_keeps_one_success_outcome() {
|
||||
let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
@@ -236,11 +313,7 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime span");
|
||||
assert_eq!(runtime.parent_name, Some("mcp.request"));
|
||||
for child in [
|
||||
"runtime.arguments.map",
|
||||
"upstream.http",
|
||||
"runtime.response.transform",
|
||||
] {
|
||||
for child in ["runtime.arguments.map", "runtime.response.transform"] {
|
||||
assert_eq!(
|
||||
spans
|
||||
.iter()
|
||||
@@ -250,6 +323,14 @@ async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
|
||||
"{child} must be a runtime child"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
spans
|
||||
.iter()
|
||||
.find(|span| span.name == "upstream.http")
|
||||
.and_then(|span| span.parent_name),
|
||||
Some("adapter_attempt"),
|
||||
"upstream.http must be an adapter child"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -19,11 +19,20 @@ struct TestPostgres {
|
||||
}
|
||||
|
||||
pub async fn postgres_database_url() -> &'static str {
|
||||
&test_postgres().await.database_url
|
||||
&test_postgres("local-tests").await.database_url
|
||||
}
|
||||
|
||||
pub async fn postgres_schema_url(prefix: &str) -> String {
|
||||
let database_url = postgres_database_url().await;
|
||||
create_schema_url(prefix, database_url).await
|
||||
}
|
||||
|
||||
/// The first caller owns the process-wide disposable PostgreSQL container.
|
||||
pub async fn postgres_schema_url_for_task(prefix: &str, task: &str) -> String {
|
||||
create_schema_url(prefix, &test_postgres(task).await.database_url).await
|
||||
}
|
||||
|
||||
async fn create_schema_url(prefix: &str, database_url: &str) -> String {
|
||||
let schema = unique_schema_name(prefix);
|
||||
let mut connection = connect_admin_with_retry(database_url).await;
|
||||
|
||||
@@ -73,22 +82,26 @@ pub fn unique_schema_name(prefix: &str) -> String {
|
||||
format!("{}_{}", normalized_prefix, uuid::Uuid::now_v7().simple())
|
||||
}
|
||||
|
||||
async fn test_postgres() -> &'static TestPostgres {
|
||||
async fn test_postgres(task: &str) -> &'static TestPostgres {
|
||||
POSTGRES
|
||||
.get_or_init(|| async {
|
||||
tokio::task::spawn_blocking(start_postgres)
|
||||
let task = task.to_owned();
|
||||
tokio::task::spawn_blocking(move || start_postgres(task))
|
||||
.await
|
||||
.expect("test PostgreSQL container startup task must complete")
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn start_postgres() -> TestPostgres {
|
||||
fn start_postgres(task: String) -> TestPostgres {
|
||||
let container = Postgres::default()
|
||||
.with_db_name("crank")
|
||||
.with_user("crank")
|
||||
.with_password("crank")
|
||||
.with_tag("16-alpine")
|
||||
.with_label("hermes.disposable", "true")
|
||||
.with_label("hermes.task", task)
|
||||
.with_label("hermes.board", "crank")
|
||||
.start()
|
||||
.expect("test PostgreSQL container must start");
|
||||
register_container_cleanup(container.id().to_owned());
|
||||
|
||||
Loaded 100 of 112 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user